From ca7cdcaae303d1c3f44812d30e727ccd50118e2f Mon Sep 17 00:00:00 2001 From: Comma Device Date: Sun, 25 Oct 2026 17:13:32 +0800 Subject: [PATCH] tinygrad: merge upstream 92b742f7 (02e698160e) --- .../.github/actions/setup-tinygrad/action.yml | 7 +- tinygrad_repo/docs/tensor/ops.md | 1 + .../examples/openpilot/check_image_reads.py | 8 +- .../extra/gemm/mi350x_uop_matmul_2.py | 2 +- tinygrad_repo/extra/models/t5.py | 3 +- tinygrad_repo/extra/thunder/amd/fa.py | 12 +- .../extra/thunder/amd/fa_fwd_causal.cpp | 26 +- .../extra/thunder/amd/fa_fwd_causal_d64.cpp | 728 ++++++++++++++++++ .../thunder/amd/fa_fwd_causal_d64_window.cpp | 728 ++++++++++++++++++ tinygrad_repo/extra/torch_backend/backend.py | 2 +- tinygrad_repo/spec/tinyspec.pdf | Bin 99291 -> 99849 bytes tinygrad_repo/spec/tinyspec.tex | 1 - tinygrad_repo/test/amd/test_emu2_pcode.py | 7 + tinygrad_repo/test/backend/test_ops.py | 19 + .../test/backend/test_transcendental.py | 20 +- .../test/external/external_test_example.py | 2 +- .../external/process_replay/process_replay.py | 10 +- tinygrad_repo/test/mockgpu/amd/amddriver.py | 3 +- tinygrad_repo/test/mockgpu/amd/amdgpu.py | 2 +- tinygrad_repo/test/mockgpu/amd/pcode.py | 3 + tinygrad_repo/test/mockgpu/nv/nvgpu.py | 4 +- tinygrad_repo/test/null/test_dtype.py | 6 +- .../test/null/test_simplify_valid_idx.py | 2 + tinygrad_repo/test/null/test_uop_graph.py | 10 + tinygrad_repo/test/null/test_uop_vmin_vmax.py | 8 + tinygrad_repo/test/null/test_uops_stats.py | 3 + tinygrad_repo/test/unit/test_attention.py | 21 +- tinygrad_repo/test/unit/test_dtype_weak.py | 2 +- tinygrad_repo/tinygrad/codegen/__init__.py | 27 +- .../tinygrad/codegen/decomp/dtype.py | 5 +- .../tinygrad/codegen/decomp/transcendental.py | 6 +- .../tinygrad/codegen/late/coalesce.py | 2 +- .../tinygrad/codegen/late/linearizer.py | 8 +- tinygrad_repo/tinygrad/codegen/simplify.py | 10 +- tinygrad_repo/tinygrad/device.py | 28 +- tinygrad_repo/tinygrad/dtype.py | 20 +- tinygrad_repo/tinygrad/engine/jit.py | 2 +- tinygrad_repo/tinygrad/engine/realize.py | 30 +- tinygrad_repo/tinygrad/function.py | 4 +- tinygrad_repo/tinygrad/helpers.py | 2 +- tinygrad_repo/tinygrad/llm/cli.py | 2 +- tinygrad_repo/tinygrad/llm/kernels/amd.py | 4 +- tinygrad_repo/tinygrad/llm/model.py | 38 +- tinygrad_repo/tinygrad/mixin/creation.py | 10 +- tinygrad_repo/tinygrad/mixin/gradient.py | 12 +- tinygrad_repo/tinygrad/mixin/movement.py | 32 +- tinygrad_repo/tinygrad/mixin/op.py | 22 +- tinygrad_repo/tinygrad/nn/__init__.py | 9 +- tinygrad_repo/tinygrad/nn/onnx.py | 7 +- .../tinygrad/renderer/amd/__init__.py | 4 +- tinygrad_repo/tinygrad/renderer/amd/dsl.py | 2 +- tinygrad_repo/tinygrad/renderer/cstyle.py | 21 +- tinygrad_repo/tinygrad/renderer/isa/x86.py | 4 +- tinygrad_repo/tinygrad/renderer/llvmir.py | 17 +- tinygrad_repo/tinygrad/renderer/nir.py | 2 +- tinygrad_repo/tinygrad/renderer/ptx.py | 2 +- tinygrad_repo/tinygrad/renderer/tc.py | 5 +- tinygrad_repo/tinygrad/renderer/wgsl.py | 7 +- tinygrad_repo/tinygrad/runtime/ops_amd.py | 2 +- tinygrad_repo/tinygrad/runtime/ops_cuda.py | 55 +- tinygrad_repo/tinygrad/runtime/ops_disk.py | 178 +++-- tinygrad_repo/tinygrad/runtime/ops_hip.py | 94 +-- tinygrad_repo/tinygrad/runtime/ops_metal.py | 12 +- tinygrad_repo/tinygrad/runtime/ops_qcom.py | 2 +- .../tinygrad/runtime/support/hcq2.py | 196 ++--- .../tinygrad/runtime/support/nv/nvdev.py | 3 - tinygrad_repo/tinygrad/runtime/support/usb.py | 22 +- tinygrad_repo/tinygrad/schedule/__init__.py | 12 +- tinygrad_repo/tinygrad/schedule/allreduce.py | 6 +- tinygrad_repo/tinygrad/schedule/indexing.py | 5 +- tinygrad_repo/tinygrad/schedule/memory.py | 5 +- tinygrad_repo/tinygrad/schedule/multi.py | 12 +- tinygrad_repo/tinygrad/schedule/prepare.py | 26 +- tinygrad_repo/tinygrad/schedule/rangeify.py | 8 +- tinygrad_repo/tinygrad/tensor.py | 4 +- tinygrad_repo/tinygrad/uop/__init__.py | 6 +- tinygrad_repo/tinygrad/uop/ops.py | 48 +- tinygrad_repo/tinygrad/uop/render.py | 4 +- tinygrad_repo/tinygrad/uop/spec.py | 24 +- tinygrad_repo/tinygrad/uop/symbolic.py | 28 +- tinygrad_repo/tinygrad/viz/serve.py | 36 +- 81 files changed, 2136 insertions(+), 636 deletions(-) create mode 100644 tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64.cpp create mode 100644 tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64_window.cpp diff --git a/tinygrad_repo/.github/actions/setup-tinygrad/action.yml b/tinygrad_repo/.github/actions/setup-tinygrad/action.yml index 372f227e3..4fbfb7488 100644 --- a/tinygrad_repo/.github/actions/setup-tinygrad/action.yml +++ b/tinygrad_repo/.github/actions/setup-tinygrad/action.yml @@ -258,10 +258,11 @@ runs: if: inputs.cuda == 'true' shell: bash run: | - sudo mkdir -p /usr/local/cuda/targets/x86_64-linux + sudo mkdir -p /usr/local/cuda-11.5/targets/x86_64-linux curl -fL https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/linux-x86_64/cuda_nvrtc-linux-x86_64-11.5.119-archive.tar.xz \ - | sudo tar -xJ -C /usr/local/cuda/targets/x86_64-linux --strip-components=1 - echo /usr/local/cuda/targets/x86_64-linux/lib | sudo tee /etc/ld.so.conf.d/cuda-nvrtc.conf + | sudo tar -xJ -C /usr/local/cuda-11.5/targets/x86_64-linux --strip-components=1 + sudo ln -sfnT /usr/local/cuda-11.5 /usr/local/cuda + echo /usr/local/cuda-11.5/targets/x86_64-linux/lib | sudo tee /etc/ld.so.conf.d/cuda-11.5.conf sudo ldconfig # **** gpuocelot **** diff --git a/tinygrad_repo/docs/tensor/ops.md b/tinygrad_repo/docs/tensor/ops.md index d102c697a..4a4e2e2b0 100644 --- a/tinygrad_repo/docs/tensor/ops.md +++ b/tinygrad_repo/docs/tensor/ops.md @@ -52,6 +52,7 @@ ::: tinygrad.Tensor.linear ::: tinygrad.Tensor.sequential ::: tinygrad.Tensor.layernorm +::: tinygrad.Tensor.rmsnorm ::: tinygrad.Tensor.batchnorm ::: tinygrad.Tensor.dropout ::: tinygrad.Tensor.one_hot diff --git a/tinygrad_repo/examples/openpilot/check_image_reads.py b/tinygrad_repo/examples/openpilot/check_image_reads.py index 53a194adf..3aa31e2ee 100644 --- a/tinygrad_repo/examples/openpilot/check_image_reads.py +++ b/tinygrad_repo/examples/openpilot/check_image_reads.py @@ -1,16 +1,16 @@ import pickle, sys from tinygrad.helpers import getenv, is_image_shape, temp from tinygrad.uop.ops import Ops -from tinygrad.viz.serve import VizData, _reconstruct +from tinygrad.viz.serve import VizData, _reconstruct, canonicalize_ast with open(sys.argv[1] if len(sys.argv) > 1 else temp("rewrites.pkl", append_user=True), "rb") as f: data = VizData(pickle.load(f)) steps = [s for key, ctx in zip(data.trace.keys, data.trace.rewrites) if key.display_name.startswith("JIT ") for s in ctx if s.name == "View captured linear"] assert len(steps) == 1, f"expected one JIT compilation, found {len(steps)}" asts = [call.without_after.src[0] for call in _reconstruct(data, steps[0].sink).src] -by_ast = {key.keys[1]: _reconstruct(data, s.sink) for key, ctx in zip(data.trace.keys, data.trace.rewrites) - for s in ctx if s.name == "View Program" and key.keys[1] in asts} -programs = [by_ast[ast] if ast.op is Ops.SINK else ast for ast in asts if ast.op in {Ops.SINK, Ops.PROGRAM}] +by_ast = {canonicalize_ast(_reconstruct(data, b.sink)): _reconstruct(data, p.sink) for ctx in data.trace.rewrites + for b in ctx if b.name == "View Base AST" for p in ctx if p.name == "View Program"} +programs = [by_ast[canonicalize_ast(ast)] if ast.op is Ops.SINK else ast for ast in asts if ast.op in {Ops.SINK, Ops.PROGRAM}] read_image, gated_read_image = 0, 0 for prg in programs: uops = prg.src[1].src diff --git a/tinygrad_repo/extra/gemm/mi350x_uop_matmul_2.py b/tinygrad_repo/extra/gemm/mi350x_uop_matmul_2.py index f3efd581a..b44b6ca70 100644 --- a/tinygrad_repo/extra/gemm/mi350x_uop_matmul_2.py +++ b/tinygrad_repo/extra/gemm/mi350x_uop_matmul_2.py @@ -114,7 +114,7 @@ def custom_gemm(C:UOp, A:UOp, B:UOp) -> UOp: C_view = C[gx, :, gy, :].reshape(BLOCK_M//TC_M//WARPGROUP_SIZE, WARPGROUP_SIZE, TC_M, BLOCK_N//TC_N, TC_N)[:, warpgroup, warp%16, :, (warp//16)*4] sink = copy(C_view, acc.after(sink), rng=300) else: - sink = C.after(barrier.end(K_outer_loop))[0,0,0,0].store(As[0,0]+Bs[0,0]) + sink = C.after(barrier.end(K_outer_loop))[0,0,0,0].store((As[0,0]+Bs[0,0]).float()) return sink.sink(arg=KernelInfo(name="custom_gemm", opts_to_apply=())).simplify() diff --git a/tinygrad_repo/extra/models/t5.py b/tinygrad_repo/extra/models/t5.py index 7a0fff23f..60dd1fe34 100644 --- a/tinygrad_repo/extra/models/t5.py +++ b/tinygrad_repo/extra/models/t5.py @@ -62,8 +62,7 @@ class T5LayerNorm: # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for # half-precision inputs is done in fp32 - variance = hidden_states.cast(dtypes.float32).pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * Tensor.rsqrt(variance + self.variance_epsilon) + hidden_states = hidden_states.cast(dtypes.float32).rmsnorm(eps=self.variance_epsilon) # convert into half-precision if necessary if self.weight.dtype in [dtypes.float16, dtypes.bfloat16]: diff --git a/tinygrad_repo/extra/thunder/amd/fa.py b/tinygrad_repo/extra/thunder/amd/fa.py index 4498b30e7..30e36967a 100644 --- a/tinygrad_repo/extra/thunder/amd/fa.py +++ b/tinygrad_repo/extra/thunder/amd/fa.py @@ -201,12 +201,15 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False @functools.cache def custom_fa_forward(o:UOp, l_vec:UOp, q:UOp, k:UOp, v:UOp, sinks:UOp|None=None, *, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int, has_sink:bool=True, window:int=0): - code = (pathlib.Path(__file__).parent / "fa_fwd_causal.cpp").read_text() + w4 = bool(getenv("GPTOSS_FA_FWD_W4", 0) and arch == "gfx950" and has_sink and (B, N, H, H_KV, D, window) == (2, 8192, 64, 8, 64, 128)) + full_w4 = bool(getenv("GPTOSS_FA_FULL_FWD_W4", 0) and arch == "gfx950" and has_sink and (B, N, H, H_KV, D, window) == (2, 8192, 64, 8, 64, 0)) + source_name = "fa_fwd_causal_d64.cpp" if full_w4 else "fa_fwd_causal_d64_window.cpp" if w4 else "fa_fwd_causal.cpp" + code = (pathlib.Path(__file__).parent / source_name).read_text() compile_args = [f"-I{(pathlib.Path(__file__).parent / 'include').as_posix()}", "-std=c++20", "-DKITTENS_CDNA4", "-DHIP_ENABLE_WARP_SYNC_BUILTINS", "-ffast-math", f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DATTN_SINK={int(has_sink)}", f"-DWINDOW={window}"] Q_BLOCK_SIZE = 32 - NUM_WARPS = 8 + NUM_WARPS = 4 if w4 or full_w4 else 8 NUM_THREADS = 64 * NUM_WARPS gsz = (H, (math.ceil((N // Q_BLOCK_SIZE) / NUM_WARPS)), B) lsz = (NUM_THREADS, 1, 1) @@ -217,15 +220,16 @@ def custom_fa_forward(o:UOp, l_vec:UOp, q:UOp, k:UOp, v:UOp, sinks:UOp|None=None mem = (2*B*N*H*D + 2*B*N*H_KV*D) * el + B*H*N * l_vec.dtype.itemsize estimates = Estimates(ops=2*B*H*N*N*D, lds=mem, mem=mem) buf_inputs = (o.base, l_vec.base, q.base, k.base, v.base) + ((sinks.base,) if has_sink else ()) + name = "custom_fa_forward_d64_full_w4" if full_w4 else "custom_fa_forward_d64_window_w4" if w4 else "custom_fa_forward" sink = UOp.sink(*buf_inputs, threadIdx_x, blockIdx_x, blockIdx_y, blockIdx_z, - arg=KernelInfo(name="custom_fa_forward", estimates=estimates)) + arg=KernelInfo(name=name, estimates=estimates)) lib = HIPCCCompiler(arch, compile_args).compile_cached(code) if not getenv("NO_HIPCC"): lib = bytearray(lib) rodata_off = next(sh.header.sh_offset for sh in elf_loader(bytes(lib))[1] if sh.name == ".rodata") - struct.pack_into(' q_reg_fl; load<1, qo_tile, _gl_QKVO>(q_reg_fl, g.Qg, {batch_idx, tile_idx, head_idx, 0}); - #if !WINDOW + #if !WINDOW && ATTN_D != 64 // D64 builds scale AFTER the MMA in fp32 to keep forward LSE consistent with backward P mul(q_reg_fl, q_reg_fl, TEMPERATURE_SCALE); // Use sqrtf for clarity #endif copy(q_reg, q_reg_fl); @@ -290,8 +290,8 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); - #if WINDOW - mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE #endif __builtin_amdgcn_sched_barrier(0); if constexpr (causal) { @@ -342,8 +342,8 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); - #if WINDOW - mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE #endif #if WINDOW // window masks interior tiles that causal skips @@ -409,8 +409,8 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); - #if WINDOW - mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE #endif // Finish softmax for QK1 exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); @@ -480,8 +480,8 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); - #if WINDOW - mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE #endif // Finish softmax for QK2 exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); @@ -549,8 +549,8 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); - #if WINDOW - mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE #endif // Finish softmax for QK3 exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); @@ -614,8 +614,8 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); - #if WINDOW - mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE #endif // Finish softmax for QK4 exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); diff --git a/tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64.cpp b/tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64.cpp new file mode 100644 index 000000000..400667ddb --- /dev/null +++ b/tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64.cpp @@ -0,0 +1,728 @@ +#include "kittens.cuh" + +#ifndef ATTN_B +constexpr int ATTN_B = 16; // batch size +#endif + +#ifndef ATTN_H +constexpr int ATTN_H = 32; // number of heads +#endif + +#ifndef ATTN_H_KV +constexpr int ATTN_H_KV = 8; // number of heads for key and value +#endif + +constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; // queries per KV head group + +#ifndef ATTN_N +constexpr int ATTN_N = 8192; // sequence length +#endif + +#ifndef ATTN_D +constexpr int ATTN_D = 128; // dimension +#endif +#ifndef ATTN_SINK +#define ATTN_SINK 0 +#endif +#if ATTN_D == 64 +#define FA_VM2 "2" +#define FA_VM4 "4" +#else +#define FA_VM2 "2" +#define FA_VM4 "4" +#endif +constexpr int Q_BLOCK_SIZE = 32; // q block size +constexpr int KV_BLOCK_SIZE = 64; // kv block size +constexpr bool causal = true; +// WINDOW>0: sliding-window attention, query i attends keys in [i-WINDOW+1, i] +#ifndef WINDOW +#define WINDOW 0 +#endif + +#define NUM_WARPS 4 +#define NUM_THREADS (kittens::WARP_THREADS * NUM_WARPS) + +#define MFMA_MASK 0x08 +#define VALU_MASK 0x02 +#define EXP_MASK 0x400 + +using namespace kittens; +using _gl_QKVO = gl; + +using G = kittens::group; + +#define SCHED_BARRIER(mask, cnt, group) __builtin_amdgcn_sched_group_barrier(mask, cnt, group) + +template +__device__ __forceinline__ void sched_barrier_pairs() { + SCHED_BARRIER(MFMA_MASK, 1, Group); + SCHED_BARRIER(VALU_MASK, VALU_CNT, Group); + if constexpr (Pairs > 1) sched_barrier_pairs(); +} + +template +__device__ __forceinline__ void sched_barrier_exp_pairs() { + SCHED_BARRIER(MFMA_MASK, 1, Group); + SCHED_BARRIER(EXP_MASK, EXP_CNT, Group); + if constexpr (Pairs > 1) sched_barrier_exp_pairs(); +} + +template +__device__ inline void exp2(rt_base &dst, const rt_base &src) { + static_assert(std::is_same_v, "Only 32x32 tiles are supported"); + + #pragma unroll + for(int k = 0; k < dst.packed_per_thread; k++) { + dst.data[k] = base_ops::exp2::op(src.data[k]); + } + +} + +template using qo_tile = rt; +template using qo_tile_transposed = rt; +template using kv_tile = rt; +template using kv_tile_transposed = rt; +template using attn_tile = rt; + +/**********************************************************/ +template +__device__ inline void mask_vec2_imm(uint32_t rel_vgpr, uint32_t rel_hi_vgpr, uint32_t neg_inf_vgpr, + uint32_t& x_ref, uint32_t& y_ref) { + + uint64_t x_mask, y_mask; +#if WINDOW + // causal+window in one asm block to not disturb register allocation + asm volatile( + "v_cmp_lt_i32_e64 %0, %4, %5\n\t" + "v_cmp_lt_i32_e64 %1, %4, %7\n\t" + "v_cndmask_b32_e64 %2, %2, %6, %0\n\t" + "v_cndmask_b32_e64 %3, %3, %6, %1\n\t" + "v_cmp_ge_i32_e64 %0, %8, %5\n\t" + "v_cmp_ge_i32_e64 %1, %8, %7\n\t" + "v_cndmask_b32_e64 %2, %2, %6, %0\n\t" + "v_cndmask_b32_e64 %3, %3, %6, %1\n\t" + : "=s"(x_mask), "=s"(y_mask), "+v"(x_ref), "+v"(y_ref) + : "v"(rel_vgpr), "n"(THR_X), "v"(neg_inf_vgpr), "n"(THR_Y), "v"(rel_hi_vgpr) + : "vcc" + ); +#else + asm volatile( + // x: rel < THR_X ? + "v_cmp_lt_i32_e64 %0, %6, %7\n\t" + // y: rel < THR_Y ? + "v_cmp_lt_i32_e64 %1, %6, %9\n\t" + "v_cndmask_b32_e64 %2, %4, %8, %0\n\t" + "v_cndmask_b32_e64 %3, %5, %8, %1\n\t" + : "=s"(x_mask), "=s"(y_mask), "=v"(x_ref), "=v"(y_ref) + : "v"(x_ref), "v"(y_ref), "v"(rel_vgpr), + "n"(THR_X), "v"(neg_inf_vgpr), "n"(THR_Y) + : "vcc" + ); +#endif +} + +template +__device__ inline void mask_kv_tile(RT &dst, int q_abs, int k_abs, uint32_t neg_inf_v, int lane) { + const int col = lane & 31; // 0..31 column within the 32-wide col tile + + // Absolute positions + const int q_base = q_abs * Q_BLOCK_SIZE; // start index for this Q tile + const int k_base = k_abs * KV_BLOCK_SIZE; // start index for this K/V tile + + // q position for this lane's column + const int q_pos = q_base + col; + + #pragma unroll + for (int i = 0; i < dst.height; ++i) { + // Row base of the 32x* chunk produced by MFMA + const int row_base = (i * 32) + ((lane >> 5) << 2); // multiplesof 4 + + // Relative index of the FIRST element in this row-chunk w.r.t. q_pos + // (smaller rel ⇒ more "future" keys that must be -inf) + const int rel0 = q_pos - (k_base + row_base); + const uint32_t rel = static_cast(rel0); + // rel-WINDOW keeps THR within the inline-constant range + const uint32_t rel_hi = static_cast(rel0 - WINDOW); + + #pragma unroll + for (int j = 0; j < dst.width; ++j) { + auto& d0x = *reinterpret_cast(&dst.tiles[i][j].data[0].x); + auto& d0y = *reinterpret_cast(&dst.tiles[i][j].data[0].y); + auto& d1x = *reinterpret_cast(&dst.tiles[i][j].data[1].x); + auto& d1y = *reinterpret_cast(&dst.tiles[i][j].data[1].y); + auto& d2x = *reinterpret_cast(&dst.tiles[i][j].data[2].x); + auto& d2y = *reinterpret_cast(&dst.tiles[i][j].data[2].y); + auto& d3x = *reinterpret_cast(&dst.tiles[i][j].data[3].x); + auto& d3y = *reinterpret_cast(&dst.tiles[i][j].data[3].y); + auto& d4x = *reinterpret_cast(&dst.tiles[i][j].data[4].x); + auto& d4y = *reinterpret_cast(&dst.tiles[i][j].data[4].y); + auto& d5x = *reinterpret_cast(&dst.tiles[i][j].data[5].x); + auto& d5y = *reinterpret_cast(&dst.tiles[i][j].data[5].y); + auto& d6x = *reinterpret_cast(&dst.tiles[i][j].data[6].x); + auto& d6y = *reinterpret_cast(&dst.tiles[i][j].data[6].y); + auto& d7x = *reinterpret_cast(&dst.tiles[i][j].data[7].x); + auto& d7y = *reinterpret_cast(&dst.tiles[i][j].data[7].y); + + // - reuse a single neg_inf register + // - keep VCC live across the pair + // - avoid reloading -inf or recomputing rel + mask_vec2_imm< 0, 1 >(rel, rel_hi, neg_inf_v, d0x, d0y); + mask_vec2_imm< 2, 3 >(rel, rel_hi, neg_inf_v, d1x, d1y); + mask_vec2_imm< 8, 9 >(rel, rel_hi, neg_inf_v, d2x, d2y); + mask_vec2_imm<10,11 >(rel, rel_hi, neg_inf_v, d3x, d3y); + mask_vec2_imm<16,17 >(rel, rel_hi, neg_inf_v, d4x, d4y); + mask_vec2_imm<18,19 >(rel, rel_hi, neg_inf_v, d5x, d5y); + mask_vec2_imm<24,25 >(rel, rel_hi, neg_inf_v, d6x, d6y); + mask_vec2_imm<26,27 >(rel, rel_hi, neg_inf_v, d7x, d7y); + } + } +} + +/**********************************************************/ + +template struct attn_globals { + _gl_QKVO Qg, Kg, Vg, Og; + gl L_vec; + dim3 grid() { return dim3(ATTN_H, ((ATTN_N / Q_BLOCK_SIZE + NUM_WARPS - 1) / NUM_WARPS), ATTN_B); } + dim3 block() { return dim3(NUM_THREADS); } + size_t dynamic_shared_memory() { return MAX_SHARED_MEMORY; } +}; + +template __launch_bounds__(NUM_THREADS, 2) +__global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_ptr, bf16 *V_ptr +#if ATTN_SINK + , float *Sinks_ptr +#endif + ) { + _gl_QKVO Og{O_ptr, ATTN_B, ATTN_N, ATTN_H, ATTN_D}; + _gl_QKVO Qg{Q_ptr, ATTN_B, ATTN_N, ATTN_H, ATTN_D}; + _gl_QKVO Kg{K_ptr, ATTN_B, ATTN_N, ATTN_H_KV, ATTN_D}; + _gl_QKVO Vg{V_ptr, ATTN_B, ATTN_N, ATTN_H_KV, ATTN_D}; + gl L_vec{L_vec_ptr, ATTN_B, ATTN_H, 1, ATTN_N}; + attn_globals g{Qg, Kg, Vg, Og, L_vec}; + + extern __shared__ alignment_dummy __shm[]; + shared_allocator al((int*)&__shm[0]); + st_bf (&k_smem)[2] = al.allocate, 2>(); + st_bf (&v_smem)[2] = al.allocate, 2>(); + + const int head_idx = (blockIdx.x % ATTN_H_KV) * GROUP_SIZE + (blockIdx.x / ATTN_H_KV); + const int batch_idx = blockIdx.z; + const int head_idx_kv = head_idx / GROUP_SIZE; + const int block_tile_idx = blockIdx.y; + const int tile_idx = block_tile_idx * NUM_WARPS + warpid(); + const int stagger = warpid() / 4; + const int lane = laneid(); + + const int num_tiles = ATTN_N / KV_BLOCK_SIZE; + const int max_tile_idx = block_tile_idx * NUM_WARPS + NUM_WARPS - 1; + const int max_q_end_pos = (max_tile_idx + 1) * Q_BLOCK_SIZE; + int max_num_tiles = (max_q_end_pos + KV_BLOCK_SIZE - 1) / KV_BLOCK_SIZE; + if constexpr (causal) max_num_tiles = min(max_num_tiles, num_tiles); + else max_num_tiles = num_tiles; + // The existing four-tile prologue/epilogue needs at least four tiles even for the first 128 queries. + max_num_tiles = ((max_num_tiles + 3) / 4) * 4; + const int q_start_pos = tile_idx * Q_BLOCK_SIZE; + +#if WINDOW + // start at the first in-window tile; clamp keeps >=4 tiles for the pipeline unroll + const int block_min_q = block_tile_idx * NUM_WARPS * Q_BLOCK_SIZE; + int min_tile = (block_min_q - WINDOW + 1) / KV_BLOCK_SIZE; + if (min_tile < 0) min_tile = 0; + if (min_tile > max_num_tiles - 4) min_tile = max(0, max_num_tiles - 4); +#else + constexpr int min_tile = 0; +#endif + + constexpr float TEMPERATURE_SCALE = (D == 128) ? 0.08838834764f*1.44269504089f : 0.125f*1.44269504089f; + uint32_t neg_inf_v = 0xff800000; + + // Initialize all of the register tiles. + qo_tile q_reg; // Q and K are both row layout, as we use mma_ABt. + qo_tile_transposed q_reg_transposed; + kv_tile k_reg; + kv_tile_transposed k_reg_transposed; + + kv_tile v_reg; + qo_tile_transposed o_reg; // Output tile. + attn_tile att_block[2]; // attention tile, in float. + attn_tile att_block_bf16; + attn_tile att_block_bf16_in; + typename attn_tile::row_vec max_vec, norm_vec, max_vec_prev, scale_vec; + + zero(o_reg); + zero(norm_vec); + zero(scale_vec); + + using T = typename st_bf::dtype; + constexpr int bytes_per_thread = st_32x32_s::template bytes_per_thread(); + constexpr int bytes_per_memcpy = bytes_per_thread * NUM_THREADS; + constexpr int memcpy_per_tile = KV_BLOCK_SIZE * ATTN_D * sizeof(T) / bytes_per_memcpy; + + uint32_t swizzled_offsets_V[memcpy_per_tile]; + uint32_t swizzled_offsets_K[memcpy_per_tile]; + G::prefill_swizzled_offsets<1, false>(k_smem[0], g.Kg, swizzled_offsets_K); + G::prefill_swizzled_offsets<1, false>(v_smem[0], g.Vg, swizzled_offsets_V); + + G::load<1, false>(k_smem[0], g.Kg, {batch_idx, min_tile, head_idx_kv, 0}, swizzled_offsets_K); + __builtin_amdgcn_s_waitcnt(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + + qo_tile q_reg_fl; + load<1, qo_tile, _gl_QKVO>(q_reg_fl, g.Qg, {batch_idx, tile_idx, head_idx, 0}); + #if !WINDOW && ATTN_D != 64 // D64 builds scale AFTER the MMA in fp32 to keep forward LSE consistent with backward P + mul(q_reg_fl, q_reg_fl, TEMPERATURE_SCALE); // Use sqrtf for clarity + #endif + copy(q_reg, q_reg_fl); + transpose(q_reg_transposed, q_reg); + + // All warps then collaboratively load in the first slice of V (V0) and the second slice of K (K1) into shared memory + G::load<1, false>(k_smem[1], g.Kg, {batch_idx, min_tile + 1, head_idx_kv, 0}, swizzled_offsets_K); + // All warps then load in the first slice of K (K0) + G::load<1, false>(v_smem[0], g.Vg, {batch_idx, min_tile, head_idx_kv, 0}, swizzled_offsets_V); + load(k_reg, k_smem[0]); + __builtin_amdgcn_sched_barrier(0); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM2 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + + // Each warp performs QK0 + zero(att_block[0]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + __builtin_amdgcn_sched_barrier(0); + if constexpr (causal) { + const int kv_end_pos = (min_tile + 1) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, WINDOW ? 1 : 0)) { + mask_kv_tile(att_block[0], tile_idx, min_tile, neg_inf_v, lane); + } + } + // Each warp performs a partial softmax of QK0 (i.e. some of the online softmax up until but not including the second exponential scaling of the attention block likely) +#if WINDOW + // floor the max: min_tile can be fully masked, which would NaN via exp2(-inf - -inf) + zero(max_vec_prev); + add(max_vec_prev, max_vec_prev, -1e4f); + col_max(max_vec, att_block[0], max_vec_prev); +#else + col_max(max_vec, att_block[0]); +#endif + + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + + sub_col(att_block[0], att_block[0], max_vec); + exp2(att_block[0].tiles[0][0], att_block[0].tiles[0][0]); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + + if (stagger) { + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + } + + // __builtin_amdgcn_sched_barrier(0); + // All warps then load in the second slice of K (K1) + load(k_reg, k_smem[1]); + // All warps then collaboratively load in the third slice of K (K2) into shared memory + G::load<1, false>(k_smem[0], g.Kg, {batch_idx, min_tile + 2, head_idx_kv, 0}, swizzled_offsets_K); + // All warps then collaboratively load in the second slice of V (V1) into shared memory + G::load<1, false>(v_smem[1], g.Vg, {batch_idx, min_tile + 1, head_idx_kv, 0}, swizzled_offsets_V); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + + // hot loop + for (int j = min_tile + 3; j < max_num_tiles - 1; j += 2) { + // Cluster 0: + // QK1 + zero(att_block[1]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif +#if WINDOW + // window masks interior tiles that causal skips + mask_kv_tile(att_block[1], tile_idx, j - 2, neg_inf_v, lane); +#endif + // Finish softmax for QK0 + exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[0], norm_vec); + copy(att_block_bf16, att_block[0]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 1>(); + sched_barrier_pairs<10, 5, 1>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 1: + // Load K3 into shared + G::load<1, false>(k_smem[1], g.Kg, {batch_idx, j, head_idx_kv, 0}, swizzled_offsets_K); + // Load V0 into registers + load(v_reg, v_smem[0]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 2: + // A0V0 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK1 + col_max(max_vec, att_block[1], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[1], att_block[1], max_vec); + exp2(att_block[1].tiles[0][0], att_block[1].tiles[0][0]); + sched_barrier_pairs<10, 5, 2>(); + sched_barrier_exp_pairs<6, 3, 2>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 3: + // Load V2 into shared + G::load<1, false>(v_smem[0], g.Vg, {batch_idx, j - 1, head_idx_kv, 0}, swizzled_offsets_V); + // Load K2 into registers + load(k_reg, k_smem[0]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 4: + // QK2 + __builtin_amdgcn_s_setprio(1); + zero(att_block[0]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK1 + exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[1], norm_vec); + copy(att_block_bf16, att_block[1]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 3>(); + sched_barrier_pairs<10, 5, 3>(); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 5: + // Load K4 into shared + G::load<1, false>(k_smem[0], g.Kg, {batch_idx, j + 1, head_idx_kv, 0}, swizzled_offsets_K); + // Load V1 into registers + load(v_reg, v_smem[1]); + if constexpr (causal) { + const int kv_end_pos = (j) * KV_BLOCK_SIZE; + if (WINDOW || q_start_pos < kv_end_pos) { + mask_kv_tile(att_block[0], tile_idx, j - 1, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 6: + // A1V1 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK2 + col_max(max_vec, att_block[0], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[0], att_block[0], max_vec); + exp2(att_block[0].tiles[0][0], att_block[0].tiles[0][0]); + sched_barrier_pairs<10, 5, 4>(); + sched_barrier_exp_pairs<6, 3, 4>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 7: + // Load V3 into shared + G::load<1, false>(v_smem[1], g.Vg, {batch_idx, j, head_idx_kv, 0}, swizzled_offsets_V); + // Load K3 into registers + load(k_reg, k_smem[1]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + } + + // Epilogue + // Cluster 0: + // QK3 + zero(att_block[1]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK2 + exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + + col_sum(norm_vec, att_block[0], norm_vec); + copy(att_block_bf16, att_block[0]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 5>(); + sched_barrier_pairs<10, 5, 5>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 1: + // Load K5 into shared + G::load<1, false>(k_smem[1], g.Kg, {batch_idx, max_num_tiles - 1, head_idx_kv, 0}, swizzled_offsets_K); + // Load V2 into registers + load(v_reg, v_smem[0]); + if constexpr (causal) { + const int kv_end_pos = (max_num_tiles - 2) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, WINDOW ? 1 : 0)) { + mask_kv_tile(att_block[1], tile_idx, max_num_tiles - 3, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 2: + // A2V2 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK3 + col_max(max_vec, att_block[1], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[1], att_block[1], max_vec); + exp2(att_block[1].tiles[0][0], att_block[1].tiles[0][0]); + sched_barrier_pairs<10, 5, 6>(); + sched_barrier_exp_pairs<6, 3, 6>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 3: + // Load V4 into shared + G::load<1, false>(v_smem[0], g.Vg, {batch_idx, max_num_tiles - 2, head_idx_kv, 0}, swizzled_offsets_V); + // Load K4 into registers + load(k_reg, k_smem[0]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 4: + // QK4 + zero(att_block[0]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK3 + exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[1], norm_vec); + copy(att_block_bf16, att_block[1]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 7>(); + sched_barrier_pairs<10, 5, 7>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 5: + // Load V3 into registers + load(v_reg, v_smem[1]); + if constexpr (causal) { + const int kv_end_pos = (max_num_tiles - 1) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, 1)) { + mask_kv_tile(att_block[0], tile_idx, max_num_tiles - 2, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM2 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 6: + // A3V3 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK4 + col_max(max_vec, att_block[0], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[0], att_block[0], max_vec); + exp2(att_block[0].tiles[0][0], att_block[0].tiles[0][0]); + sched_barrier_pairs<10, 5, 8>(); + sched_barrier_exp_pairs<6, 3, 8>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 7: + // Load V5 into shared + G::load<1, false>(v_smem[1], g.Vg, {batch_idx, max_num_tiles - 1, head_idx_kv, 0}, swizzled_offsets_V); + // Load K5 into registers + load(k_reg, k_smem[1]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM2 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 8: + // QK5 + zero(att_block[1]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK4 + exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[0], norm_vec); + copy(att_block_bf16, att_block[0]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 9>(); + sched_barrier_pairs<10, 5, 9>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 9: + // Load V4 into registers + load(v_reg, v_smem[0]); + if constexpr (causal) { + const int kv_end_pos = (max_num_tiles) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, 1)) { + mask_kv_tile(att_block[1], tile_idx, max_num_tiles - 1, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(0)"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 10: + // A4V4 + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Full softmax for QK5 + col_max(max_vec, att_block[1], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + + sub_col(att_block[1], att_block[1], max_vec); + exp2(att_block[1].tiles[0][0], att_block[1].tiles[0][0]); + sched_barrier_pairs<10, 5, 10>(); + sched_barrier_exp_pairs<6, 3, 10>(); + __builtin_amdgcn_sched_barrier(0); + + exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + + col_sum(norm_vec, att_block[1], norm_vec); + copy(att_block_bf16, att_block[1]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 11: + // Load V5 into registers + load(v_reg, v_smem[1]); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 12: + // A5V5 + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); +#if ATTN_SINK + { + const float sink_l2 = Sinks_ptr[head_idx] * 1.44269504089f; + typename attn_tile::row_vec sink_term; + mul(sink_term, max_vec, -1.0f); + add(sink_term, sink_term, sink_l2); + exp2(sink_term, sink_term); + add(norm_vec, norm_vec, sink_term); + } +#endif + div_col(o_reg, o_reg, norm_vec); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Conclusion + if (!stagger) { + __builtin_amdgcn_s_barrier(); + } + + qo_tile o_reg_transposed; + transpose(o_reg_transposed, o_reg); + store<1>(g.Og, o_reg_transposed, {batch_idx, tile_idx, head_idx, 0}); + + // multiply by ln(2) + mul(max_vec, max_vec, 0.69314718056f); + log(norm_vec, norm_vec); + add(norm_vec, norm_vec, max_vec); + store(g.L_vec, norm_vec, {batch_idx, head_idx, 0, tile_idx}); +} + +template __global__ void attend_ker(bf16*, float*, bf16*, bf16*, bf16* +#if ATTN_SINK + , float* +#endif +); + +static_assert(ATTN_D == 64 && WINDOW == 0 && NUM_WARPS == 4 && ATTN_N == 8192); +static_assert(sizeof(st_bf<64,64,st_32x32_s>) == 8192); +static_assert(sizeof(st_bf<64,64,st_8x32_s>) == 8192); +static_assert(alignof(st_bf<64,64,st_32x32_s>) <= 16); +static_assert(alignof(st_bf<64,64,st_8x32_s>) <= 16); diff --git a/tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64_window.cpp b/tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64_window.cpp new file mode 100644 index 000000000..b1ee3e843 --- /dev/null +++ b/tinygrad_repo/extra/thunder/amd/fa_fwd_causal_d64_window.cpp @@ -0,0 +1,728 @@ +#include "kittens.cuh" + +#ifndef ATTN_B +constexpr int ATTN_B = 16; // batch size +#endif + +#ifndef ATTN_H +constexpr int ATTN_H = 32; // number of heads +#endif + +#ifndef ATTN_H_KV +constexpr int ATTN_H_KV = 8; // number of heads for key and value +#endif + +constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; // queries per KV head group + +#ifndef ATTN_N +constexpr int ATTN_N = 8192; // sequence length +#endif + +#ifndef ATTN_D +constexpr int ATTN_D = 128; // dimension +#endif +#ifndef ATTN_SINK +#define ATTN_SINK 0 +#endif +#if ATTN_D == 64 +#define FA_VM2 "2" +#define FA_VM4 "4" +#else +#define FA_VM2 "2" +#define FA_VM4 "4" +#endif +constexpr int Q_BLOCK_SIZE = 32; // q block size +constexpr int KV_BLOCK_SIZE = 64; // kv block size +constexpr bool causal = true; +// WINDOW>0: sliding-window attention, query i attends keys in [i-WINDOW+1, i] +#ifndef WINDOW +#define WINDOW 0 +#endif + +#define NUM_WARPS 4 +#define NUM_THREADS (kittens::WARP_THREADS * NUM_WARPS) + +#define MFMA_MASK 0x08 +#define VALU_MASK 0x02 +#define EXP_MASK 0x400 + +using namespace kittens; +using _gl_QKVO = gl; + +using G = kittens::group; + +#define SCHED_BARRIER(mask, cnt, group) __builtin_amdgcn_sched_group_barrier(mask, cnt, group) + +template +__device__ __forceinline__ void sched_barrier_pairs() { + SCHED_BARRIER(MFMA_MASK, 1, Group); + SCHED_BARRIER(VALU_MASK, VALU_CNT, Group); + if constexpr (Pairs > 1) sched_barrier_pairs(); +} + +template +__device__ __forceinline__ void sched_barrier_exp_pairs() { + SCHED_BARRIER(MFMA_MASK, 1, Group); + SCHED_BARRIER(EXP_MASK, EXP_CNT, Group); + if constexpr (Pairs > 1) sched_barrier_exp_pairs(); +} + +template +__device__ inline void exp2(rt_base &dst, const rt_base &src) { + static_assert(std::is_same_v, "Only 32x32 tiles are supported"); + + #pragma unroll + for(int k = 0; k < dst.packed_per_thread; k++) { + dst.data[k] = base_ops::exp2::op(src.data[k]); + } + +} + +template using qo_tile = rt; +template using qo_tile_transposed = rt; +template using kv_tile = rt; +template using kv_tile_transposed = rt; +template using attn_tile = rt; + +/**********************************************************/ +template +__device__ inline void mask_vec2_imm(uint32_t rel_vgpr, uint32_t rel_hi_vgpr, uint32_t neg_inf_vgpr, + uint32_t& x_ref, uint32_t& y_ref) { + + uint64_t x_mask, y_mask; +#if WINDOW + // causal+window in one asm block to not disturb register allocation + asm volatile( + "v_cmp_lt_i32_e64 %0, %4, %5\n\t" + "v_cmp_lt_i32_e64 %1, %4, %7\n\t" + "v_cndmask_b32_e64 %2, %2, %6, %0\n\t" + "v_cndmask_b32_e64 %3, %3, %6, %1\n\t" + "v_cmp_ge_i32_e64 %0, %8, %5\n\t" + "v_cmp_ge_i32_e64 %1, %8, %7\n\t" + "v_cndmask_b32_e64 %2, %2, %6, %0\n\t" + "v_cndmask_b32_e64 %3, %3, %6, %1\n\t" + : "=s"(x_mask), "=s"(y_mask), "+v"(x_ref), "+v"(y_ref) + : "v"(rel_vgpr), "n"(THR_X), "v"(neg_inf_vgpr), "n"(THR_Y), "v"(rel_hi_vgpr) + : "vcc" + ); +#else + asm volatile( + // x: rel < THR_X ? + "v_cmp_lt_i32_e64 %0, %6, %7\n\t" + // y: rel < THR_Y ? + "v_cmp_lt_i32_e64 %1, %6, %9\n\t" + "v_cndmask_b32_e64 %2, %4, %8, %0\n\t" + "v_cndmask_b32_e64 %3, %5, %8, %1\n\t" + : "=s"(x_mask), "=s"(y_mask), "=v"(x_ref), "=v"(y_ref) + : "v"(x_ref), "v"(y_ref), "v"(rel_vgpr), + "n"(THR_X), "v"(neg_inf_vgpr), "n"(THR_Y) + : "vcc" + ); +#endif +} + +template +__device__ inline void mask_kv_tile(RT &dst, int q_abs, int k_abs, uint32_t neg_inf_v, int lane) { + const int col = lane & 31; // 0..31 column within the 32-wide col tile + + // Absolute positions + const int q_base = q_abs * Q_BLOCK_SIZE; // start index for this Q tile + const int k_base = k_abs * KV_BLOCK_SIZE; // start index for this K/V tile + + // q position for this lane's column + const int q_pos = q_base + col; + + #pragma unroll + for (int i = 0; i < dst.height; ++i) { + // Row base of the 32x* chunk produced by MFMA + const int row_base = (i * 32) + ((lane >> 5) << 2); // multiplesof 4 + + // Relative index of the FIRST element in this row-chunk w.r.t. q_pos + // (smaller rel ⇒ more "future" keys that must be -inf) + const int rel0 = q_pos - (k_base + row_base); + const uint32_t rel = static_cast(rel0); + // rel-WINDOW keeps THR within the inline-constant range + const uint32_t rel_hi = static_cast(rel0 - WINDOW); + + #pragma unroll + for (int j = 0; j < dst.width; ++j) { + auto& d0x = *reinterpret_cast(&dst.tiles[i][j].data[0].x); + auto& d0y = *reinterpret_cast(&dst.tiles[i][j].data[0].y); + auto& d1x = *reinterpret_cast(&dst.tiles[i][j].data[1].x); + auto& d1y = *reinterpret_cast(&dst.tiles[i][j].data[1].y); + auto& d2x = *reinterpret_cast(&dst.tiles[i][j].data[2].x); + auto& d2y = *reinterpret_cast(&dst.tiles[i][j].data[2].y); + auto& d3x = *reinterpret_cast(&dst.tiles[i][j].data[3].x); + auto& d3y = *reinterpret_cast(&dst.tiles[i][j].data[3].y); + auto& d4x = *reinterpret_cast(&dst.tiles[i][j].data[4].x); + auto& d4y = *reinterpret_cast(&dst.tiles[i][j].data[4].y); + auto& d5x = *reinterpret_cast(&dst.tiles[i][j].data[5].x); + auto& d5y = *reinterpret_cast(&dst.tiles[i][j].data[5].y); + auto& d6x = *reinterpret_cast(&dst.tiles[i][j].data[6].x); + auto& d6y = *reinterpret_cast(&dst.tiles[i][j].data[6].y); + auto& d7x = *reinterpret_cast(&dst.tiles[i][j].data[7].x); + auto& d7y = *reinterpret_cast(&dst.tiles[i][j].data[7].y); + + // - reuse a single neg_inf register + // - keep VCC live across the pair + // - avoid reloading -inf or recomputing rel + mask_vec2_imm< 0, 1 >(rel, rel_hi, neg_inf_v, d0x, d0y); + mask_vec2_imm< 2, 3 >(rel, rel_hi, neg_inf_v, d1x, d1y); + mask_vec2_imm< 8, 9 >(rel, rel_hi, neg_inf_v, d2x, d2y); + mask_vec2_imm<10,11 >(rel, rel_hi, neg_inf_v, d3x, d3y); + mask_vec2_imm<16,17 >(rel, rel_hi, neg_inf_v, d4x, d4y); + mask_vec2_imm<18,19 >(rel, rel_hi, neg_inf_v, d5x, d5y); + mask_vec2_imm<24,25 >(rel, rel_hi, neg_inf_v, d6x, d6y); + mask_vec2_imm<26,27 >(rel, rel_hi, neg_inf_v, d7x, d7y); + } + } +} + +/**********************************************************/ + +template struct attn_globals { + _gl_QKVO Qg, Kg, Vg, Og; + gl L_vec; + dim3 grid() { return dim3(ATTN_H, ((ATTN_N / Q_BLOCK_SIZE + NUM_WARPS - 1) / NUM_WARPS), ATTN_B); } + dim3 block() { return dim3(NUM_THREADS); } + size_t dynamic_shared_memory() { return MAX_SHARED_MEMORY; } +}; + +template __launch_bounds__(NUM_THREADS, 2) +__global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_ptr, bf16 *V_ptr +#if ATTN_SINK + , float *Sinks_ptr +#endif + ) { + _gl_QKVO Og{O_ptr, ATTN_B, ATTN_N, ATTN_H, ATTN_D}; + _gl_QKVO Qg{Q_ptr, ATTN_B, ATTN_N, ATTN_H, ATTN_D}; + _gl_QKVO Kg{K_ptr, ATTN_B, ATTN_N, ATTN_H_KV, ATTN_D}; + _gl_QKVO Vg{V_ptr, ATTN_B, ATTN_N, ATTN_H_KV, ATTN_D}; + gl L_vec{L_vec_ptr, ATTN_B, ATTN_H, 1, ATTN_N}; + attn_globals g{Qg, Kg, Vg, Og, L_vec}; + + extern __shared__ alignment_dummy __shm[]; + shared_allocator al((int*)&__shm[0]); + st_bf (&k_smem)[2] = al.allocate, 2>(); + st_bf (&v_smem)[2] = al.allocate, 2>(); + + const int head_idx = (blockIdx.x % ATTN_H_KV) * GROUP_SIZE + (blockIdx.x / ATTN_H_KV); + const int batch_idx = blockIdx.z; + const int head_idx_kv = head_idx / GROUP_SIZE; + const int block_tile_idx = blockIdx.y; + const int tile_idx = block_tile_idx * NUM_WARPS + warpid(); + const int stagger = warpid() / 4; + const int lane = laneid(); + + const int num_tiles = ATTN_N / KV_BLOCK_SIZE; + const int max_tile_idx = block_tile_idx * NUM_WARPS + NUM_WARPS - 1; + const int max_q_end_pos = (max_tile_idx + 1) * Q_BLOCK_SIZE; + int max_num_tiles = (max_q_end_pos + KV_BLOCK_SIZE - 1) / KV_BLOCK_SIZE; + if constexpr (causal) max_num_tiles = min(max_num_tiles, num_tiles); + else max_num_tiles = num_tiles; + // The existing four-tile prologue/epilogue needs at least four tiles even for the first 128 queries. + max_num_tiles = max(max_num_tiles, 4); + const int q_start_pos = tile_idx * Q_BLOCK_SIZE; + +#if WINDOW + // start at the first in-window tile; clamp keeps >=4 tiles for the pipeline unroll + const int block_min_q = block_tile_idx * NUM_WARPS * Q_BLOCK_SIZE; + int min_tile = (block_min_q - WINDOW + 1) / KV_BLOCK_SIZE; + if (min_tile < 0) min_tile = 0; + if (min_tile > max_num_tiles - 4) min_tile = max(0, max_num_tiles - 4); +#else + constexpr int min_tile = 0; +#endif + + constexpr float TEMPERATURE_SCALE = (D == 128) ? 0.08838834764f*1.44269504089f : 0.125f*1.44269504089f; + uint32_t neg_inf_v = 0xff800000; + + // Initialize all of the register tiles. + qo_tile q_reg; // Q and K are both row layout, as we use mma_ABt. + qo_tile_transposed q_reg_transposed; + kv_tile k_reg; + kv_tile_transposed k_reg_transposed; + + kv_tile v_reg; + qo_tile_transposed o_reg; // Output tile. + attn_tile att_block[2]; // attention tile, in float. + attn_tile att_block_bf16; + attn_tile att_block_bf16_in; + typename attn_tile::row_vec max_vec, norm_vec, max_vec_prev, scale_vec; + + zero(o_reg); + zero(norm_vec); + zero(scale_vec); + + using T = typename st_bf::dtype; + constexpr int bytes_per_thread = st_32x32_s::template bytes_per_thread(); + constexpr int bytes_per_memcpy = bytes_per_thread * NUM_THREADS; + constexpr int memcpy_per_tile = KV_BLOCK_SIZE * ATTN_D * sizeof(T) / bytes_per_memcpy; + + uint32_t swizzled_offsets_V[memcpy_per_tile]; + uint32_t swizzled_offsets_K[memcpy_per_tile]; + G::prefill_swizzled_offsets<1, false>(k_smem[0], g.Kg, swizzled_offsets_K); + G::prefill_swizzled_offsets<1, false>(v_smem[0], g.Vg, swizzled_offsets_V); + + G::load<1, false>(k_smem[0], g.Kg, {batch_idx, min_tile, head_idx_kv, 0}, swizzled_offsets_K); + __builtin_amdgcn_s_waitcnt(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + + qo_tile q_reg_fl; + load<1, qo_tile, _gl_QKVO>(q_reg_fl, g.Qg, {batch_idx, tile_idx, head_idx, 0}); + #if !WINDOW && ATTN_D != 64 // D64 builds scale AFTER the MMA in fp32 to keep forward LSE consistent with backward P + mul(q_reg_fl, q_reg_fl, TEMPERATURE_SCALE); // Use sqrtf for clarity + #endif + copy(q_reg, q_reg_fl); + transpose(q_reg_transposed, q_reg); + + // All warps then collaboratively load in the first slice of V (V0) and the second slice of K (K1) into shared memory + G::load<1, false>(k_smem[1], g.Kg, {batch_idx, min_tile + 1, head_idx_kv, 0}, swizzled_offsets_K); + // All warps then load in the first slice of K (K0) + G::load<1, false>(v_smem[0], g.Vg, {batch_idx, min_tile, head_idx_kv, 0}, swizzled_offsets_V); + load(k_reg, k_smem[0]); + __builtin_amdgcn_sched_barrier(0); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM2 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + + // Each warp performs QK0 + zero(att_block[0]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + __builtin_amdgcn_sched_barrier(0); + if constexpr (causal) { + const int kv_end_pos = (min_tile + 1) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, WINDOW ? 1 : 0)) { + mask_kv_tile(att_block[0], tile_idx, min_tile, neg_inf_v, lane); + } + } + // Each warp performs a partial softmax of QK0 (i.e. some of the online softmax up until but not including the second exponential scaling of the attention block likely) +#if WINDOW + // floor the max: min_tile can be fully masked, which would NaN via exp2(-inf - -inf) + zero(max_vec_prev); + add(max_vec_prev, max_vec_prev, -1e4f); + col_max(max_vec, att_block[0], max_vec_prev); +#else + col_max(max_vec, att_block[0]); +#endif + + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + + sub_col(att_block[0], att_block[0], max_vec); + exp2(att_block[0].tiles[0][0], att_block[0].tiles[0][0]); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + + if (stagger) { + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + } + + // __builtin_amdgcn_sched_barrier(0); + // All warps then load in the second slice of K (K1) + load(k_reg, k_smem[1]); + // All warps then collaboratively load in the third slice of K (K2) into shared memory + G::load<1, false>(k_smem[0], g.Kg, {batch_idx, min_tile + 2, head_idx_kv, 0}, swizzled_offsets_K); + // All warps then collaboratively load in the second slice of V (V1) into shared memory + G::load<1, false>(v_smem[1], g.Vg, {batch_idx, min_tile + 1, head_idx_kv, 0}, swizzled_offsets_V); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + + // hot loop + for (int j = min_tile + 3; j < max_num_tiles - 1; j += 2) { + // Cluster 0: + // QK1 + zero(att_block[1]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif +#if WINDOW + // window masks interior tiles that causal skips + mask_kv_tile(att_block[1], tile_idx, j - 2, neg_inf_v, lane); +#endif + // Finish softmax for QK0 + exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[0], norm_vec); + copy(att_block_bf16, att_block[0]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 1>(); + sched_barrier_pairs<10, 5, 1>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 1: + // Load K3 into shared + G::load<1, false>(k_smem[1], g.Kg, {batch_idx, j, head_idx_kv, 0}, swizzled_offsets_K); + // Load V0 into registers + load(v_reg, v_smem[0]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 2: + // A0V0 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK1 + col_max(max_vec, att_block[1], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[1], att_block[1], max_vec); + exp2(att_block[1].tiles[0][0], att_block[1].tiles[0][0]); + sched_barrier_pairs<10, 5, 2>(); + sched_barrier_exp_pairs<6, 3, 2>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 3: + // Load V2 into shared + G::load<1, false>(v_smem[0], g.Vg, {batch_idx, j - 1, head_idx_kv, 0}, swizzled_offsets_V); + // Load K2 into registers + load(k_reg, k_smem[0]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 4: + // QK2 + __builtin_amdgcn_s_setprio(1); + zero(att_block[0]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK1 + exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[1], norm_vec); + copy(att_block_bf16, att_block[1]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 3>(); + sched_barrier_pairs<10, 5, 3>(); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 5: + // Load K4 into shared + G::load<1, false>(k_smem[0], g.Kg, {batch_idx, j + 1, head_idx_kv, 0}, swizzled_offsets_K); + // Load V1 into registers + load(v_reg, v_smem[1]); + if constexpr (causal) { + const int kv_end_pos = (j) * KV_BLOCK_SIZE; + if (WINDOW || q_start_pos < kv_end_pos) { + mask_kv_tile(att_block[0], tile_idx, j - 1, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 6: + // A1V1 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK2 + col_max(max_vec, att_block[0], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[0], att_block[0], max_vec); + exp2(att_block[0].tiles[0][0], att_block[0].tiles[0][0]); + sched_barrier_pairs<10, 5, 4>(); + sched_barrier_exp_pairs<6, 3, 4>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 7: + // Load V3 into shared + G::load<1, false>(v_smem[1], g.Vg, {batch_idx, j, head_idx_kv, 0}, swizzled_offsets_V); + // Load K3 into registers + load(k_reg, k_smem[1]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + } + + // Epilogue + // Cluster 0: + // QK3 + zero(att_block[1]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK2 + exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + + col_sum(norm_vec, att_block[0], norm_vec); + copy(att_block_bf16, att_block[0]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 5>(); + sched_barrier_pairs<10, 5, 5>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 1: + // Load K5 into shared + G::load<1, false>(k_smem[1], g.Kg, {batch_idx, max_num_tiles - 1, head_idx_kv, 0}, swizzled_offsets_K); + // Load V2 into registers + load(v_reg, v_smem[0]); + if constexpr (causal) { + const int kv_end_pos = (max_num_tiles - 2) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, WINDOW ? 1 : 0)) { + mask_kv_tile(att_block[1], tile_idx, max_num_tiles - 3, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 2: + // A2V2 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK3 + col_max(max_vec, att_block[1], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[1], att_block[1], max_vec); + exp2(att_block[1].tiles[0][0], att_block[1].tiles[0][0]); + sched_barrier_pairs<10, 5, 6>(); + sched_barrier_exp_pairs<6, 3, 6>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_setprio(0); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 3: + // Load V4 into shared + G::load<1, false>(v_smem[0], g.Vg, {batch_idx, max_num_tiles - 2, head_idx_kv, 0}, swizzled_offsets_V); + // Load K4 into registers + load(k_reg, k_smem[0]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM4 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 4: + // QK4 + zero(att_block[0]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW || ATTN_D == 64 + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK3 + exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[1], norm_vec); + copy(att_block_bf16, att_block[1]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 7>(); + sched_barrier_pairs<10, 5, 7>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 5: + // Load V3 into registers + load(v_reg, v_smem[1]); + if constexpr (causal) { + const int kv_end_pos = (max_num_tiles - 1) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, 1)) { + mask_kv_tile(att_block[0], tile_idx, max_num_tiles - 2, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM2 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 6: + // A3V3 + __builtin_amdgcn_s_setprio(1); + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Partial softmax for QK4 + col_max(max_vec, att_block[0], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + sub_col(att_block[0], att_block[0], max_vec); + exp2(att_block[0].tiles[0][0], att_block[0].tiles[0][0]); + sched_barrier_pairs<10, 5, 8>(); + sched_barrier_exp_pairs<6, 3, 8>(); + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 7: + // Load V5 into shared + G::load<1, false>(v_smem[1], g.Vg, {batch_idx, max_num_tiles - 1, head_idx_kv, 0}, swizzled_offsets_V); + // Load K5 into registers + load(k_reg, k_smem[1]); + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(" FA_VM2 ")"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 8: + // QK5 + zero(att_block[1]); + transpose(k_reg_transposed, k_reg); + mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW || ATTN_D == 64 + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); // fp32 scale-after -> unbiased online LSE + #endif + // Finish softmax for QK4 + exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + col_sum(norm_vec, att_block[0], norm_vec); + copy(att_block_bf16, att_block[0]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + sched_barrier_exp_pairs<6, 3, 9>(); + sched_barrier_pairs<10, 5, 9>(); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 9: + // Load V4 into registers + load(v_reg, v_smem[0]); + if constexpr (causal) { + const int kv_end_pos = (max_num_tiles) * KV_BLOCK_SIZE; + if (__builtin_expect(WINDOW || q_start_pos < kv_end_pos, 1)) { + mask_kv_tile(att_block[1], tile_idx, max_num_tiles - 1, neg_inf_v, lane); + } + } + asm volatile("s_waitcnt lgkmcnt(0)"); + asm volatile("s_waitcnt vmcnt(0)"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 10: + // A4V4 + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); + // Full softmax for QK5 + col_max(max_vec, att_block[1], max_vec_prev); + sub(scale_vec, max_vec_prev, max_vec); + copy(max_vec_prev, max_vec); + exp2(scale_vec, scale_vec); + + sub_col(att_block[1], att_block[1], max_vec); + exp2(att_block[1].tiles[0][0], att_block[1].tiles[0][0]); + sched_barrier_pairs<10, 5, 10>(); + sched_barrier_exp_pairs<6, 3, 10>(); + __builtin_amdgcn_sched_barrier(0); + + exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); + mul(norm_vec, norm_vec, scale_vec); + + col_sum(norm_vec, att_block[1], norm_vec); + copy(att_block_bf16, att_block[1]); + att_block_bf16_in = *reinterpret_cast*>(&att_block_bf16); + + __builtin_amdgcn_sched_barrier(0); + mul_col(o_reg, o_reg, scale_vec); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 11: + // Load V5 into registers + load(v_reg, v_smem[1]); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Cluster 12: + // A5V5 + mma_AtB(o_reg, v_reg, att_block_bf16_in, o_reg); +#if ATTN_SINK + { + const float sink_l2 = Sinks_ptr[head_idx] * 1.44269504089f; + typename attn_tile::row_vec sink_term; + mul(sink_term, max_vec, -1.0f); + add(sink_term, sink_term, sink_l2); + exp2(sink_term, sink_term); + add(norm_vec, norm_vec, sink_term); + } +#endif + div_col(o_reg, o_reg, norm_vec); + __builtin_amdgcn_sched_barrier(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Conclusion + if (!stagger) { + __builtin_amdgcn_s_barrier(); + } + + qo_tile o_reg_transposed; + transpose(o_reg_transposed, o_reg); + store<1>(g.Og, o_reg_transposed, {batch_idx, tile_idx, head_idx, 0}); + + // multiply by ln(2) + mul(max_vec, max_vec, 0.69314718056f); + log(norm_vec, norm_vec); + add(norm_vec, norm_vec, max_vec); + store(g.L_vec, norm_vec, {batch_idx, head_idx, 0, tile_idx}); +} + +template __global__ void attend_ker(bf16*, float*, bf16*, bf16*, bf16* +#if ATTN_SINK + , float* +#endif +); + +static_assert(ATTN_D == 64 && WINDOW == 128 && NUM_WARPS == 4 && ATTN_N == 8192); +static_assert(sizeof(st_bf<64,64,st_32x32_s>) == 8192); +static_assert(sizeof(st_bf<64,64,st_8x32_s>) == 8192); +static_assert(alignof(st_bf<64,64,st_32x32_s>) <= 16); +static_assert(alignof(st_bf<64,64,st_8x32_s>) <= 16); diff --git a/tinygrad_repo/extra/torch_backend/backend.py b/tinygrad_repo/extra/torch_backend/backend.py index 85466aa82..49ca13d1e 100644 --- a/tinygrad_repo/extra/torch_backend/backend.py +++ b/tinygrad_repo/extra/torch_backend/backend.py @@ -604,7 +604,7 @@ tiny_backend = {**tiny_backend_out, **{ "aten.topk": Tensor.topk, "aten.constant_pad_nd": lambda self, padding, value=0.0: self.pad(padding, mode="constant", value=value).contiguous(), "aten.cumsum": lambda self, dim: self.cumsum(dim), - "aten.logsumexp": lambda self, axis, keepdim=False: self.logsumexp(axis[0], keepdim=keepdim), + "aten.logsumexp": lambda self, axis, keepdim=False: self.logsumexp(axis, keepdim=keepdim), "aten.roll": Tensor.roll, "aten.logcumsumexp": Tensor.logcumsumexp, "aten.lerp.Tensor": Tensor.lerp, diff --git a/tinygrad_repo/spec/tinyspec.pdf b/tinygrad_repo/spec/tinyspec.pdf index 7dec94f0b04948bfb83d1914651ec732e868e6c6..3e062de52767deef755d32ff2ebe0ac500924825 100644 GIT binary patch delta 66562 zcmY(K19PAYv}I$vW4mM9wrv|78((bOwmY_M+qTn5M?Lq|)KtCs6X)!`R_(2DsK^+o z1P*|Mokc(Z&c)T)+}Iw@bMr<|)`7eYwf9VOT3Lh%L3^aU_#B0pIUzf@$J}N9x`re0 zs4ejGOYK3P!^ff-d5g)!R6|9PQ@iF&bHs$GSFj?udjlbeu?H-wCHY?jBag*0_x8s@%vb1_c@ zfo51Kjp4;2sLd|sJeiT9&&lQcw$oO}m1>HMS@=hIsGgG$?ZJy+E$RajU@0D8h*t8O zKOlJ63KN8eGUR|0O)ivRI;|OGS)j-eHeSZqU&7peA&;vtU|n4T-p<}zl%@W{E&$&vzXnC((PW^9S^Imyl=z5*d<02q={Kf5L=$DnnMh^v4~3x zt%BKttS0cd7)gdpJ|#)}C%|}QaY&Co($uce{VD%EQ1_|6E!Q03VxWL)+>pA1QE~28 z@{C7;({UT3zT2O6mFP%GlSGrvXnbY!$=AjhHMP$!F?W4DghG^-!yyuwSg6|1o=GX? z;4yA?Qju8u$i3)#3B%Ca@%Gb__Q{r*F79c0Z~#Ad;{|(}dC>h53xJNY@xJwaWc$Ro z^zqu_ky%W~mffSwOG=jI11`ly-v_68RM=2gtxY+-xol;fvlslTujdu5!_Iq8Mq_0@ z5>~M@p-!TXP^g}gmGVMnvF@Wb2_T8iBk_V+#0wIR6k~Pxa#hOB$w6^K!ilpDm>|t) zE1b_|3ik_L~Q<)qv2KhixUidDqk#@8gJ~;$w+>R{J zR3*Yvp9@l&C~pRA`Uy(wkzKni?YPN5O{mc;i_4PM)~Gozu^Xr&n2ztQ1JdQ1Kk!!V zLU?{h1p>=m^B0Dko~wOUv^p*!;y0YFB7vl$gt(kEGbjd0z{r%vp1L?H{5kAN{ev`q zw|R#O|Mz_ep+sy$qlb*s>R=12`oXAGx$&nwiFtmqkczkT9)W{NMT7e%Ty!wWKldO0Y$-X1D95zt!gOxBo%KuK;?vdXB z?jrbxP?>(1dC!XS*ERX(nK*NAM9?=`I5Z7g?Z=8&XM@ke`u6Dz&BtV{nz z8!g^PnO9}RkbR)>u+|Hc&+t}AlJeX%ED=P4U$|^dfo{-JY$XK+l{=r0x{vT6hz2(F zFZfO=fUF0|$b8*OWx+C(ajJrn$ZQNl(k|cExjyR%4%MW4^sA{!tDH4I#`{ae?V+|S z0YgnBUukteS!5XM#upLTP{o|HsT$d{lAL2PY4bVmt;uy5^%EW^Z*9TnI`iO(2R@`9 zw8?D6qyFyJD<$_cPm79r{*^tg<#bCZBIxlifVk8pN6oldwz>1cL?p9{iqmY-8Tp#x z+slOHSXNxXvmKKSYBy-*m#2D15xA_PW#%dGoy0e5IzlG{4^J)X^<4D%xVA!&>(K+M z^y^M<0^@d8QIMa01P6j9*xwx%OG%UXn@04Kr6AF}MUu>v0s=Bt!5|c*1O42_@9|_8 z5Z5&Z)n6L4V2tf^s?b?2OIiJ^BEiVMt`|al(oHw|-AdpxH?$_AB0N|6W*jbD^)p0o zdQbTJ0`_Bmxp{k~wUA$T)Q?`HiB*)tI!F z@Bq{*^z@a=A-n*#<6oiBkRYvm?86#t0MhIPLyTkNFhe*tc!LDthY-+K?>9S_?kAc`7ThGjzJaw($O9?B6lA@ARqx`9JznkjWHZ_1)B~~?7 z%`^LX&-p=y;-Nst85Yr@nu~LwO+#mo+A*n6W${pRTbjKGBt-9p6k`eK=+w7yXJ4Oy z|NdTe4GBO$Qg&8srbFv-ghDp-?`?k*{Y9P;0ad`*FRs*w0fD~ zA)msWz0g&pztcd~$}2YStVy(MVnC-GkYUP@^ATiZ*6L?RtU07dSo?a)hXd7+I`7?4 zyY``jG$VCmqnx-(<0=F^?m#%CVx=LhKBqfW(x?cL)>MPVsE?3QWqiW@C>DZsER;>_ zYBZ^2^G}f>_5MkR3bk~2d7>*5i47Kt2IpW)zx`XjbS7h1{D2~71H&IjX%J=D3GNS{ z+eZEL0Q~WQ3Y={3Lb(+3>gX+^Nzka&S*{)=4}CKto1Vwf_jVdUw_5p{soP_9n{q^b z;!=cnLR6c`@E46^w)l4{n$RY6-Ya?;f44ad7mt)vS?v}+~s zY}0x1wkG0Rq7bD3l+G&DH^+|E=7{37$A<>hK;0nypCza5OyGSyUa|;@aGTyyVd_t69R-LhH{OO6XIYUkxzQM#@m@TdjJ3DZq;3HAbMv=*c%l2u^ z-U28zI$l%VaG+h07DeuMQs>P;c4wL5SXkqH=QWzd({wp-ApK(JQ7VxTWjjp#O(X)hE4-k7pT!UUq6I z=#T7F9#}cRc`#&449VnC8{s-;NMCxe63cI)O|lA^)1L#s#)ajM^O}HP z7sxb&176ycJ(6Ofn1gF?Vmz5yho+e|1DCd3D##GmY0rLzqm_+LV@dNXUeiW*sIs~T z2~o8}JW_>?g=xK2-|j$Z4_E%x-#fm3z;q6>X4_9KIVY2o+=sto)ymyOg$il#F*D7UppyXD1&5jC2EN~nWB^>78(;H;;9zlzqdm}8_pGU40hD{E24 zzcv8WG;R4Kn?f*}Y}^g(CHE%`LsZkz-|{-$=)52r5s{fM1>&b84#-v4r8`J(#E_SI z?25DIo(_Rd>i>cuDtgJ?-dLq5xNM*mj?ufl;(o~(k5OW@A0%@GFAA~8YWBKC zQs?tv_*T6j)mt1Pnv&=0r;_jGK_NCJ!EF{5HwNQ5?uu#7nQ75ovgp7zV^}j6w1-hHQ4)I8uKYyl4v0p0j}Te$QS zDZaRFr3WIi@3IFbcPa39*M;LowLQSD{l7^1>+I6|aeiHb!I20fw$NWVgKj^{>)bc&EyW(u-hDe(@tZPU`E}2^g)hETY7%~Gy^c;VjM@)k1Zu3 z46x5+15a;XWh=`KSBPKc(uyGq)tOdmzppNMk<2|D?$)@G%w3J@IWVkce}w^xp8A(% zALh&NTPKdg9``Lfj{Ni3`M@lQm;!GC&EIT?G5XWln6AQN?)J>e)Z&i1AjnhE@}5qPk&|t=l1AN2^$@f3vz?UipE1 zJn0HkcIRc+6T=%?<977##J?9{Hbbq0F_S;YlK4s*m{h^VoVVWsH(h+*M%yXm`vjuD zwJ>Ytkwlc9_F~!bCac|e_uiT+l$nQ>W9!;alLEM9Y3CB;(_;KdOMAdvjvb z|29Y^sG!OJJuLPVa_a|xwlsKM&XKyVE@zG9>z?pAXxp>kQrCcgk%oR^)=ii{FRN+(DPKmYj@c7 ztBglppRzkXY`qTHH&7&;P*gdN_&%BG-SWY}@%NJ{tBZSRN0_JAin$ zDAY=G*BIcZbr}r`)|f_G1m`@Z$M0DFOX?7;nKKBT&qdoIQAnOkdBKpoLKMd2&bnO* z?Mmxy5Qk7mpp*N4KTB_{cplFqbnPa|9=S+wH66>mimZgH?9zr z2{`vgFzZ|*CVfVK89ay~u30cF9+3NAzycsy9 z$Vo7hzT3_7?@m|Ire>FdjaNBbGFH@hs!01RE@AWZ{P(B6tFOCOWw=B5iEbCWk#f3L z2|L<52WNl0iTZHkKI2rgoWogY47jM&JDBsuGHwSSB2YZYPNBu zS$o8T_1iRkw38J>+PLoQ_qhXmUkPq7yd0gv#okX^-(ofL_HrCbEF;G8P}O`aHcaE7 zRuEofh#P8s*#QX zR~%9ufMBb+%CMcSW)XDFP;FCaYX&mEdf(<32A1s+ge<+w)Wklg=BlRS&fMFdp>(?w zu~W&*p}csiol_Ins-vv0ij^xGVe^d+HK(kk)b#*udMFo;?X~PazU`&c*-gO;`2~5kJRvVMB#;`SotYS8>vd8PuE|$ zXO@;QlFQyX_`h;L=MLiKfx)&+|9+TJW=V0u`bquZIn=Fy90$beT65VK=a>mpV$pWxfC14JxH+wmO$ zC5$Nhc`>n1Fj!Bz_erC7s*DTnl;s6A#o}f7h-K)5@5?W7;{0-JO@ho^~EPhdU!Z33nufg5* zAO7foOG{?GVcv!X>lXty${(4^6)WAJ)3|;l#>;F|q^vB+a5A`F{NnlwBltz?h4R-h z?ZPfgZuUByCXNF-Mq-EAKR<sW8 zF8uBs@b-6;=GJR$L?6<1zSzHgl--u(5T@7$eCI1R`VtZ6)jZGq>6 zSwZU)L^Mv6eoHig?A)tk-_$l#q+t_iaG*#Tnf^_dK}X=uI%-M#F1^PEC@i7yWX6n( z?cZ>_C6oraR3E(M^UJW=s0va2`X-*>UmG)TMt1TRd0IvZCMa|Y`$Z_Lr<=+t!j-HD z=x|<$BhI7_0Qz8Om~>Lw!)*{>DiShX#VGi&<+!Pu^IX{T;4+!tWR{2A7L9ebaxp9V zBsHZJ$>{E787>+LuTr!?1Ep5QmV-{S#r~|xIqQpg?a1|n520|eWEc-uz53JptYJletGKboDP4B}m18p1gewTc6Y{-4Fp=ga2i6BOxG1gX z>V-Z21<%hh*j(J6fQ>+=No`Ix`!Xl~u$oh;-;MlfC3?g5Qjog9t>o~%WLU8Mirv7n zXuvkCI_1#6CGN<)leE&blq;16)CrxhTpe-FvH~-K5D*yICw=e61<}iu^IP39YuQlq zDOt!c15F5gL9@YXbPm%+isG_qgDCo>bwi{FY^?Jhh7~DgPSt$18pZSA4cufWLgaovAZpN!u-_U1BUwatP(0(M*p+ zQ7Gzh0=l)exA2}aZAIePU(XPaE84N;)a;k(PnhG>xiI+zlp%utBMna4TPLRQbmhS+y;h;KZ@nhrR=TkR&FZNW`<}P3yyp6%0VvpXX-8Cz zj$2QAZy9abb#!#IR;>_YK!`x=rBd!YSp@tD3?x7aYjJ#T-+pxNFIPerm3UxqI*^5U z*vY13#CZ)+b#7QIC(q^ta|c>B#tfmZpX?$ef|R>GK{uMZh8|Uomp#nuR4dm0Q||a9 zN5$YOt+Kb%Ni&s*K1+MqEv3HAyU_WlnTDjhSgyO%dUO|A6|05i~u5NWh07{pg zd#iBAx&*L{dKTxo)7cI!dXCeLP;oJska7?q;?4mOV3ayWy79ZW4Tw3)P1+XyCJuCJ zvsLa;TAIUoW1&K@B08AzDhw1c-UzF| zC~W|QA(=2(`2$u!W)x5^$%*E0u)fe8z+d|7*td0Z8Qdow_RF3}B#Np5+uMT%cRB;X&x+aczokoRu)UyA2avvDLgA z%m`cp$*7HSl{u)r-+UOx$SJdEYoWzXs7!#96%Ii?+PPr<_s7#w7#eqNkaGs>+|F6$ zce4@8)m+npdVY0TQ7`UfZu$cZFrGcau5(LwT`}D>@}A-UJ2~iii4)%@_tY}el|Zz% zd**Q>ZSBY3%Vbkq^-zbeD=2E=KNpoRJDBoqIcKh3mex*Ec$tTlmY6$ulxP z)kyTVK8?YwUxTEm*?y`;9^E7M^R$5Wl}4eXZbdwDe%pOaBytALhNyer(e_t_AeQ9D zqX5IcI7db<8rg=^2Vh+SuNucvGqjbhE_3pjgxr&skIGqoyBRbDh}p z(AClMX2IYT?MuarQ!(y=Dyv_>I8pnxtDF^&Oav{RTHiC@KKbDwZY@{Abu-?0*Nm?)ww&rW1Q)t zrO-dumo+*RD?vInn3XkEhhdixTN9Sio;jnv^kcs@x@YemzHSD8?dR1Km2CLxb|T(i zgk}}bYQYlK8|p3NbW5W4jUnLBr>vaSE1h#)wg1W@c4cy@tVRLk2iEJhwk?MtR|3|! z_23SK7%>1{>EU_>C{QelBsX40OS{J9eLi@U7keZ<^Y!9LoVTG#^DCC{Thq6rtsU?i z(RZ6b8OjbXFIL6IGm2jH#s3^PEH{ne{Ck(PH>T0pBXsA5h1By)rxlp*j6Hf_$fb4J zv*Wh3{)FZ+iH8H!kc=;;HoqxPkg?DH@wzL$G& zq*M;4qyM=DqLtj%X!`zyAaOfj>T8CY+@+;J&P&*Qs2I$P8AH-QgMN!ZNTF1O9@p5* zuIp8Ej)Cn=w|sD{CKbqEbubvi^8d|yT&doBwbEusBKnKus3!FKzE`k5%R-Z$4C8+2 z_sStq$-`MYL;%AE*2#?S|Y{a zEyrk6(O9S(Q;N|#GA#Zz+pJe~zxgGv{Jzw9}s0>`N#(&K_L`h;(+o^um zsBO{%f(*E+4M;sUSb)}d?|<;^rQP?TMFoDwc;NEnA#_hO8bgKb{p8_orzb^XQBu(N z-`m;<#6kCiQ{rF3?o_nEUgYPPBo(5?CJ&g`zd@zhSppo#chB?8h-F}KVq|UvGgW*# z_~QKK(8@b)=|^;!i&KDCjPBY1E&SK&MUooezwX(w?64)@=2u<K}1^W zRqmO?S<=js668cz-b2gMr`&YWO#ItKHTw4wY$_Cq=st8fwUf`}zem<$w0@Zbd#3*+ zWPun{*ez#ugb7qmwQ!<(zwD>n3UTNH%JrD)PG}85Bs(MG zH(KY#1q}grO5&;JY9pzjhEQUm)!!+A{sbExu7tCE(rYqrh1!aLs-ht~pMxnJWc^{( zzi%im&oWzc7LLs-oAC-?zCV>Xk$@i1l{TgtDQpczi#Cc7e@0Jxjh!^*_5=a)gHfcp zL}YQMya@UG96pC=L#kxkYZBZvDg^gtAM#@0Zc@KH=(X;SD~;Er@c;Qu7uf-WBji<> z`wyY@b>g#ksm#mK@`*X(hNBTxws6d4N)a4uI&Xh^&@->P zk%%Kmjo)Adb{e4Djy{kOY}=W*ibcrKzedKtcb4shxUQP3P=koKcld@G_k{ehv4lLC z_%2^{1N-46w+9WD;gZ|}5^aFmRCI-M+y$`i74IU{w#vk6*Er%D_fu1eY*ySo0_07r z;@wDOa2sEYC%o`Sl>ip2y2b*Wu!Todw+d`13m)_b=i-`Y+a%DV%>kTh!H_R&&Fi_% z`;D!=`WWu~wyz7t!EdI~R?*?UX+wYhk1c$HmI6n+hkt}0Qh`#YJ%hlr1_xipZhBwl zj(Fgb%fWL4n4jA^TDlD!Gp!N@Z6%Asl|+-oLhTla97PSca29#)IUi~>{~e`6dK*Lr#M{iZ+em(c6Qc-aA|4Hlb`BEm@ICX9Y4e)n3*p&tTP+hOC`UOb1>|Lm!`= z=qtFp;@eh?W>SyL6iV}UF6Uw8^oZ7*n3tLweK^rERdA6W48Yn8H3+KL76?ozc8t|G zOD=jk#(-u)gSSG*iSiIPKG;)O6_Y>9?4ycv5Mo#qV#FN?GY_}UaabxLRs^KZa2=mF z*(;&>dnXZxX+<>IAYowA+|`cIR)_2JLr}>a_C!j%9`q39SD?YYEpz2~&i5?GN&`AF z2ubdd4=DQ=$(bop&I`&6rc)%Z&Kd8hs6mc~W?{}P)DX**Nlx!c7HW? zh_Z581)FqZSGV7K5yBzqF;9>l{QGl%C|J>q!QV8kt;K&471cF~;1p%Dt{-0tDz4el zYGA9kYoK=#Gp-S-@+X_Bb%0~lgeG01;-7#h1e+JgTnOPiQ3R{07uRz$6 zxiDb*7njIx+lngCKYfU+;UP|+TV2>#hx~e(OCh1x6@74AZ5+Fnf(mOETS6U0Ptj7|6WDjgu zbzJrykhF$m-mm2x-oYNKc((oyV;N@-Hpi=Rv-nugxMmo-B&nw`b})e7GvT(e;^vr| zo3I<#n#PEW+3YyjzHNRuJh$HB#>BS`Y|^vF@zF*C^N}GE(vh-KUHK3->`sT0B?eX= z3>l~hsD(+WD7;kV+%-I_7oO8!Ly*M|T?YcW5s3m3CU}Z17RW-e8p-9?o=dvXqMW<4cBH4#(k&vH$Wr9)d7DN6RD?B3qYn=lz)m9dn4mZ54h* zK$Y$GsisW*5h=5FaH|t4l|47&vtx*I_%#M;cz<i5g;SQD#}7iAQV1NosSv>umWf5M zgdI`aCKWw;7`W>g+ty_uOEG~BOB+hFPG_yiQB&@iwk2qUCz{ZH)oL_HsB+{FjHOp( zMwDK&6Lp)13@Yp;W%3<7v-36{?R{bxj+t};^gsAz4@CkfuM|P}MaB+j8kQ9&AvzJB zv6$mMJ$-ka>p>rX8GwGu62}OuI1w?*30~0wfd=Jp1F^5cRlkF*eOZBo9qUl$IKM&R zCE3)gC%)lPXF|li*$-rdUl=HX*P}u*Lq`o+|C}H3lb!O=NRB#>#&idg(CF?Wv-5JK zGe4B+G~Ce7nZ8a|8Da@}4nKSc<~TCYu_#pp^t)|l@fE$yX@Eipha(e~0EZv*b*EXL z4kOG-ZZ#HLSY?H(u_cQQ&qiTCH0%!W=o#dnsm`Y+DVq|ERT^bo)lO|}ZZaAJZ^jfT zT@ct>Iz77~n3%#Wt!U=#Y(Kr2mLlu?jz{tAVYyde+r?=5ABfPHVUMDecrJ+lg?7f7pNY+*oyeTA6*yH@w== zO}0$wD>KOXvqV4Fz=_b%&QxV{tC`y>8`U&x(yZDD_yM(;za2Ek52|#q=9TAVu%+*%+W6!&e-8!yz+vG+r z&`-10Dh|-p$u1lgFOs8GP%fqiaJeEW|8BP%XwoltREalkV=)}3=QlYh1cD;!@s2?G zWaidthHvC&ve`d@ehY>4zyj39U&--l^Ov?il)%=-_ zxiedmu(z-syH!ozdYXQMcIk{b;iLDn%us*YKS>Ln|Gj93F0e_tNI;Pbwmf*Ycmz(g z`RVk1-`ssef4w?}CV$fK08Ia%skyR3*V0P;Y7Z+0GEZtr{8Z`jp& zy{U@}d+$8?EG6oY+`POz{xUV?xAAN&#eW>y;u*or4X*8D4%92R5=?Kunu*$;1R-9L ztrW1^y7>yYw}ZpinUPM-8DsvgY3JU5wT@S9kSqlO-F)37N0?>S$lfS1_na+ClQ z1_D}L{t@FF6{cxi4UP%NWDZ|+(U=&EbdUn!hDB4Dn6d*@FOVAe2QDM%`Y1NoqnS;_ z8;&{7EiM^0UXoQQRFduPoj41XVVpCuEGwXkpx-W_Nej0Lrt$QE#u==D&R~QYXKrUP zNy#!}Ivcxw76VmHw-?FOg)-)`KQ=uaF9qV*`5zt^R#{^F5IPq|V-*!2{~4>9^nlSf zf$1N7Cc?v%n`vW}?g5 z7y^MuNLCLZ(mXiF<{|6u3y#XMMGSUFkeJ%1|K%@bs$M>qxoA@*DFDqBh7G_OYFmm~ zL_^cE=ym_Fkgr*+)BiGd$&d1{Z-Xo!z4JS-8o&#xxUwOzYJwE?vs5rPGxgDN9MG~Y z^BjY!J3+sVp@Smx{8+dlES7aF5^~yWL%Kw?9zuV;q-;%tx#`x_Ct4fbqc_<5XBv-h zMGB1@(GgI3{IlE`VI=5^02Z(!`Z+N(oAngr80yis8h%8wI|BK30G}%;s7>&3d@h3X zg|ZW)Ak67M8m`nH|Bf33P(}YC8E7na4Z{?e392S)!p;Y?U%Ko=hWpwzL%E2v1(6iA;*tMai{?tjPp{X#WWL&sBWgG2o35!sF=$+XRrQa1E=?xSwz& zjgH*svly65M+sDtXvz7FM+qkqb_CzG!{628_2u3U2i{_pA#df!rYDXz5i4Kaz44EO zjRZdkJr9TYR48Nu2Q?D~Q-6r3K(tSu(TYl%-FQ-Hp2sA!s_G%s{5H+}Pal0rD>v~f zT&C;%?iupSDU$+s0X&ej6i%_{MX4=tDnj>9%fRUX%@zMoVN=*Aj|iuRj+TZRPw~-T z_iF-a*avn|dTv?8`Tm|mj`VgFPcj+ATTm~Vf6zJa2}LO=H$~yTrly2GA;t>LY?F2L zP8_uOwAfo{KM(acvizOe3^qC|qKnOQQws%MY~sntHkL7y?B0RT>_uj05lNZ)vQFcc z=6Y#~Mn;yNQDg1JyCBEPj*4#G<-0M8E78p6$K=fXpIf;_*)T_jX?%_YF=(uK%e*4Q zXPY`*Ih%G>!Dp%l8R2obLE=u4fd=(N*t{{kDxJe_Xx*yR`@%UGqnW z!5?7b2+VqLMzjDY9RULSr`Ixhac*H^es7v=csCS{%qE=^^Ry2y9WS#&MJpXEj*%xx zcTKsO> z&|0QBV>R8qa@5z|O8-3TadxxP&1v%lgChqu>A$^P9RV(#cY4R6Z=$)vL34>8Xm~L0 z{I~PtK3@A`xTL^DWmW&qjb-O9@Y$fIKsea_ihiQDHqSruYl!B(ST8_i!9z#yg^96?2J3B)3@9;iF4BabBfOePxra3sTMDV0sG z7RAPQ&jZT1my#*xb$`rmnE8dR|3sSBqK>JcfQKWct<_C-h&9;cQk+CE*2%pKwmG#e z>K0J7Svl`qLjGg6Z zq15ivXWDVpVKaedv5jC21MSgoK$qN-Wsegtj}PpP`g^bu_5BhEb^gh`GhjGM$o|8$ z_dM{lJw97u0%zZSg+krA)LbT_{pSNflHmmfwQE?0rbv6R* zw|@uIZ#KHCjFDlkVI6d)Hd4xNFfqAA-k+L5?J16yVoH*dRSmq8#1N>&VN~p4Tm!MV zIYIyoynp=tA9IYh_iGN5u_+eFCygdr|Y%etb9)K81qWRO(?&vC&a>|aeTwl=!c z9T}+d?#Jt?Dun%4CQre*_(QT2^RFmBrN{~CNwGh6>jYXTznZy2M+LK3VcX@Ec_A>i zA2AL}arxNW9|KePv=P~|g{uNS;-U;gp+Q1>es|=Xoap=Vmr^)Vr7G=~=>y5cc5oo5 zv#<2w^?8 zo)T^HWNo0tFtnlPueBPpq`?;akaq-W*fbr%715WmIyJq{N}kJ zH>GY_uGty(XJ-x6LkOq`6y1lMgvc#R2fH$=uAw4Cx@Ls8}h(ytMa&@da~;z9?NMO!^%)&O(~VCVIpIqLS&&Mo}xC%UY!M9KN&3E zdDW5V&x@+S9Zdy4&A`TX?m4i?% zr(pC_&LydbQ+=4!mN@kF_a@=zl&df8>IbVl6r>90}tghNk zbx)mkV{rp1IcrUkAacoe7E^IF`pAWX1#>XK{kh@=I~gu-IZ5WBJSV3i>5`(BTr+tZ zzXf>~sXIA>YdSK3Nn=~LpOS>AZIj?2e2PS%aU%dV&OFBEfZ?!SZZmDAOJ$}iY#>0J zlj6g3kz+T`u3g8ounZLSJ3H=?H+hI~u!>78bL7Ra4?7mJr&UIpCY>h#t`g~|K6!Px zr?dB85KE!SiT| zx?*!qZZBK4x1DIBi%nTUKSRYm=483uV%6e1eVMcNf_KIu{{?WS=g#kHTp?%@tBa2w;&~zh~;jFu0=*|FVl5mL)O2|eYq-m zfhM-jfKOgyKI4PfJQCv(KaxSN-RmGbTSsN&g$s>cQ3#^k3LAZup}%#aS@*25Wa3vT zfaKfFt*Zr5D+J^Wvn*yy2YtdF;FRn2etu`b-H)byCgL`6zikDPG;>Th9LAj;Jp5FZ@#U<}ZDd1^R|8;o9 z^PLc4lKe0Y9xKuovOH|;>bH$nw9u#nV**u;heo~uV#I=y3 z328fR|M&$1QWmiqliisXHe^ILsHLwGGLG%GF{wK?E6m;!oP|UFK^qILvAz6`?=wTO zqZ^In#;LEm@q~*>`|~eRy+JlD7WEyvJ1v#LYq8{|Y)jnZj@sgOv_cR1)LSykpF!PR zng>AE!U_G@P2^7>sjjjp*RWj=(o4RZ2}k~=e**pKX0taqxPwoVg}hn!Y7Fa0rlGBx z@w#zGgw@FoTP0Z1M>N*MH2$0CWL4E*EU$Fyq(|kZ+uLn9pI<~z3v?d+ zMkbda$j}wX%X$9!QWCtDoSM&}GfK*+{4vllkk(H+VN&?TWudZvtQ&quX@mdm+!2+z z-HxRSEB7w#PD#%b2BdNR+Dn=bSudTdF$b1l5FPBb;C`KLj5t9RvA|A*%aMAv#2%w0 zt6I!JmqdmRtTn1{79O5#8*qY<{c_3kk-~}QX;V&Mg0Yu$@R;C(z~Vxk)?vPqGXcU! zFHCG3jj?`E*Ntnh6|y*xjr3YG=-nFpdnkA6!)EF%{!1Vq-zv**Pt)h(^l$6xRkp`c z;d~p^Z$Fjs&_JRTu6jgl(*>e~{zX(cafV-?E>F z?*Frs|BY_{^OO!IHgN3B#K}`UlmOHJ7L_-;ye|LEac941DIJBNz(Hf%|u zrs<&D^y((lN@=(s_B@UjgiN8D^mY>itLat=?O9Giy|)i zPX1zb|M}tHTc4l5xwnUTP@ImE{*SI2bCIVK>m)RJ^6eyD9FU~FZ&m&}>`Uuo5TpZq zb$Z_wDk0Y#_op9S3(r;#L)dh}PcA;&ps8zz;}aLf-@n^w63`qfEvfl;S>Bcgj`s4u zyQwNee-nczY33;g;|r84ng^(~$1MCtbcAy_L^xH0%6=-=KsFy>W6jyT-V>mRlSh~Y zJ;P>{#(?s_0w`u{^PRhBsEgnznVix3DAh(y4Bo5nNP%sx!dg|bY$Dh;Rb(i3F{p4SX1q&0lB$Tc0 zSJuX!`*ZqEya^+eAy>Fvg)<>@(s!2&J@o_;9|v%MbyA(^?`4Wwew~PoOm$Da*f#5F z7iM8(z+SJhz(s|BHM!T(nRjS7YQa5xrYxvho#9WG1S7K5*eUQNA5Mi$g4c8gaf zHelj2h8EcS#SPoNFq8FCFtKOD&TAH)f+$w+;wfgmk9;FH%+B1Kl|;7vWlG6K7TUM~ z^bbn-FVOtU3F<|a`P;5tB2%EF@8n%u2oZFMc`x;?bBAL5&eg!}N{Tbyh__`FE-W%~ zYVFaNFX9UI$YU~@hB_;H-O0@YMsI((rx*hgiz2X7@`g~fV!%qA&()ns@san(JS4{4 zNbx7k(-=8frt6>6d!GDdl}G~5p+YSU+aSuY73>?_;nO$(_|OtkXEl^qfoJp^|e z+>_Wnee**-;raR3v6Y~mEvp_LN*=x`oJ0isk$^V$1fjGyir{B4ung>ipbQ8Zz-oe6 zKo>kEAOgp~t@hZDC(}fm#;XiIe(7^C2!`RXKC?iM9tE2`&BVy?d;cf^mWsIlF(^Ra zLR>g!8NwQFe((dPi8Mk=8t|)#kO!4Y{4Xnulw}eOG_wUd$FR}SGB3vrtxcHCl{|WA zKh>xr@#=55m??MFW*YQ!7a<=mB(uU~R^k?yQSXCeDmvno zc*~(G__dMxZnQLs838&vRz%)1LJaG4-ge%^LaRT?}W7f`xoOHYHGP?G@5w1 zl?Nra-;kWOsJK>RtHUPq*M^K?sa@X^m~L!(`fwnIlCWd*UYWf+AXJvIzE0HX7ojwP!(kXUxQ$87dPS^|hlzaZ`z_y@GemJKD^`A88{K?A{f z{JB9;OIhGTE5Q~s8Zz=qaP#!`(fDH)Q6e8A6fyE=YF|gH2fJzNBy+Kr0=}I@0H!ZNS zfcP;)g|&U5!k}az88nVzoC-_!v{L^LjY2gmVTB@L!IoVgh0nl9C+ftxvkvmh8rw)_ zTXc3bRxg4BaL7Lvy9?!D6#tG zomZp8S?;}m6(mVg6QzGcHHQ<-+C?LU;+GX!(ca_v*!Ann^#gH*SRmA|~SNE3SEbe*H zlBG4~%RnhYXGxWvv{x%CSD8O{LnU&dbkAPzP&D~Wmz7f)<_%bbny0|GddmsE0X@m4 zabHCMtfPs>u6~9G`j!#wxM6R+_iB*%ZR2Xpz&21^u#5hbtG-Dx!S#B}Zu1NX#eX$6 zKgI|F4DRkkZ=TQGT#}u6pV$(ud~#K>|70&v`3&k^miRVKH-0V!cKm$Ea`v0I@Mq9 zAf#@04K!YO>Ec3WRTc-`?AaJlUW1MS*wsaE`+c|c?WQZg)Q+1+`}{WC#-xov_WG#Z zq@c^L?$d54Fn+#NxDa-5t7|ZmrBSXU0}&B}+%KmLHv!2DVWgaJemre>Cb1MY1q3ci z43>PT3-?BrB1W<){OUZBsFU}Y)uDJ1(m5L)6f&;oml5#E6JAtH?`&lQl}meoQf3TZ4Z4wn6tK{V0 z+c^~h67W|N>g9^;P+Bhu+47dZ!0E2MK9tT|Ts;iZQYY)e3|bJH=y;#`0x+;$;00=B zznb#H`oZ~TfBhZ$Kzd?19f1M@oc`Va&dG#ss5gROKhevV*R;tw^y{k0LGByd@#93% zu%-Y$+}E~XB#w-`dOI{Kk+G|rDsumdiiz>lc{#J&_2ZiR7TVfi>Mg z3Wr{S-kPh)doGiD9*sHKROia^7EGFFJ28;9N$f&JPhzli6AXdg=)4Q>(*Z(-E{~}) za-HO9w9Eo(n`RcrCziXzIA8q016srYwj^k|tiulhj%Hb~8@L4rV9Frc({^sCQBM!1 zkQ5caNHQu;wX`{>@IZ){(hokkrtN4^YT!JN-qt$}`tD|aqhdJ}c(=Z3opUvyv!6j= z_=#ClQCD5tOo~51Ua@qAm50cFV0H^7#}BMkyCVLqoy(kJnMQSHIDvX$R{<&PGgM4 zIX=WcfoTD+eclN~+xhJs6leGGki$Yt-CGL26UVt^lBko8v$nReNlHakqzspFr3VqR z9C2?R`CcNXc^5;xPSml5MWultl4sIVjLmBqxX@lTGIRK)gNw~wp1SVkG%tVa#@JM7ZjNfZFrd`oguat8u;V)%iBN$Ut;|Wn1^(-^-b>jJ@bhNviQ0#O@tD zYl0ohPP4F0j~K}t7e+#hG~G`1V-M{}3f!lu>)GtH%6x2qzY?^OCqvVfVu?5OXR<7v z$qHGy7`~Z(`B8-ydK8<|kBYuwk(P275DNc-huRo3>`UT2f;i2ZNybR0g=j>tkgm?S zt#LoMT|Xeltv#?wm6HFu;5az`8|;?KO2bei`3fpDYH_o2eOF9|V|y|XKCaeUECSPXdH$lWpcd8fTAO8#4v?D&t|sA>jw zkpW?8P4mDK<&%?IG%vcjHt`F(cPw+7-q!RH3oxFamwN9j2NZJ*7Lul{tjwHhyAzF8 zRPAVujzvi45m06U&FQ_cE_b7ojf(HtmdRMnmhY2U@2qtf7j5*&WQ95Gxf(yO_iZl# zaX@m1Jj6=^e^NURT_R1vtb|T(>93db zG={zf1C}>J`X1%{ixoh6?S4q4dItB&ARya{<`35~zX~zVP|+W#`JNEL?09u>2}=n;aldeH%EMT)vnXi~cv&M`g}crOvN$r)4$*`)v||#5<}{;B zX@G+=Z>mnJPOYh%{Y`=IAr}lm%~*BclB(P6E~oHKL5mAA$*L-~29ye7nK78nj{#k_ zSohJS6+^xS_LZhJ6$`dD=uKLyJ@c1EJ~kIXS5?uo2R2iU5D`WOHBQWl?aT3y`NOqi za_s!vWj!(yq<5x>;~N>83L}Ix=!pfc0sL&K1$-tr4W+myUdD8I(EjdMo3M=UFnqv8 zB3Uwm(!3tp%`^r=GAHfV;299XDS&&Snsf*~y_8A|ClhZVF~Pa@c+%AY5IXg45Q_?! z63(IC67D9xo6YTwP?DN?Yd1>dWwN;pMBpo{Kz#E24g^FwjaR&^+0*2$a+7Pj{xL4- zqB(uKHzFr8?rZIs3AdRn@Puj?3wj1@%mlP#XqQ%NcL-_Y*Ok20+vO={aqC z{4;qAWzoJ0;2RUv4~>AOy^I=6zvxR0XU%;}sY8MbS;6tR{h(Do3)@f^X9pRBsuhVF zMElDw$1w_TzP6DJTQzZ#aJ`r8S;|ZT&(NnmnPpnHlwg&PZTG>Dlaf5?<>`dxe2Q=1 z*F|rA>dOM@q|dfFMG?{OTop!IdzheuN@GW6)W zpDP89x3R~pRv7CXzkTSkc940eTyXvLh?PU3ErS`wF|wju{Bj`0`>Nes2zoO*v*KGj zXIIs$?bqElHGMGjpI6NY4p+}U`{>D@fi_jVwij1jcZBnisW*B}9KdYCAcKn|7kp~h z)KuJzVe1iFXdhkMdJxuCNE0wiAp=wgQ7;gvL0IS+3{sn;+WKJ@jYKYo@HPo+H9A z-_g}^G4{`pk7gc)3;>STh|S7qCOFTY&{V5CE!}B$92vs2ai}eDuX5)K0qz~bh{6o*RRHvSvTS_+-5IzYqlN(X)L`_+H>Vd$12 zzcX^FBw@gW@_jKbiQz}BtLR(x1C2iVYrYAdFu7Hk1xMr8hJt%Kj6UX-S{quvv`LD! z$&xPH9J>J}UAkj9Y@CdTaGv*?RE`#R*@mB4nLhfal?}`XKaCIlSr5UxsoR}+acwbbOP?zMM%3hWG9(RXwh5ij z38)eCR&#Y3RRGJ$`KR1C$ z9Gk+(uH9yBe+B*rgF*4l_=fqY>j&lm^c+kq+ zGO8;ltF(m)pE$P=vW~u+#mbsi`fti&6=ZdYge}M@gbo-SNXyS^GKJ#M5!#Ht0$9gG zy7OzNO+a?oNm0(+f;$6T(JhgnWXvGJ_BbqX;Q(e5&GYaMD-*YUf~Q%L#~2GC1FB1%&aR(-oAEk^Ufec9y9L`M z5~#o&w&3>U4wfrx7yNSdO>=@lr(?%{pXJ#NI6(3dyAC4g+*5pW{d55wb7`(=K6J~X(jmOs=uxq z%$yD{t)o`Qtt-luw7-@qwy0Z}J`CDP<01C=5TXA^fO!loPr|g=H{I!Y~fyKB78u?v9XXSj~M(GpGmR4^vOJxfpfspP()) z>dLjRKeC^b^42n(2p||NII>77Bzfp@pa2h{gXKRl$s5Rq9=|mLAV(UPxPn$gEOJGP zvnT9<-GJUp8^#_^M-I%u!9uJmUPhME&(3#Usg&W;h~*-GFCV0y?OLSIVJ>$VzU7Ra zT6@cRUc2dT9PY`;9llEgrjL1-v-g6Z`#ai{vkgw$xKrZATZH;@kO$&%oy5`@vjC(R zyii2}h0AhiOVFmc7or;Qp^23`?)^*#X*P42#g|2zeE4RPY$2^9C8J~Ko_ow8=%KGf zSV$HlI7JF+bkPBA_6Q@{j^*&y0S&r=g*?*ZEBe^?TCG`)peh`Bb=4mhE1#1{zX)ay zUrBlRMDafYdGY-Op9s6{+e$L~^8tP_i0F~`DsaZ_w_AGKXbAH;D zzx3{dX_VZ3K8eR!h43t}vHz=Vd{GV0_)D`R8WDqTR;jBPa=ttxV=?RG;324P8VJzZ z2HaHFw2c>NQ6Ep5qA#S)TBG#?jud!` zAP^bW;$tz9?-a9CK50v%bV(vzTEGa6|9cXCcuUBda}|#v8F~Q)vqKq2TUB3qvV=ec zlGEsHQDQ8bAoPQ@CMS15EmaROa+sCn_ru#;{sE_Jnbt*yE{hri?3q1B`viybIxLnD z`Jt=0%;|+zlPpYbqyTu>3|(cSF60R?griY6awwYi$3_maAzPvy2~Jt?_mve-O4tGP{A*{U?*YcXD*W~4ud zhaS9rv-BV)AOTH?^o@a-@S>hQ^7wIKMVE}k}oZE2Wj_Vp`etG7Y)Y~z{le|Qq3ar`YJ{VQ2?%jV- ztROZ6ub$MZXT+|O-_55GVDl}AdU!qDfy;_V(8(7m>tlKY)1)$@Y}+RJ=C1l2`jMT_ zmpn{*Xk~B9liLaFGFFFhPT=lRy`pf3W1WP|`JBDSb%{r-M~nY`YQ<%jd5M6D?{>vR z8TLB^dl@{WWu%ZxS|)0!cocbHtdQhi&%)vbg7RcAV0xnaR2mUeZ4Cl$VI!~Ug`HYa zUv5qANk!x@*!X=xSa4z;rx<*jd0>gyXUUGKgMBz!8Rmg80zkgip&w$EZ3r z+BK1bDm`V=J!+kSlk-jxN14`YU*lx6 zq{Tq$U;(KW^grPh%+@xB*DytKVVgx+cBw9AubukX6=s@V1Qq~0ntSYeM5kyGss z(E()uqv+Ikgxk}ge;NWeCb{F< zm-7{RMbrte31--rctzQaz#F1JkU7x92^3jFUgeVE6wA(H@3MW|YU@a|-L^HcOKF93 z$GNIqt84C)q%S#9B>($XJe=g8L!0YsB7Kxx1;6LGK0l~*)4Y71D|XGQPQs*Po~e`) zu;}%%CdRrL4xnFh8D%Gt(C&3saV2~mjO|S~*)H4cyZUahQ{4465X%^c4CzxQ*DIBF zOvjUP$59dW9UGrjnfTfQ5rEmrRg2)VR7WI+p!fBG1FyJ#+|2W?d-6#Oh&Qe~5xcgh z4*-sU+Ebu=vwDDZ!I&X%u*1IR%JWCA?H_LXYxFk+2;epc#c_Q4M|RuF!L}FZh6;17 z-B?PQ810kEZcT-X_qry=VSOIE!79r`pI4C}f&lJze`P*3z<*-=Adv0-w3mBkjElZA z=)rRCH+K!Tg2W7yB~3m8uou2at|fJ*z7yh7et|SJD>FF+FaKts7f*v6`^f;ardljs z-+(*DEnsPXcJenh{ZjI+!KV0m^__fi!G0(z*Ef2*8wVz>z5(?g7bjo*#MfnQK1Z5U17= z@?d&Yz@j@&%Fc;);5-_({mE{0C-&{+eI?>8@a)hW7}LdrM*nDi>0FV?(;b>dqK^?! z?YY8dk~J=MC$NArC$d4opival6UI51C(!9WGHfcm<$N*$JZ1qDxJrx5q^}X1Z7ADq z2EY|1g$aK}lM&KfCz0z;{NZb)uP86R^BqHoCBaZ3!5%kn4(k2U`cRRuZJz9$_tG^?05)ei zac{uXe;K;>I7SJ5Md;R8+AW?xw=^lBfjSv=|0=!W*~`oIGI<1t2o6l<^Z3CA4S2>$ z0sUBfYfqC6`3g%%Ruon`*1eI2)B3|0Bk2>HBqr8=ZmV%CxgauMqbLYP5 z7OzlB1zlFPtFDq^5w{RO2BXDu7HRd}xe0&oGeC^H363=c@M*I|9!TVbNvObfQ;f|X zh;ooF4-b=(4G%Aqv9T^KvoUsu0jL<-S%(HI7SHZZ$|0<*wf=D?tMIz;Y*%?4<8aNlum?JD?u8hYgOS}y34R~1-snyO42Ki9y+LX6C zE~s@lRdV^oSr4Xh`k#IEN2nopl5(Z39MNKgrRD-FyDP7zG)mm&0AO?xZXhb<<~YH^ z!2=ZZjt13)8w4Okjr0{M6H86R8iT^?#B7D75H;ebMQ!u35E(1E8NmmZ&6i~}i&Mbu z<+ib~DU-qRVq`0%t68qg61{rc&rrF&h$x=egFd0V*cnljkMbnWlr)pTTuCsIAR)B& z{io|lf_Jo=7z7eBfVV>;8E!h35fO7eHMl!w+Qr^NjzmlOj+_kLw!Y%D*^3rlza$`B znUR)XS)Ty|Nk2aLrA#+>x~6a+$gX&7?8=f3Q6iF(U?Hq>?L3OewlOowuCxsz{^;EK zYJLR;G7kJJtU%Lq8AkBM6N_ue@_uM7igCPfGf!g2QyD| zPJ+|xCg@5b(08OC2~X})^qG;xH`}=zUKbHk=fA?bcI*Cby=;~l#(*VTgs}iUM0e3G zYF?jbTD!`@aaNeUnE}0a4Rh(0UJ&U)gk)# z8gu~HIt3j$<^W|KlE6_gxxiIdzo3ix?#%nulg>U208xnXsKOZfB_rV2j~%~o*I?Hp z6JdK)TfwsY;dyWjJ@p;XX_L#gT%8$5kr`wg8gCdLWm`KRpuF~HC~>-y|H(+D?|`yu z^9Y>E^={t${@lZYUXS)=cVtiY-XLeQzf%LhAf5qg&@k3+ygGKp&Wyr7ULo_ha9@uW zb2V!{VCRZE;jsJxq38nK3t^sn5`)#fCHM8@cQhUKU~ruH^6$>tj<=Gy7%(wA-<>4k zNv2Mv;<&gJQc`kEp*YKGiXS}bZ>dguZa-U61lp^Q_)BGnDQ_i`$H0c2e)>5Y>iU z{9e(Xu<}DbXNHgq6s}S7e^%!}>=+e24jet=dhz7{I7Lvcp~C720VV7)Koyb(2sw3C z15mpEwfYvlflo%uC)}kF3K)Z}2jKgv>FlPA1-^FcFcgv-b65V=$bx^N$}|}5P2?3s zFY$Oc@Tpm_CrQugj8hkqRBQ}ZJD0{LOMvt1f%6m8d-G;$9gMw5tvL=05KNovmu`f1 zrOnJh>5hYsg@33miksKNUXM@zv6#N=1n657$SoJuI@|=fqmT1D*A#Y##G%KN##OI}wtx@xiwk7s0 zdF&`(sqnhXbCis#(#;UP*pF+wg|)_ZG@1w*QsVT+SjT8AMth}8mMgV5*fvmz0D97V zI6UiLFE?+F+Mc8zhOQ=_R+3a%>bv)FXpXVg>~63uW8jgzOAVDoGr083HJOOV4~8_=PY(CaWe$+2msNaf=z&UkfCl-uZYfj-QijwF-0WS&w{ zprT}92NFSE2SVLHH~0ATvU<{DVJGSfotdwt3xB+$&XnF=WZFE5m(Qadn}N@c8P zxkq(D;+|Q_Daj@UkJ4a^q+wuRJDdwD_PU=#9R!95BgzRPVOe+2m@z`e_P_W6uBSU$ z`|n6+lDiunfc1Y6@E=|~1Tn&f-n^l4xo46OSm;0zxfMv_ZpamF=YTH~#UO%tpY<8w zdhlEd-X9J7A96yO>aBQddfwVoEdLq!5b{7jIX6eUh0%qW9tigc(#EF|sBB=PjSpMF z-9++q(qwQy_(;60|)%{1OgyRe+#h|j@bR8w~ANwQT`1g$qi+g8>6aVCP!)llmf^!WJ> zVJgw}ep=Q}4b#D(Hl~739py1ax=JvJAZvX5SL=|fi+?FL@ulW1J4+HDLNaj{&gPt1 z$=x2hcRN{^i{%)?IikB;!T2d>{;GL%>Lqm*g5u(FpP@9RTK56AW9AMrp)lOm5)+QV zF`CHa#a*;NiqjkX`0CPkYg&2Q761Vht$hj&?(#4t_0#@*0cGeHV|fTyQ00TmbI;HFwT2*pp)^Sr3mAL$$~G()BF&A z32P!lmMEU}2BgrtD|KyGtlH#kXkT#7aB}bb?456Q0RB|ENciP`OeIl-g^`~A?B4k8 z{`m9_5+57@Akm{z+Ks*jl2dR1Nr{=Aw$IgT6c16DlWkq0DJJID zRzmu{oK##Fq*4K#0#Q@uRJ9u&vhjD-uh{)gl#EB zLfPX>kzV6|I&VWk1bKv57M@WbkP*)%qXq}An6v<%61NLA6W+E8fSl?zF(v*R?4SwY z=rt;O3fhWoTuDr9Y)njCX^f3sS%K&OPFoq`UqE?B8}vOHgd;BUdA74+7nM*zcbsaf&o2$t;@ z)t=SN?9;INb0x1Y10dv{t-$!akY(Nt#%k5P}d_-dFa1<*(j5I1&W6*Q5FhVD7~vXM>ILu9!|q~N=7fifY3T)b0y%gv2vH|>|H7U6`zQmwK9$%L6@M{)rDNT;Wu}rl)03=rh5|1 ze!IaZ|8p}3%4>R@PV+fcq$q*zupndoSFrFlFr&|-&&*+So!z$#BQTdcUZnyE-fwhb zzkyc-u{uQ&jJu^Y*0k6o&1oKh?%EU8!O!lr7EYuGSv{KDo%HslCy_HS2L+(IhO;z*W8 zQ(1`0vauixv(U08y<_^X3V5E)gavFG5^cK~?%4?1FqS8i9f--S&}zMSACnVWC}90l zt7BS=&wgRW@VJ~enY00Ze@<4UVI>CA_8>Lso&wXfio#;6-|B_KHx*XzN5`#OUL)vI z32Nh*0Qn6Gt*3ACk_xKiia1TFs7B|{*99dbV8`g21ohWK414_YvpkAT6#3>jjk>_W z!UxkKo9Yu7jiwsU>HD1#4udoKW|7L64AJ21voWs~O^!wLg6<87i_$;IuLys~9@f)w zPw@&juKilrnHu!y@)D^11A|Ttl17VU85)n&M`UUVzOX+Ss~pIx@p~J{y=mH%Vt(+k z-{+6OsV)I3eF34)y{-FB>Q$BMIp!m(`~hQf7~@DkBKE)T)YtqE_$+({z+_9C_Ldxa zS_RFDFFg@Esfr>%v4xaJhjc@e6=xS`REW#@I*o|+o7h)sC*3dBD}45HUou(XqKP1l zSS1!8irRS|IYKv*RMyS=r{XtEc|zETi#r zgm_MGsdIf8a(k%{!?%^vrixf-;_(SE37@%@Jo( zkFQsNn46w4g&;h#VIHaRNnIbvDf-#@2Z(<#dXY89Ng*f9H!}0SBQLhP3n!@25p^RT3WfG7?IRmtu()cWPc^lS>u890LhY( zF(RAc=ls2DxgoLZd4(MoS_c*+RFs)FRqqJMPpE^5%Plo@xQ*L(kO-5%6ae0?m$=3_ zr~%oVu?o-l9@2O9gu7QH&wc>ffiWM!9+qv~}-67-QBFT-s z5@>BC>W*CaI|=Cf3;>zQg>t$vlDYtG`kMHwPnSS`N)cLUd>`;flV-mXfBDq`p0Skkw6KbB{Hw z=C^_fUd^R0Gjn(rJ+0Fb;5eV(>nn)g)qPDkk`_Rbl;w&VXMa@-uVA%Olqdj+7{Xv3 zH9S9K7Cn}o7d%2jbv3Cy`t8mv_IK7EFkp7>fJvIa)c1|=lX15U13CSz z0)EnQ`|E-2blPJDQ`8ro7JhEe z4+KGc*JE^L#Y*(@NH75GcoOZLR5Dn84zIHh>ZRQb{f|!18I(reFh@8tBFxxVZXhr8 zia9gxxVTw{e1M4E=Jn^`8kzzp;3)EMj9AnP;09Wwie`zm9wF=w(Tm?^m*!K$(8c>& zyhve}&XUfKzA|wv8Y8fvxpc5FZ znnLX0^bS-St{e8CpKqg8K0KaIZ3kM(Y52KCM2ln2#m{m;i2qS?)4FFsGJ?$x1WF%ODW znCHoBoF6^=wYsx&e=b%pM8i4>rY%L498-?_*ZPt`^NL}wF5ok!D1M;Zbyw=mfw_XfSw z@e<=Kk^PWuzL@g(rCUXykOSX{^a17(MIF<)k*{{OG`>XKqDZneLwkH)$%4)R>I`T) z5M%%g199-!wOzA$+DndV;3Rp$U~X@gFHSX=OQ;F4JIZbvE_?Nx!wVNZ%Cc@Vw?joc zN-o~_a_lN;tONGld+F4(|1?z)4_V=MGFo1tgK*wG0e~57$L2Sf8heGy-+B6$a}F6X z^S#_19_AvDN4t;?Wg(A995Tc?G{I1i=HnoIGVtA0&o;4yz5$#5kLy=2Y5G60hT)rU zG#^{4;kYw2Vomg9-MU3_vA zE8=V`bz*(w&`9E(FJTkP4lk{dApbE)iVvp(I2}5+ks_abwRR$qxcRtT4_l)_BFV3A z7`b}f)!Vx$H}Vca>=QvZ~8YWbAJPEi&T&iZv==4rmb!KtqvHFqf1N0w4x zR5^R!|CW7FP*Bn3$+;l7Iyt;E=f)Sa%J!k=l~PO;lo5onl(n&hL9CQ4#cM`Y6$YgP z7+YdN!Y4`zClP|hM0}x!f2FVO535YT%OLVF_6ykyI!eOK!LX7)Zw9W#K~a_P%%Njb zM&+jv`bL=w2QDHhR8S26f(SWcaZM(&iO7tEVq-C-o6j>AM!;&!&3wj2!9~{v<1>}i zFN6S7z6?7;`yjFz_Ltp4O4QgwLTOZ|kP!&yb$XDT`FL!Tf12*Fs{dMeOUWqfVtZ^$ zYi)a=`}qL-Av<45I*Q;*%CA8LDE$AG@Q>tcO+I+H@W6o-u7rY=)3Fww5vQn_t8Qcp z^ZoeSTMz~fE#P6ko3)<9$Pl&QUIZzix2wt;sbZlcvHE?2jV!_K!5(fr?>;?t)Hhv% zAIP}7GU($+goG~E!H=7})4I9yoZ$Oi_;6L`KKrpvzED_#66o;~01%zhMPL5nUMOiU z)2B@fK-^DW(Ub7NK&WaJSUCK=8EiS?_CZJZ=)MP3ZZdw(gGj*e^ZAh}d&VbJ9+{;F^L)gt~RDUZpyG9k|MxvGQ9^cmx0ah1_a=5I@n2 zCF8}@OG+PKx<*{GHyep9c8;1358m7vq@Z8kD$!nunyFT=TWpq2>OOK4VTXj2!D0F0 zA9XUQ~lFfV(og| zLHTdg=qMNgC_VZm+uEVuQ17pv6s%+s@HEJwqwE*-pvXRW@RSjmROpqpKar;d`z5YU zNkpkANj_<;0FM7PjGM(9NUAuYjsi4Jucuck^(2zLJ%YX|9;_4A4OHsu)eF(Z&||M! zd2F+0swNkpO;WR|T9#;~%lI@S=~FV`l+OC=|AfIZc~AJ>7C;b0?xyU@c)Vjm@Ego< z-SQd0hp=LcDkXor1{_bVZ+&^CZvFek6}ELQPj4=^1fhY0l4e#`rX*vN-#A};!!-8-U0Z_Xz#>~ zQ_gCTu6%n(=^jpN=o1WN?DUg(@@(KGezRm`Jr@IY6Xr2y14ILYE9`vEW6{=@awr;s@H>NU*OW}~WV(N*|80SeP1 zgDP@8|3q)-RHuaM9fpYfKmGwqGgvTkE4aoR6ruEC8gmhRue7OzECn zb8qy>*kO>O*RC=|6v=-qChd3UWR6%J0PM^YIF&9+6$0 zT^Nu2b&<!j|qkCPPne)A61 zgZ((Y7A8tcf?wWpFE{&icWA!zB3S}e0rOuMrF=(c?LG*cgDt=SKJ%Yn_$=^6?)-f| zX#S^5mhmr6viEY?+HSG{*G~W0O&d)}&8v5Ljh;*`*tsR~N-Y7@Usuh|s?9%S){UL6 z7a-g{1LR+z{$%`u|Jm(Lp#NU624+e!p0t8Zir?~?d24<7+F?LSTAdUH)M1PcV$M<> zZNEZtGkCrWS*v;KsWgGd0^7wWc2o3k4^`zak zg>}32*#rm;`*Bw{zU)Nl zp(3rkWQp0Dt0fY;II!H1I#^pIL}`CXn>sMEcty!US%-?Pq@Y=qH${+4X|n>AI%Wbn zX(%y_;rK$j0Y{B8G5kR6RGbdi3~h4gWoySV%7uqciw-M(Cc(NyU}2d#fmX?VW_z<| z;>#y)8Gkn#9-j{2$?pB_H`UD)cEs$}{CT%FM?^a6H5{AFZqztfg*TKBhUB7r)*~S7M*(*)W~aia!evb z%lIG&hS6=n@bUu)d5x!m9xDhLqay*~Z&M9~}V4YKa*2@(^B`IKt@Z|(29ae;n z@`6A!PW)=2L`{EGg`#=9g>KB)+8)T_Dbp5U6@Kpf=i%A&ryCtd`6!xM_Q;KCdrh-G zCyvGA5HMFxqklixd-w~Tmnxgt<0rDWI(YdOj=S$I=)w$4Ed6wpGo|E zKI%b!^SO{gYErig6@TwEmE{z3A*rhkn^-*61ndPK z-j^~zH&>$8j-p$ePxHN+h`nox`RAu%FRc100A|DNlLPyCe|T1b3#69g=A%vBvaZH= z>`wNP6?<+l69{bXg86;7x?o3_?yy(qoi_$x`9S3zMn(y}g0G?_yGx_i8nO)><3+V- zqpj;%%zs6`vMQ-Y+nd1e$jXk7Q>vR|n|5-{!@Tt3F#Q|K@XNsv3NPM$|JJ>=?P)fy zWz!n8fKA(q%q6_KDz~RmS#7SX-OHt(xq#|e;Ld-3{cLh~ZVlgF-%_#PA}cE?E6q_F z_GKMBxUaE^!!ffIn^E9Gj;W>=qHCv4p=)1#g?|D!^WZi#RuLb+7JGBd$~^2HccJH# zk5L^DQt8d>IZ-6E`N&nQt4|<8C@z961S8)G4<Q1W zLl*a8jiHyxW*^xj4_1ANs9BE2l%Hg)(+?Nl{ft7tC8=MovWmu``!{_{I z`G3$h;)6iUe+j-`cVQ})^)s!|HNhDs%%+(S@GCBZmxkeaefYp@sEs}J2{N|ujQ0lN zXT|Y^&KPatwrK6)q@Lqlj-#CX;lCN>E%<@Ac<{3c&J=oG>7IT0JhsCS6u=fq?Otp3 z*Q}FP87147AWIsrrtK}J{DZuS9`e99B!6&sz)2gERVa>)Qz1(-Kf*)W(*6i_v)zx7 zv7LANz&84)dk}-WB$l#@l2T>fMwG#kMP1*Z9=7A#;jr;qnH^3Mi;LR+nDnw;S$>il{k`6w+q2-Ee zL*Cha$k@xvRZmkV9+VA}Hn36Ms$&^~k8M8A@*{ohIwKV_Pg|WX3N35q)jHo=FPp&YT?MmUinTFP( zN0PifmHGSG{gpMfdwA#=93?TkT$ftf`q5$ba6dA(5qJCv65AZxoJXB$j(-s{Y#n(% zLo-(&PaUB|U-Wy}ZQ5Hwi95M3b_Uh0VP{8$Z8GpC7$s#+;3ZF+>Gl7K9?Vq`-G1-m zBle6XzbD8Df&TaVBYbW9g?}!xT7N~>6=Ctq61ZV+zWM*Xx%q$5^`qRL!ExQgx*GyM z;=yM4Mgj4dgK0Por{l?ZI)9GCaiGFJkV5`c-~(|S_SvVHk@v!eKrSIIQoV!?3GMtK zhTn9v_-#EsTV?ClTLTiW>}Yi(iEm2#r7N{7hR(QV)<7TQ3k0M`F| zJP$8g_#B>qC-%*Sd9diK@4<`1o;8XW5Rp*@o`Asp4pE`TZtj@U!YZh_u3KR|szk`3j)se1R8pLvu#pEBy_j>);g{Ui#*XpLpyuVKrKW z{W;9TIpC!PAB%%rMZj-5VX8xrId$D${iF(&;K_^={s!p^;QYSEZ20=j#J9QkXy(B$ z;a~mx3PjQwttZkxWPf4uqrbs49(zu}Q&+5CqgLBbCvi|kGtZnka<2X;d-+m0_9m7e z;vI&^yE8>V^@#-!T&So(maGO2yMdb;R>DdpY#}u#C$%XhAjX9via#^6b1mG28}=0zv~J2!Brj8D~#5qI|X_7v+{B zV`Dba73}83EAdJt-mnorkKM3@%gr^JN|m`bw8v^eO$RxWbNVV#xnQOILaxK$gd+~< z9v%xAa0j0@@L?JU|J0}0Us#@#Wh%-saT`jK3_5mA(WiZdeDS%W&oJeX0o(1k@xJnh zDR*%gHzZLHNPn65CtbMTRkWX<6zvF%rjK>yrrPv>Rl5bJ}(5 zm1@Jw*ne#S<~v|oA=h9+<(aGjmFamLPG_N_x+=5TTEo@WTKBfL$~IrTlS5D#aAjbC z)N>VfUKI;7fmfVKb!0Yh+QSKz>uO|cYG1ECqrChzY65ZPsx%9i zW3DFS^{~5N4trZ{La#TD$af@5hK z2kFvUO zPk)*Z?SzO-uOC6A>4h=9%mcv##wpHeqUhDzlapRYpEh%F7<`@B7a}yYX~w1*Gsf=7SZSrQv|N zfqgRO4^L1}H6hZa2~UQ6H=1$az^CY4PcPl_+LC3Zt>-z$8~#Bv?Z3R6Pf!0e`BBSf z=(dwHzbIvvk@2P8Ak1=#ge$^SS3Ge>q-FoZLUy#-ONtOOmXN8+9oJw59G0ASN`H;| zRVal`UW+WLy!vArRt|YdvaFVpX12cCa=^;NV)&<|uBxK?V^-cVa`a&P$kE9;By?>K zvc!>r$NDbHA*Yw-^HcC%Ohb?419EtJI62&xSieM<7w*ZaD3FyG*QA#yOLLdvwU~m& z;Z^z|vMO;CvTWzcLYO-=UXoo^QGaA-Yi;O2H9r6i1Grw2VXY;YwU<{mRPits9!vID zl~-LPDeXqx!e}B$!eX(o9yXF8$fqzP0(-FmQs@w*60*-yscg(gSseavX&zoi;Pzdm z{RD{F`%>IFL`w?tkg-UaX+;M(VYF24`W0oIzwy}Z zl=E2E#~T~+FPEUy%ncydo0$fa*MUXbByreGlMOPe*wLY zSWB0&Z~+sOP1>7(FgZRx3UhRFWnpa!c-p0U2UHa2`u;4->|jG>T%$8)X05UJn#3sB zyJA5_fZ78hg*&=jp!-f*VDGs3GE+*YhNM!eGC(&^%mZ3hW6z_&mqG9hHdUS%yNM4 zIUKXI!9-XOtPeH-dmnp@{VtM-dj4@EN3ReM`4TUAaHL;k?BwwiCr+Ig5*i!s=i?hG zn>=yil(Ck7_gONh@v^1vo&h1z5dnTO_h2vC(($qt<7F#DqJ^V=vXLRdG7q`0d!V;0 z#9OvbzDcHtkcUUee8NK%p%J6T%hvh&MaZHB~t`@$-`xlzH-^3 z^=sG37KQ{z%9i_i%7Y{1vT@^NvIx0c<{KFq`r(9s36Tn)@gdguq|7@cT;?T@boUF482=Y&-yT>Q5+39phzVabHV7Mn z4aG)dW3aKz*_7V08HV2!FIbu%OJZwI;09%MH!WLso zv1QnQa%=^*8e4-oV{5T>*m`UOwh7yeZN*$LSIiA_$2>4k%nOra-k2}uhxuaxSRfXJ z1!EytC>Dl=V-Z*+7KO!NJFs{xK{P-_i+&ReK@+}~VaqJxv&EuCgeb$t!A50sU;Lif zS$y1ff$d>jhSy4Dk~Cr^QEfM^+t6;ux=-(azN~v;_ZK}pdYtR&-^;F-cds)0$@X`9 zZ|Qxt&)`1EeJ=H#-S?COaGI$+T7L7%^$#LQtnV~4W0WkE7H*lF;_!6mT4z9SiMa5Kkmixzdb_vd1N ziB<7g4N4XTcHnqHnl>wrj?<{((^%x$`BUsB&zD;G^56XTsMKYr<7ZfAzr(2D%V`tzfTDGqcxgbMaPs)Dk7bqj~l8(X(EYS#M&KW`SmTklkFr@Qtn%sr>M9_~o`k`wo{eq?2cQjr(D`;$q=9 zEc)oUl+=%%=eK=}N8_>6t@q%pOvb)rlK?@R3GU?_AbSs`{)?o(rk&rhZH%{NcR-t>+`5R{qV^Cj-i#Dr)h1GC1LE1iT9lw~h=T9n4 z7hB z;WRkoZtKaM%Q~M%)nx8mzc!RRWH2~ z(cFQ#qQt5PRDm6o$k$?L(B!pCd$^cx4h!a8P{?TVJ z;hDH&BW#yKXGFtEXwJKn$3_A_h724DSF<1VN8*@OK3ka3Xy-K}=n-8x!tWuhY4l8T z>8fJ>ad!XeoN2QuYqlu4GaNiE(r>LB1()<(+ts{S0YASLT4%l|@<#eYf_6i74$ zyH&5JQTHLEkQ9-BPeww?m4mzM&Fm6_oC~G4u|TWb!H|2m25nck(c68h_e8K^Et<0} zRI}ki-9d({*wb)|ZfxDTon!NM0Bx+0GP8y1xxn4$;0(dZ*UW=RNC;GGxV z0#@-BurDQDNfT57olK(Idoa1}b=IU9(vr2R6fHB%cM?dVH+$}Cj%0(5fVL(I)=NxA zQ1NvAW7HSfO&BJ;Q@@Ue1hhX0#bxXgI8cg`Byd#bx8~jSi(Eu|38|@3)OPeXi&~FW z9D|x;VtyNc{8-wgJScBvVk+Wtm+iCrqWS2FHtNyjvlBn~&~bdMVSB3!lb6^WZr)+H zCV1M~k<{Sli+_dqPxpRf=ELVwXRocRd>Pc|4!&)ddUpwR@zd7pH?C})$5eDmrH*qq z&YCvw(!F!_SH2UV=sg>+jX4~&Qb2gHfbd}s9uoS0)1Y?G_S@~@W;oPLHuJynTcuW* z4*eA-38^@NfHSD`9z zNUf(~?`sbWGBp{gEZGeM8jz-8lB2goFQ!R<8_aWCR)q_jVpIb_k$Xlafs{BF9VKEx zN@144n3>KL=QW@DhCX557Msl$dmFqqQ@&tTggfIE;T^V(c5*&>eg{j=ovSk%b$N6{ ziMgt3huJ^gH+36JRt%VSb+j8Ju#n?B8>!U_-?p;N40{3@f<(g z+Lfobi#sAAhdNZH`-ZdB0Y7cTt7)g(7Z zu?ah4{e7vx0y!7WtlAOa=1I+ax(?dI?!O$r)fj(#Q!P`wVGUSGhr0oNDjSf09>vvD z>7cTtoC(?6qA8<)yn6L!b6j(HCA+(+DWjAo-&2AtVAL^6b(%7f+EI{LoLiPv$)NcN zlZNU7c7`*FDUl(b^i=SyvPNHDWM;`L%KYrg+I{r*;DbmO+I04iuAjeo)QE-0PafLS ze89+@m)wW8U>e&+k1la=cUNzJJ0ZaD)}MnS>^bFTkK5?uL{wyIM|Cv3Mye(dT+z4$gq;9bi z9Ax2g;+N}Zu3XMr@64e1Kb?cy!~1a6IZ+-g6~j8XN_q~Jt|9E<@2>~npms65-=7O3 zKGwWO$e4GVskXTPvEaVf$5!{NiQC54?rW4EGhr6rZVSHcy>0bPnA?`d6W9txFCD)7 z?ZY628JA>S_IDdu;j24`Mau{Ldru}a=yYZ-jyYR{1V@!@8UB@8lRd-AkpX_25H zj44VlFRY=eW6MKAVxz-0o(Vbo)uk&B8M0r;5Zk}H)KT^a@*R-^Nb9504+OC&4*m!i zL&xmwlgGc&59A1cO;`X3rG1=%(T@!z_USSslc~6*q?A+!wV@L@y#D`{sPhPcb=klr zr~=PS&Phn5g1kVqh5t8N_`f}G?wTZFmSla|_Ep>7Hu4^yEm-TV04L=vBH2j3`RCIiQqQ&gN$R6h-39|9s`7AO* zo45J=g1B-Tzd5*dZ7dbDLmgxJ95do|bvkXGP~^B!B3eQAxO()7CH2><#GB9C^Q`kS_YBj3S(AMcpAopy3@TM@$kN|4{6 z!GCblivQSv$fNIWLH}EuvUKjQx~S!6)e3F9tG>HwP&cD+^B2G+lp1IeJ|dT?n$uSpFzc0}j7n zx|;KUpKv1U{&Q!33(M925th3D=yI8&x?`rM3#f5wC{i24B*!hAU5^LyXJq4QCWW!5Xr{6s6q-gVEy zX#AzH-v=*IFVuH55(#x7{bda727CSdL?A$aGqa^WV7RvZdt!w-Oq zP1Ig+s_Fo9sOE6RKKj6ZgJx0IFbg$_lb8TY7y2ITqxXGpoflROvZoqe<{TPdfcwT5aP13! z^0y9KUL}1p@(1)jgI@ksc)h!kCBxYS>SGLplZNO}e5dR~r z9}_u5Tt@5quir}e{;1c28IGRv@|IA4MiB1$atcNeWWVEeN9&H$H!e;=xM0IT6dUzw zXSC?)IWhberAm#!u%m=QeW61DTm#2Z&}k2Jg1up1rXU{}j8vA|s7ji&2$8FqO~@t@ ziDyu=^H8tT2xn053-_1i(wXVTtUp~C#tBz$g+}QuCG52h;({(&HSNvc~;Q%ath7y78q9_Y@AAAnU^UQH*lMlt;QdT|QCB{$wRf`Mh zyCMp8#?YCsJ{d<=+=E(3)u6n>J#gP8Ji zP*%vm23SuNmVt7?1@r6pjJ9rn7K*Qm`6{$Q`lk1uRS@!tcx{5m<&S7&H)J&GkNQ3y z0qw4}?5{JiXwjeIBrx_n(ty|yMFbO{5D@A@JFRRNgkz6u_rAQ-I(36^t3dGbc@#YS zaO=a*O%7)sRlMBt()p3^J#rkzPvJk4p7XqkMj>p)qNNNuXs%yne5GZ7i*?XlaP1^_ zuBerUBCyDdCEb*(<6>iDnI-cS6IN2&1(l-8F912EOd|mu9$#;)X2|5ZHRTmetrU`V z7QPF@NP*vQ@T3Wz=qd&9qMtok))D%HG!x_{C_!?nj+w9WjSZ)_q#XY-o=vzDdu2<* zy0U?ft8QJqMg0>=zC|B@FzB;Cl?xgCwqTmBTSmcktDbyQFqx1bid%tSBfG~31TLnQ z0?!sXLmu~OIj!17-zvFV%OV9oMY=HJLo|{>Vwf+f1o$>CJ+mTEM|M+(iz4SSvbn<+Cer`4us*hAXAdmfW3D&Y*rAzM{zM)H&+f zsf)MZM)@^BMno zuWa{hyR~_X3(irE)h+j-O~#}^1EWvQ6LhcD;-9Ssr)fcdVgjR4Cnc*jcC~U3u#-mi z^2KX?i$iJ@Y<*a*>Wg?g@`g!$H*Oxa$=7rC44<>##V`dLU3$`MO9xmj&Cu)1${2$I z7)s0R+z%YoRnU+)^w}5e+wIxk?%j-n;;Perx&F;oEyJh>OqM>+key{00%k9BWKvTs zO~kTV%lqPg4z#n(B3%@J|7N|=ljc7X^RXQRq)076_S%9nH5)GRM;^FYVl2+NT##>< znRN>k(9lyN2;#~3Bt-iob~u#;wjVkK4(2k)iPOf@*{7KF%EJ2xE*dSc4&P79Up`|AUpYO&`oISlmCgti@Y~~ZH;EC2qEkHG##&MRr#5iot|KO zVAW1j@vfY`7hI3bMEwwrq$muPL%aDuvgBzf{^=iZ06R%9-3m7E@L^U3ZA*)!mj_)a zQn9gz!JSg-R{gUF#fQKOjw$s9o3>c&c-?=`@b8KFRUN_7yyP5ZygDgK!=Qis%Lz|Q z0BL}KQsyM*6{hFr8X5SHS5A0_5NTjp;tHj-G74_sL``Ox-43(O{RVY7@|&bFmR%8) z#2|%$6`WRB)BwlOrPV;2%T}m?GLjO?-4DbMVLUCHiRa)DedvXCGvP@L0foCQtg{>x zmZmd?yc$qO!!bqvXj-9VTqw=(qiC4I!R66^4!>LtH$%!yzJlMuKGL({>j=jnx%Xo5 zu!Mme9Lk}g78Wx#T%wy5Bpao+QxGKg9jcR{-ajTbU9Y`j!-vnU+nt+_Z^DJY{>(WX`}N7M;34^nUk{xGiI5hQ>YB9E zE+usdP|#?x!sNBbfU%@8zok01+BZcDbQS1Qx5{_)Fa3uJyyL_7~h+Cfi+udc0+_ z=ox{AjQ(h>3+ir$&J5r8@6}u{0u6?CNVq5@9{dO;C5*8gb&=By_2Cl?*%N+8kS$%G-U6D*EE-MB1{2}f zGY4uf)iIL@a<1SQVyZ=!cWL$S$zgVIyb{+Lby>!Do_q4E^fd2$7P@qH z;K@c~QUP6P$SM#_oW*BQJ)s=DEI= zE5_a6H>Jn8f<^)3H;S*S!&%(n-=B*iHb!M|Q7rO-U%>a_HdFoX`gY4I1c^1c=T>0! zv@SwSibyI+t4&ohNP=b~DO`_hjcM8GRKBTXzutnRuagDI{(@y?#Obos5Mll=M;w}o z((vf`plu`QF2&w|eA-Hkh;8~EHKkOJF+D4ffkU7-oQzU%gW9OpP>G5iZaV8)uPP3V7#H4(^u0!S=0zQS7A&oD56Yt4Hv&)zWxBuRu;smsO03Gp*n`t zbgfNl_Ug2|tPI=TRQfG2@vfXh*|+4uzm^fFE8Y=KCF!(jiOdM36WE^#SLbTe)2Zra zQ1*Yy=tvoV@D+-7bf+kMHNH*}vTzwq-b{-Gr7f`!i0nF9F%=Z(-D&I}a4~MHP&z`y`N4X+98g-XIj_--*_*$T%E==2d~WtrN<(>jR<3rcMEhswJNz5mP85`shBp&ikFDF5n&vdYThUFX-e zI*vy@HX#x7-+Y`*9Lh@YU{U42t+U>kq~;V>lYH99$A2>DUxbeeSL^f!HCq_q0hII* zBw9Q4zvAihZ=d1*{9GyWw>Z-0$FC24$&g-uIW-`;mQ7a2C52J3**i;Af=HTdRiO`r zSHIAAN>R9}|NnH%{?$&HZ4!Z3apG5TFk5Q+SDb-q6=lNfsN_oWvi4!XUe zy5)trnVF2bvU=kDG(%pMs!|sf|<;C~1 z21}Wjv2Xzsll=F*e>Bi2MPt%ss#<~|Dob1@6B8FuM2X@8QItjjK|qv7c5p*ByZbfG zz9}jwxPU^GB~oeNDQnl#`aoV7_L&LPQ^vUVkxrr;|lGEa}-ribm zT3nnqAw4~1vafG?M!b(cHQsj>S(E0w=EDwR%UQrSuZFL`-Vo;INE zC_RCAjpD~a;e(DSJ(VV9zg;*TKsV7hn9)o;Q^QVW&)Sc)H{0JD>_7O@klZ0(ys*h( zgu_yYR)MRtV$pb1rzx#H0gf~@GUu=dJx<414*!69 z_|99%&@SR@P^iY)3^342tK`!~9cv|+e*nBYocF?Dhy{kz$tU)xcW*+Qgb*{T+pU2l z_V&zAu-782+B#uHd-mSew#M!Q*$sw&5}|xwo4J<%yghoFn8?mF;EH@aO2g%ig`qhY zzm&SVn*?t98Wj19aE#eii}r{(5?D2NaIEj9)bN$rYq&mGvyy^3e`xkQCqv z!{H>jDXm2iqP{ojGyFo80MyP;&Rx@dKk-XEN$8)Y1FHWvrgU;F1C#%C>w$=W1K@^w zYQ(%T0``Lj2xj_rycsF(#b2s7BkY3R1w0py!eVBZgrW|K2cuDFo-v15n_KjpPOA1x7eXEuNSo#wO!sE2CNbM$HtyzI%q1-%L24&|s>;bX8Vd`AISc3$^Jeb|;&BP~#$GrYM+13>5||G( zH~}lF*{Rd{Ft{KXRu98og<;$sITZ%0>o=k;IVR(#VqxyOi0p74iwHb>Pa|*z zb$f%sP$zO>IqQv?`I=`Hxryz|7YbY>4)Vt=w#P5s9S1(}w*z2bA*vpKGf0we51>0j z9uA&L>xb~28dC5WP~&Anf53A{XE3l7#*qA0i`dhw#){30la>oX5fj#j@YqAX4Ia#= zUuOp%7u&-tgM0{zoQ*=VvI7=Mx>L5TKp7!Xu>8PPV&@g2Bcl8(%k|*E0$RtOuRyUP zUcw$(x}K&8^}xmYWU=J8o~zrkJp4JXRraymSnmY&jx! z@0Y4u`7@nK$xWilY9M@Rm!Mj5COdM2NfLNU8zklgjMKM#Z_t;`K#BMCPKTU>=V&!AtLnaEfWFMtfU}t}Zw zV-}AmPISk^u;!;RFbT%@JZccffkurBn35yr?#6@moh_ZU`*|2#f`=!VMHT!)HuX?~ zlnw>yMCvv;DdjnTvQ^z3e{_l$rlK(|cud!Qjvvm_vEXc>4_`or4pEv24}q1iV;>8H zzNl(0D?`Q#VMAU)cD5#}J-g*VTVt1?lCR*%4j6ZD=DoCg>pH;h-imwi9r0bxPf=hfN88jM){_mM-cOidJW8Hk2;w zOcFL30y5`n!cR>H9RyxKd+9*I$FmO!+=7&Sq4UvLeqAIo7mF(@@(%CNRF`fmuM{|k zl(gRWjLkf_pSpfYN{)LcR$5k zCC`4|NN!5m(^bL@=?O++4KZFQl8^L4mKae73~45RM#oyZ9-v;n_W?3=k@{XC_3a~R z{S*A0zXCt30*Y&fBd*ds*^}kY-~d}HX9rs3ye*!}LUS)(l>5{@3GGo)2?nnsoR^0(c${hrdXC|yXP8WSgTUn((u_#Yr|QTl%i%NV2+tY1-?yP7rLtUj zPOiBmM@+Vu%uO0uGPBU!|A6M2v#M%~YxrG%mD{RHMQD>TZ64T&?H(O>P#4eTb=|eK zg6i?2rwBhOeDsq)ml53Y$8S6-|HJSucfTb|Bi9LcvV1LZ{KQ4e=O&3jvD^(j{r3rV z{x3}U_W1cjO<7Z)GvR++q6awmhdgWq-2W#6E?yMAPOwG2_P?Y4m4DV>^SZ4}=YVK` z`GdF8+6mvQVGSMyZa5X^U=feTtME$D;#VLGazTJm5G`QuZR)_Sg^T(*o#NrQN~Z>C$jl(>#0`u;r~ntGd<2r! zpz-a)ui?M@jlwVCp|?EY4S4h0K5!C$uuF}4BEl5rFo%Hp_9%L<;gImo&hEzJe0TTy zgeI{u5rst%!PrZ{sqBK|Pt*tw+jbErZ&t&g(cj?7LjR;^OeMzU0WmH;qq%e+34M*xPj`nu^B~p@!qYGcX2Cn_+8zyjX&f9dMKAK~kslBVZ7?V49SF!3kVA zdrH0!X=-@M?=E(qzIaVex^SH3s;K9*j}!aNe=0)=%LU{m^t`*Ji-$2ke+Qz7U0m_V zz$L*Ob?D@BA+HXZcWNMkFyd#T zt}mZ(`G&h={qYBN6Vdrh)0RztCP7!6oUxXlpMSZ#STxoYTr`z!ycE_NwcQ@<54`uu zrUe;<`R)e;N?5EMsMdpu8!d-_$ad-v-rm0? zaOKju=>pI!cNI_m3uAwmq8aR^W>EbkpzPdaO-x(X{{3x@on%~bc@jkR$ZU_|!hN`S zUnyUP6>3mUzTvl1z@u8}l~tjKKGO8;JO9&{eTjI)ZXvA{85{XMsBE_gZw0AX0uWLj;fN9ib|okmtOYiogz}$*UtmpM=)4c)RxpC^KH(DFT>`` zT(&#`wCbL$_Jn;x?Ea+kDRuU%DuQby&G|cM=jVr$!^(sWW^?g>(>%W%NOP+ZWycD2 z>k`UW@@ugV!oyeYO4*ewb|xGyy0+22qv-QZ3pCSxQO3x1S9>-K^~J?#Kn^{>b8{&veRQ9x~vveAkuYRWa3|!6x|XO63Hkal3|o*sWWZPXA+Z% zi)e`fx~lpHxbxTguVZh*t&_wVoaN7wqv0fBm$!WV3l?6fegiYrB%4s;kN!`B&@AVN z!003xB60U+XBevfKo`yiC8g;Xq6@nP*iRm1um^e zj&G>8G**i+8DzS?ytLv1N&m{gt(}`;1NI01P*^<@RwG#L2L4>2wf9M^^tGlmeylB4 z)p1jv!BAu}h;FzFGjNbTO@Bf$20A*r#JpbAon;%BA_LNFwXG4}#9=fy+YM@HF3Psf z$$%BMyKq;}GT_}mgwCxf-O|iAmsU4ah@9yW)IP$se>w^7@QCL8>&i+i&+-I^0n@8! zlFupSuUw^>EPfLrNNHT@`nvLpuX%~}HMrX%)YlpdYsIZa$hdXuYk%G^dI3v!k^~$L zco4aKnI(p5E*_3O>FnHi6v6A*?=26>1ZQ)y%gPZV{Vv`I1q{NjbS!yhcCdN5U$Cx2 z{*VL*mRMN!c_Gr)J}U&0&}L2XX7WH~B30P^UsagjKQcXnj>t#~n#pf8nk@xlRq5{f zcFn=GJ&F3{?AUNA=6~F^56_?3D?pMQN!M3cZ1s^VQKxK2bX5*e$}zB<_%@IH^&$90 zei8!*;5WLmrlh1w)3_1k2>nWCGoC?u^I>^w$3y-b2DkN7bbcN(=4&$Q&`v=f%BXBV z9SxOv2alI2iu#-Fx1d$dc{TEaDp=lFUBd`N?85X8gOS4v(j^dJUN~s6n9U`n=8B3# z=NufQ^6t9w8cStqNsYz9@&5rg%fG3Yv2Xzsw|WQxdk%jxGd?~Fb98cLVQmU{+Lc)e zSQF>Eo){(rim1fUnz1v%s;vuxDB5~J#eJ`JJ6Z)rKtNeTKtPrN0h0Vl*u$n^B!EOk z1rJ5*QmwUDtqZjawXKyNtGAoAp0?l6Kj%L84?%BRd++mH$TQ)Y%*_9N@B4l4_x?mg zxwudwk;s2*#mY4aabfZE!xEFH`uVJhj7o_OQ#w9bB#ahbK<=EtQ35X#6oUkBu`tF3 zk9QY$xVzXjxN|i9#cy}%7br?(9_Bd02pHl%hWz#XN%ygi6F<6-qg)43B5D{lk{V5U zQ;$Tn$W!$E9~TRbNZ1~^G$Jxy6{AY`pW^2?eSUvJV!ARWDq5xR_w$=J*>QYTF>i`u zSy=e4gx$%zViaNV5sGC~6f37FRwe8vXJZsEC&VkZM@EOm?ocG`P^^pGs7OhUR3g1ifEN8ah9*IDkW-4f-=f?2e~HMH`aMkvajPlpGB{&TIaK3>HLMO z)-If)N>eFzBq$XTk*ctm*yJgH3Gp9Ju1ZkGg~d|jhb1_Tp5lMFg1@_L@lP4P|K+0)Cy`PwVHo=omxw+qc&4psclpkwVjHlVyInIEEPv3 zQc0AON~Tm)s+f>Ketak&2OTa{76qc+qDO~;s9=ykU!*&l}HoS8LKcd`|@>F@=m)b_g%DjJN zUwK8qVUNBG$Oc<%A|V@E#C@e+wsdBBZbeHjk2Cuu^gfNgID_4zE66V7agzO;Jk_31 zfBmRf#<0J`*ujFUmbNh)RZWeh$XHUvzXe`2)Jra6AG#zP>9q3rwJ3>;WlTrA(Qy{D zP$k1!33$LA+C%7rAS-;mTXY2cU=@FC6$_tuS(&wYC2iR}o-3@T>k0}>)ohk??Ew2! zbz(;73f(9rW3(JJSlByS+RB6(wMQ!C<;ZyC9tYtP%UZg8cXdpGJRu5Ya7yME6Mdvz zr*Gu(R@flJuSvdv0@|amillta0_`>t1l)nozZdtdg5|O=XZ7G=)!g^{KD~e3Bft0Z zXIReJJz^UrJw|mDZ*OFxbn4k$a;z{>0^E1^zTvSad?{PFItbGo_5;1-u(9kE2csoF zop^OE--PeTf?o?%aF~;9(Ul$Mp$S5m$7lQ(@E$@Bvcet5~y3o|Y!)lnw+-;r{#(VTNr`Y;@Y@%C_g-WZq zF9fn=m8dLRjS6!093GdC-TsDM*lBq6IXDb&LD=z|zy9k^TSaGk1rMuO zI0g%46;GG+)&#sZlih?THA4a@!MCyN!@GaS(t2KFG!#^_)u^PNhajd7mFE`db-DRm z#75eZ`o@X5ES`jAcnE);a zA71p=bb3f&0gQu}VKx7f*ZB1LTh?!&<&+B;Ksr*Joc}+JHmkXFW?F<@UMS(y_y-8x{FKMvu%?v zJKIT=;KB6s*dyuf+>&0yTM{f`Yt3K4%+&(2%uR`;zMD@d$btMs9o)anMuvO_bHH{+2krUuGvWjFIt)$S!^ z+09DHlAPVa+(LNOS(!gcq{k_6ZHmSsGg&wdf@u5ePv8*`z$1h(QcCU77lLdFEpW;P zCvBc2-SB(hYlX>H>Bc^9VY941=;%pSN+nX66O&OA@NHR-6?LRq50kXx9k2eh$+Ev%N8Xh_5;X9w ztUv5Ue-4}UqX~vNv`G4JZYDazL*RAhBXjCfClE{3slnV5nC}GQTL}z0xC^^fl2Uz; zVuLK3FwTGGN!oGUy*>A&3;LR%P-bJWR7-F7N2Vw~wvCZ?eTTAQ_~i^9ihWUzA6|0~ zj=)h^clwK8etH%A9^j-d&i0gc{bhr;#;(5P&Em~?7M_7aa0tADXTsLoy-k}67 za5Gzh%4>O;NAP>oj4pE?eX=Yo%(230tLNEX=A zUfIBWQmhW=<4BAulpL<&vQuI-@$4!*_Atbdj-J$f`ok}O*U!?Q|FWY`?ATyg$Knat z3lGLVcmj;T9^lvZQOSjNKC7ZN_DO&=(_UP9jw6GEkd$GCNUO*}61p{Vv~Zn_jwOYq zCwPBB!kj26z0O$~OldJ?qLoZPUnKW@nDN)2J?&5iC=*Z)gq=flAkn%;JKeEu+qRRA zZCf3?rDxMT~Eg3r*)^OXKoUmKGD)fo*{yR7fQSmDhMH5Cp%#?@4ITg=%=GpIk4c8 zA=CG`Cb2f;`R#<6$p09^NgDPyX%JWtSym^o-RGQ~=(P&!MFwu zgTX0GM8WqOOcL^(`HBaPFhb%|oIZ8A?Z9Z#f5zzAlQ+*AK2tolK(jK?g9E488P9Yk z1+#E_AGyG%i7VHeVXp8g;1|cZwGpJ~#-y93o0Lo@FTHe-5bKm=~w6}{W7G|>wT3dtjJnj9m5r3+j0ycQwoZTqn{Of$-Ah$*AP(<@QIqp zS$yT3Egth(c;!*?P!2syDd;&kdNl^?FcOr)I+US%{Tqj9A$y`r>%&gbn{x9^H;P0N zw}?e&Ukes4fboUCZ7OYMb_OoNmhK6(is8V=mo`$;<+u^Qv1L-}P;{4mE2`%_EfM{`_2(UgB>Bq#$y)J`wi{OrQj}H`j&cSEIvHk2*3O_80Wt6V=1ALwAh#< z)V@hyeAO7V3-_0QiI2=Pl20r3$8`7NbRq}-7;GTx@F#SgbPYe0MX3@o-X$&YVuyL+ zgSN_;qcCrkUqseK@xVcsbWth?q+Ydz zf0)0tK+=S1E<91;S{@V+{@ZiO?vcW~Zk~+BzXCnlxz+8CAtdH*LB?2a!13mOja@X! z;6lHk$WUmz=Vc=$uYSoMC8hwm$j#tM7e_(##b{Yx=hja??Re(26cEH>%R#inVjzL5 zWNAd;1@Wj&th?x|xw|)5;4RS1?cDMqa~kKDf(3P87T9iYp3nf`eL zxXO?+#BbWIbxq7V0mH8`{iugMe$X?0tqk}du*PQF-){$YG-ZXO`*C`KhwvbVTpe(f z&aq)xe3`j_jPQJ> zm_qiOdu<|7@W#db@ViEh4P?jm3T`$eL}u`A9H5w61p8|o-RM6cVOEH@KH(m_?e-6BnjS_D-lXk(Q&cs_DEuhOxu{t+!&?cBE$*-}Ral z{)zWYpRBlmfU|Hfz-Ra_ut^ECH&dX|b6W578NN`5QXdG+=O$^@9o`(ZG8U-#$`2a$ z476Y#^tXwg99JlfXT_nx{`WtxFBto8&9-|W1)>xIvQz?X?7nmBXcqx83#pg)Pc?t) zhQ-%TV%sEgsTxDqBh?I;=l6F1v^_|6xbu5D)m`x~EEdwIA^y!{^x#rj_% zFYfv*OXPreWz<%Ve|*c>a{m3#I-S?<=n48q%ZmeW{2Jei`p~|YzY!VW5u|FyrlwH8 zsIA1_{>EHdqLIo;)IoJvGggQ?pHZ*(P@}IJ6rDFrZtV(9;^KR!W_Jrb94fv@y*ZYMU@G zr#y)65+82q?73iqA?bFzhY90g9WQU`BQ&z2TU@%T5{o!mX{bH2?$e~IK_~CUTuBN= z30(smykacGep*U8yZg=lR_@P!RC^hXo?jMfN9eeqK4cYBxh4&n{Shsd6F-WEerSe{ z)`}kiTedp&N~eq4_&76Fj@!*vop&sB`f@I{;isZrW@;;`=(MZ$QXKja5x<1dVzVc_ z)>!f|S$**86|j^Rz?#}eOQNdmso-N;)@uNi9gLL4$B(Va|PZ9MAWHX(4~zY;Oiz&b6D`1EG*KIDkcikne&1{z$R&2YWafSyP!-m=X(s zdNa7Ab2H9imE{!Q=NvbjaU#{^9I4CwCECxA*l?AKSSN>yq9?LSmUNDi2w!E@6UeP` zvJ)*Wu{NcxoVFrrwu?poy<&T^|} zR-wz5-bNWQct&PYz+-KBINE2-1c+}F3Hc~5XA2c~*{D*a&t>E{ zfl1UHY_5Gx2~;;AY-aP=$lmVes=MKa%%8056e^Df%_1`!kD`CHeMV(7E<8F9!7uF+ zO<{GA#SIHE|Mh&Yyg9p>38+m+3Ff<@8t?)~xx?amLO2}F>NO6X3j2dY9uE$Xg=e9o3D+sMW1Y48f zjzqp-G;?a8=Fbq_hFGo;@HXGh%RXUb)oj0&&<3U*=W{vJ`h_TT1T>5BKct;{MPX<)5-vB52c;xq$Bnr zEt=eMeMb2)AfVcu4cVB}_p7dQ(TDkFVep>ZI4|R}-D>bfRqhZ?Tjo!VjD6s=ZOR!@ z=9>gd{evE<9p@6&C)E;gr9zpZa@1KbF5Uz8XO@2|+|@5>$)nK0gKpKl>6=7n5`1F} z`^LG^+n3QE-U{2+E+Gq8!bx8byJ7O4$CRr0)WR+D%u27Fid?m7c9g_3s4oW+iX)4* znVjxEjnKtN?24l8C-2FB101D)sVZisg;+c`MH}2z;9;r6!*+r)jhv%qsm=j&X}z3= zjvp_@?d*G{C5Lr*x1)!Q`s?avrqagjW#I^Mb4o%SouG#IN!0+4$!DwwhSdq)*{kcH z34wdhgn39h?BC=eAv-I7aoA_mp;f~77nj=iJ{l)}BGEn)BQCR~Slz`x-qb?1?1E{C z*FeIN^5kd?(`Q|}!C3IP;heyZYx0g!1x12e0@g)5yKSdEv_E;%;R5vy(fGN-t>HD+2xA%ZeZ` zFA6jorS->8_U4_Ad@K6g6}uF@B(o0V6mh8ZO_1F+OS5+=7ES(MKQt%u%Yu9Nr*jXL z4+CL=5?UgaM0-E25p7gGtKkQCZwpBwMZ&0A-BY# zEyG=7iaXWrONC)QSh%^SK`k#Kvq&*yoI8YFN||v$QeC}5kuV<0TYcArHaNE*35#4lb#(I0P7tDYbj8b^j9GCbPO|qtqc%GY2O(yGFA~m<&1&E^R`E3KJkF zol?G#_XurzJcrmVe?B)dD_1A|Fiq#Mv47At(0>v2pr^gGR)MTij~Jwcj#>!~84=m^ z<+>c*(uF!!>MIfcCE?yOk$+ygvhe-X)RL$vR7-e8S-Q-0!Er3^73M&sDo>~M_kxGz zkES?}h}*)=;?2B&9Ffw_lBOwYRS0-d`dq-L_JuCoSHVP+6jg+#4qg?fqC)$}78SzC zy+EI*p)G^>(Ti1>AA0;zK{zay3(F;Q(mQRHESU{n<1Lb|!3-;FW3nZeBf>>pCvYd} z)8yHepli?VR{l&wzqk-cW$OL0Nse0kzPK%tPQpNUBea4z)qH%M_ zCb<8?%Ty$?eJW`5Bc?2lPAv1JiZ-qAd|zPMP^w#Oru z?s9O4#x1oq28#Bp@pHutHK2Z06(rxpe&ifmX^h;uYU_}KTmov6jQ)!x6xF-^frk0l zv`!~|IaOOxaY23yWS6$eg4;JVRf0aWxgOn6uLQEQyRe67QDpM|hxWII&EDHfmQaW2 zg-kkL@{5^mTc4qv0cu=~#L(2Al6gLy+NW4<_ikStU5qV(2eO(0xa0VvcbZ{$?&RjO^Ed>cI_p+wHoLfcx;#?Dnj&KFSDyQoxbi-Xr-DJkWq!^8&AOJ zlhMk9PC-)t+_F-eT5WNF5@gpf5CuOy8gB6QkW5)mdq?@5qo2>^eMKb;N(UbC>v5@+ z&Fcx3>?#C(69K4w)^8Zyg|gvzm2e+4T%!It-c^B?In&GLvkR(0!;c)nQm?|<{;ex> z4;6>^033mN=wN3=9_zj!UqN6UDat8<1!mVtLCe;^Cc=3U zhogsQ4TLzB9dB5(va^(^Npy8iWtoc==MCvdc7G6U;C5E07Uc0*;8cSKb{=#a=1yi? zvO)vLc^#;srY3Gxb}y{#JQn1Vy1AE?+p>6gFRQ{V259e<6W)MsA8B2;dWkok*7K=m^Znw$m% z_2_Y}Y_9lW)+l6pe21HDs(L+b4Cx6?%ck?G&<8x7=1clx>^56p6u;2zep-|^ZPzsI z%sT~NdS}%{zhu#zqaZ|J*OJRb@9oCXSb(xkAcF0h@KdnC9q;_@+7}ct^A(u97wTo_ z2C0ZSH6AZ1?wvz&;uh7H_P}tCDip)Q>#JjBv0x2xY;xPS`@yJd)jwb_WNauqF(p4D zv<56{H=rv0I&#_~ZLrU_wfxg1*zL%^&Q{&cmjSX!A9%I788@_i!K5 zfV;?wAoTWNh@DGs&Y&!a00Xu>_^PQH*RSx7na}Xt?4iR|_<hxArjze-E+2K&k;-@2 zE;f*VX~Pl`2=Xp$9QBsC0@xB7G?o^J4gKMy@V?_ADG=Kaex)G)B-2ps zliYtx*Ia-1TxL~FsD#t(Sr1M^ZD_N&pa)-wO_-E3Yi`zRx9`gysd}FFSs`nP37V4r zDk~tpwKNswHgYU(f8K$^icjis>vaZFKgYM1LmlEC$g|frqsQjIvyK;@ot(3J5oXAa z4lbV18mGi5>X!D#$NyL)gHRN@VN&h6f6LSEr)^koqp4si`@yE#l!!~s96@C@#YI_V zRefsWe$9bo_xSX*f^C0DY-F40BVIsQ$cK;+QIP0R6ql9V9Aed9p(S4kVJaQzL3RbAlOCq0gSl!>X4bz}f zCSxWGXs9Vy^wr#aIs2HjHDH0wzNhqmllri7rs~Hf;gOa{%Bg|z;6$wEBEbigFB2I8 zmLb_Iu-EsU4~agT;&3b{AqMcwO0-^60j}8$i`&e8@Qx3x#UY_$>jV9QUbqNIZ91oD ziof4luheEo>x`RbdE&8+tR0e#&zXq6=jj?3KJ;;p-He!-Y+|@=MNj~ZS5b;7y-%B3 z84>D#gugOAiRY*Bc8A0J^B`~C&V2Ro(NfV11;v%`nTd(1Rq@7|^_|fM_CzcON@3Ek zJ!!E}gNCBqFV_@ZuZ?SKg$C{sp0ZA?s^uKTN&kgOaL=5N5VAgV=*4KT8E63_5T^G2RhBVuOX&;qq1@%YvcAiB@!bp?wdDzy(-|%LW z^9XHv-8^gVl&RvJ;U@-Z@EeLGf?TVR_KHi%t9ujobSd9l?7ZFxVDRE>!F zp<;s(MTSg11_UPlO2Nj|T{3EaqjSi6)&!xB3~gXHz!c!)nQ4!3ZEloUlfv0L(7fi; zTl@IFJYZ1w=J$?^y?W?HrdB=?H7!>*K{ffykPT@eHt90{g(#3r2_i8*Eh~k1ls>}l zB9PAj2CWl|jspx4m=BcR9X^RWdzphbn^$cWryq9gX3O{+HuutVi_h1x8n@!io&0pa zolOne&&*!EJK(GnTL7Mnfw({58%b%IE%mhau^qA$q6Zm=UCufzO{$W}(6 z%>#JT4#VF(E)9LLM#ZDR3hKo$nX&L^ugU(SPyRYNnkl}F-;au_t%9*dy~w)M93vDk z8H97$&cbc*yO$$Xf*v_@fFLMaMj-IJ{6J!l)|HAKjsn`clbB=o*LewJo0+5umJ){g zA$v3woOZxW=%jS`m>8d*cEx+)z<~Qjc@#j<>Www-PboIC;JojHbwzoza`&h;I2Hfc zMVagUHQMLV(B9Yg3R$M7Q9q!T7@4G-UVqw;MtK^@{z5{Py7B7brgO11C)~9Zco$-^ zn-kVbm5*vuiY%65UCK>w!L$C+pR%a<7Oto?>sNTGSm9hCb|F7WB`Ev($tfhYR0`-p z>Ap(OJ(XDlk4F}3q(nqJ@ZH+fSJaB$o1bIe1unF#sfvo)3)(@#Zw%JdFgG7oZVbpI6LI-bsqHs(@4triH&J>A}x0MrD4{fkTrY$$h$r6 z{p`wQxUMoFsUDWSz3yuz-Pdg8QU=Jp;q=W-iD1+sB3B(A_+Hy6fx+2Q{vNWI`+Rxf z5L>{(Qlo}~3m@|8m@t5#ExLn;oO}?)K9)D%)Rv!rZJpsArMG~-OXB#Fd4GbygD)@a z;iE`CXY&ic5{dT&8LJt&AS0ROjX0Z@kaE7)Vq07|`K zp*`a0T}o@N48$3Xyhy^+?Yo? zZl>FL$mC;zIqA&)_^sIm)0Wmek)2&!cfe#fU6d9wZ9!lyej3;dVdYTLn6;&=a*fgI z5s+zAb-LR+swD(--U`7Z1GIYCPNj-4aiD8M$b#;}BmKcM90m=tvlR)Z#}?VoW2_9v zkeUlN^D2z5fpa50$1bQkuW2=}k|pCb^E2k){O1k~SpQY<-SGmnG4ErPV?~dPKM2~# zu#MoQZ>o>E!OW*5i)=D-_*8~nU^wOZ&^;Y(t|$nOD9!M_`|uqH@X^j~vIxL4+2gYh!i(IWzCD3MPDYQeZ7SD`_1k>Dtm%k}Qfm!-Ix)vaTq|rG9Ex!$vdiQ+ZWG$;@J9-Z{6hvo!B` zyNCpWuEak5Pq+vDa_)`zdzS148RuA866Kj~!t9v*Eh+ z38Ja1jg2qtyDG+neNXbBcCRgDo+IN<3Ru8#XnV)a%A8jF`x&&YU1**f#&)aVb4J;} zPQ^b3K-~SL6K+CKTHrK5XdyvZQ@9O4Z~^iQE{863Fb9i`?B83Zz8W`LDb#_1fn-pm zM6@78wBJ!4Wd&VUkQ~}d<;xE?pqmfdRrcmiFDe%4m$W?JR=K+Ku3V205j%DDMy>+6 zewi_UU44D@ypZGi50h}>Mz5I(2vmu63|LhVPheu<_HV9@boO;Z>re#~W2X0y3;|5a ztrIHQi%9{zx@{@C?g4-hfk4EIKFyb^|8}8_NzGR4&%860_Z+w8Gv~*lqiPn&5(m%_ zOnJz#6iBL@P`4o>&ixULD(J;vO7kFP@Ntqxa_>Mb^h&6t7mxAxf7!B6EtKwxIADphuZs1wr8h@L4& zLxPq#8EHHWZ9vM1tu9(a-kQ`atR-|ZkN`=-nyeW~Xu!n?;s(Ah29jt$FkS)4lGs0# zm^Cn(5wT8$7vpD){#~fMHGu3k;IsS1S+uu4d=UNScQ&Z%uyw@T3D!xw$?%@_t`#r- zse2usD`7u$VpI%Sn|<%gZ0fvO^>ZBy4!O(?nTuud9caE^KyG49QYk4G zEuJ57uT3YAPybCw+zF*J8~am@Js?_+L}B*#)^#3-=Fy-!mXMnWODV3qR6&w`24-7= ztilXb*If*c=BUok6MC!m1jc{u5&=%u`YXIst9i5ZtCK3f!+D$uXK`V|cPTN+TT$#G zMjt0YY1kfaqEs(}egS2j$(9PlANw+`0#V7P1*0AM386}AIuD*YJ}|gf1tK(yQQo=A zt8dW;;D|IFiuqH4wXVG&C&{r>C|&X~f*dq-f$JJ(QDHwsm;XunTZc?t`XL=` zlpMtHgE&6~y#LryLZ}ACV>=~{#-}AENfPbS`CJ-SRB0-~wgW2Exl;3*-cH7)Cm1iH zqbqIbWuuvxEjSch8kc00Wu#Kndln1cyJY6-Q0>+ zuHj_w)K^QKt@YH1g7T`$@;oN$U)Htw`kW=;pEzH#$jre&=vuy@r5Cz{Tt4h;Eohtb z>G_u*^YhVF4&b#lk5yHf8|j5VRST9GT@~Ry?E>~5H9Amb}bQdEOplc z10Q1*cks$B>8r1vWG-Kxq0)I8bCrX<=0m*Z*sNkP&42+s6|-Rd*ZA5!uA2^S@!GcJ z2iB#pu#)qeUKK#98`3`5k*JNKI0AQ?n}~XI*;KhPJ_N_GvdxKegu~Og+9tysiTX2T zIfr=javL-%+#d%t%Yw|(5-e^1xmxOVfsj!4&&&oKwq;rA--pP?j>wE!lujPP*W4#KXEmDw%fKf~rU*g#$Y{!vnYL3UI>m22C(ES9 zw&v!SL)coJs$20nFp1|4T23T6D?mvS1_ZVDx59v5M{X#LfMM(I<;Wpq5k0f;O7+>X zt?gCr3`D2?1`6?!h6K;h zC&1K8c35C5gg2Qea(3ut0bZIZ_@-7K7NZ-UZ#IXLl;A*cBi8{|AU89R0RSoa6U0e3 zLWS5iR@xSMBaIw?6bI^H5rTN%F#5#VNB++d2ya9-2g1^nQWxJN#gqPX3G)DMttgbo zlijgI^u&Dkr@TbwC71I(<&v*N?6vwSI-oh*cRDTHDnWY+>*0Db_Mlz84j6QJ8E<#> zfGS!m4Ww*?;v9YO)ZU|~K;cbED=}xJRK2zchO^qw#)*?-XOml<^)emrztJI+>||n% zfVF*a!pQl@&m-WRpxL)$CdEHl@^gRs-LH=F(L~fXTX(p>k(XqwOqn8Mx!%+%0Ld0H zqlF5&3ix2AtR^G%1-Thw4elF;gh9fvX~^J-;d{CM!7-!Zd)t_kYQ{Nm?M8DiMztJF zM*{@(Acq_&ml@JsEZVW9IBKiIrFP_Lg}vL2wS`t>G+K4kR(@&OP;cAMM3@@17ObDJ z?tA%7QF8~2WJ{7(>D1ijMByEvK#JadkMkq)y3c68+0)z1G`}R_Yb2Ekd00l&#TuL^ z3#(MQRt3GXWS3f%UM?@DVpR!USlR4s{5H3ZEqfCY=lC+M=C$&-;)3DnANqeR%voj> ztR!b&8ckwT#3=B;9Z)qEk)CY}N%B)NQ07fcUL?Dma6IDZJ8NhmrysxG0%PmY70S+r zx>`YvdByEW%S5}O0go4Jb;kPg`ug(IZBBe|{-rsHB*`^v9CuBN6rSxoa@pf0;3FZi zKhWpivubv%&<(ojAx*@ebC zTw%hw8KZu6*`vK|R8?`29# zCC=G76V0o0XGY)dA_~?DZ3blzUhc&iLlLlTQCXFidKquGhTwnV z8QgJle3N|SJ3&*tLR_hm zvMMlu#bOfvDeQ|kYe(aEY$-jJ7mLwS)-8beUGeM-AOVA)zt(rr2v{jGn~cp^Mi~(I zs?i*QYva2OqY(OG!;MeCIkEX7 z!=(w8PJ%o~JBp>U&)p`MePA-hWF{^tWBP_8k6Tk=lM8+{g+VG7^Ze|;gPM!ERB5c} zEa>M7oQ(F|n&%ITC%~{Wlp9`L`PXK=dry<6v!1TL8|A3(ajMm3Y`Zw9TILmn3MX8J z2%d_MR&i^8Sey=2Z6`6A4k>Y0V)>;he%Z)h=`x z1vE|?+_l;U6tQO>cg_4<$62YaDclb+hVG=!@`vF_`YoN#t=0X`EN|1Ow7PDlK#Mk7 zi&m3o^L%7{WDHlNK5Ah>u$dA((Qm9UmR6hO&Jhxdwqk0wfGOp(=G@UTf-q2e3Q#OK)mbw=;x#X`1Z|sf9+O8W6}sI(OpFY z5=E3(0Fc^CPvYqlX|nJPe=tQnte>5xNh zd@y(-NYaNXFY15P59PIU#6;Wu^2;a>H?3VXH*wY>k4PJ%2eYG_DzME9%Z#?3&x_h( zEq2Kiw<=Ex+?(<4uI~f^4Ai|baR;4Z0`#x7hi&U{hnOpS9g$@N?+78}j&d-KjdgI} zlu0;c?C$lx*m~+5{#r)y8oE}akh)OM-@F*(>&r!3HbxlZ|CpJ{5iIc>2_vWi{soZx zOe&h0{MhAUvZ9=@J>O+>5hcmDIpbjxs1asXz&LSFZu0?15*K|X| zE+t^J?3Qtat`GqoHAz+KdTe*4Hw?3|)zSLmW?K!G#|!Oaq{Zxr;s+EXEsGUpfldN) zg02cn8x2n}t6q6j%>3xuQ&!K?)6P*9t5+1%K7l1j8J{bSVWyys9rStOm`aqLy z&P|5%Ef_d*3YDyR%@c``&rw22HVj6rHab={2h_RpBgpg7QbFhrH~Dnx}S zLGeKf&?)Gq_CU}<(9E+^9-`Q@uvMZSsv7EX5lnzjv@vU=>k=yZN7n$U`BfL4J(j&T zYCMiEXY^b~Uzsp95;O+1#q|N%SS~bgJo-l!#xCxLW z!P=#Q=5(dsZRol|E%13klbdIha#MB{1cgn^OX(WXeE~5DII0^a9T6M9SaF>GJiI^_ zg{j^2z11HOa0M*InV{>rMU*HYhUSa(BbNGs3L;~2egjp8fyIa%G!dLgAS)uT zb@LR9tLdL`uH- z2eyg1P)j3h{Y6&RB;4HA`d_*SxVdVe)BRW=h}l_<9oOF}nev=$S#@35c- z%Bsk@W#y0~DvO|m3XC!_(wPdjp0tA^P&orR&-gp`9qaeLQN3Z89}*5v2gRHkJv$ulxGW?r5nA_ z>XPN8iWeoE-1y4Ai-+RBg3zJ7DPTSEoDb02w$h3|YRs7b@E!>q*OVVRo4hP;>o|D0 zl;UuGdOE+XzZ_p;VQm$)Y9qhDxklC{W<*_Nm911AAIRsgOOD!z>eP27X&=WsL;cMr zw;tT0lhIn#(evx+(k&yDQZ&f*-Z?3{&Vt!7NRv!BCo3_fzjt%rGbURrJj=f_cn!!& z4tOznwbs*{pxE0Pboy7MeId%_{rGV^y)${Re0DV`kbTn=2JiyeS_l+-0ll4GF4wN3 zMQFIa1#@yYcvD{xUjp(!#E~#oMe?2??t0A~gNFISFAyDw1&a^{?VYneVfDSy|HW{z zv=1*jzKWRk9V0n$z&KIE)DKQC^8vgh{L6?-x@<*}n15eewiBcr5q0UU=liek6?eOJ z04s~PgjKqG9v`uR7bUBh49yy1okLffIn>_oGg`=Hq^*7n<0PDC2OCL&i%>5qWm2kL z?}1r(MXvoB2i7P?C1XWmt-{DWjdhObK4Ce`f94u~=Rs2&y{J$no-ibuDxlS7!GEUE z`j>jAP?2Lp!JLqFb?kHLdkFkUmE( zL1XBBK>JyX?dyNfXCthClFim$V>;~*i7l2G{L~cCgYMn$**ecrOo11U=&7+*KrUC@ z_f3AqwJZRmE^PUR`fqmMU!b=D(z}q*0lqy*hNf8%eg0?soOtsB%iKLpo1g&Xe(RZ4 zx|W!`0Z;t5X$gNzx%EOWhlImI8iJhJ1^hjy1gQz*z=|P`zWb!`plWCgm}8s>M3{_h zLw0H%OaB$4I5DUbO$l{YZCcUxYO{W1Uv~T*^iDzjwd|09mKn8>Y@lF~P$hGYI(<%w z^JKRX8o~P5z0e7~d(237p032xpX4{(Sb_>`{1IObm8!aTjR$;0a`0^8R)Hd8S8BU) zN?1!%^pL&Y(C`RA6dy)R^z+J)J~5p%*zg-vA-Sqf3@?VRA8T~NUgNABWM`-Y@@EkL z)agCtJta!iB~}(J6Cg`8RzL3D_i~)@rdFI+XH4B{=_t+`o2LxN>Z%fwL9*kJ^9_>N z{h=HO1W{Y{$9$&)E5UJs#*RgG7k-jMXd?3)#xhzn0L`}bOi%#RVzI~C7Y65u|qffVr_fgX%u!sZgF#Qk|RiX&^B zyo$rbkFGbF5SQttBzvHsQ6(3k@6o9KOl5Eb(X4$#vF3&5A1kM;CGqB9qUdLePiO`X z{!0by7Au}ODL|35?Fi{dpj?IW@S>!+i}$vvCI_cwiQ}Uwh3GHrmhS6a3bBo9W}bz~%r+`gYB>42;%c_S?j`k5 z%9~cc?0E)@&`X73x>)^1_;Pj=r%HA`%zia&;crH&a2D$dASyqWRiA{x>np!4c47Zf zq9qO(sA)AFZ#4ZSMSel8yW|!prjatd&+H7+3B>2(Z0<7Ex(;{rX}S#fWG%p)V?fbn zjn_7;G2EqM;^WP*@oih%G1BbK7c6&(=Lmd0Opc!HdoZHg9yNBh9=|I^W;RbN`{+b;fq-2RtNLo&XgBn4P$@%})9d z^)0204S^bdIEJIG+OrPM1$?qSp1&mFc-GkW)uEdNAVe;KxbALsJl)tyV822NdKW8e zF`tG@BPAc)m}!8Wh%eZhkEi^(i;L}czV~Y!od!Nkn-Z7!%P5^47*o2g!hiyhO6vyJLMSx26t6ePh%XkVI=6 zK-*D?4BqOrPGhy}s-CU{5 zbHqHKU{A>x$dIr5o57sTDa{`QY;MeryX0wkg*Pf;ZjPY?A;uQLjCbu4f@ zX_S$9tW@eY%5I0p5ePOf7({opov9bCS|aH_6>$`3G$Y};G*Lxq{Cubc1-`rQ!j+=+ zWKvhlCC&Iz+w8fTMXVY`oJ}Qf>@C0L#IRU2?`!OP7|lrw3=&0~DDk&r?0XOVMsufCgYH~wAQ%PzTbur4qz`1D3f|3NcP&y- zJpE%rxSAOe)i)@+uJ-|u<;sE^N;wbK3yL1Axl_rA=RG%b<}v=FDnVh#O0oXDf_meA zZ!l@N#kP+kH&**Rb73^nFAj(RlJEX@l$U(|cns35l~${#d5nTv3Ds?2%BaO*H)Ub} z)p9QXm}MwSN0t1;UB7@!lOr?ZZ^X!!A%omGFA z&K7(QtO(Fpy;sx^wbNNXJ3BQKAn{->o)! zkgU6=zl4>8;M_3BGUOjSM*%H0)0D6d8Ge47YMe*ueOzvjsK}xB!Q2Q?yfq=9?xo2# z`)TY?gOd@s91P3$FinD-9B)VJ!yO?^X6AxZ)@+5h`Ba6-Fwcl@)%_)>*WP7)`Al!w zc_|gncF%w_9Df6N&bGS@^#pW}dQWyI|7O35XAj3<;p6AGRk?)%)`nz)>Tj$rez}V{ zkgMGO{g?_<%=ebi8#w&x0n;_T!|L0HAPjY$p3dFCBTg~*TY;bFCCK`o^tKz%g|m5K z7M08Q4;;iWh9gtBCI#^e4^FY{-z<1~DOiqe+)n@){WWeY;gWAZv?lSZvKo*}He%p1==VA&!f5-mGFos?QF(R}A&)V9i*o75Vqkb1yxgoFgjf>X9xXWwse- zp0(`&_1rG^ugk@8lN|DyzNP1(jOXAPTM8rbUrAGG*e=C!c1-StD`OV!qt$lecRQ2E z5FG%f9D|st%m-DG43$rkVlhm+x3_2A1dFjZW0u_KzXDN}59Tz$c<+|_(~YaBs9<#Q z+K3Tb{_ypYnDm;Cetl2#F)CTcD|K3wicGu)oB68_kDB?0g=wbnU zXq1frVRe`6pH&%)pnTU=oSFs_heR|X_Xfad@~2WHJLR2yosU{&dnQ^7>=v8O0#2By zo1oFn=ibXp#-8eVJKNInLxU0Bm;7zG!U3AzNpNR=80)SOml#Ye)-UIe0S#9!CR_Y2 z7SnTZQ~Ya}o#hSn;bB0fzBA44>cL-6;A-xP=gRS~sahhEEfPybCUYZ7Q&IlCM}~oC zVZI0!+NE0?;lFEUDhVCbsd^!%(>aBt=5UtBY0%Do(Di6+&6T2~sAsMezE8U<(@> zva*^QGnsL+Fq^Wmn3%G0F)^_-ad5F38*#Flva<5?{(pC*r22t~0qp-p4A^G2tfCyg z4c|=v!ZePo%li0ZX#967_{GNBnmyQX$lk^Je*)A2EB+nAoeL5dF8nG{mXs)qmK#iR zD{kbEM6%pV zo}40kn_SJE+P#%&8WJadtfILhzefLx2J`y|H4ZJ@!&^XhHrbRBpxQ>^0+tmL=S~t|ez{i@Uwhp1h|yv9 zI^uV}dw)JzkaTYfecoL~mi}?Kk%^4Enyh=}?jTcT?geB^g?oSKbMA#?M~(Yn=u7Sz zGTrR%3q9msL}ov_`^h7@|Ff7p_0zpUHp@=ALw3pCvRn3$myZMg0HwQI>I!9UWOHeO+Z1?o>+b2n`MSoIHzh7v4V4YH$bw1aATR0KfVJ!wv)9Zx7E;klLQn)2t^i z!2M~SKEcl&hTon$k}l0KLLC8uL=OP6sD}@?L>G5a9Zx^+*KaiaKAI|E_Fdod%O1|; zA^0+y=_i%xK74lbv&i@>wswnv4CN_%YiB$T=iG9k(gA=-qb2cVpY{7Q?A(|_w__1=pzMkfK!KX9t zmeSEZ*Tsd?xx>@UE;qJhIPxv2G~b5QQcC6CUFz}JQUilX!3y~0-!gH^{!TEMqXkAq zh%!WZ|3J}j|kbT4}FqqUz2_fZ}2uc?#FVkMk?WA#h zNxATZb1lhFZbqZgxOT02e68V0(`#}$NdOUe?cZamS zkC)&BrD^<#hM*4}WD(}fBBv6*0uSTDhGU@$dbMbCcGCICy5is*brc&jh3!WGN0FsbT8kD|FWyF{zIwy~*AK|XJTBB$Cy1Xc?pzfy$rsJJfgaU$?xRWmDl|vvm%MCCo z%g;OR#|Q6Bvh8L!la&g4gEc?sCs5_-C%y{!u+LziF2gMwF{_%j&iBjd=Iy{=r;|hk zhOq)a&1pC$ru4Q<5HrU#XhQ^3b5Y-==HLEoDD{vWg0M*zsgwetDt}QaMU@_YvBr=F zz7IpqR_f0|8vS4(sx){1ckpZEqyL3N41Tp89_j1 z^q+S@Q`BV2RZBIMP@+5{zWwcajQL>;qWFesEkqU5tN?6Hs3TG$AN98jS#{na)8(pt z(o<8Lf%EbIt_V=^IAqkJ#_jSIbSi;V+vX^ze>J@hX+9o7&B6_o zK+#1wgAKOxbK(XjuT@0vrk1 z$*d(BZ2>VwX<-ELkzK>sAdZW=Kdbu=VH#9Cm(8>YfmgCXv?NwZShN&n;w(7uh&R3# zb4r-|HZw!QI;lAgtnK(f{eMd?UrCp79SiZcFwiDWHm|k4Mq@DD>86Y`mJe$ukKI?@ zZB`fPBpuslyHa0g*mxuZER2nc6j?z_} zzY3&9Im!CC>Jzr znj`GiKhMhv5R@qW>1S}{7s^;~toG$YU;UQqqk7^FO<~DE(?N0n7}QPmL2;MpVgQZ( zbAk`BiP(Hk3Bs63`5GEwB-1@fa`x4cu$@QOOO+-&+p_Rzo$!D2sNgLAajG}V8aAgQ zHaoQwD-;!y6Kek56C=M`YBD8AiJcVPd24EsJ9^^Ce$@4NGSPJ;O+{>|k-jM~P?cp! zW~*{4y0E4X4er|MDw#S@-RpyhN&zJHE?*wm;t{qsGmAiJxM0@YS_G3cAnKZC-hSOp z4uZ@6VWP0XPCx3+ft4)wgN}o(Z`Zdsa+;kR2ODJRTpttD483|(Aqif9DPFb|oByG{ zVN|ASQhqAH3$LSJuiWMp~d&&NS-ym^(d<=r=AGBd{-?O3$mU=R3^sK} zK$C>NqYA1lbswKvT4ji+{3KdgF{$R3F?q1T64h4rrJiGV_5~m#ZBlbd98sxdG>jZ0 zCou!yu{%dE5E34=h`jfII|lji^ApTrfGp2GlzgKYGU0$2^WP1+mM~;@VbeN5M5+9yUIMr))0B73g+G}NybW|Y3jG} zuG-w#OdXYk!3C^mC_^cCzQ|0YTM*h`PN!7vcCk0y8IH7=CZsh1$31#~e0UV?*!W-y zI2B>E4q#!!3ckYDA4?t8qDP6~?^Ensb}gU^Sv}??owOan(!94mW zwP*B?A6S;{E#Cxc1SSE$z6HQvM-WDI256M1K=95j2>=4;RW9qFgX!R@zBkyG#ff=D zv5fF`U;l_0DC#Jz88&yq!5OAqyuhTGpPu=`RpP#|ot&edAW(-M@<6e{Ln68lld50uFl9!WI1_=crW1-?%3f8T{ zs0ILRDAO}Kk#|EdISD=W5?GQ)*J(2wF&S3oDr!1RZ)nLi>H0G8%Bx#4J>b$ju)Ba- zQ`{)`g$Vw^6PntsT=Tf!!F0kmc(@Wg$~6YDCBOJf@WfABS#_HF0O7Bdii<2Y^tja` z?vahZced_YUA#7htD~h``CL_94DV^~<^ZZSV@af-QmkzpT@==`>lA$nq6EQ2f-FIy zg7!sp+r8LcnN%g82$f^6dk9WFk^7f}>)D2rM7p@_hf7{&>Kw|d%#uxu0Pjs0G-Xja zR^#;yOJh@o#ZDSWH)E!ToL|Bgu!(q22L<6B?%}*Gs^z+0TXtDL1L!Ze=levN*ns2j zS`aa+;Cu3t%~lMpYoi5a{#QMfxG3$~`I!%+Yu~@4e|p6=l=BG}V(h1NucsokYzHPh z0UIApIvWlb^f+CD7`$U%WkWYeoWid3AR)U73Vc6F9%VnvuB>q%6|2KFKacwq!uTAE z=18PA;0(~|QQHO+kQYChtN=FMQGi3yihz7r(VtYBZQsWLZ_Z~Ma6&PwLiHQ!po_ng zrSYh5pg#;AMk3xUx{$k3#8|gU_`4&x)}GcoEgASI$!NSJ1i^jK_-A{k2hhD>X3TdZ z-uY6WkeST2z28hDDu|ipl*nNmG+I7cL`eo{+z~zEeOeH&emV*Sjs7&e)qrt>h4Niy zO)NNXnGLr(xiKdK8iskLs^7M=(?-h@6~v3g;+@O{DN8j2)lao0N*v=l@w6<6CZ!fW zmunsYU-?HVeh9bwyUv)8M)|VKUg)ctNG!!dyIu05e9zOEkJa2x(Yo8~t7>Ei1Q?C4 z5p`9)7B69P4N-fETdX;rAVAfQD?f7Bto_wlGzZ7-v`tR-l_F@aud*c`)}>m@mE zG|$>XccS<6p20i<4j>(_8|)a=&(#@22*i@9ZVm0m$d|GMnWvYwE5xdd3cH`<62_{v zK{krRz$w;^&9=--atfxR9G9DPRST^*i`XrhJ<4L~yi;1CjX_sN`R z?B<|aNmpf5g1PaYG`he z)a98?n7%}UZQTvO#iSJVh>l*Gqsa^ zN4-&DMSI-3TkaXTBeuG%ZFtMwtuX+2l7qXv%B4UbbruTioYWPK%fQVS3{h=jTx|BO zA$wnLYAl$2AF|1LwAx??Ot6||6Fh2`ycZCa>N-9`7~i;NJ?VpsE)gztlA`l?DAm^jCz6}CGWWR{E{m)qGflR5Oh z)HS?n)t{EPVq771X|~h(-i9UoJd;LA1ZX zZj&64xd1HxKd_kA*|aBZL-L)~m{K7mQp}>=G4b-(BV>;+_|actxK-M!g|G9kFX?RvU3`R>Im zx=*1=>iv1|>fbAi>w5L*s_O-?@yuY9p{}}nyta4klhYYtkVh+absZwKMOR3SfHH}{ z)yBQ~-3i<}1S|be4GKh1H2>b;#B~X5p;--J>l(3_wfMtoie!XX)s!0qK7y1c{aLg^ z$4Jr?z30=4#%iVT$U;vER`>!gV(!(#`0;APw7~4x24qfe%?)%YdA|ahQBu*b$Jm2@ z*S^hOeUVdc^G)QmTQ?ZrNYP_F+*TykoHSKCLz0nq|+rhYlPdB`dWvFL+_Kc#| z7~3Xpr5`&@el_u513#U@W>J^7ua&qb*`EIfdmw6cZJ1)&d03Hmq1pzd7-m}3+zh{6 z=&a5(C{6b2>!Euf$Nlf{&XVULxg`a5#G{Opdq=XH zv3WC+>{&%7;<*6A#SwpBrGu#SlcPyZ*^AXxIL5=NNz)S26MaO7DDWpYmqe*D2@L{n z-rrv zyQz&)p%zDs*0N$gl@YE{65R3qlMfdC&)#^S)1H?ox=9sM3%W(ch9FZ44=~M=sme&- zWwTehN_mgwyJcMR5Hngdp=c7N)Pu3_N!87cb4)f+-9ETV)fOCZaxS~u)kS@=YxVaa zt;`YztnvW5B+95+{KXZCh37;;zB5LRGISkFPnGx76(8B;AXWaG)giB+nMpDD1C=7v za8RA$_nDORZNheKkS4)vwEF-GiI`U^-z!!QP3?6t8v;QK67EwV2RPm{k{Wmm*|Zf9 zjy=QEDseO>Up`Ol#SYL1>w~3yv8M{#Tnyqa-E4qI-!!yc**rQsShiUtQqktTB2{o6 zEExxM-*zvGPikdfp^XxESY3G23Twe4?~)XvG5NcXpty%P0OVCW9EZGT9@92<#zs6#R?<5!!_VC*{Pa zVGIy>0{isZsxF5fu5d>B0pTGULC~q&+m>ut;i)M!ggzLQ#@C9uQj+e_G!qNymtt#fq2~pgtSt>?C^+>$rQQCS^szqGp?bQgqs!d zBI<-1edwWmaS2Ui!Z|~>MOK}#vNF#h@&ZIdVeU1}y9G-TKWMRnS8Af<$Mf5Rab^!H zP?Aby99_6!IREi|XtYIkFzYt@E4@+sG-{8PPWbV8a>F2YlAGrw!@YpkdsSOv`*Iyj z$}k76_+y+Vty{N{lYl;@z&Ju(tKS`GXgV~*<^`<=PYkE3E7<+5LHhGd-TvXo zCQLXhB&9@@BN~Vi7h}{k>gGo7ovuR)fsuv&{^3nI9=I|7Ks)xVhD{TgVu|RyNM%u9w5mXbc;ZZfa}!iiV~cYiyu( zYT=yBpI}pm=~Zsa%4#;<8!-*Qc##pYBl@deK7(Cp-!$G03eTywM)Z0I``coJwR*P4 z$y*;DD}wW3>EPNT>6)8j98p!PI~TN}n`Ay8{9;igH7f(Q*ae?Bvv^isIu2MvXKFPf zZej2FNe-{Y`AT4B!bCB65zRph%FHOPn!*pvsj)`DqqViS$xE82?pKzY3Zyo`w74#r zX|ga)Q=%e-Ll;qp`*H&NW?6BuCxR~~v5VfG@g=B6<_xPwCSw?Aq$|+>cXg4*gjVzK zFXYV#>sut>j`O@Ar^DU)5iZ~wKm3jZ#J1qw&Gkb^dBObu)n-8$US z6!$?dD{w=13r*z73`4JC38kLJHxsyP$`C856M9?_SYIA3*x@23O2ED~&Gi3tGlP#S^tNT=bg zFrXv-VdKe(XdbJBi5Ne!H=+mqO+!NFv`7hjq9i@zKgQ9w>X+uda!AHRLUVEzRxK_g z0jLC0MAqkVSWL;rl_z#bVTda;cn-6QM{b4&|6qEgew4yEZMY1IF;9%2UfE`Pb*Sy& zs_oaX6-O{TXvCV3>i{7BCIv7;Ms-ALscEPlHpLf2FvJ%u%|;!Wrs7Q7PP86IM5VN* z^Ep7cmD5%6stZ3q86kgotp(R%hjDMQ!sj5_gNn|>cIay2F4^pFBvP)DWU$r9y|GyM zoi*V~dtS&h$&)a~LOYe#Rq{YCN~8%mL9+3esE|&u`jVPSLjV|&rIBKJL}fa1)b0xg zMVltGng)fVX2+xZ3vMI#C)n*T>^KU!p=%D5m4-XZvfNHi;c-q{xHQfbMDmQ_)l=;D zo2?NRx$W(#D<7E+j6A({xD?zbhkWWF6kPCgHI%}w=Glc?+4rQPh}DT2kQlJX_#zW9 z9tqVM&UAPY8Ucy<`$CN|AW4*BK^Q6a(#K+^3n;4NH{~va2OXOuX4Emv#*Kd4&G$+* zZKP}k=ot0@CQhkd_hvSsyZB_klRL7Gvh;B=Wc-Tl5B@qV{;<6SNOjzGmi0fHP;*D1 zys>{Fvv^?b;9H7SWJT>$z)l{tI%Ym|ndFeWh*;l%e)i2SmN4A|gc7|9fHtD=u&_D!_v3T1 zqY$Kw#anA+YfC?E^VB|Vz|Z%2S{^>RH0bHDQ*#;vnlJu^!u|)uThA$19oxG)g5`Rq ztToFJH^9z(wY-7_L&7W93&ME6Xu%E#(CP37!y#R~iC9U|B*m}3=w02jv8DOW@~d)e z;NoZ7q(TOwA#2Dtw|WVhsyBzbIZ$Zzn;D&8ePx z0@QM-F{~Sn)b1gYgUo+(gcZzp=%UWNx$Jmjmg(xMf1+$1X#m^A#L^NB%Chft0F2k)Hkcga{Wo!ZWQegE>U?0cjtFZ_AKQQ} zxJ7yK2s9`e23`F;ZxG>~rB&fi7PLfy?m1J?FvRdC8q#L8neV7CjM$YN3bMnVvVDQi z%RDao5Xc@}i+-B|+Y-+@P`@`B~#m(^^_@dg-~vCQXIj5<&wiNadd7HhfP>FmX7 z7rc4iv@g4EB*a$LYK|%U9jIa(Xv54Op-7lAoyId4+BPQor1m;`uiGp>-Rz0v0(4A$ z@Bp%c0RF+#N%9C5ndBqJvLfSDKE%+KS9&l03poAXva3+xlkEua@LA;8X( z%if7*T$XvX(BD%@d~iSxo7v;zuylrd>ig#sAGvl>&pUyGOxj>IUBiMS(LLStb{Z`Z z<(Ac>X5{5$JBx)b@Z#XdYTxfP0NDBbfiww%fD6FP#`3>!-D92cSdvkso==*KHMpRX zsc_wEb&ymes5`lbEmFMB%)tQQ7`t_d&qurtQTk!p#XEJVtvvw0}3vv z4+D90a+>d}LBQgpPUlM~|L*Z@*2DKDtBwQM+j-w2uth=4KdFs%$cn2ccYtt-Igt;hl(O|&|hh>3Z;*Mf(0hztW;x^fFO7-kpJTTR0ELHoL93*Ps_X= zv?@$E_bNOqNwkEV$4AIp3=?Wk{ehX+dnhu>nf}B{VqGv{DPB51K>5JMAO29#i)+q= zXz(*xRquA3!#Zc5Ll1S{35YqBxo*97+=QSqXUI8-sXA@O)g8&wP-xC3@$K^bt#a_? zp&irG+}HEcz5Gfadg02>SuRd}TiedNfe=~OO5j7yAb8}=-wczqnvuporBK$!mlhUC zRhu!-*d3pDwI**cR*PaEgenQt@|;C*S2=h|{ynttCi6#XV$uJr3<%2ZS#j;t7DHs& z(}YW`{v*H7q8gOm^2dQc8~Y>1)tZx|@NqezZ_AQoHyZsk5D90UvPMZ)AQRM?mxeX& z)m1>5;d*7BzK)9Hn`QQIn=R#PB0B7X`04~45VG!OhsJt_q?|BpW*w;6$NXpsS{Cm?_*7xv%wzhAYU_)O<}zsg zu9zM8iFX$10sV?X8SzLbR&JNZ*+VDy9unE4orU8!iOc1<(MrYM=V2IF5T>m4NtmmYnOL zQHP>{QrenyyB*@hm$}D9HWB_I$Zj>~u z;q6c*9xz46OgD*T^omu!F97%*#r%Y#<|EWUOryOIpxjoh=DyP8#eS@hxm_)Qg4~R8 zbN9clkGVF{F9A7@Y^lfNduDXH93{e**l{R`dw?s|d-j?`O@jgI85LY{!mdIj{j%kS zmZ_7RxOKyQG}F?9GzcO28>_s$g|GKM!wW`Y#!v@c$x}^>GqF#|*4g_z?(rUsi z+rD1FXpw9|K>sWoQq8K@uG>0D#*MQHUgfB@5CG&Qbq=gVv_z#mpf-^i$qs7au=-eb z0+8Ny&siQ!Ts8o<2`!0ikRV@Q<(?^kz2qR%gBa$PRkT_6Ub{WTuZE zT^PsvlVaaMX*Y+vwPmtJz~D2UaisF!mXZs)2--mV*BU$gv}ZBM)iiY^;36}8`WTg= zTn)3@1Rg6S-S5rbZGYE6S+qe#JtYFjC7<5*;vAe3k}cjLJB@|i3}5M}nNxbAav{u) zCw~i6zCk8Qz{{kZI((wUHSA8lh9Z$!QVgu3Jo~{(`7BeNhD>OWpwSeKmo>uB2f)!i zIzbkXBh|c4Z4{RVJT6t7NG)0so@GPt&{q6Nn3^k2x;C$$+g4IB$s|LeM^_0)Dg{23 zA|6GLS1RNBx&1LceIkqx?AOE5+qvy--&zWm0=Ci$i-95AjXra*P6HR6bfS;yA+Wve z5WqW~PT=hZaP)i~{e%;i)V!530KEGJccTL1dFo}eVQ07!YocaKXvl+vld6K!^G);z zC@0(6bMz@qq?g2Medh($%C+iHqb7+}Y%?Y+!$N$>sA%HM$E>CZ%?c6Rq(pS&KE_K!<+tV&;YjNTnvy+lgirc>(EMprCb|A zEog;WPNc)3$Kr|{qLz7=0b()4<2L0J-Zjfld`g@tytp8*XzV!afIH>YZd{Ttb8TWM z+*Rc@mSTt6JSE)NEA+HPU@@7ruaEiqaVrjrO}LC5N2ZdQdB}2wL=0ssqj?Uld@lhZ;4?U!_VLFPV0B$NR_WHD-g$hk#yAz4 zgLO;OTs4F3VWqCe1z09afxqjl79y7$Eu7GeFNLN*tH{@qVZoyw@Ay`Z>e7)#T}%}Y z03Wq`q-~W=mwFZqo)*JnYZ{y=kjd=Jz&}-@5;QlV`9u#qE>?YzKiqJYP07$aW)`*7LJG^i|Y-~j> zErB)7G+s3kTdx(RG^TpAFI}*ye!6=?gO;BcMo3~Zs{I@!$gRfJ8m*qCWy^&u)~H@i z5Vtw*qybxtb)zkuEPY^S`5b8kL~440poE@OdhSCsY-k zFBboan9OG*s)Z|RAlcDKJDlQgz^&~0b&xbcQQ;C-7AX>=9OvX460D7tpec3LyM^^8 zzXd|oi_=6qU7?^?>PPD4jS{7v9SBY=v0UhZw?T&sx)%|p?i*-@jG=8lKQcO%pm*-b%hg0{wb0|=5` z>9LVRWvMr8IBR}aN2aj>rFT)IHb5Z0o}rZw08uW9&9VC$nU>s*-GO6{ZS0lu(Ijb6 zcV&LucEK)YP;BaotCUxtr9eOaGngk zAOxZNkq)i(KQ><@ZhEr{EoQQOtZrII7=dR|%21k{2PNpYcFskOi8vIpsYdjP=V3+d zfZf*J_1M;P39(TIxSVN4J}M0cj^b>Buc;*M4pI3q5y6=n?nyiv9m#-P8-@zaV;}8W z^aeZ4@S93q6m(DSwz%gr@XR82Iuk<{=xmkO%^%<1w==3ySxV6dvvmBIC+RBnT+mgm zb*w`L3NqA%TSkQaQ^#cGbT=1q)R%5nK;Pkl$|KI^jF>&bolVIvs19mAB%BX7F1p{z zCT<%tA9xwN@l;zb48?Rq3eHXpRsHRRTIOM~a%WMyWU|vRll9Uu0tiKk6g?iXW)u1Y zvJDd5>yW7557f&E>Y37Djf$-)Ly^i&VyDG^_D&2-wHG(Ya!#v-O74dFQlPbk4v$CR z{MS;bqBz@Ic`Er?Z>^$vSl^ZEk~6WmEu#YtSv!TPPL=#c1Kek??P6NAh_B}|!oD-W*?^bjP`Et=aK5FYUgc@7o^)JYPzPBuQFs02}-No;TOJQZ6{7 zX#c5ILXLW6+}GYATJ`mcHMI_|`RRuMAW<gV`k-}Db#ZO_o<o67{?FW%syz@@SEz(7Wdo#1+kG{dmPMFw0?XEG^yU<^J{&$~1ka z+_ua6IXQA3@pmhz;?UwCm+8=fY;<=e__KBphiU*sTvnG;37_##-ab*U0!Nf{6GzfQ z^ryLN1O%;m{+-U%$zQHCqkZ>Yt?L813x6$u{?IFHRE*5;BTHe*LoANcB6J$22VBWQ zNv6EJWSPwj`E1TaGIIKkp|{my)J_xQH-*8QcQ_c)xw}a4Nwf2u*hE&lUnjMb80F|p zCzJj>_w=L~ACrZ7W8^#QURf~N5L6sF6$z1-0R7 z@qKmwW&1R@#z{rS1jPQQ*v$ZvDcd>nwt2?PQtO*-y~ z9w-ag*f0_NMPcysZi$QXuvgx);mSVMwPXG~uy-y;QT_S}VmZq+k-J8PYirIcHj>9+ z5-^2F>a(O^R8<-~-EOu41>+k_Jn55Zf`q2-Ng+}n;7wu#=g@Q6sxdC^_-5a&x&*!+ z>R|U)*?q^I#8jsI$>8`kw{r$K^rRh*^Vpb>m&G_(05%rV?7SEi!^qA3E=q|ow@fds zan8)Mq+e#pA(lm5uAN&lVOI-LKz5}g@6^#a5LrTwz(^ z{L_V;+VWyFrhsDqPBH*6?;iIBV+D`ib5l4Ch(L_~@urU90D~d1d{NrRG1%7`^kB^d zCPp`JnKk-xXN$>6Yo=Lv`My5Rg+P*08Ke6L2T1ae5L2VjhJ?BaWB7NX!Ib#NJJ}^P z$BLF#hWnsqjG|?9<>fhyKKj?oSK0>M{2_K|?P59-BpQQtNQDLnvlM1$t1aNC2xG#Z zijsxcRH3YeWzZd2Sh<-LiIGX!nLG3LVM|Cz6ZM^#l40C|MlI0Xo<3;GJD67d=V>L+ ze+8BElv9zL^djRO^adIvP|=f=)Vr|5aOK3A?Y%-KR#~p9*w~9`3cK9dei!V68lK0| zDP}t0vAtYz>}H2X-SIWW7M5w zpS=c;{AWl`7eO6)vvsv>o26_&R&;;Twq|q~_{$N3^G`>(+TiWjpI;8+G4|)ih3|^# zspc=QK~B4zIAkDnErxlL!NG>9$!NcUpx*RTBQ-uugq;Bf=#7NYipa*tVI@HYAp-K5 zSIwu(S!j8jn3xXVXif9)q`kEe##blU2(d5VjHO(LXNh-I z=$Angq?qFYlj7oPoh~Ky50{lU57CnzH&+EOhq`=T#P@@hYFx3=A?1S}bWQ~Kv#-?4 z4aZLF_>lm%0TycKPPa`Hw+?$>)Swj!Y>-D`+?E)n;mz9{_|Gm6fml&Vp~94!KbC}t zE8(W%l4yZ#9Srkj#wkJad4o*)G_w$V#V*Iu{Q0q9d>*uOK!-0GoDtI@^-D3x6qGeZ zZ=f9kt;Sox2{*H^BME2LcUN5p<4v=1uVFqU8ZCg7*`Q0kZmt_%grtxDP&6y()#6wB zt~{b=il^=gWvqD{dQ|SJ>^@LOIin3sO0hmB4)1yGGmTN3g>HQ6mM>&VPY>K7K^QF3 z+j({JEijo)f*897R>-WQ(bQ0WdzxS^SE*A;=FIgq%TQ8m!DzDU8<4nX_wz~$7|$ci zjCugppL>5EEeadCO>Vp_(O$V9d@Ek39nZ`sd88d8{bF*qti)c_z?q1=`kaKOc=zoB zYL^QArs&c$LsZRfXYL*FWOK=Jn@6r#(T+6RmBdA%ocbhD4OOiMuLkD&0j1!nOEpv6 z-k<up=TZE8jeOdo!cNr!@Gt?`oLy;U3iw(FCY@=uw z;^o4AIyd*ssiXEPA||{+Bw&i^lI*3Ve%{s~s0U|kMv==b{zQ}SP8cvZ^wLy(8A6?Hm)C&J`+(6q^?2c{)%Z?)xFcLi(>HTW!;;2XCw#vdjUWYH>u~ zp5*t)_X>>4wam)Im$vuhYR<#p*4Pre(>&LmUNE}JzxFsOIAPHw?m_`mg?ep7ngr`s zWY(BU)#?m)7AnM8#Ie0J=h%a;$ID4JW2{VYc*OrnCBH^xdeu#Auks4MQcJJ0*opN@ z^+eztRx!29QUA;@Z#Mc879&7;QH^onxgzvGg}y|-Nk{jq@!E4tmJlPsYtXs_Lamij(e@wd#+Ba4N0{(q1-=iN48@^8%rGhVf}k#p|F+=Z1?4`TqrDP!4`<>+y|CDZ~9OK8xThY_0u)?&p2H{ z&cM}}C|F6j=!uMN-Wera;}EicKqWN_U3ffT3(}Yn!KBfa8}We_(nnp{OO-@!ntyC& zDoPX@iEAUUsMGX;^tAw2)Lj=>GxcMH?pTKSQFw%U*|Lq!W@Rxm)mNToWh6Vqwj=!| z+2ghVfoc}P1jo+sOBr5-ixOp?ztY5t;>ReFYZu+-_wM%tMIq70Dv6g-7_m_AFc^7z zsb-t5B#dfF!khB>nbsE{M4Pt%Ocs>BhPeC{Vmg{h9gUs!)lLB4X^eCB$Ys${ZraI+ z?HuJJMW!+htXT>X7m+|=ux=I|F$Qaj``}O)P4>1YQR1P^mEPVzI!h8PY*R5D;(rzp z=!|gHKdOE-L-@B4m#Nt2hPDf~L#3ADuGh5uZChsX*F->#XxOtXM*IcCX{ql_RJj*O~98#;Tk2;_!%!-J_ z4BmSEgxNC}3cG4lb>2LpOfMdM)PzE|Cki^XSB3xm)1C-6)Qr_)zI^osGnkg(UNXvN zj}I~K{8KPCwzOX5bZMZs@F^uxlKNqJ_48txASXo)WULFIZX`1PXJ#qy_Wtx*@2|;- zex|yJQA^-o30F{C1OJkCVPlO5)pE6w27c~H!jp+lh*BX6)qJX3imX)IA#K8e_b04< z$)}6*8_33BI`BH^Y|mI>;N6*&sG3xhciu|AF)gdlucQ6x#f-SwfQU5^L|bj;bPr~% zdutRO?jZw^c)EPTu!{w2WkduEJQ{^Gsz@RTqiUdLpeKRiACz($b?M{mY%gOK3!EK+ zg5lx_OQGLve?-e?0;CryC<@#frx-Iu?57M+-xC4clTru@agyZrsYvXIG5u#Ffs~_B z=nq5%4sLoJ1JR|&^`H!y#f6&--3N46WAd~Uc!U9oCXDwH)$T}16Xf^8`u=N*A|DYT z;6q_215)^$O9F~A6Xq7BzuQ4)yKyHo1%Q09_m2Qpv)PyhPgl#kAutcxKDP&2>#%iQ z-#g>grW;n4iMDjwlp4N6p11~@D43J2Cs;Pu7hG;r7Lm!2B-rOn=+=K#(hBrUiC^5k z1E&CF^B0ncrgSX-F}-y_tev(t?ZXW_{rnlUMIrSHvMAM!{iwljU=iEEi@|SUg0qf> zS(g?)vr8b)6|u68L@|@9i<PHK2OvQdUNhs^qsuRi92RAXD0Z)v|0d} zR(*jMYR}pZhmQ?qJ9n=G<%npDm3`q5k>1uEoyrd{ow8Z<}Sp436I)P*e zr*fvLYG99*kJBA7H2^=KYTT}5&UdrAC_HJI^!P@M$ZE-#ywzqbYh`<3=MxCTIJQYL zd=Tu{g2;!X44*AQy znY$Rdi<*O{bC~PO(!CXVwAPva{S#O+5VUK(Xuj4}Qtc{pS(YOURB#WN)PTT`=PYR8 z_nGJ)(OEO;$L-x0jQ(eI-tFb7%jf5&5fl}rWY{lmthv$*jmwfD0 zRk{40RVZ$dQ~Dc^eMklvaFYsb7`uOzLuQxBE!7vZV4w6#< zKZa@9d;4j4z}3!nzp@cthx^xaY0cU`s3^jZ|JbIZB^HKtaqAnvoy0?j)O`oc7pJ^EelD1pwUBm%9z_( zA}5H@fYTNkCxHXNZSkU~$INdbLEv6oI@D{quilXEwC}L?$jQo^r&vevBj(W%WB4$N ze5{ONM=g0u9?x^*G_xw|R;e|vYw*;>lIAIM_j%mfdnR`{q$dKKpNy78QV5k zt9Aerdr<{B)Lk;FKF;TOoMX2V((9?qeVAM-Fj5i++n)!pwLZJR3%BRj7zUD=MxO|L zP3BhZt}zcjUrZjXGL50kbDVOfpwIR9v)NJVRQziY0tX8fy=t!re7wfZ>J22H)&`rs zf^nmc-6#k9r_~2noC!SsUCjV6xa9)f!8D)#J9q5xoC((mq6R{ho{c|`{Wj>C%?|9hc9#gcwQ%fi?9Mpw|`(@+OUfSYw0aMGoCJt2Xls3XEb%CmWCFp6z)iznfCvRPs?v!`euV@3rG zH$MZunAWKG7yf8K=w0Vm^)#_e(T1qiD^rEsn9o{*Tnf_>t(aLSgiKqh4eN$SWy;N4 zI(*%9-5$xBzVoOvZ`cO>soRiip^}+q%AwS}L-{rgu)UX=Nk)M&^vV?Ep^5s^+VzWo z65V+Q1rFt{j}|OHJ6syUz7$|QWG=LkS&RW_{beUlP4Ny$uHNMSbrCC&(c;(;%san> z^q?As_b@4M9iyr0yY|Llp)RxsngVeKqoj{gGt8v@QvnktgnK_?x@-eR8LX+os)73- zkg!pK55jeAT@KMzWv7g48G>zLS(Z8d(pw(*NWP4|(R4vw*vYT8b8`n32*5A}=8}5jdnp71YG~B_t3Ul7Z0@Q9=FJ#sly8uRAwZs(*jVzi2zxdrD-^ zgL?tQpC*~!!FX95P3rb9Zu^=uKzjlJ?K#rALq}s#d?&J;+uFVc6KiGg__t;FLqVKY3JiCoJ9oPKs$)u|DqG!~Vd_}xz zPEv)$qKR0chXih~EztOXR9$0tB}}u8Ik9cqwr$(CCpfWf+qP}nm`rTjckZ|M`31dJ zSEH&n3W4xktht3_psArlxz;KA`=~#{jKv$2Y{ogv9NSX(6mZia$XfiIedNIEB*H_n z6=1)c-|8X)5Hl{~KsI4^VS>(QARI?X-vf3ng{rXVZP8W*JJSYobFv(OD@o@p1%GH# zxCrP{cbUuHcVLd*eB#iXex`fQx%?-ofZT76{tUuRkTa zqfkjO#Kk;Lgmy`%>y4?y0?r>f!so8k}p{D~drv4jUm z1|dx`gc?pt;Lc6m3QgMi?^?}dW-Vxc^PAyo0S{S02e1oW53{|v;!mrcrc)cULzA0J z2t8-Ir$~QtDZ!Y<*{y>ail7nc?8RoUvLbLS4N4wHm#zI%6cEvu9qm8Z8% z?TNVDbpUm+%g*Ts>GO=n*X?dvH5iZj`N}w~dSF|#a;gv8Ze0t=i&NCIRpA%zmFQLH zV9US7=pLMyHhjjT&4mDZ=iXGY&22J{1Fsoj(=R#9%li=c_O?{F8ja+2`JC4s!WUZi z?DY;cQ+_?CIT{F0elKi%Ppt@X@wt)I1)4p0-U$e#C+E>#85AWwrE*USacWp9rZ+%tm z{>;UL*eO5Pbd|b)oJ|9`1)^u{0Gl?}IzGb59K(%XAKm1h`w1+s%!<=BUr%0Z?J*s1 zz_+U&%3C(C6B-iP<_q=}%z5`*p%XjwrKO&x)wYhc{7WmF#UIx-(B6$SM|SVJE-zss;)t6ZTJ>IrtCQe7Q|S5Cb(i-M9V7f5R-DRKL- zX@;Cq8%_B=Gl8x?$DjIUi=DqOasrR*0S5b=S-yc!(9bJw0!oX8&;}j13^vtozrbh> zn|ZgeDS-IlWGYECX_JWw^Q;LL`Jfn$JD4GC_t*RA+yA0K=Oa znck3%HA797q}koO`gS#yLn{+jaR&*vnaDNhI8F?~p(B7Er!a!PJl)Ti6KDKxZcQFz zW$NmSq9}JE>(Vp}2%ABy6a-UFP+y)lJd;=on+5_GrnfeH7_2A8+t^^`koHUQr2R%- zv9Pfx8>EP=p_AOg`J*NZensjNK#Ag7-E?Zh)kfNE{r*rSc}pSKdiv2)){tN!5T;ax z{zBm`BU(=0nC;3%@;Ewya_sr6aNz2j<5Pl|3+4_vD>fTSdhTqYnJ zOnFH+aPlbi`h;oQ)eL1?X84y>vg{u`@YvsYRVcMhpeVut@OqPlhTo12j5FnXJFs)3|^$6Z2Vn`?<7-KjwbCq8aGx^ zXC9Q%_aW#-Y3lnNppV_pTl7f{SLfeAx>s6Q_e3bc5@ER{e>Tk|@Y3Bs&$A0leD@(8wC_j&N32ID!1uG#aF0Lx6OY?CP*?>ZwlAsJ$byKTh-KF{v; z`B&#hPMZFp$aFYxI@`8vU#LmE?xB3gqm8o;J_iHFY~cLbvY|DtOL24)50lpW9~#?p z&a_y)mh=pC0HqgOC8x!+wuf5d!9`){)vZxx8e^P}PmC$-gP1m~Mn~J!W*ON4LXL%(IE5FUcCZ2(!QWF=^Qy8$=e#5B0Oag8d=45&tQ9FeMFht3ws zx&ps=$IhkLyc{7z+N&gv@19%mK=D*%9@a?N@Qn-ss?R3vbRNlhh^DU%z4GQ)Vl9wjZ zJrNgUfY%)Wun|epdQ!jCKM5Ykf5@dPxKpvXZFjkK=|#bY+@=k$kr>`k@tEUuq4o_3 z*fC>JAOwj)37@`^ar;#pEIU;z(h2qJ8Q10wnl~CZX9g5r$d3lOwAs~rr1506BXS8cujIwLSr8U;)^IV#H%|utkSg z%wB<&*Q3l6^JF1LPw-cORc|}o9dw{ z!GY>cHNY4&xb2-SP=BEhY&KPa9bg{epwaMV!JyexuCWy~?5`g7GN3AgjQ6l?xk1`* zcX)XKX!!KtGAr^BiuOiVnZQ?~w%6rdW!L zUY^m@V&hRGYjt?ml+18aW#vOShp8b2N+O2=s1;Qqh-Ds@n}`*e_@+GW5;bj5Y|bj- zY1KlQz^8+OfoZp2BcM?>TOLisHp4e^)Nf1ovPQ(^OU(AYLYF7CwtFa3uQ2VFja$@O zD<{gN1HLhT0ciu3qa&&*`uV*0ktk9YN)!x8wzO$nycp4jv8~#P8&V`HI_1lj|F7!}1*u)}|6VDgtoq z4AC#zLi zQSfC?aJ`EqWotO(JJ67@4je?QC!Qt(a&)v(EfeilecqDgLX_iZ1$_e9I5TC&`3tur ze(U7sYy?^Qqnb7qo3YF6cW^>?*E7ki6`F<9fRupTLvx!?T2C+2dXqP0y!70i7g^AJ z0OiqieXka^aDWW5>l_%8Zp0*x%qHRtA~{4ilGjTjT1YYR24&= z>$t*4heur{Y|LSN*^jJI8NbDI^8-}y+y^@b_f~PG#*>nt^V$<0EoP`b@z>je$o+4o{7vg<8us$-YPQD6bVo&j< zbJ;gAHUk@N`xs2^xEQHjjWHkv#D{i`_NzcyWOjDkEI6zOlOdFYAV*P>4hh)4GpV@M5}$5E8|=Z{>9jeBtE%g%k{e@ z6eP?V?lPO?HSPug{cd;8p*OY z&@8`FN9qLtnolQGpDx9rK8kqqwO7$itu@aaFUpbzi8A-OmRs8{?>!-D_T4)By5Yxf!AW% zROjE+QB_S7>)T=}EGQ5JGm9}df?L2=#jNhG2Pl^&A`1d~5pxmeMOm?5)lsA^(U_t1 ziLlei&QY&l+o0fImT2V%#W($@gha}$M^9TKgDwnHFK{1}&8@YOxvcFYOy&?!HSDjbmeiz#hl13@hS67f-)f}svK+1+XfkR^CXI5 zhSgVc=N@uiV&wuQ^1M#^`ANh@K~Oi1mNoO0?M*tMI{o~{i9ccqb>DL{Sju$zr-!_L zz1nk(8-=w9dwn%H+B~+3I-%T|<@!G&^G0A+0@xyeeN^o)w`i%A(UWDc`Sh3s5cOy% z`{D=$9gNjE`2`EIP0>o5>BhHsiM@HtE{UObO^ndy*;C2c7Spcs zD);K@zU_QF1fpN%TDn>1s@d=;Vs<***gwjhxd{o<8A2pDB7({R;X8*K&EX{#khq zi3OXD^=hm2Shm@pUq4iiCK)GSb-xqQ)SJ@uN?JpT~XV@oX_RfveX1PphG zA_Td)1#L@;JHaK+k(Nor%-f3X3Fh@3CxS-h>z+F91L^_oY$86!fe(rUVn~jTk|#GH zND^LWG@%)kCHyBFzU+()RpIY|ZRX>mh5#eU&$<*H^oh`b8fsem->xQwhcR#upgZLO zF^e$ZZM;F6A;*flMil*|;rUfrP+$%)wjD21+C>!?6DB-|R8&Z_vbN?>TDsuS9|aA^zi0&x%@vlc$?}7Hp#?3}Faz1B0otI_Qcn3f zw|d#nQJixNE_$?{d@eg|LJ_7@s(ox?9*!voUmeAcEmOk~sDvlfh85>gf#3*Wk<>SI zQUS+A`#918_f@pUm0fgy;aK>`6X=NV^izDZQC?cQJEL-a0cYgy6C1NFBxK&^Mnc6q z$bDNAr%myFV6tx8NoPj{NOR1F%|$D$+?SkV#>k%<1-^6u?D1@iJ3RdmljRRKc6zBF zP;aUg+p>p*_3%?^3&R8NR9P!vai~FJ@F$j!ZeWT8(9W0RJJvWiJ*hsa!b05zRw*m?!u)iJR}~eKiA&fgA-%& zGo#>Rqj4sa)IEgpFy6IPZi~UHjTiAxT}?|9GhrD&LN=fTF3q|47(ml6rcgbSQ*ag| zOG7NnshWo=0S?QFFq#8Y7VB6W)j)WFfLq({_8CnkNu(Rs+pq(`{Cqq4t@kVXKUL|K zH{;yOV7$Ty61NR$o3;BQ%ZX1$a$*T~M%?~puO(M$_}LdlBbWwp~Fz)`}*H5aN#+qrVCyw=u`6gxnd#5QFa! zTLY^J!bU)L`c(vwR^CIR5qS!TXQ)vgn?^35RAE0+Rz?orT$k+N&v1&2u++7FpqqDO zeS0k2q%Z>u9oFv^RkC6qklJX|B#a?2jN-Rz zft%YQIF?ks*rp`PfYABw%jJ=FXmXIzt5rFbA?bj=|2y!TJGJi@Qr%|goJh<5ugp$x zRpy1ILh}sO_DdNj|L@Nz^xVY6raRR$*HsZIr_m$8$IcJXftzwzlY@E;0)Xv*S*?1B zN;W}^i2l1@XkM3?L*?aYqLwLvwt6ylMs@65JVv94S0X=O^jvFUe3rdF$IfZlep(tW z=N88VGnClrT5B{`hS)daKwX118v2OL8&O^v9c`G}S8qqXD|uHy(*OZKU)<-qqi@vP zwkaJsXT!B2X}bQ-`nfXu zEUQNizz~Rye9EJ15R~l4cZzamQ={9yLL-Yn|l-_>*Pz67_+xmR{`< z>lSGxU;?#2V2Dt0@#8qeh?2}nx5%RSL;<3VgoFs-qVezN3(iqB>NV;kdli3FN^+_7 z298eKc_>#1PImcBOskn1*RWmu93XNiK@P)S-f6Drt_k z5!mf*^qPOfU-TQ;xcK&IxQvtiUZ1zdH~OgU?bNPnH0Y5J;Xq?u-XKyWVWs%sFO;i& zJRCry0;QCDMi;d(afD8Tqmi-kejuUJV4$Yc>)69TEb=-@wA~hi9BSkvq<|jB*o2hc z04ySy-Xt9(y5_=I%t<*y@p2yvhZ=RPUK4MZ;@Fv=aN9N&nI>Xo7=JJhsu8X&q7`k6 zd;<|oN|ulu`FU7%Fyb%DkJx93VYu4xSTF#xA@(&{Q^A&y9XTs}4q9O!b}(vP+Lojn znqV+(UFee39mSyW`P*E?5lUwt!UDR?-(||YEhhH^{-v>|%C5O)vh|OB(vI36>OW-8 zRBU0)N(|(*xwjXh6T?|agA{sAkRLSg*YLl`>|$Ri>Y6EWC8~})JMp>C+L)|-u`gn7~0 z6WK56Fmi5}N7!#{p|`qsSSZlT<4EH^z#Ts{T>u0}mQJJEU$8gUv$tOKol!29^MT|g z$O>ZeN|reJ2*6&r68M(PncD79tobF8_s!17K<$85vY6?&ROR_W5QhSgCOnK$|@i!QE0I6T=A00&aIu_CLE-;2MI17u`HR4vI^VgBe3KB>`f2i&oI2WIwTbP)j zp_!Okpr4*vS^!VKev}qv=;0dbe?M$yGo_Yi2X+T8-Z&6d>U+&_5OdM<_cGUJP#NBY zXzLZzub^UR(V)mVh`?@gT1W^S#@33oo!;LS`nZl6$`Ob^r64 zXa-qs?MdGq6++o+FUPbwV_(B~AZXju0u(1WT*(`IU;yIG7d+L+4+M_8Li4qF*@>7% zBC*7t>GU?F}bH+U%Yy;}|(YB5P-#@gzb!f@gjFK01!-mvZ(fQ)UTXIpDuG zWrQJ{L)(H|uWQ#qm+#@cIU*`mWZl$O$boSu?H*MKtj?*-9@bpwT@whS5A!&iWE%7v z#G1|XJOLD7iZu993{2H&z^B$rZYo@-B){Rg06Y9Y3_)4)EVpW7c+K7MHR=2rLZxlp zHiAZY@+3j!`|tvAH4U8er!Wx~QNCVNSHO$W}g4EgWt zxC3bU15#l|g;U)%OXEVOJ1a2{%d|>;#2knhvXyyQI{3TFCHij~c)hcLaOFl>Av1jf z22p<45th<_Oz9g#eX+RXuy81g+eU;)NrRMAi8l`z;F`rPq&e|-YdImW<_kMCcPlsu z94^DiPbFDCR7$99p(*-ewTmV2A<8;=C;>KxvedRpzesZ zr^<3xC-+iP_F%X?m=YoaYw9W;lPxsve0(_gKZt@d)=}-HZ<;PbXx<+%f>nt}N0Wp} z(o|E`lnZDps^;v$%n^u$2r%&0zZM&fTMUOnLO5wTw^ipeB|!A1zk2s>ph}FP!~#0)Zo$Ke5S^Em-acmf@A+{M{$0LPJ`(@h13HS9 zW$-_UjUgqydDpoXa%@${CK1PkSwf_^hs0SI&3(VvnHNlxIyGE|m&$tBgNW%xSxYn|Rc#G;9Q`z6k_Eee=mA*j*qS%) zc;iOHXJH9Xpq~cBr%y+@oogsWrX zG6(wyzwVB?i+TgFShy#R4))+37v2}<&e|NTj@o;~pRi&U){AW4SlGuFY)mYRA<8sV z>jnG}8inGlswsX1l0PZRWdMDBt;=~=Sl*x}d{p08C#2t1Y}9^pG~{FVujuZ6CjgMw zK1f#h{n0!^Zi67j@o01o(6F9TiH!kL7_mjm&1|Gg|FTYFcyF>Z3xfK9O=XLxii@G~9F)cf~eH#=VqQUt_XhrVdcxU=+51rzSYD%;;EAJq4RTJf8E-d{1& zUGOHY7ySBgLckHq>LpWF$%3md&&w|$Zci5@>+n3)2CeFdAcN=a?&^sviZg{7T(ckJ z{o{x~5ZV#f3*(`~fdB!U*xfgB*qU>DIXxYY<_5{&<@76Kcq=o%#RCk{ShFB4z<)9CoG)Sp!@Htdly#YQFP$RD7PLcC_8n=(~R ziK|E`tr^;&TNkMpRI^GScl{avY*S!&auvd|WV;XR7Wp!x9f0bYJcL{2qwS^X+K&%| zKTBazMi?=Ua8yv_M)U79d)lA})uamFa3OZZzqJwS7I>z6N@j}rD7<7e1(KtOjpx}3 z72`bdDP@SO;3C*nKCy!_e?Y>^!}*9yUJ`%hGhG+ExuGaqabXBeFq zw5M#bdu&E;3ITM+xz-)yX8QPKJz*D8yl+b84<2s2dgikczLL*o7_CTtwWgz4iF#w6 zArxBX-;!NPmd{zK%MpkXC*4+3IOAmA{12L`Ydd54-gO@KfyN0yU5Ql#a5EOKQWkrr zdSAO*$t|{Nc}B;lv}d|g$gH9S)fK!nk#|CCr4(sG=*DC(uf`Y0 z29{PaxO--s>(VUkf=H$$;*>UJWQ;;|tvG`~A1^J|s)6wiM!h5*2(x$qW8d)ktTw0l zEKyWu)TSc2@8j!Tu69yKM^E?GIjcUE@<^1sC3Cg<d^1R01DZb#mR%}w) zHO)pV8h^f2rso_6(}L2gPTbnfL0zr5zSBvt1Ta10Lr|5?gn@N0o%|Lru(J%peT=r5 z>=Y{B5H!CBb@-lmke!u)Pc;|+K5W1KtD}EW*KC%7T@c(%x-yh@wtnJwQVlYT%~CIy zTWKvDT_tC!>?N?tCAO`?NTyCeW)NdE`gB2@u^za1H-K}a>uCdOdJos!6aI(8i9Cv_ zx1Sg9-IwlN416>cfXJO^cl%6MDrv3MciV7ramvGStZ$%2jd9;pvf6D z?T`gQc!7X6+QYx02XTt14rhEO7sFPjCH#HfwqM{{*KXS~a))=XNhxZr!V}va4|$n9 zFB!vugUT7CJQDbM@pq!1-Qcjfd3Zn5+a0j`X@_Q$q*!NffIF$P`|Rm%S1zvy?a$G` zUs9#rto%ksvAX)NQ)PMTl&A_NCun?Xgj4#D!WNgbfC@qb{62C% z1{OcuZ7t6rVgJy74)7IO9TjITEog=ic*?6*psp0{B;UP%jq1T;i#WHT|A`~tGvhem zpm`#-0rwA4k@pWek=)$c+T51+bZs5beXnc?b-x~*D%CuNS{1UZ;?Yn;i@hxcz;N~B zcFbF}fwgKI(6z?gc2t6*#y~CyWsdid7gy?0T!SRt9B4Qi-knVjBB z95^D~Ntd8L?oX%i1uQ`D^|3eQpg9x8C*O~KQp6%?m;-^T_%U(t{yn2=ao-5o)u`{; z_7txJb#9f}ZTe$EN@{RaK=IN=N=*6xWT4#a(^_I}+j#T&q0iMu;3(e7s@9BJXujD?J?Wo^@-t?ZIw0TH0=?A>=r{}+hl_m0Ps9E zdMO#tV<~%P6pqd&PHt|BN$)QRz%qBIwk5}!WjGEkdW!WMaRPbzJx(k_IL&?}7nQ~U zU4If6e(sRZuf8h+Zl8yscN{|reYSKfKFn$J8ghf)%aSJsJD(C;*nBS*h&x?~(5zb* zz+lp>?)T%Xn;^i?1t|=3oKXu3Fl9}^8BR8uj|2{YTTs$Z=E!_zBAENhr9q>t8pG@- zfT>T9&x(P)jXrVw^+L)bKL5H#L=FUp-d8ev$sBO56Ubi=H8vsiL-P8m%}ilz*cJE@ zchV01rUi@PvLtDQ0{%0O9N;XVWbc5(FEdYycqG3cCEP9PZ()2)gX`i3I6}Jy4()oM zn?1(wUZHz80{5$DL29>+6GRE7JA^iJLCMbqa!Q6s)&d`f``gG>Lmy0F3V99mBkO2a zeoF>19BDDKvE^R7Qi?p!0_Ieth)a$cHs~4OUUl$wo#o>W^AR(@V*J9s$+ueDaN806 z7T@n*{7KfJNiFw>FHRQ+K+>O~HpnpOIZ5^k;>&u2`vNp`g+|FLaM47Ns$7ZHN0$CC z6C-uzLgrbz`UwM}{|kYXOWeLt&lDB_b7k|pqgF5p9A;RF6CLDd^l*5!@~YF}RARFhn19?3+SVt2SE5ks>ZaK5$>-v<)3RB7LkmPX4NQL=_$DJyRZ;JSC&WA_R60a zZ&i|X5uc|ClzRpVAoz!gxS7z*G?=r3Znw5Y1W$9|tei}&xpMF_o3ZYPIWK2;6k6uv ze#--yjoit@rSC6(o{RC3w=7gFkGz3<*~=k*2)?C#?M3AcTsAPyy>-86)qpXpT51io zUK^}pK}F1CY5mL<&moWD!|&(keNQF1V9^556%FsHQ{r+;06SWdTDE~DCAf>2*7-kI z6mb7mwOm44f}6pmOazHt+|vt4SN?WEg-NyxI*34*3A=303!s~EtLoojaVez1QfR*%G>xH*g!9wJG!J$1(S8XuJ9eEe$=V?O5=cR0EI&o@VWl>>pEz(q!KFtjmLEU z03I}rep@-L3QY(9${QMXs#WK6r?1iKG9ZC8=H0q}}Kv``&c+L<7NN|_eTkI4`StUx|QW7MkZCn?d z0#`Se0Eoldu$^DYr_#~ZHLet4lS|8smAaHdJe@AFew$$yUcEIyvZ0ZS8qRCBWErIo z>@4%=Fzk6mVEi*BVLb#*&s*h{`~z};H=3rBz z40h^~9!)o?pIlcFS%MXFDjO724H9z}41AJ%0bm?q+CLvC?t$Z2#9W3FLuY(j=63Bt zJD}=ST@6pudn^<{;5gA|zsBXLO*Y!&UmtJ2+De8YclpH%yMFRM){6g{@BCi#?1Gj} z3;Tt?7;DW(-?zl#pQKx}Q;INYM0VP5shm|MHCjI3`5i69+~2Ym@}Z6JCjY5vFrxgX z4oIAzq&8pxeTp@g$SKO57S-qcksfpDwG`3NaG6>t7=nyT7+8d1kJ^>(jK$xeHEhT* zV6z%ljG6SEd~S`s?UjGa&2a@TwnQ$)3(B z5t5_B0>{~ABdVGK46k5+_l%>thvdw}0)BghttjE(b>2Y=g63B1LYQ`f*y#g@iAIiy zhlqoOp0zQ=1y-%>pz&^m5xiVBL54cg9s_fqCYiF-6Xl>#CbEf%qeo%)OhV4PtX*;uPLk{O=UTPCnwxlbD6+ZyPR12}$_ zd0YphDn_!ElyIqCNCxjVl&zrE_Th4G!j{*nXGvXJDXZBLA;)#)h4sB*U`lSz?r7we zCg_YlepWc6GMw0Ne;+u9!vBRX&Td%sC$lFGf>wweO)26S`g&fTTBDhyO@?^N!~=2RO5!{L%MAQ5Zuo!WNKsA*xkRPsR3)3ox-uKmYyp zy0WLeXfE&8+v#qLo1W1@ZSh4e1ru-A77t*xsD*^0$=ms~lDCbu|Ew<>=qcMvtlB{e z$F+j)vfVfC|$&e}OB$GMJ8NG%s1at)fB0 zdl!1jhhZnG`1E3aRUagdN4R2Zb{7U?OZ$*1_9Elq_+{Y}oVOKL20uSvWn1%%A%34z zk!$NA4)-fRHkF0Ml?Hch1rRAE{+)uz#}|nr((PjL=85r^7XNG@&kj5VCxxMGcZ1|o0I+`@VJK}{_jK*H z%$?Qa4c!ePjBaJD9Hm~Un>k6QPN5Tx#Xt+*EXFrK);}?Rvq}HX(--x!p(POevKsgD zd5=pnP;e(}*mcXvmZ4tMm-v3;Y}Md}ucd`C-;e;Q*5VsELafxtFMhCIw9@ z%BtM?+_#ckOb}*+65tD`F)6o6eS=lJXZq2#V8iw#u%*l0%elVoRMs)R9MZX{3a?fo zh>Zm*F^O;xGW0qm#2x$9LXF!VyEJyWbqDSDfx`cdnTOc9nJLf#CT|%~S_u*)3?bIG zgLQ3vN}!98Jbyt%(V

RlkviaJm4Ot;p`qDnjm=HM8{f1PD=IT*LNey}+zMXW%+ljE%GWEmYJ%!DyVKdW!Rg- zLE=RkK?~au&13L?7zCl;@xb8YI(}2UEY4h?SY4-DK%BUdwQ?I7-H*He;7N&K;BMjx zdjmG*`d{c;FGNq%e7$kpSn-u@khZ}x{@SEQB=Aqu=K60468L~_9@Q&;og&lF0PPumJcnLlu|06 zEFYY;l&u*YN{Ms{UK@tGAOIb})cgw!a)Pu#Dn4*@*bipdPxkV5pDKozOb79UPUvzB zOrQgJfPqesVLp95$E1-5#X9v`P{%U=mxYuJ;w*87;+$1P*>aL`V9+3+={SSP5t%VH z`S7%&SHveOiu1VQ8^&)ND+fYQ4lj*S?Rpkxurr(Q)0Zv`|7T--W7%RTr5tevv;5$8Y%5~!f%J% zo^m0atnCg?S9kQ}_9ahNA)JUaTk+FB+j`Y~5s$kn0adl6gN2kFVM9|F_G=sTvF3TM zfL;S?j*sunQEgUN;oG$$iUh{Ak#8rQQ|(O^iqt8=aL4V{KM*g73RR^%iw}=CJ8XyA z-slRuZ-0I^c0ay_H5!A$18E=5j8!!};C9|C$u%}uz^5#Im>afZ70~1DzH4M+?8a2S zR63}5_2c%GNDrn}uBCz!F>R12fywew0SU4$3f6Gi{xI_}AIgZ2tSWm{6GAwm((~vE z`N=j)glAMC!q$itLe&<6)sTtVX28M{nb2CK)(QV$8YlEqsdiEh!pD3e>cLHPFuobE z9@J}U&ho#t%8thHi-fABe=R^f(0r5}^+o0wg%k$#X39Vy*%~p)AM^i~i6ewUi=E^v zwwUv^+i1o`JpHW|0)#?an9e5%NkHIXEhmb9C*Ip-*Iuxi160&^b8r}X^27w>mu4L3 zzG7^4pO)xIqE14ZHye&EVVRN&56Qw6FmG7KD%D1qg1JWf zr|=||bR=TS;|~exTwPgO+n1fwCZO!`+L?ao7ip`^KVY~OYKyH7{)snwU89q&9}`%z zw|k$l6N_D8k<}B$nn1o#r88)zq}-$=xB6=Y{|PGj&?{LcEhjjEQ$$cq<4dHYNGwCA6_u<^Pt%Mu4SI(l2q&P{P-`td z)8>^4@Hl`UtJc;28wUKYOvK?K)9Ap7KLPFx-<)|QR;9_x zocrN6vLq zWsVq0k*4P9Nz*Mhj?Ef@m^Kt))9bHC$+MzrVt-D&$Mw#%`?i7X4fG~RZqPT*Yii4u z1tUe!RSf1yJ1(_HZ=6!}9~?`2vPFFV%V&hZI4BC3kl&R%K26n<=*|BQq>xL02BzhT zGyhFqKw8ZkOl5_3zjxQWENRUOJE(@)xBniL7Z_o5rh*9yF#a3pHw!5o-Vwle1hVVu z`d%c)m)8vtMqG2ASW(E=IJFEoSZrdvy_hIjV361Yg9?gg)r~$taeSwt0<{{~mmN5Y zvRIO6i}QtCBHR|_`0I{AJ96@b^+5IQ5Sz2*Ayl82S=@v2D)jtWoYIrGC*x-6Ys_TQ zRNyw+DplFkEVpH?!p?(+^9@@6;fT|ek>$h)%$Ni_ZS_lp@Fmx!_sX~LWeP3He_9l9 zgE2OQIY)KacbCQItB3LZmn)@ELWx*fJf;9OIq^Le1nFsbylR2lZuDXXVl82Kp|B^9|)JSJMj zoLFSCsQUhjsW#skE~%DJuT=*l=gC9P#V5|@Q7fH-k5Ux5cA=vG__Ag({C2ju|7kLj z>H1TgAa?d$>+1q)0HAZWKRrjQ(vQ+SDw|uZGIq17JjLuGC!*T9S>yV8p@;-@lPfiq zZzK~ODa;fdt1u|ZRG2M0Tce>#f-4QELbZg#6b)9isDhzM)fD|HZdJCH7O|@Eqz;fS zY*nFB#ZDn3k04_(m|ROS=BQL8MH-2kjnU?uq)v^v>FC)*KXKP?)Mh5kAz6_I6jzxM zYZYE*ceQz^e7xXS@%3Wi@oEEZZ67~B(!EUJC(Q0G-wqlIL?mM$BJrr~22H~>c*2EX ziI0otO*B~1@^^I0u~s-}h$60azLB0I`tix4SoJpS@UGKu? zcT4Ylx(#X2A^m9|HcxJzb2)kX01Vue-FHo=RnJR zG(8?3!LL^-1eUXn90oAq^fv}DG^@u)*-2zz5hh$&sOSEs*2HH=K-qhxjD)NsD2y~{ z!m`}SGX5K}KnRJTAKL|p>I#Y2Da5}GCcH(2y&IO(!Jpi)6j-_FkPv6DY0RkGMAlEzeyI zg?9uVV4nTDA{>$VT@EAm;0e2v-q6AqKcZ4f*H8Vq{W3SybfUn)9s7R(Wk8z0@=-Li z?2#MO_L^pWP8^HJAz-eYMt^^>_wW}wFI6_P$4_K$b@1{n$O*kpsCRmiaJ*r0FU;!A z>t!Afz6s3=jTFx`#4f?hSsVbpm!$uw(JtkCp+h5Gmam!3g+o0ANSWg>*Aaphb$qWh z8S*7VKa=?TeAI&yDKkorn~S?4q8(mwcy*F^ruCsd=5rx~)TC|~Du3Q-D$6P6LQ+>7 zHnDiB3D^rfyf0;bZmvYF9Ywb`pXPft5qsAX^UqJkURd>00L+HjCkOWP{_v~<7f3C~ z%}1NMWnGQ$*q!VnEB4%ACJ@-%1@rrEb-|7<-C?iJJ8ulY@`1`bjEoX`1z$x=c9%x2 zHDnt&#*1pvMqAgjn173WWmQs*wl{&_k(C`Er&Kq`Htpn?hk5D6Vfr_e;g^FU6kfdh z{;hj!+tX}Z%ceDG0h_iJnM-(eRc=qCvf5l%yO&Ena{<+{z@7j6`q|{}+#0^UzNKQn zMOIc)R+^(U?8`cMa9?8+hht_bHlx6W98*m#MAuH8Lf5|f3V#J|=D}@ftRg;sE%xS^ zm3i1Z?n2KeAEP=Rq|%$$bD~IS^O37qSD!$HP+SCC2u8jW9!zisLOuPR_FX(`+QyaJ zcXK9qDE$>$hAsxGX-o4Bbcj846B(O%vLk^8>aXWMIntEAl*4Vmxi24@11tMPHaI7S z%dk?>htK)b@_(Ui#0Pt|Y_Yl1UOm`yVu;8$D*FAc-<`tX6*P#b&b z6J%`R8Sf3k&x+#-oiWc`3HFuJ>-FJNPpn&fRi>Rt56&pr$UxweuRg# zrTr1=X1gCDV>|Ekfo=3p_aFv$Ni1a*C8f%|jVOa7i@Lr+J#5Ff!(rpK$f-RG+Mm#g7W};X&H3C`m#5r3_ zSz&2$sejSP;}ASeBB*J`^xJAM%SO_}G__rOQV}IxZ>!sDZ-6m4z?)fCRfx)vlF7@@ zBprNsL(3J{hP<=;kg=DStDdG#JSZC`ZD7$((!7IN73ne0I;A+mD2c_=rx_70juesX zb74r*&m4iH1U`d2&0PCEUj9hTwf%88F>vIGwSVcSg?`dpAJw`p$$CGOiSA@kcOW=mR`R4!k=H~xJ*N<|0 z2FG;|>uw16hzFbD8wJE;4yNHWoQ@~s>3=v5$AJp_KnnR$fe*xS*k_+&M&1h>0=a~= zNc9pnB((E`7=F{ul5cM+|Fit=@vGeR6E|8fv9~`|Z)xX`thI$rSIUXTDjgzMN4J5e zSZD*C0$Bg=@jSd};d6Kbp4c}R=E0(`z6UQ3d)6pkKtx6rcme|VJ4A&VySZcblYb3P z_Ji&e-G2T+9E#ecl8LgsBce=G1k2^^+=8f+sUh_#31vfb;trv*GJA z6W`|EqnQW4gn#w#D-cO%w4O-&kbi~AkNyVJch<6ws@6Hqf)h8A_aG|0CS+W{9>;`UXSP3hYu;E>J9^62}*=%NWh0>Ob zN(zj~kji0SOb6jm9IV8l*KiO_!7?u6pas>idu+(|%d_+P#Bdud3J48^Ab&gsWSl+K zi1OKzT$EdijE&huSFoECuf!{rc*92gJa)qpE;rX?Dpls%&>pJ=H67$g&grW}<${&+ z3%L%16OK5bdw48hz#V+rz=vrZ{8OJ|e_?q}mZ>Pm#BC@^GU(VfMW6N+^2O(hKEsql z25h(E#{0@2rrgD0+>k^)Ab(}zpLF4VSJ8fcQnVv5nm*Q*n`-Af5>Q64G8k#kCUDHz z`1kIZ?Cg)W`}=s@CHN^KO$+f$9Cn9p7%hcG<%`2fQHgN{)Y^k z(Qc6K&1u)ISE>y!V}G{=nD2mTgZ;6UYYkUhYu(%0D%*VV zP7Xm~z?Fdkl8?S(2PivWa>wNkQO{M_c~vaT1YU6_)sfl2X%8nO3U(!O3GyQy4cD%c7Hpbh46$aFfOCBgfD4x z6j!jv3XY{|l&=IJ!}`$r2RiORvB|vSiKFz0qZU+ufIHYIi9h*1YGBX9cpE&+%S}W& z?h$JLnT(mUU1F6bgeo%0!1&MU(oCE$>6L=WTKxt1vbTrs;n`{_Y`4I6qr{143-|s? zgKzaBJj&|AJ%4FFv=bsSy?z9drWeNaG7kg~7^gU=iK16;PfmIrecH^yVeoZgUx?7q zrWu=N%qWv>Bi-{h?v(o6(ck#&#D&i|`JG&+AUqj6rkoNFemXc&K~!A71nF4al3^8Z zrrRv2tc7ncHJdV&L*r@u+_bgW1Ft9rMcU*%N za9DEQDStKUSD_R(c`dS}^6HOiSUKb+$+B8Xn%Vkl%KnX}Of%Tk59)(aZyB>?Adga=L za5pP#Jy2tlavpAxI!|}xIojy~qS5W;=JIm0#avxgd&|?ax8js-kHuP5Zn0K)juAe5 zX0S!^FO%TfvVSo%K0XR_baG{3Z3=kWrFsWg6xaIxEX(d-LuFi}GiGKjF~;7FNsNNM z_l{DR-eD;su#{!X*5{aCaEngEF>=80);(xd`3SVWAM}+n20%3^oCg{xy zEFo|rL2f4u6$``c(a7H7eZB3wT+sd@65Vqod6OVS>3#9L-SGYb=sgVoZ^Y)_Bdiyg z-fs~Oc7%xNMf4*E5pNO?iQh$1QSUz<4+QL&M@C{C)kR z4O`IsPxuzPhj?G$Pm|h`?VAF@zXK z3@1htV~DZDIAT08ftW;0CZ-V6i5bMZ#7yEn;(g*nVm2{{a3)-cxx_qTKCysUNGu|j z5PwUFWyEq~6|tIFL#!p%5$lN!#3o`hv6XNo+z5BVgYYE02ya3`_z-@CKM_C#5>v_|MA0A-Bl=B@Hw}L+C6-#Z&z^`8k)ljHCp)#>J@Fgj zHR9v;^X(5yiM*nXRmX;1Nu1h@#$0UIN9+| z-z|Nw_8ZzSrQfCgv-+QMqMgE>GMw&Ei>T`Z3WCbP5IkGbL8ZF#U94s{v2acC)QbnHw4JlxC++hRnW9)AP* zIC5n|cB6_zL7fswVY)6mo{8706Vf^4)%9cCCa<0A^@D@RpG}a|N_cW}gr&x0$TJwZ zL+}j=1k#geoW!6psx@@*S`fwtk*4NrU=M=^z+4$pN})ZxC2DiJhwMzmT5@%SS{0xMNL6^ z<2D1=RC_ePfjN5Cdot%kZqm-zE(>;;;~%lnjiyxZJ&d^AerVs}a+Y%O%Bb}?%v4@1 z`ievEAD2;vv2*>mZ}Dt8cDn5@oSDTsc5cEDw432x-U)Jd;kkz{PpE^P9DiIY`*!B% zNL;`^EWdi?Gy2;JchEq#%Py!|def-#<+`d#U!CehHZn*UBBi3>;QJl_;7~vKwQT8@ z8A!~cx1e6yXv{v$!hTZf=EJ7x8@Ot8S2l0s%n>Z=FLl*rH*&BBt|BS>$8F;magM@C zmHA>DJa2|RZBEbH7vF`(yMNSSp$rDeAob2;B%S5%8W+#PYa|+>k_3_0a?-YNT|ZdJ z=1Y>O;$BqwKlH|i| zn6e*EgEQ{5oy@(g_kV5TJhkfJL^`}Irmm)VCXZ^y=P*$PH>dfM^=Y~L`H^$R7cj=S6A1w=$8FouHoN8bE4t;XCk=REQW3d z*_FsvFTc33@bJ>G=HjBPynNR9LFfuMR|mCPuh-CNnVQUk+kdwqi9;z9@SP%F1lfB> zpTNW?;?9k*Lk8C%22Mh=UmrYnQuqO6;V8I@dwU=f$FB6r4XtI^5p!~?4z9U3rUsyDO6Ap z*&O0tvz|dchkw0;WQck+3QDgW+}&W|7L(K*D6@|PI@J!A+PgJ)yV9N6?pw1bk_&Iu zo^7RDj2G$;vV7&9##2mF+s^Ggm%jt(;&8~!!r60yzsJKF*zX}7aEk?wv6w94PC3k% zMM%;3h$t5{he3m1M0~}4@?n19bj||HBBaw)L8IA8Fn{euqR6ChT9J0j)o+3JP5_*++sLbhLWZ5ovg1dyP2nXi1Egusa4i>_O*!G zj#VCm+J9qWVH^BF)~q_HXk%k5+D@i~4;j*|tlwyOh58QQP$!S2oUNE4ySe z=Q$f^PMdq_?zx65-(V>E&XUx{9u8iC5gv*WK7Y)^L%2Q->G0~f-4S7d!!1;c@SCtz zW`pU_Uty9`22;~5u$O6l0JO~V#<0_1mXvy@dkcC? z;b_ZR#FlW5@I2rJ2@j`!onsb3`}Xg@6~oTXzOojd#trkQZy3rV@`Iq;=q9;c1s36A z4}Ya5$-cutyN~0eNoQStGY3D1QXchy4&--7jvO!*gS28cNfVbCK}VIu)YcbQRI@ge zq1~e~cj0C;eBbvQvB%LJ(;q`w2Q{FdupVxc!Co_;AY!X~yq_-%HNdo{)?HQE}QsX&v zl#BzZMcGDERt8&=-*W0J=7eQiTn<;_WAxrk`+=2_9;|nyPxv;*WzEU+J2-029KFe; z&u1D-E!EXKECC6AY1=rea?rG^qj5`f`1zlZvigCyiRJ}mj0m2*E*1jeOd0>Suz!`T z09o-VT5B~#7rOinqT{CFk)nq?JOg&_m+bS)=4( z0x!H}tIE?m#GR2aUB;8hQ7iG94otpWPy>12r1IgGlE@xWDWOZymL^+$-rRWMw+EBZ z+ZF8Jl;?)P^`}NLXa$;%CL<5z0e`2WDR9k=FKf>4=9Go|yz-e`kWh>|;g14hMXxG(Cwtn8`cSbHa ze)7cmfr_b_p*3I5`KuDE-jx zDpXvhZU_yy_HT>;yYw7fOWp%~P%~Sa5Kyt2nc=q5H=fhOpQO~oKYz1_(pSkVF+qDP zsavE12RXQm{N?(YE0^=vuVGQbpVmkm;XSzWoG2fbiD5llDLaSCR+EnK_ZN+CNQW5S z8_0)~A8217WXiwI)>z&D0K4z~fzACI^0w)P`&!ipY`E38+t|0gw{5=RzHKc$L2Xd< z%;}ro-fO2U5Ma+1YkxBLa)Owo&LX+hg^zO8xT~38nP+E`G&Sk;yE$smI^ic-ZB$LL zKOR}as={~euBfx%;LTT>0kmHa?UiOBns;i2_DiAtPxq+Jf}&Gy%_^Bz;xPr76kJd< z`3NNXab2$Dgw1O=u+%X1LU7KPqeh%4*j3XUS`o@o^OXJxZhssFoUgC!`1!`Q&+}PB z4lwSfg;iv05x9)0#ChalF8NM~w0Yw7{qf%xw^KhtpUpBsMWV4*A8zJaw?R&BPnotf zJqi?tv&9(|MYVKITt#STTuk`JGofcczjWmROAY88X8%{0I?Mk+zB3Agv_3lHKrn~m z;SX>Tbk4~+d4K#X!(g7&h6jQ$#&-=c`Exc634lum67q>Rb}D zE<3mwRZ6l_aubv2U~dp(<^N4q{_iMQx;sg@HCUguf8M^gow_UJU~7Gp;G}{>6kI6L z8`HB)bjI}yc{%Lm<88)Dg9A1@BoTx%zRQh<1Z;M>K7UV@NXH~3SjhNQjsHvT%A~en#v5eKQgJ)g z07(2`-mb%gc4Jxs8N6_XL*a4jb^8%U`B-A?1LB zU6;Pv?v_t5iP-xw(6(TtKC|tk%KW0@;C};#P=7}|PyJ2VQS9KpO>oZN&bYX`FAwE@ zC8@8_&_6h7<$o+d)X~?Mp#QB*Su1yUUepOQ>u}lb&hH*t)Wd`y{uH=|ab2@X^h_v` z6r|`4u?+e!s9suJq1P94Ra%2SmhLM555Sd@ndt_7hFTcg^>9#9u0fy86oKq=PB==| zf`7wL+3w=}C!DDI|6G}0!E*I~gr)vJx?HZT>3r8>0ZOtxP#RS+QylL7XAvROktmig&70+Kys2czl9Ud zk=4K$#V%bw1q3odWW{+M%HX5Y8AeqozI_SC@3yi?OtaDUJA z6s~M?y9+Q<}%Kp=hVLZNb8#KvosbHUVP=e=@m4DMdN@mm= zjamzI<#F7jQD^a7ub|OvAI!M=AtyF53Fx%6Izy9{cmESeooA0jy8x9vtB3^&ENXy_(jrhE69Z!LPmQRNOo#;8H(hZg>6pD<(pQWh8&s*Ge45(C3r8J86ucyR1>?e1q^woToD9~A_i z{TvMsKiK-<6SLEq?<=2edA8=Ikgp) z&22Q2cNM)3!YIsdczDtbPjsgOcsjt5D(?*YPL>7ov5u3Hrf27={o*2+Evd(UNZ=B` zjJvX>ab5Y~ht;<(-hZP1iKJhnw^{V@pVEa2d6h8D*R8GKx^13(nJ`(9Bum)wLWsMtB-#jI)BHL)Tqa89baS{f!@e! z^)X4|bZF&HbD5=}!CIT~KsBF-v-yiWe6LP)=E}!c;Ib=XK?7IFo-DpIqk=^Pc**i& z?^EaKYo{*Wg72HryW6u^wE?K}nL?0R&cSWAAu3I;PfBC`)=27uJzFO;IGv^pefOlx ztuQWMb3EzVj(^&;U1@bb4wd0~0Ri;9&2wjj%+5Tzfn9w{dqv;ukf%=tS|$;s#c{}$ z+yT<^vkk`lY!;X@8c(wR{1x902KM;!5wyebuqv~$To_vJ)OZAX9*H_ay$ieG0NKgl zT}cIOK!bOVM~=hV{6&T5=%$+1d(bX(Qjn1~q~znd*MDa5Pd07UmLaja{`Guu}Y?fvk z4CUpl(FlxXfKtAuf#gJeh}MUJ|{at#+D4M3g}i_}z- zd%3W{AuIb9C}g0Q6c5CcC6f>nfVj8lWU&3vA#gB{Jx-oBoz6MMW>gj3J8%gyaLu4x zTplZFn-(DqtGfGOpT()|K6JhxdY|gTvX471>VK2}iNp(iHf@bzt4SR7{hE(gx2gTj z>`pJRJ*ax8xnx)F-V1I=-bDitgT=LQQ~@34{lHPDq4>vtz(L$3gKR6P(YG3&+%Gu#i4suDEG$yyfu z<9}Z+lJrE74ror4`X|121Z}z?=@4V;L}{(^=Rg zi?wP+P#TMr7%MofsJIc1VajTNE|06!096!?)7>8=3}d~l%EZs$5Oe6MO_}f_hk>Hq zR@PYoipnxrV}30tXW*FP05q-0+Agph_kVs2Oy%LS7^h#ZVttCXP|x8vu%GN~#5&SB zSmCn>JSb%$4~O$;xRu3BjgabR2Fu^kI4DVq{|41dQQseuny=SgztXJtwGny;eQKmGjY-v~>62+yHYjBeNbj^PdH9xouLZe8dyLv#29OZA4|kW_2;r?-H1GKWT! zv%o|+_RN90OZDs|tTkiD5L+X%zRPNUPYEaS1$}hqbY4{6xmDa5VslX!(ne-+7*q~3V%k4x(H+dy8bM%uy|7RU4R`$ly&t-vBfZ1eHW@(r24qyY}Sg8RbU*gzt4_{o6&{TGk3*OU{<<&)du3YJzbR@x{ETtkYTC zxwacAWr!YYde(b4>&NEr6WNI?n=r;NC%)>=HR8^IfqX2vF*=iv=8!M^6ut?!nHzRDbXcVjB*xO| zR$%h72_a@>6rHTorKwmXMYE6$u9xUc={Xs6fw^?Q!HT4xixtU%*s^l+ba`4R?*GdW zkKRS;l9+_xZGR(~?uotSw2c^%+YCEu%jjHFMs_|6he01W8Kp{$8k0s#Cnus{g zjhC{B?eqljP6dL2(h7w*!KhX1R@V&Xn_G(yTjda4crZ?h;0;DY zA~v3qSL$*S@T{Je{Si>7?m>;dv2sKA(N-Hd;&Gn5JKV`poTau$Hj91esDLcdT zET!$1O=|b*b-Qc~+rwP;H82Zqyi@tt)WN@|5w9=V5kV*Gb?Hg$NTkQ?&n(g8=`u3t znif$0e}B^GNICGsNjthr6tPOOP8qsjDMQ`Fl2}hq$)HGx51yY%3Giv1Eo^b(A7thf{SlDoqv#xt%G;*LDQGr!c`_d05P=#{KBo& z0iS_1`|V!4%1xn3XInznW`;3ik;6tra7g;^Cn91K6#p^h-Gn~><+Oz4QBQnY(sAVQ z!PlOaprm?Pyf!8=lvW!eYExKkB1lQ15sCbvgnW3X?W3c0vEY^d0I7~HBY=Ndkm9G8ABSq~8x;9Dn+ZrMz=%K}sE$qKQuqr{i*Vma6fPG}$(V z-p8*2VXqxU;iiHA(>2HEJ0U(A1JC2d&*Nc^%>1u-CDy`uVQTmJ@iKgV>?z!nU5GsB z{*vle6y;@Qu?C~j0Dv(;LkFhKUACM}N_s_i?|uP$vX0h2AL=JCy+`6d0wSha|2rTk z*a>Uab4y`JewJCv_OxFY+Peqn-HY#ewU^mowB9h=Xf$2y-P_V?$}(n|Os34NjNX0z z9~1^lnUg`=6qBR)yMHjyC`Dt^WvW_(ASz2-CKD4EP(+F10#TGk0YN~JMs{#RHoN;Z zjqISPpx^=uQI<%lNnD~aCKVv4bcDWluR#YDz=h zr9eSFUBo|D9}_nxHZCP2F(Y@pkDuSf8T!=Rw8Z#?4DEP7zX{`P$Jez1KHA{ul}Y;S z^rS>>bV{r?*ndYm&qo`g&n9ORwWIYZ+L*Y6=;T#e{VHu(T(~wfJuWR>8=t1nOilOl z(S{`?rfakHX-Qi0ofelI7o8p#tIbS_jZ4#JB*bZh-VP1Z&eEr3Xy+!bj7v$6(|UVr zwdrwj+JuaZ)XBcS8JY1u`m}i8Rb))MZ}O8t>Atr6yeenS4+-<0J7-2Moxxy zl|D@y88Op*>4w42hdIQ4Q4bG&(yF}*|YW|?9KMK9Q++F4bB_<#S0r9 zhdVBHY;}C(RP1!k>8@&~>SyOQ&bNn5f`cln6-nSB$;-19)_=H3V^~~erRkRZO(>7o ze{U(wEfTT0|3`X#v7vAppQU5K!9pK7jSTOI5{t&8I!#&a32>sJkvWGw=y5v6a`*?- z!*||7hISEOgCaG~VSs^FS|y(@?pPzi`~%?K;j$MTAQl)-C!g4(-n|iR6hh3XZnp-K z*xNHd!Cs57YRiNb?Kyi}+ZwwM{Y2ho|-vtWB*PpNmD(hLY<)a}i7w}v-3X7Rt5{fz`9*jnz`5Kk=O1>mNmJ}Dj zTQ(e5S}P!1y{pKax3K`_nFOpU!cHL`{FK-Be6&Q=o!RvHcbd}{pg#Uo`1F%+Yu@L9 z`e7R8n#3H7*|?i;F_*T81gV{+p!{CZwXgv&l6^3$mh=bMp-$w&a@QHN3N+6$aueH^FBG^&9ORE#Y>!{MI}Uu{ZwJ7>LR39we~={K9zb`v zJPbUQ)(_!3HKgJ(pvKFD{($F@&R}3Ej3M!@7O|&UjTM^~CoLC(A||X0;jxE&8$6g# zzs?CfF1CkP2Kf*aIU9v!Wd|&jbf;`vfvto@!SVw)iJe!3j)?NFEZ2hr3uzsDz5>OH zcnN!C>5}r*WV(?P@Cf|kfA`~H^hd|`G`EV&SZ+Ef?6|$T8)L$z@mPhx>C!QDvgL@_ zyrt?{r|tz)o|sUMH$@pv4xpJMJ+ zNBN5q9MUoV1yN&!FsO!c4A%vdS^}`sc2Q!Ip_u+nVT0gn0)f?qf8hqHz*IL@du?`Kr*u<2bJU*OV$+&5m}3pf_Xf`1TVcO!YQVy8trW@zPj{;*s*Hy zj#)gOIME#s#hRbSz$6&o^Qb`_2O2dlWJ-^iyBiPMceZra?&raw6c0-a#;j=q-u)DF zl|1`-A&peZo^BFmNKG&jYl!hek$j{Vvc!lwU`R1Be>&FE^#Jwqy$_I~i{$qT$!{MC z>!0A~@)h`D6;NC=9C4H8$(}5C1_#(eIXh4y=WXk$EHwAxMY&Jilh7U&m5>k>)t=DP z)85`g8s;CufOC@4I&6UVvkLR@cpSvLlbCpkF|5D}w2WWjfh=+2^9sMj4unE(w;^Ot zf_Mjde+fEPp5-q71rKMk>BrZQv_g_^l4=|M!GRCtR??=r9(Rz^`7|9lWjTBX9pO1c_xm<Wko zfBxXDw06SxYDmVTU?fh%xmd)b@hZF$wD=XshCC2p6hsTydz(6N%lPP7LcsFC%(wXk z0sFp)5m%otynIu0!Sc8qza}I&lN-4=TV_DIb9p zHE4YM@N4+*exvY9c*reJcmv-2whx>If9zVLo`^7oIm{uTzCDWGYd9plv$MPLIN#m9 zE}=XIc|eRy4=JC+BHj6H zXHUuZAzckG`Q63t(-$Y_W(dbwu8Mk2`#7=R{HHX8uv|cXLeIOKyLcG$^LHSM*wqb> z2wW1pL5EH*7xL?nd8Y;vSZ+lAPP4hbUg+w4zr2b6>~sirA@SIndTR%?4k*=d1jBzO z>iY5t*KfEx)*pXRHxZrBGHu>ye-d;hDVb~d`30A|OGIN$;YHKd4VS`Nqqf_F{ekyB z`Bc+~2mK2#5c-D@;{W?OLLCoKH{bnWKnVy!t>D1czl#!E#kK^L6QG%cmR(8|GT%MF zrHlV}-FMqXEXiK#Q1o~Fx`6G;@WBjBDxDHSyhzG3&OQPPfodJ7xY2Tme{84z;O+fO z0#`1bn;`(ra#!)>zcBW9DVo7vY6jI$0?Nrt(ZsZ6@893n*h$(ImoGt7kIeQcF5HKU z_m%QxSfK{xb+e90+rZ5#_`R zb!!vKSMqDH55mJ%?n>R2Cw3+rF21(GzN7f_jSDo>eNpC!wO4yK3H2o+$+*#RYkhQm(MBRPS0>vmvWw!-is^-}yg2 zbjc=Egf!XP^XnV8f7Wd)7i8zhKJ?aFRDnpP z7A~eG2I#8l8{p1g>%We@3AauXXKrTPubRFh~zi9h;32|}}7 z9s;A2V6ep9mtA0p`U71!8Ekdz8H#&OBfV4J~R=uOaWZWU{ zH<(SCntmsiv^>Q!tiUsjFzmFz`N!m}B?!9NsM zkAT$(R*wXKF3{ThWUY)frgVO+ZLO-~rhJ2;*klk#;wsF*LHaaViZRg9(WT~fqV6o) zxD*+XW{Yi%@FotUx!EJ3hUTJdZC>sdtm}|JB#Q$}EG+vx5$S85B?3uklcr=7d7v_pEbRWT zEX?m8kr6>hWF`g87K_qYid`(amCxxr<)@<6;3}aofWehlR%`MJ1*AczSzJ zn-_l?h;nCqCiQ!?2 zCGntGL*Kux4PDKBaY7P!zfrTPIaB~Oms&tAq!v+2sHN00YB_(k zids#rrPfiKsjbvDDumijMN(1JE-IReq2j3oN=YSBNmPoM&_EuZl&75#Cn}Qy(QeV> z0igrB#IwYWPI9MmdKjG{c}a4D@nIG+70fN?Sm&!Q9xjIl1`a$mXzrl$LEpHBxLzEb z?l#Ts%OU!qTZdj9Hf>moyUbndj-`KRhv&$q%fKTI4!QN7M^@Nk6$x3;EbcAwu%_>uYiKL8F{!bk1R07e z`FFvChC0awJee-eLRyVHb}dTaq8a0nE_95=EL6&{Mgnedmv$3+!QTSk>=u6=25(pa zTgAd>9u{V8ZgE={kLL)h=-T{(5*3^2Slib&MHQbOyi7Yn$r#KndNX@hLtB^-gXVC# zybKu*-{&AyVqQy^?XHT-m&Zk*bWX|qVx$kZYjq7g-U=IJc%|f9$fwMaWGQyJ!!C_9aMO%7^heilu zp1$W3$h!&M$O3!J5MdF)z-v(Og;;0=PWI8#^V3Y+t(ps+7v)#xU&VhS4j)!Zt{3NT z=WP;ZdqMspP6_835O>|^;>UnN7LhRbRSr9WpQNMM*v5S?c~Ik@$k$_oEO_TaZ*IIK zvcTBE!z!4-+-sU0!n^f0C0lzZTSd2@3KbS{ZxCe4Do|;b3gzeMI6O8FyZjA1u~YG? zvv3IBg^*)6e|_=>_B?;UXBzaHDz+9G_w$g$>_w%j!a|gt$GtU+KD=|q`7tcsg#GX| zycKT)@(OQ*|1U5S29aF3Fd`ST@5seM<@U^n53O%Ml4@M`G% z_}*JsTF0vm`uqyE3KiG!;Lp^evYdRKHYbk@+en*J-abBu#S?$A3=hJS@FXCwFcCcN z{Ocbi!7gMyWfeLIvD114&!5D-8n`cmwfI%=#tZ(MPB#ez!dQ46R`IWSj7yuhWjz

vKBZ(@X3Xo{ zcu@CO=c^K!a&RMlwU)O?;vku}-F=S8{^yANC~*_clM;VTwLqA~{lsl_yX$tf1tb=Y zMfylZYITEs*}>_FoAGcKQ~hPOyo>pya`z&#>}I88QTFZtZa%!}sLUr4=`jkN8zZsE zL>5j3f7-VC89d^Cc!VHEN~zp>gOD|@8BST@q}83I8+sqSEHKF;-Pk)y*evS{`uYn& zOQfo3MjC%;Z2KIwvr_Bv2S{^-x9(&np{h0>bl);RRz~|e5b@299?wmN5{Xpiz+{93 zyqeczMJ=h;LnQ53`?D`?vh1&x;rFDE1vPvp>kE6qpTj2oXoMklEs{Q(lY!pj!S6cr zi792V1Bk_{lmKoK%yR(oodgE7?7~hJq*Na!TOohbDvY(dlXhHpf6sksU~eN7$gB*O zYUu4g$QZ##w=vSr?@?wHzm&m)u@}nr#%s>PVK@ToPJi{wlQ*%~0Z!`VXisV9Up8oK z^r}mvSiBj}#M5yQ4uUuE4A^?7r}0u7pIp@uYGTV#Sq%?!34ZUG&}Gi8SC(mk*%ml$ zaX)|411EZdZc77&hpm!MQ!RH+{5p@P~$<_NV8C} zBlM8qJ2ZoOkcr0&A3&=K=kk1^fGL4UpUmtk&Hhan%%7~mOl~LDBmFz=cOyme;J7CW z+{6!sHuCX@4KsHCe69ju=eC298`H}kxYR#rn zmUeh@Z5;WGI(+vJbGXuz#}-kfs*zc7G%M0yOCB0N?Bktq1JDNxC;@S>szQX8ofxpuMPb~IZtCoeA~gw$@4C+-dR4=ruvgLxd$>5V#)X{8|jKP3Gs z0>fWm0{W@n7B`gu3~PJ@^LLRfJD*_Ba*n)ER!yozc(=#?))-d+@&;qP;-X1$$uJv* zn>LKXQk5CL_YV+5+lGmb)#mf)+R87a=e4dpB?V18p2B6k{f|!*0&V{30q1=}w{Gap zkao-Z=)^el--Y>@f6@EaFbm>pC+5#SfRb2ugxkgS>%=+$uDfOa-udccA=E7tJJnT7%R=n7fLqXK@jsz*xiFCzIN)ky2#iYWtux)Xi zB+pO$U>$HQz0@kP3R?G;qWQU)rKc{twI4Xh&kUURW60$?$ldF2%;gNxBX>LzTZI>h%%_KDtD@ypch8auLJ1zvazrl z2Tc4XOrdWM1-Pt?&sC!WgQsVldA|E|Q{qIM_8Ggq-a`W#isZp}L;JA9btsXz5#dP( zqL?v|1Dy99m=FV^GpnQ|5Vn0;eS*BZAhyJD3eN9m>x;u<%B@)(Y0qNCGs<#QT`U2#U}CqUCk*(jGOy6 z75|t~_K@6LT6l#3Q26vaO^_x;e8edpThUkGvHA$^yfdZ5UE)r@4{~$Pty_!eM+>P; zc;{n4SzVp)$cJ}Gf!12?AhG5ZW(fz`Zk%~|5(sY8=wp|O{2yN{bn%2pHw@jW1b2>U z+U+`I?|W0TvEqs4ipXu)NrYMD#swQyaoP^_L4ftu8wG6okHaZ`LmHNJ+j`5gOs6JQ z*?aNcHWyEEZ%1J5!MZWA{=(dL?3S1^q6v=Y-L4U3w?inQG<7>X9e7 z0zac(>EejFSmS21-SBZoM``f!p&<&?yJzG28thH>UbUJj1vnqx^@`7KUL&$8@iC`H z&B5oxpDN`wxN~_&`2#h*&@??rm8s720oj?|4d4-!~9cC*qh*x5wZu_UGu1x-73Qi41BLF#Dz(O9+vY z;&H{lhKG+w>Z-Hfp!(vTyakU)1+4wBQ1F}CYY$9Lk^>wZro-jLG9{CAhw9}duGa1z z_Ni2PvsKd$GaXs0X)fmS(Jre!les?tqJMAq3*or_9sC-=^!+X{HTxCTz z(x%pfUV;~MyzuO1>e{GbL%;gqiKUwwiZ(Tfzon{4*nmk*9mQp0g|ac~@2pTn=p@{z zwq_QZIbFD^vDdHGFb6SdW?=gbk6^ifJ0|kj{6)nPKjxs@=8VwxG8$WR+z1OO?_26U z`*r@HXkAQ#Z#_!gLB(J0LL8mzR~o17&DlmBEb!!B^?rT5G0>g0F(8xq1inIfg6w9q zftw{nsLwO&gycjSOhmRmVn8gAqs`T8{6)%HPR^LHcT zWz}eJf@d%b9XAXtq5i6_*;_xrn6WdD3)E-kpH<_Y+K4zNO%IAEt_^SW8MAD)3{@iS z2q%YQ`Nqm2HSdzs53l&6ezKNW-cIms)){l%p~-a|OIJupUfihXVg5FM;;Q;(KlIxM z;ktV2SjXfXhM+WOvT_bz(kU(L1QmmQ*df&cTBn!L$WLjLoxV-ZMSB%+s9QR@XoJfo z2}(J|tc=zId8C$d`gDvPmqSfoTTVsolva66+MFZz9`Uv3{DSx4j-mxFD?WJeF{em! zEr-aCjFKz+^Fby>EniB9d|OrP5JF|E2eY7lYYsOdpOj1-C!AT-uBbdTnwOt^IbJ5| zlG7T-T%`15^zq3TKs^l*pHZ9-I@i4Yn-}m^dG#S{D#Z@dZ%TR@Jl(o?<+q@^N1J#c z+i5eKW@-%h2)aIc`VgG*HxwP>R+67M>T#G5!`=Oza?OF0`AhfHZUIshshUMOJy@aHvvvUw($jMzt>+$q4~v)LcF zJ$X99RPTon&H4hMneu>#e8Xnng1wy0ZY>L4h%v!YBtd!i`$jZBBxsy|>NIzcZ?4>S zESIS{PVd*-G`or7rqoAGr_Jk|Zbm|xOlXWiAdfL<4`K^H?MU&&Bi%6@+cJ*~rbust zAC@TyF1*T3xF2JW<IeSB%uq9xSFKr#NV!2Fq4k%Y zNtYFf3k+PfLUy1QOh#2-se2S$k*D;-pQY{4KLS_9My?Fk!`sfDRZBJ8gA~(^?ccKM zkp8ACx6%V7mNrZN^#*2hQWr;Q2`ZjPRousgGQw`>ZwV}VLeuF zP$LaDE|eWeKZ^BH@D+DHy&^C7(zK>6i$wnjT>aODDKJaSJb`8i0_B)n?BUsY$#&?f zl@qswk-e($g5f`jsS-O_iMYW(Dz|da@t7}MR_soyE(?wQ7Bzy;?06M%o-|4FG~#MM zT4U+U+jcH&WE>e9RNd6}srOQQhNYH-zz@PSWsjPnz2-R^GpuH;xYjkX%QhNvd02z@ zG93kEP+b3GOG}xvlcM007NmpdJ>i5;X>vWrKZ%~`SLS$Dp6sGCdoP~JOOdsgDq-Of z`kQVk*F>u(m7tO|w5;Qw=9w`<8`gpabw_n;$K}>W2g$dhKpWEZ(Jv+3`RAW&c((88 z7hG@E$=4FrHE9?)(o)nNcclvN2HRaTq0s=uUiRMB^)uge=6P7&*L*sJgqR?ILxHSX zZ^`cV8>GtFQ)jCO_z~+W{Xpuvww~ob{>pbZb-NUf2kE<0TemF#3t{ut|HB>sfFst^ zQ|LmY^;3k&)JHK*n?#{?t1Xk$DSj%6N1p!w-80%6@=4CYOCPus3jeI9STl)zR zd)c3?t;cp(*$G!!>HYSXS*0ek3E$!CqaPkK#G*@A)sS{9xe8`szxiu(H4Hv?WkNtk z5G*Y_-Y7}abIyR?srQ1T=?|+ez*jVoA~P*{h;Nu?5Y; z)$7cN7;D;()o%t`TCZ?b5Ro2}O|L_Rt%7s+fzB_E=mL-n?zqo;!M6=SsOPTy(xxh0 zPQ?)I9t3wyw2PZiZ7?n94FWUHlXtg}fi)$+%HFv(RmF}2Gj^<)n0V2}?hnL$OFw)v zQYT7!Hw`c^oNw1j*lsXSFAzBm#LRz}v1oz$04!|(6Ts`I38jfT`sW(BG+a(`L&;S3 zCE_E@a>;XEbS^>|2t~|Eh+Qbm%@On24~^0kCg}2mxg4ypZInzLYl*bzU)Z`Q3K~jm zJ9g97Vrkor4;w9f&&PFkW}Q!EXUF*x*AHN3hb$i!mNHO~v44134WAxhVB6`aB50^u zUMKA11S*gUfXQs+mAT6aNI$<@eDtyj_EiDax2mjTV#9pw%hI^-y$^LzN+Dd-_3`Bf zo|3H@7ig$EgS`rcEM=h*;gc4`QaN5SwWfZG836GMoWID#{XFsqmj4Fcn;WJMf{R@a z_mq2qGXH&|DtzOY_zbgIshI_lsNht`iin9%HYB&rgc27p+lfV`XQBe;fOf`!09LAP zfB@mj6mZE^hxJQ(x0G<25_+Ld&;b(AVi*kRl0;x&yXM(YeChNDU|&OuiQo+-pc;vq`bi3#V(#(hTV)(kHJw>P(s_0Oo_a@G6kx49Rt zE_%pphdhVjVs3pm|Mg{!1*a^tOA(xzLpPZXY(>b-JB|iqv5(P)ss2|%h7=;FC!YJW zbx1wNrUVl59F^cGTrdjyFHfiyy@iw1ov6)2N&p={WJyT_QDeXoAynxxF!&1N%W8UW zORlVo&EfOOm`xBFO>Vjo5qrCl$H!$8zGGV)A#iVe+5Vj;^WT8gV65LD3!^)L`5C(~ z%7)xkasq{&zdk#8?_DQ{k@1(B`yLVM!RZ_O5^$u}?$#N}zO2=#HtzbZRc3W;Ih;Ed z%YYkqf{P~3EcuP-0pV;Vv{mj5R3Qq=2m>8Oa7ZuKA~2(yK0bRUT$GPc&S?{i01Vmu z%uWH;HVBueSCu>XAB6PAj6h36tC3yEWo(A4iM>DAA=dp2*pw@$50Qw}5D#QO8GjaO z=vgDcEtDiy0tww??JJXmhh1T#u6cvbvjAEzzBVBXmfx$C9OW{NbD}GQ*TpB+&ipix^{M>wgTn|I9IQAbqD5e&|Po|Aqe#-uKrWz3eUquZLlf8Y&tchyOhNSp~S#de7 zGpR?W-Ytjg>x1#prM!~1?Djxdj};#57GkE+2YY!!M}LWvXpXbNl8c>b?*b$S4PQvg)@l4>#nfXz8ef#LzYH?uN)HCJ}D2j~$=agATDYE5am?6+|lXIqyo``s@b ze{ntC58h*KEI2w5lt`rE{sOFFKxW6YlCc%w9x(DeWO)fI#o&aPQSk=3_qij2#)$<| z1R2*AC;743(|1Sr$9LseMhQJ=@S+RjEKOwcxlGUUk#ftAl2*Vg`->TKMLgroAH$fy z@Hwt`5O43gLfc#rv?%=LDXxT=*Ns$X`VCy)jF!gtz#($`DMN<=^8l*;L;du$xY@I) ze^x6SKpr+Z;;&9if3Naco;!@;ZW^V5i5GV|v(_1OIJpg3@-MWP=>;q>IajjrWIP`w zVh))1K3iC|Dfz{h9${*Cg9?C!o{1WcOai<^)n179TumT4@{8=)QW#m{FgdsWZKN0VNLH3aFU+H>5HF{hm$r{YurxlLAK=vm8nN@f6Faq zN@nm@jdjySjlz0nt43yHO9IN;Ytwgp-G9*6+{Rh4nfK>XZ;}GVVfA6SS@dd@#UU2fKEHpb0Jihr=Q4biG54lu1$RG@jmja51V z#P)#MTl00i)bCY(YJlUkjO(Nw;vGybCbOXtj&a0MIn41BR5{XGT}TBkqB*>Ld~EOd z`~uBtG5nnN0Yh)me7|+H@)^;6Hb16*+W|gB<<1I3+-%icsLs=20}$yB#4{iohIX=5 z_v45>RD6*YJN^L%Oh_dV$&75J+k8t7U_`A3nu2rDY}n${H@H|+*w+Xu&^qy6(h(oG zZA@EyEWFSr(KT-nPCBZ=y7*{h=^UahEo`jKvB#i@082ud?5&oe3X!J}T&DRDPA%UB zG9|H#^7hHIkTE1qcR2E^`4-Q`#2jw(`M}CVRer0KkC2-VZ`Md!4c<~e0BHdN-Kw6Q z(;jk$B><(2+2SqOwAvDNOqjg^{*=tffBf6|S+ej?TUZF0n3(gFN97hKDg_kQvl!od z$e>>#pwE)omj>&5zJ75W6So?J$pSJimw!bQ>1Jx5RH@pycrllQ`%6GieD&=RSrmmV zVSEQq|C}{fA`9}!i`9M4B_v!08HSkj!cEi$Usm~48*rXcwR`l&+6H_4%M7yWUYoS? zKFB_f!5+Q)3@hnus|7hfg;Pw({3%TIebsIhpka3l9TjVaHBF@2{xQwFdBQg`@34rr zp_W9!<)A6Sa>Zcpk=Ln6v@^Dk`LpL+q(?=Id(NRv&R~_F3Z@2;K0JrA9j4b{;|j(f z=W|Zdho+0IT@da@i`H5AqS9KQt@0wykl0mI{zDro!xfJ2Z@(PgL(em;&$XuHZ{S=8 zVA<|YqInU0wpS-{yMuvawIM26Mn|#zoeXVl$O5U*H#G|*i16s|4}o5761g_A{I|dV zX(pMV&Y;)pBCF{N(RX^u5S=8z_P%TL*P-p3wRU!iag(xg<9fgmenmrj7?jt1yEBcL zoV(do!~QkOWqxkWsaspVBBTHY+ac^YAXk^UIH?swz{xX4j8^@-%p1rUuVk<<)RLO< z67U(KI$6UX_pol2@Idi(*><3=OpiO3Ag?o&ni=YJM2r9PMV|ozq0Z0|2wFw3UfGy5 zAYB#TbnzT!LSP#y(=c8!oV&TshH-6cpVW-H>?a%c8X}o1up5Ob>&}w(r7;CDgwmB@%ZNX z4~&%6$*wly){EmdVf{yzO5s~m3q`?D69WvJXhJva1ys~JeqNY zr}sJRw7^Qtc`u1XP8Tg=&4=&Qlp(Q@L0)N0V;!Pn+dru zFr?@~UrTmQc;dlUy0?$ji!E{R5Q*mt8& zE=Q>=N_~zk#ujA(SYT|Hgj5TMBQ(st-qZ=TrTngoycBT16x6|(^b9*(Ey5m;_{}y* z6KR`bG>P=SUF_!CDTS~|bD*PN@TVLYgBMTt8%8&y)3^V0!y+!NUH+QSm(0uN@?18L z%FEULr04vx*6R~hW~8jie+YCcoW<&R{G#z!TJZ8f0H|6yKyN`ixOjB7Sf+sq+D_Cl}by*J&J*RXJS&)me^BA ztbRIPv`TSh+mRRFwv9sAw9>%^c+CmkJp@aba(At&5mS&29Dy;%$;YweRZB!al~cpdx&|@KFiAA4=*O{OU4xK1!xMg zzFiTYEvq^<%$#U0POURAltZb~IvRZ~j$;c_Vr@qAi-NaiSZLg*ai3YF1*lf8ZnnB5 zP`P3DA`-f8zenGzae*=L&mGZ-49Uj0^7=e zu@f8tV2Wh?O+=Ig%GYbF1kRJ}gTRM6Tp*TENb~;`w0a2uOJdu&F#K{hB44vUE?TC9 zEt!d*?0sO`f-dW)`(g7X_GgR0^wF4A-ivPYIA3e(f4LY`8g{d-#;f&E)B458$0~%& z9}AVo#O&ufVi-*3xlZdQ@8kJu7Cmi$VL_uaR9HINemqo_9)Zc zsMoWtmSCr=I~dfsH>_-IuFl-S`StRIi=AHG3;RR4%uJbn@BDlK9;Ww@K!E!X?zY%J z-p-mk$nQcqP7mFFzUZ~{X9xefbn7DDp7_}yuB{W~H*1G>hk9aR3wBY(L8tZ-`l8G-)wD@glxBYB`-MJ#fk2&L+4;+Dh-_2_Y#*AY&oO9 z9jEo})LWyB9M0v*M)VvrHhuv=>~+)GD}d?;fJ<&G(+Z`=S>aKEv`Ibr!tR$tA3VpG z+x$l`3Djp{MEWl-rqB#&SrT)kI>e?(t`Lkpxhb+^6qrb{A!cK)h9FI;YV!Y#xr_~A zxPxfOyd?pNtQle_j#!hwB>YSKj6@U?q=;lKEFy~d0Yz6p5hDtVG&(wWu*z$U4@d_C zb_#eE&>!b~apF4YIq|t${(O0@HVh0AO^JQlsX&dCyn)ex}?(TLijf zPs*lKr+3!s*G|`7hRC5Tp~y#4L@UT1D(Lbz`0mfzo3}z<6TdLNIPgXBUJs}l8~$v} z%rHJ=X|d(W*`&LR27+6bC8x%L7Q!81Fj zq#e{5=la7VrE3$D@mRdkYKE7hbIsF6u#K-fM@+&BDSFAtLd5FJ!N)EDa`2dv~h;!BMX%334-~LLP87ZkIB~xi3YoKqCmC6|eJ2{PN zmb^gwET#=;P;A3*?d@-@r1Vftg-w1>jjH+AmteVKa_i64B#-HvA!%W_xN34@W^BPq zTsGluufZN2KfQN5q)axS;mvLWl97{OQjyK`i0F0sr8D0Il;MEyYrwe2Hl9+Yb15ZN zDQA}U{RqgA#EeGzmiJ4xecL^90F))2HT#t~R<+Y7f5|`H<5nt@# z1a!P(A(UgGg8(Y)W&n7TR3uW8pTU;-aUaS8;}?X&`>Z1*YI4%%i}fc-ruGuqcF9IZ zgt)yT0+ljx+iI6P0|rNzY52iqR~_Em@otk8qcXIfzF!nrXYfIbHMTvXTQR!LDVpj2 zoP!!;86=Pu#7Hf@qJhYb0}ql%R?z2Yd_H8;dFWdYjz%3^H^Aq25Bd4UMZ0HMXGvmPX})!|!A&~ID~o;LPi-*HEq%CI-DWXi5F@075f;FpZV26RY* z=TON!J8W&Z>u+DKxd#hquQ;oZB)nlU2}qGn+Y`UTfJB2kEwn=^8@l1rVo}4!g|z0^ zUAHBhi@Lu8tAJ;eoFQEL*_k5iM%#7vs$}w3s*)t@$_lFvF3?Nys!yaApvc}_aEbvd zI|XV?rU2wNcxg{ZjFj8GQCfoyv~|>i1qt!;jmIlJIlC)XZ-d(^8{m+QIHuf0lgKMkXT=`<0GjQtdz{oLwIr!(@*5&&^+$5-Bs5b!ACis^~pegGNf zAm~Zbw}1H~-*bNnQ&GXCZrmsi5Ti%c(&KbMPE`F|xz% za|DGh1wEb?3p1jXG&>H5okO>V%{N<+3wl2M2>{a-Qzj;KN~KADN+7M4H>@j@<)3^S zSklD^vgE-4X)llzmRlw*e-K=YHtWk45e->x1K4Ns=bXo2vcCKG1wWUwR`Y1&$3f{2 z%%>Srlore!^Eg*JQ!f~GGrW3679$2+fnT8EWWghOWDdH`Eer2k8zI`29SvQ%NJVQh zj({_j|6XbMQ(LBsGYU ztOR5O*+(xUzXq?}6 zclY-8$&A|#Od&wDTD8G`XG%jo@BG?Yz>Plsnu0(WUvnzLVW{5A{O^t?0e@APpXPY9 z%7fJ-kod1f=NfjxXcnEAW0934RtWb(xKr84&&!66x8%#a;`<%^@@bcdYGX|w7t#=zg#&8G0P*y3Q!;aL3`&z-v&KhG^JzpD+#MgmrxzOBr07fj zF3ucBZ|W{SRXWR&jCYhmeNz0UV!&UaFh*no*5sO0cAN`0?johKDh_sz)m4u8rPnWi z{_Lm|wxBOoZID?(1x(7z%3@Rdk}z*1@WFVNXboI2PFyGy=`vNthe`&WrAZPo z6)E4n!n&!qjI@q(IKviOJAiCh8&@10K6DL8I}atW-^KoM-*#NrKmaP(0)>gC8B5`+_)Ar7xo;D58wenkfK>%BlTPfwi$gj3puww^aO9Px5X@mY{1l z!;gDEX;=Az5F)s65BQPnPq2DrfUF+5Ty!)ao4$ZTCN5Z|Jq+8z_Yww1<@JW$K2-}{ z@B*&VsDp32yX`MQsMY_NbNDtyN_M9l`DTZUN5D0|@%fO^>Yt3-*TqfLjsfQkD_S*$*mZVaFuLxty^$>uZ5tz1GU%6SFCbxRae_3R! z82Jl1RaBF9i#zC|w%y8kf9$`$*_h!>o_zYjcbA?V1dynz2+2VW%9XeE1z)XAd_}lD zJMS0C#h+gA0McJ1lsaHltc9htowz|M7PPHlc!lRS2pld;oyOrY+mZl5`_=l)Hy*g?5XbK*qhnCuf!TAjq@EjxWW_T$-7nF|w|eb|*X0IKgs=fi`wku}Jed@@!Qof_P*TFL7# zkCSz4BPFtWw23gXih1De_Lt&eTZnAhZocL_*e*%y>I!S~P)hexPi46m;TPb=I3~9q zcUB}GO%zuI;=Gq5Yl7_fNOJ_vO<>$s+@w)k#1v_sB0=XME*j$Ozo{jRR^@8>t$~3J z0LMVfr31P)8TDXqG`e=Ch{ML=BJsO_Ug0u54Z!32vj1qlAn7LG44}p0_R=pq_Oq&4 zYzR&*5R)%f_Rzz^@<9u`LHT$WU!{C zA{6Uw>TIwKXI3eX^SP^oo>CBLCLgsKpp%i6mS&#lv{bZrG8-=fj2qbyrR$R8E=fE} zd49FS_(83_&~*9)nzbqi#|7;-H#5QJf?t2L+juEGO6+mWdiTOvH#G@_u zf;2LB!qxXXgF-c-2>XheH6F<_GrnMv?~pWx`STthV!*bpu_^E$iy9p5g6;7cTZSYx za=@7cjZ6d2to6o;YfW*#)SXog7-uvgNm?=HCvcly%TqUMJ5+T7J49s+zlV{bG4(Wa z+S&z!Rkp?Y^ZZYJ6^7Rncf7dJy){d+4DO%GY(Y~T1P>W|+^ z-IM>V77VC?(La59wvar(N{WiUCIt$`rk3H!-iZ*NXR3E~${YZ~7YQl>;7HC-^G^J8 zGav1FfWjZjd@55%omz+L~rvjax{!iYpp;a$;!j0e7v6hQRKT0EWTfjjv>&{s=w%k(1Pr)I;P`04H1#?x4 zveqFbAoT59w-vXUp>9XJ#;y^7Vlqc z$%9<}hK@1){Se5geuyiE&52}nq!bATy{#P*S@>2Xp0+j^Brm-0N^TeKZfdQxS|iLA zEJUK@c;8jBg1?J@Mbn$5dzicxMbr@WY1Z$ zU=R>$seB_zb-^NDK=(lTgo$J=EFuV}BF2VQIm8hVsYRPdj)h_pFdGbPkYro>9I)nm zIX?@!r&Ab(emN`}Fk*4gf{?86jucx!o>DC zlng`=cR+hCRJFYLm*GHC-wDJ*^5vE{D{(#N8FWrNAspAd&@XV81;wZ(=wNF@c@h|y z3@#@H&M}D_BMlHdYf1RzWNhVR*}$lgK}A!}_}U6ORC4PD$dqwLR1NG7N{|Uey2#22 zA)qmZtx!EsfMp2k^1{M+j4XJ8?en6-+n#GgJr%kFpc@IF8W9=C9thqgkq3tuBfgx zO2}UVT8M))GX0loGR9(;JrO6I?NNg1%Gvv#cccPy?cNWCOOfjLUqyl7&|q3=%Zp}w z0BXG96VE@ska&oSGGL3v5}qi!bYG8|r9KkZM#SKE-NnIIkWy4nXi~tNYuxXH}KnZ58P~6CGpsBmuoJ4EeD! z#I6?W9=8$Ok-}JiuiQ* zvwEw#j`oxN#%5(Oxo3sXNa=NMAOl~mffyazGpZ*R4i`Z!2ioIr0C43E9`ugHZvf^< zzJz3xFkD;VZ|9D9)r#<(CW8rZY+=GlTZ8#I;xW+8L@-YY{5_s zr-;5?sP~CST-@%!KNBys(-8E72>|hhgs=f=oUl|+Bi~t~i=|sF2q;{>#>((Z0;)P! zQnaT-&Dn~C(wE#zEm{3@aV7hBY5f=671hvXa$a?6%t7MyvemIAq9k}?B9AL0+LS2L zFe4zSVsbUx_%4j?K-RQGP_};gUwsiyYvsQPY*c-Zd>{%TIby}LGQ#}O%mL~b;P>CR z6A^1{>he@(jXaW7)vz=5l_CAza!Uc%0DQbabD#@EdmzAIf{$?b%)n0 z1}=w^$Wmdyfm-O^Rf##tJAx1U)<)mr@C>P(cmM_4KS{Q`%LS%o0A1%X~;{X3x}&Eyc`4PNdpaddJLJ8S_HM}D2U~~ zl2$+r9c0HGqtfsLXHh$0*S#11R5`)<7>dXw$*LM596z z|3(^DE)R*gNJKsAuvoc6$qs1PG=))PwEQqjb3<-FY-hlIRr53^HlY#2k znf4;1*1Tx^D7lE}{$zfC zQzgdLkB`2VGS_@TTH8HCB}amq$_A`xf|j7T}0r084W1wqjhTWz6FQV=3K) zyR6K$J6I&+6IQu<)E!xQ^CErt^j@aA>Eq-meBSX6Jh~H3?@H;qJGg{!vwtIi&V$>f zZ(w^iJ^?wN-kl=-fY{`1&(o1>7WVy%on9E8FLNe3Z|y$Myag~Ty{(T>8|D64KC)-2 zJEtYu4oC$NjRwclVJ+`wUc|dC&{Fki#QpMa1AOe^g>Yrcmu?yfK_G?DR-$c5>h$vG2Kf^3ht61>fn`sk=Ml&B z0nY4cbhhZ?O0Wh(XmnTF3waz%8Xjt8BQL9!hF!}h9Ag#QqwNGz+kJVMM{gNMU(KX#AEbW}#G_dKIn*5T zHl!H}#)Z+Si4%)Tp^5t!xV$ft{hD!b517Z3VF`iK5#{@Xrf0l4GQ*={#?t7vlN?yq zazKrJn``lhw-$f;IQokr7>gbDK_mDc+r1`4fdA`hIXju)S>dnqpZ;H8cp$3B!|-!S z@DQ-&^9A;ksRUZw0j$Ho2(FyXx6Ll=3bdv&phCc!r&9cF9rWf8{iYcPi{mG4uYj%y z4VEN|Ui_V@hu-LQ);#$r-@8@TAKv$-v6Er6U2Fvkue-eLe$F3O9NkYPTH%X(?mq;F zdH2!285nP7-(&U7E4Of2#mjUs%Tun8aBOz9}RKpA2ugt3*PpaGAt3|lSIC9}{ z!;k0I3GvZyym|4E4-9W)c*q!z^l$n-c?5@LT<`%J&zMI)Ge4AeN=>51;n?ikx>p~; zFAOs~^kt1Z`yI3^C?a?HLx@Ozn;o{QbZhB{HfJ`ed(pvbqmfUCijCp7!`sM@lPL^;Q=D>28U75g2#_1t z4>OX<@rkAADGzOqoKE02AboB6i2guAO3VuQYLeG%lE&&Ppp|&f#RWNRDg`8}EbR9E z;(_K&N_l)Tx5sypD1hq@p$9bnCYzwwGM0JC&1^(^)7d1;X{trXcx`DDB-_i8FBbUp zynR2y3MF)Bp;X+2KDJe1gq+161jhf@b~^SocylH|SdR<;wsg_h!c}l`F^1zXfC*M< z0{A+0`CuA{K^#7+4^oY04^Z`lASB*!`!>Swlrht09U7oKDyLq<<^d}E%T{*>l_Y`) zR?++!{Q14Sy2AUP{q3S8rniAvbIT>9%;|WlftJi83IF)5BC)!G34jVgQ=LHhv;Etj zUjux)0%VHjn=wH1s85i4llK)=Fyn7ynAZ*Yu?Ee(BF}q-6Is@TS9ezDdJAJPQL%S? z(WVzmKGsxgQB*}N1fW+tsqOPh137uacFZ85In>R9yt?{w{>B#Jb2!ZHq;b9U>7#=x zvrVp3K_v|pm<~!WVaO-T@M=i=2yqyrM}HOe%xW%<+QMSh0+nlNg41sUxmrHU#+WVg z%MyIhIl!^m#|i4p4Zyon)AmiK7?$U_rz?(Q)n=jzdt09N1Tb~r111^7o1&eNlmXI| zCQ$~*$#-G+2HU#TjZ=<%8PlbJR4V~J|1A2R!{mKtUkxtulGpvaQ)9;8vKVhCcdpzp zI1qc(e?CHhOxZD&bB9XVayo}2!bi&l^19`pQ;5qqBal`(B94ODBU-F>2y1i64edvU zg1#%A9yoar0gyXggRRhR!&x7sSayn~z$Simn(pf~_GdjdedX-RFo5`_{l~Ajx2gkb z?`>?GrFZ=X_n2&-8(FL-p-i6~`p>R^ zsNid>a+*f;6&^$R6CQ(GD;`6%T4^C{>twlBX-Yw=c{v?c$$?!?&88sMnFE*ZZYwT* zvr7qh|36QF9K*8NVpaRgjII-ouG4n={rmLz^(IFN?dBGq+C`_{yO2R>edXfFa1u>? zr_(%$zI5_Cd-`?b^zxR^Nu5RmTo&!>zz+yC8BF(ol@S)fz}UiE#*ADhEX<5%tW3si z?CjsQ`HLimx1{!ne$R|^^Ypaj9I0sj!Wgm*Si>*v>x2u%`%!0S5S|`oh zW&TW{uamz_xO)lW6Dxw13+dOoo{SHQ-WOp^oQZlhq=)2Cbsj1atDP39nD+3}$B*l; zldv?#m;0PlIhzvAKTk}mYKgk4`HZJF9l~l6CX3Fl-lCsg#Xrl5vS1XIEzYwj6j?aD z^1DR1yg9WyZ%^v(SEgkH=;n2}=KBriH&OV@LfTMz(ZWW%r%0}kM~?Jk)8B^t_y&VR z(!qx$?R3SH_0f?La1QQI!_$@b??T&nu_EKlie{1LPgZ?ru=j@pI%6pJcmfmgm9mb<1xAc>3gBTcrcvoOkB`pWMT^9(EkTYYol%e diff --git a/tinygrad_repo/spec/tinyspec.tex b/tinygrad_repo/spec/tinyspec.tex index 1a31a03eb..9241c4d5e 100644 --- a/tinygrad_repo/spec/tinyspec.tex +++ b/tinygrad_repo/spec/tinyspec.tex @@ -87,7 +87,6 @@ A value \op{Call} is void: its \op{Sink} body stores to \op{Param}s bound positi \textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ \midrule \op{Permute} & $(T,)$ & axis order $\pi$ & Reorder axes. $\pi = (1,0)$ is transpose. \\ -\op{Flip} & $(T,)$ & bools $\mathbf{f}$ & Reverse along flagged axes. \\ \op{Reshape} & $(T, \mathbf{s'})$ & --- & Reinterpret in row-major order. $\prod s_k = \prod s'_k$. \\ \op{Expand} & $(T, \mathbf{n})$ & --- & Prepend axes $\mathbf{n}$ on the left. Output shape is $\mathbf{n} + T.\mathrm{shape}$. \\ \op{Pad} & $(T, \mathbf{o}, \mathbf{s'})$ & --- & Place $T$ at offset $o_k$ in an invalid-filled output of shape $s'_k$. \\ diff --git a/tinygrad_repo/test/amd/test_emu2_pcode.py b/tinygrad_repo/test/amd/test_emu2_pcode.py index ff6366372..071d6a940 100644 --- a/tinygrad_repo/test/amd/test_emu2_pcode.py +++ b/tinygrad_repo/test/amd/test_emu2_pcode.py @@ -83,6 +83,13 @@ class TestParseExpr(unittest.TestCase): """Test parsing hex literals.""" self.assertIs(parse_expr('0xFF', {}), UOp.const(255, dtypes.uint32)) + def test_24bit_operands(self): + for value, signed, unsigned in ((0x3FFFFFFF, -1, 0xFFFFFF), (0xFF800000, -8388608, 0x800000), (0xAB000001, 1, 1)): + with self.subTest(value=value): + src = {'S0': UOp.const(value, dtypes.uint32)} + self.assertEqual(parse_expr('S0.i24', src).ssimplify(), signed) + self.assertEqual(parse_expr('S0.u24', src).ssimplify(), unsigned) + def test_variable_lookup(self): """Test variable lookup in parse_expr.""" vrs = {'x': UOp.const(42, dtypes.uint32)} diff --git a/tinygrad_repo/test/backend/test_ops.py b/tinygrad_repo/test/backend/test_ops.py index 5483cb35d..6967ea6e7 100644 --- a/tinygrad_repo/test/backend/test_ops.py +++ b/tinygrad_repo/test/backend/test_ops.py @@ -920,6 +920,17 @@ class TestOps(unittest.TestCase): self.assertEqual(a, b) self.assertEqual(Tensor(-1).contiguous().div(4, rounding_mode="trunc").item(), 0) # NOTE this is trunc-div behaviour + @Context(TRANSCENDENTAL=2) + def test_sin_transcendental(self): self.test_sin() + @Context(TRANSCENDENTAL=2) + def test_cos_transcendental(self): self.test_cos() + @Context(TRANSCENDENTAL=2) + def test_tan_transcendental(self): self.test_tan() + @Context(TRANSCENDENTAL=2) + def test_exp_transcendental(self): self.test_exp() + @Context(TRANSCENDENTAL=2) + def test_log_transcendental(self): self.test_log() + @unittest.skipIf(DEV.renderer == "NAK", "MUFU.SIN is not accurate enough") def test_sin(self): helper_test_op([(45,65)], lambda x: x.sin()) @@ -1778,6 +1789,10 @@ class TestOps(unittest.TestCase): helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=3, dim=0), lambda x: x.normalize(p=3, dim=0), atol=1e-7, grad_atol=1e-7) helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=0), lambda x: x.normalize(p=0), atol=1e-7, grad_atol=1e-7) helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=-1), lambda x: x.normalize(p=-1), atol=1e-7, grad_atol=1e-7) + def test_rmsnorm(self): + helper_test_op([(45,65)], lambda x: torch.nn.functional.rms_norm(x, (65,), eps=1e-6), lambda x: x.rmsnorm(), atol=1e-6, grad_atol=1e-6) + helper_test_op([(10,10,10)], lambda x: torch.nn.functional.rms_norm(x, (10,10), eps=1e-3), lambda x: x.rmsnorm((1,2), eps=1e-3), + atol=1e-6, grad_atol=1e-6) def test_normalize_int(self): helper_test_op(None, lambda x: torch.nn.functional.normalize(x.float(), p=2), lambda x: x.normalize(p=2), forward_only=True, vals=[[[3, 4], [6, 8]]]) @@ -1790,6 +1805,7 @@ class TestOps(unittest.TestCase): helper_test_op([(6,6,6)], lambda x: torch.logsumexp(x, dim=2), lambda x: x.logsumexp(2), atol=1e-7, grad_atol=1e-7) helper_test_op([(6,6,6,6)], lambda x: torch.logsumexp(x, dim=2), lambda x: x.logsumexp(2), atol=1e-7, grad_atol=1e-7) helper_test_op([(6,6,6,6)], lambda x: torch.logsumexp(x, dim=3), lambda x: x.logsumexp(3), atol=1e-7, grad_atol=1e-7) + helper_test_op([(6,6,6,6)], lambda x: torch.logsumexp(x, dim=(1,3)), lambda x: x.logsumexp((1,3)), atol=1e-7, grad_atol=1e-7) helper_test_op([(45)], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) helper_test_op([()], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) helper_test_op([()], lambda x: torch.logsumexp(x, dim=-1), lambda x: x.logsumexp(-1), atol=1e-7, grad_atol=1e-7) @@ -2145,6 +2161,9 @@ class TestOps(unittest.TestCase): helper_test_op([(4,3,6,6)], lambda x: x.squeeze(3)) self.helper_test_exception([(4,3,6,6)], lambda x: x.squeeze(50), expected=IndexError) helper_test_op([(4,3,6,1)], lambda x: x.squeeze(-1)) + helper_test_op([(1,3,1,6)], lambda x: x.squeeze((0,2))) + helper_test_op([(1,3,1,6)], lambda x: x.squeeze((-2,1))) + helper_test_op([(1,3,1,6)], lambda x: x.squeeze(())) helper_test_op([(4,3,6,6)], lambda x: x.squeeze()) helper_test_op([(1,3,6,6)], lambda x: x.squeeze()) helper_test_op([(2,3,1)], lambda x: x.squeeze()) diff --git a/tinygrad_repo/test/backend/test_transcendental.py b/tinygrad_repo/test/backend/test_transcendental.py index 20f008154..68788ae1f 100644 --- a/tinygrad_repo/test/backend/test_transcendental.py +++ b/tinygrad_repo/test/backend/test_transcendental.py @@ -27,27 +27,29 @@ class TestTranscendentalMath(unittest.TestCase): atol=3e-2, rtol=1e-5) # sin can have bigger atol for very big x @unittest.skipIf(DEV.interface.startswith("MOCK") and Device.DEFAULT in {"NV", "CUDA"}, "crashed") - @given(ht.float32, strat.sampled_from([(Tensor.exp, np.exp),(Tensor.log, np.log)] + - ([(Tensor.sin, np.sin)] if dtypes.ulong in supported_dtypes else []))) + @given(ht.float32, strat.sampled_from([(Tensor.exp, np.exp), (Tensor.log, np.log), (Tensor.sin, np.sin)])) def test_float32(self, x, op): - # wrong nan behavior on Vulkan - if (math.isnan(x) or (x < 0 and op[0] == Tensor.log)) and Device.DEFAULT == "WEBGPU" and not OSX: return with Context(TRANSCENDENTAL=2), np.errstate(all='ignore'): np.testing.assert_allclose(op[0](Tensor([x], dtype=dtypes.float32)).numpy(), op[1](np.array([x], dtype=_to_np_dtype(dtypes.float32))), atol=2e-5, rtol=1e-5) @unittest.skipUnless(dtypes.float16 in supported_dtypes, f"no float16 on {Device.DEFAULT}") - @given(ht.float16, strat.sampled_from([(Tensor.exp, np.exp),(Tensor.log, np.log)] + - ([(Tensor.sin, np.sin)] if dtypes.ulong in supported_dtypes else []))) + @given(ht.float16, strat.sampled_from([(Tensor.exp, np.exp), (Tensor.log, np.log), (Tensor.sin, np.sin)])) def test_float16(self, x, op): - # wrong nan behavior on Vulkan - if (math.isnan(x) or (x < 0 and op[0] == Tensor.log)) and Device.DEFAULT == "WEBGPU" and not OSX: return with Context(TRANSCENDENTAL=2), np.errstate(all='ignore'): np.testing.assert_allclose(op[0](Tensor([x], dtype=dtypes.float16)).numpy(), op[1](np.array([x], dtype=_to_np_dtype(dtypes.float16))), atol=1e-2, rtol=5e-3) # exp can have bigger rtol + @Context(TRANSCENDENTAL=2) + def test_log2_special_values(self): + for dtype, np_dtype in ((dtypes.float16, np.float16), (dtypes.float32, np.float32)): + if dtype not in supported_dtypes: continue + with self.subTest(dtype=dtype), np.errstate(all='ignore'): + values = np.array([0.0, -0.0, math.inf, -math.inf, math.nan, -math.nan, 1.0, -1.0], dtype=np_dtype) + np.testing.assert_allclose(Tensor(values).log2().numpy(), np.log2(values)) + # TODO: WEBGPU produces incorrect values near infinity @unittest.skipIf(Device.DEFAULT == "WEBGPU", "WEBGPU incorrect values near inf") @given(strat.sampled_from([(dtypes.float64, 709.5), (dtypes.float32, 88.7), (dtypes.float16, 11)])) @@ -62,7 +64,6 @@ class TestTranscendentalMath(unittest.TestCase): class TestFromFuzzer(unittest.TestCase): @given(strat.sampled_from(dtypes_float)) - @unittest.skipUnless(dtypes.ulong in supported_dtypes, "Needs ulong") def test_sin(self, dtype): if dtype not in supported_dtypes: return if dtype == dtypes.float64: @@ -116,7 +117,6 @@ class TestFloat16Log2(unittest.TestCase): np.testing.assert_allclose(result, expected, rtol=1e-3, err_msg=f"log2({val})") @unittest.skipUnless(dtypes.float16 in supported_dtypes, f"no float16 on {Device.DEFAULT}") - @unittest.skipIf(Device.DEFAULT == "WEBGPU" and not OSX, "Nan handling differs on Vulkan") def test_float16_log2_special(self): # special values: inf, -inf, nan, 0, negative with Context(TRANSCENDENTAL=2), np.errstate(all='ignore'): diff --git a/tinygrad_repo/test/external/external_test_example.py b/tinygrad_repo/test/external/external_test_example.py index 37554fc22..4ce98740d 100644 --- a/tinygrad_repo/test/external/external_test_example.py +++ b/tinygrad_repo/test/external/external_test_example.py @@ -8,7 +8,7 @@ def multidevice_test(fxn): def ret(self): for device in Device._devices: # broken on OSX USB AMD, why? - if device in ["DISK", "NPY", "FAKE", "DSP", "NULL"] or (OSX and device in ["AMD"]): continue + if device in ["DISK", "NPY", "FAKE", "DSP", "NULL", "RDMA"] or (OSX and device in ["AMD"]): continue if sys.stdout.isatty(): print(device) if device in exclude_devices: if sys.stdout.isatty(): print(f"WARNING: {device} test is excluded") diff --git a/tinygrad_repo/test/external/process_replay/process_replay.py b/tinygrad_repo/test/external/process_replay/process_replay.py index 2c056aa58..c8bed9c39 100755 --- a/tinygrad_repo/test/external/process_replay/process_replay.py +++ b/tinygrad_repo/test/external/process_replay/process_replay.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # compare kernels created by HEAD against master import os, multiprocessing, logging, pickle, sqlite3, difflib, warnings, functools, base64, codecs +from concurrent.futures import ProcessPoolExecutor from dataclasses import replace from typing import Callable, Any @@ -107,12 +108,9 @@ def _pmap(fxns:dict[str, Callable]) -> None: finally: cur.close() - with multiprocessing.get_context("spawn").Pool(multiprocessing.cpu_count()) as pool: - bar = tqdm(total=row_count) - for _ in pool.imap_unordered(functools.partial(diff, fxns=fxns), range(0, row_count, s:=min(PAGE_SIZE, row_count))): bar.update(s) - pool.close() - pool.join() - pool.terminate() + with ProcessPoolExecutor(multiprocessing.cpu_count(), mp_context=multiprocessing.get_context("spawn")) as pool: + with tqdm(total=row_count) as bar: + for _ in pool.map(functools.partial(diff, fxns=fxns), range(0, row_count, s:=min(PAGE_SIZE, row_count))): bar.update(min(s, row_count-bar.n)) # *** main loop diff --git a/tinygrad_repo/test/mockgpu/amd/amddriver.py b/tinygrad_repo/test/mockgpu/amd/amddriver.py index 0ec6d9814..edd1ef35f 100644 --- a/tinygrad_repo/test/mockgpu/amd/amddriver.py +++ b/tinygrad_repo/test/mockgpu/amd/amddriver.py @@ -128,7 +128,8 @@ class AMDDriver(VirtDriver): if struct.flags & kfd.KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED: self.track_address(struct.va_addr, struct.va_addr + struct.size, lambda mv,off: None, lambda mv, off: self._emulate_execute()) elif nr == _ioctl_nr(kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU): - self.object_by_handle.pop(struct.handle) + va = self.object_by_handle.pop(struct.handle).va_addr + self.tracked_addresses = [x for x in self.tracked_addresses if x[0] != va] elif nr == _ioctl_nr(kfd.AMDKFD_IOC_MAP_MEMORY_TO_GPU): dev_ids = (ctypes.c_int32 * struct.n_devices).from_address(struct.device_ids_array_ptr) for i in range(struct.n_devices): diff --git a/tinygrad_repo/test/mockgpu/amd/amdgpu.py b/tinygrad_repo/test/mockgpu/amd/amdgpu.py index 820c595a2..c12cde835 100644 --- a/tinygrad_repo/test/mockgpu/amd/amdgpu.py +++ b/tinygrad_repo/test/mockgpu/amd/amdgpu.py @@ -338,7 +338,7 @@ class AMDGPURegisters: self.regs: dict[tuple[int, int], int] = {} def __getitem__(self, addr:int) -> int: if addr == regGRBM_GFX_INDEX: return self.grbm_index - if regCPG_PERFCOUNTER1_LO < addr < regGUS_PERFCOUNTER_HI: + if regCPG_PERFCOUNTER1_LO <= addr <= regGUS_PERFCOUNTER_HI: assert self.regs[(regCP_PERFMON_CNTL, 0)] == 0x401, "read mode should be enabled" return addr << 16 | self.grbm_index return self.regs[(addr, getbits(self.grbm_index, 16, 23))] diff --git a/tinygrad_repo/test/mockgpu/amd/pcode.py b/tinygrad_repo/test/mockgpu/amd/pcode.py index 8d49b6b42..f3f454cf1 100644 --- a/tinygrad_repo/test/mockgpu/amd/pcode.py +++ b/tinygrad_repo/test/mockgpu/amd/pcode.py @@ -704,6 +704,9 @@ class Parser: dt_name = self.eat('IDENT').val return result.cast(DTYPES.get(dt_name, dtypes.uint32)) return result + if field in ('i24', 'u24'): + val = base.cast(dtypes.uint32) & _u32(0xFFFFFF) + return _signext_from_bit(val, _u32(24)).bitcast(dtypes.int32) if field == 'i24' else val dt = DTYPES.get(field) if dt is None: return base if dt == base.dtype: return base diff --git a/tinygrad_repo/test/mockgpu/nv/nvgpu.py b/tinygrad_repo/test/mockgpu/nv/nvgpu.py index 84893f581..c9c413218 100644 --- a/tinygrad_repo/test/mockgpu/nv/nvgpu.py +++ b/tinygrad_repo/test/mockgpu/nv/nvgpu.py @@ -57,14 +57,14 @@ class GPFIFO: self.buf_sz = sz // 4 def execute(self) -> bool: - initial_off = self.buf_ptr + initial_get, initial_off = self.ctrl.GPGet, self.buf_ptr while self.ctrl.GPGet != self.ctrl.GPPut: self._set_buf_state(self.gpfifo[self.ctrl.GPGet]) if not self.execute_buf(): # Buffer isn't executed fully, check if any progress and report. # Do not move GPGet in this case, will continue from the same state next time. - return self.buf_ptr != initial_off + return self.ctrl.GPGet != initial_get or self.buf_ptr != initial_off self.ctrl.GPGet = (self.ctrl.GPGet + 1) % self.entries_cnt self._reset_buf_state() diff --git a/tinygrad_repo/test/null/test_dtype.py b/tinygrad_repo/test/null/test_dtype.py index 35c157b5f..b6c17ba8e 100644 --- a/tinygrad_repo/test/null/test_dtype.py +++ b/tinygrad_repo/test/null/test_dtype.py @@ -56,7 +56,11 @@ class TestCanLosslessCast(unittest.TestCase): self.assertFalse(can_lossless_cast(dtypes.int64, dtypes.double)) # half has more mantissa bits self.assertTrue(can_lossless_cast(dtypes.int8, dtypes.half)) - self.assertFalse(can_lossless_cast(dtypes.int8, dtypes.bfloat16)) + self.assertTrue(can_lossless_cast(dtypes.int8, dtypes.bfloat16)) + self.assertTrue(can_lossless_cast(dtypes.uint8, dtypes.bfloat16)) + self.assertFalse(can_lossless_cast(dtypes.int16, dtypes.bfloat16)) + self.assertTrue(can_lossless_cast(dtypes.fp8e4m3fnuz, dtypes.bfloat16)) + self.assertFalse(can_lossless_cast(dtypes.fp8e4m3, dtypes.fp8e4m3fnuz)) class TestInvalidSingleton(unittest.TestCase): def test_singleton(self): diff --git a/tinygrad_repo/test/null/test_simplify_valid_idx.py b/tinygrad_repo/test/null/test_simplify_valid_idx.py index 05f831fb8..5adabc645 100644 --- a/tinygrad_repo/test/null/test_simplify_valid_idx.py +++ b/tinygrad_repo/test/null/test_simplify_valid_idx.py @@ -236,6 +236,8 @@ class TestImageSimplification(unittest.TestCase): # same thing, valid has a div load = get_load_image_uop((10, 10, 4), gidx1//2<5, (gidx0, gidx1)) self.check(load, None, "gidx0", "gidx1") + load = get_load_image_uop((10, 10, 4), gidx1<=9, (gidx0, gidx1)) + self.check(load, None, "gidx0", "gidx1") # 10x20 image, not out of bound load = get_load_image_uop((20, 10, 4), gidx1<10, (gidx0, gidx1)) diff --git a/tinygrad_repo/test/null/test_uop_graph.py b/tinygrad_repo/test/null/test_uop_graph.py index 577e39afe..7d6846d22 100644 --- a/tinygrad_repo/test/null/test_uop_graph.py +++ b/tinygrad_repo/test/null/test_uop_graph.py @@ -3,6 +3,7 @@ from tinygrad import dtypes, Variable, Device from tinygrad.dtype import AddrSpace from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite, GroupOp, AxisType, broadcast_axes, KernelInfo from tinygrad.uop.symbolic import sym +from tinygrad.codegen.simplify import pm_reduce_unparented from test.helpers import full_rewrite, to_uops_list from tinygrad.codegen import full_rewrite_to_sink @@ -423,6 +424,15 @@ class TestReduceCollapse(unittest.TestCase): # Should become add of two separate reduces self.assertEqual(result.op, Ops.ADD) + def test_bool_reduce_unparented_keeps_dtype(self): + for op in (Ops.ADD, Ops.MUL): + self.assertIs(graph_rewrite(UOp.const(True).reduce(UOp.range(4, 0), arg=op), pm_reduce_unparented).dtype, dtypes.bool) + + def test_max_reduce_unparented_empty_range(self): + n = Variable("n", 0, 4) + ret = graph_rewrite(UOp.const(1.0).reduce(UOp.range(n, 0), arg=Ops.MAX), pm_reduce_unparented) + self.assertEqual(ret.substitute({n: n.const_like(0)}).simplify().arg, float("-inf")) + def test_reduce_shapeless_const_unroll(self): """a REDUCE over a shapeless CONST (e.g. x*0 folded late in codegen) must collapse before the expander""" out = UOp.param(0, dtypes.float, 1) diff --git a/tinygrad_repo/test/null/test_uop_vmin_vmax.py b/tinygrad_repo/test/null/test_uop_vmin_vmax.py index fcc783408..9b8a4df91 100644 --- a/tinygrad_repo/test/null/test_uop_vmin_vmax.py +++ b/tinygrad_repo/test/null/test_uop_vmin_vmax.py @@ -150,6 +150,14 @@ class TestVminVmaxProperties(unittest.TestCase): self.assertEqual(uop3.vmin, ~6) self.assertEqual(uop3.vmax, ~(-5)) + def test_vmin_vmax_xor_nonneg(self): + # non-negative operands stay below the wider operand's next power of two + self.assertEqual((UOp.variable('x', 0, 31) ^ 16)._min_max, (0, 31)) + self.assertEqual((UOp.variable('x', 3, 10) ^ UOp.variable('y', 0, 40))._min_max, (0, 63)) + self.assertEqual((UOp.variable('x', 0, 200, dtypes.uint8) ^ UOp.variable('y', 0, 3, dtypes.uint8))._min_max, (0, 255)) + # a possibly-negative operand is not bounded + self.assertEqual((UOp.variable('x', -1, 10) ^ 16)._min_max, (dtypes.weakint.min, dtypes.weakint.max)) + def test_vmin_vmax_cast(self): x = UOp.variable('x', -10, 10, dtypes.int) x_float = x.cast(dtypes.float) diff --git a/tinygrad_repo/test/null/test_uops_stats.py b/tinygrad_repo/test/null/test_uops_stats.py index 315f2a657..02ff07a55 100644 --- a/tinygrad_repo/test/null/test_uops_stats.py +++ b/tinygrad_repo/test/null/test_uops_stats.py @@ -131,6 +131,9 @@ class TestUOpsStats(unittest.TestCase): unequal = equal[:3] + [Tensor.empty(256, 129)] self.assertGreater(get_stats(Tensor.cat(*unequal, dim=1))[0], 0) + def test_xor_gather(self): + self.assertEqual(get_stats(Tensor.empty(32)[Tensor.arange(32) ^ 16])[0], 0) + def test_simple_matmul(self, M=1024, N=1024, K=1024): a = Tensor.empty(M,N) b = Tensor.empty(N,K) diff --git a/tinygrad_repo/test/unit/test_attention.py b/tinygrad_repo/test/unit/test_attention.py index 7a3947378..aa49c4adc 100644 --- a/tinygrad_repo/test/unit/test_attention.py +++ b/tinygrad_repo/test/unit/test_attention.py @@ -118,8 +118,8 @@ class TestGatedDeltaNetBlock(unittest.TestCase): x_float = x.astype(np.float32) return (x_float / np.sqrt((x_float * x_float).mean(axis=-1, keepdims=True) + eps)) * weight.astype(np.float32) - def _normalize_np(self, x:np.ndarray, eps:float=1e-6) -> np.ndarray: - return x / np.maximum(np.sqrt((x * x).sum(axis=-1, keepdims=True)), eps) + def _l2norm_np(self, x:np.ndarray, eps:float=1e-6) -> np.ndarray: # as in the FLA kernels + return x / np.sqrt((x * x).sum(axis=-1, keepdims=True) + eps) def _softplus_np(self, x:np.ndarray) -> np.ndarray: return np.log1p(np.exp(-np.abs(x))) + np.maximum(x, 0) @@ -154,8 +154,8 @@ class TestGatedDeltaNetBlock(unittest.TestCase): conv_window = np.concatenate([conv_state, self._linear_np(x_half, qkv_weight)], axis=1) conv_out = self._silu_np((conv_window * conv_weight).sum(axis=1)) q, k, v = np.split(conv_out, [block.q_dim, 2 * block.q_dim], axis=-1) - q = self._normalize_np(q.reshape(B, block.num_k_heads, block.head_k_dim)) - k = self._normalize_np(k.reshape(B, block.num_k_heads, block.head_k_dim)) + q = self._l2norm_np(q.reshape(B, block.num_k_heads, block.head_k_dim)) + k = self._l2norm_np(k.reshape(B, block.num_k_heads, block.head_k_dim)) v = v.reshape(B, block.num_v_heads, block.head_v_dim) if block.num_v_heads != block.num_k_heads: k_repeat = block.num_v_heads // block.num_k_heads @@ -214,6 +214,19 @@ class TestGatedDeltaNetBlock(unittest.TestCase): np.testing.assert_allclose(recurrent_state, expected_recurrent[step], rtol=1e-3, atol=1e-3, err_msg=f"GatedDeltaNet reset recurrent cache mismatch at step {step}") + def test_kda_qk_l2norm(self): + # q/k are normalized as x/sqrt(sum(x²)+1e-6) like FLA: tiny vectors aren't scaled up to unit length + ssm = SSMConfig(conv_kernel=4, state_size=8, group_count=2, time_step_rank=2, inner_size=16, kda=True) + block = GatedDeltaNetBlock(self._make_config(dim=16, n_heads=2, ssm=ssm), ssm) + block.attn_qkv.weight = Tensor([1e-4]*32 + [1.0]*16).reshape(48, 1).expand(48, 16) / 16 # x of ones: q = k = 1e-4, v = 1 + block.ssm_conv1d["weight"], block.ssm_beta.weight = Tensor.ones(48, 4), Tensor.zeros(2, 16) + x = Tensor.ones(1, 1, 16) + block._init_state(x) + block._attention(x, 0).realize() + # the first token's state is beta * v (x) l2norm(k), with beta = sigmoid(0) and v, k through the conv and silu + def silu(z): return z / (1 + np.exp(-z)) + np.testing.assert_allclose(block.recurrent_state.numpy(), 0.5 * silu(1.0) * silu(1e-4) / np.sqrt(8 * silu(1e-4)**2 + 1e-6), rtol=1e-3) + def test_kda_channel_decay(self): config = self._make_config(dim=4, hidden_dim=8, n_heads=2, head_dim=4, rope_dim=4, v_head_dim=4, ssm=SSMConfig(conv_kernel=2, state_size=2, group_count=2, time_step_rank=2, inner_size=4, kda=True)) diff --git a/tinygrad_repo/test/unit/test_dtype_weak.py b/tinygrad_repo/test/unit/test_dtype_weak.py index b8b022829..dcffd69db 100644 --- a/tinygrad_repo/test/unit/test_dtype_weak.py +++ b/tinygrad_repo/test/unit/test_dtype_weak.py @@ -130,7 +130,7 @@ class TestWeakPromotion(unittest.TestCase): self.assertEqual((out.src[1], out.src[2]), (UOp.const(5.0, dtypes.bfloat16), gate)) def test_weak_srcs_commit_only_at_a_concrete_lub(self): - weak_lub = UOp(Ops.ADD, src=(UOp.const(1), UOp.const(1.0))) + weak_lub = UOp(Ops.ADD, src=(UOp.const(2.0), UOp.const(1.0))) self.assertIs(graph_rewrite(weak_lub, pm_commit_weak), weak_lub) concrete = UOp.const(2.0).cast(dtypes.float16) # the weak arm stays bare: its sibling states the width, so the WHERE already derives float16 for it diff --git a/tinygrad_repo/tinygrad/codegen/__init__.py b/tinygrad_repo/tinygrad/codegen/__init__.py index 1e2bba733..4c7008778 100644 --- a/tinygrad_repo/tinygrad/codegen/__init__.py +++ b/tinygrad_repo/tinygrad/codegen/__init__.py @@ -269,7 +269,7 @@ pm_implicit_barriers = PatternMatcher([ ]) def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: - if DEBUG >= 5: print(render_uir(list(ast.toposort()))) + if DEBUG >= 5: print(render_uir(ast)) if SPEC: type_verify(ast, spec_tensor) # resolve UNSHARDs (multi-device UNSHARDs are already resolved by the scheduler; this handles in-kernel shards, e.g. fragments) @@ -350,11 +350,12 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_decomp, name="early decompositions") # late decomps + move gates from unrenderable INVALID where - sink = graph_rewrite(sink, pm_dtype_decomps+pm_commit_weak, ctx=(set(), ren), name="decomp dtypes") pm_decomp = pm_decomp+\ get_late_rewrite_patterns(supported_ops, bool(DISABLE_FAST_IDIV))+\ get_transcendental_patterns(supported_ops, TRANSCENDENTAL>=2) sink = graph_rewrite(sink, pm_decomp, ctx=ren, name="late decompositions") + # Transcendental decompositions can introduce unsupported dtypes, e.g. uint64 in Payne-Hanek reduction. + sink = graph_rewrite(sink, pm_dtype_decomps+pm_commit_weak, ctx=(set(), ren), name="decomp dtypes") sink = graph_rewrite(sink, pm_move_gates_from_index, name="move gates from index") # final rules for the renderer (without sym) @@ -384,7 +385,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: if os.environ.get("DBGTV"): try: type_verify(sink, spec_program) except RuntimeError: - print(render_uir(list(sink.toposort()))) + print(render_uir(sink)) raise else: type_verify(sink, spec_program) @@ -413,20 +414,18 @@ def line_rewrite(lst:list[UOp], pm:PatternMatcher, ctx=None) -> list[UOp]: newlst.extend(ret[1]) return newlst -pm_lower_calls = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.SINK),), allow_any_len=True, name="call"), - lambda ctx,call: call.replace(src=(full_rewrite_to_sink(call.body, ctx, optimize=False),)+call.src[1:])), -]) +def lower_call(ctx:Renderer, call:UOp) -> UOp: + body = graph_rewrite(call.body, pm_lower_calls, ctx=ctx, name="lower calls", walk=True) + body = full_rewrite_to_sink(body, ctx, optimize=False) + body = UOp(Ops.LINEAR, src=tuple(line_rewrite(linearize(body), pm_linearize_cleanups+pm_alloc_to_buf)), arg=to_function_name(call.arg.name)) + return call.replace(src=(body,)+call.src[1:]) -pm_call_fixup = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.SINK, name="sink"),), allow_any_len=True, name="call"), - lambda call,sink: call.replace(src=(UOp(Ops.LINEAR, src=tuple(line_rewrite(linearize(sink), pm_linearize_cleanups+pm_alloc_to_buf)), - arg=to_function_name(call.arg.name)),)+call.src[1:])), +pm_lower_calls = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.SINK),), allow_any_len=True, name="call"), lower_call), ]) def do_linearize(ctx:Renderer, prg:UOp, sink:UOp) -> UOp: if DEBUG >= 3 and sink.arg.applied_opts: print(f"{sink.arg.function_name:<25} opts: {sink.arg.applied_opts}") - sink = graph_rewrite(sink, pm_call_fixup, name="call fixup", enter_calls=True) lst = line_rewrite(linearize(sink), pm_linearize_cleanups+pm_alloc_to_buf) prg = prg.replace(src=(lst[-1],)) # isa renderers need to allocate registers @@ -469,7 +468,7 @@ pm_to_program = PatternMatcher([ (UPat(Ops.PROGRAM, src=(UPat(), UPat(Ops.LINEAR), UPat(Ops.SOURCE, name="source")), name="prg"), do_compile), ]) -@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(k.function_name, ast, ret.key),ret=renderer), replay=True) +@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(k.function_name, ret.key),ret=renderer), replay=True) @Context(ALLOW_DEVICE_USAGE=0) def do_to_program(ast:UOp, renderer:Renderer) -> UOp: """ @@ -486,7 +485,7 @@ def do_to_program(ast:UOp, renderer:Renderer) -> UOp: elif ast.op is Ops.SINK: assert isinstance(ast.arg, KernelInfo), "requires KernelInfo on arg to to_program" if VIZ: graph_rewrite(ast, PatternMatcher([]), name="View Base AST") - ast = graph_rewrite(ast, pm_lower_calls, ctx=renderer, name="lower calls", walk=True, enter_calls=True) + ast = graph_rewrite(ast, pm_lower_calls, ctx=renderer, name="lower calls", walk=True) full_sink = full_rewrite_to_sink(ast, renderer, optimize=ast.tag is None) prog_info = ProgramInfo.from_sink(full_sink, renderer.target) # instruction selection diff --git a/tinygrad_repo/tinygrad/codegen/decomp/dtype.py b/tinygrad_repo/tinygrad/codegen/decomp/dtype.py index 20644eaf7..548a9697e 100644 --- a/tinygrad_repo/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad_repo/tinygrad/codegen/decomp/dtype.py @@ -2,7 +2,7 @@ from dataclasses import replace from tinygrad.dtype import dtypes, DType, AddrSpace, truncate from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES from tinygrad.uop import GroupOp -from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite +from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite, shape_to_shape_arg from tinygrad.uop.weak import commit_weak_consts from tinygrad.renderer import Renderer from tinygrad.codegen.decomp.transcendental import exponent_bias, shl, shr @@ -83,7 +83,8 @@ def l2i(op: Ops, dt: DType, *uops:UOp): def l2i_define(x:UOp) -> UOp: # cannot decomp a Variable if x.addrspace == AddrSpace.ALU: raise RuntimeError(f"long decomposition of variable {x.arg.name} unsupported") - return x.replace(arg=replace(x.arg, dtype=l2i_dt[x.dtype]), src=(UOp.const(x.src[0].val*2) if x.shape else x.src[0], *x.src[1:])) + size = shape_to_shape_arg((x.max_numel()*2,)) if x.shape else x.src[0] + return x.replace(arg=replace(x.arg, dtype=l2i_dt[x.dtype]), src=(size, *x.src[1:])) def split_l2i(ctx:dict, op: Ops, dt: DType, *uops:UOp): # l2i does arithmetic on its inputs; rules enter here to split them to 32-bit words first, l2i recurses on itself. diff --git a/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py b/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py index 0cf4c6102..70841c562 100644 --- a/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py +++ b/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py @@ -249,10 +249,10 @@ def xlog2(d:UOp) -> UOp: r = d.ne(0.0).where(r, r.const_like(-math.inf)) # log2(x) = NaN for x < 0 r = (d<-0.0).where(r.const_like(math.nan), r) - # log2(NaN) = NaN - r = d.ne(d).where(r.const_like(math.nan), r) # log2(-0.0) = -Inf. In certain devices like PTX, x == -0.0 won't be true. so making reciprocal. - return d.reciprocal().ne(-math.inf).where(r, r.const_like(-math.inf)) + r = d.reciprocal().ne(-math.inf).where(r, r.const_like(-math.inf)) + # Check NaN last: comparisons of its reciprocal are not reliable on every backend. + return d.ne(d).where(r.const_like(math.nan), r) def xpow(base:UOp, exponent:UOp) -> UOp: # start with b ** e = exp2(e * log2(b)) diff --git a/tinygrad_repo/tinygrad/codegen/late/coalesce.py b/tinygrad_repo/tinygrad/codegen/late/coalesce.py index ec57edd4e..7ec5fa235 100644 --- a/tinygrad_repo/tinygrad/codegen/late/coalesce.py +++ b/tinygrad_repo/tinygrad/codegen/late/coalesce.py @@ -111,7 +111,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: # collect memory: defaultdict[tuple[Ops, UOp, UOp|str, UOp, object], dict[int, list[UOp]]] = defaultdict(dict) - for u in sink.toposort(enter_calls=False): + for u in sink.toposort(): # TODO: this should handle images too, it's just memory coalescing if u.op in {Ops.LOAD, Ops.STORE}: assert len(u.src) == (2 if u.op is Ops.STORE else 1), "memory coalescing does not support gated loads/stores" diff --git a/tinygrad_repo/tinygrad/codegen/late/linearizer.py b/tinygrad_repo/tinygrad/codegen/late/linearizer.py index cdff4a64d..0838d70c9 100644 --- a/tinygrad_repo/tinygrad/codegen/late/linearizer.py +++ b/tinygrad_repo/tinygrad/codegen/late/linearizer.py @@ -7,9 +7,9 @@ from tinygrad.helpers import prod, getenv, dedup, TUPLE_ORDER def linearize(sink:UOp) -> list[UOp]: # this is a toposort with priority - lst = list(sink.toposort(enter_calls=False)) + lst = list(sink.toposort()) out_degree:defaultdict[UOp, int] = defaultdict(int) - priorities:dict[UOp, tuple[int, int, Any]] = {} + priorities:dict[UOp, tuple[int, int, int, Any]] = {} # get consumers and assign priorities # NOTE: this requires the lst be locally toposorted @@ -30,7 +30,7 @@ def linearize(sink:UOp) -> list[UOp]: case Ops.RANGE: priority = 5 # placing RANGE is good case Ops.END | Ops.BACKEDGE: priority = -5 # placing loop exits is bad case _: priority = 0 # everything else has priority 0 - priorities[u] = (run_count, priority, extra) + priorities[u] = (run_count, len(u.ranges), priority, extra) # number the uops in "ideal" order nkey = {u:i for i,u in enumerate(sorted(lst, key=lambda x: priorities[x]+(x.tuplize if TUPLE_ORDER else ())))} @@ -62,7 +62,7 @@ class CFGContext: for u in sink.toposort(): # get the deps from the src deps[u] = {} - for s in u.src: deps[u] |= deps[s] + for s in u.src_without_body: deps[u] |= deps[s] if u.op in (Ops.END, Ops.BACKEDGE, Ops.SINK): nesting |= {x:u for x in deps[u] if x.op in (Ops.END, Ops.BACKEDGE) and (u.op is Ops.SINK or u.src[1] in deps[x]) and x not in nesting} diff --git a/tinygrad_repo/tinygrad/codegen/simplify.py b/tinygrad_repo/tinygrad/codegen/simplify.py index c9878b8fc..46caa869e 100644 --- a/tinygrad_repo/tinygrad/codegen/simplify.py +++ b/tinygrad_repo/tinygrad/codegen/simplify.py @@ -1,6 +1,6 @@ import itertools from typing import Callable -from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, graph_rewrite, _substitute, range_start, AxisType +from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, graph_rewrite, _substitute, range_start, AxisType, identity_element from tinygrad.uop.symbolic import symbolic from tinygrad.helpers import partition, dedup from tinygrad.dtype import dtypes @@ -84,10 +84,10 @@ def reduce_unparented(red:UOp) -> UOp|None: reduce_parented, reduce_unparented = partition(red.src[1:], lambda x: x in red.src[0].ranges) if len(reduce_unparented) == 0: return None ret = red.replace(src=(red.src[0],)+tuple(reduce_parented)) if len(reduce_parented) else red.src[0] - if red.arg[0] is Ops.ADD: - for r in reduce_unparented: ret = ret * r.src[0] - if red.arg[0] is Ops.MUL: - for r in reduce_unparented: ret = ret ** r.src[0] + for r in reduce_unparented: + # an idempotent op (MAX, and bool ADD/MUL which are OR/AND) applied N times is the op once, or the identity when N is 0 + if red.arg[0] is Ops.MAX or red.dtype is dtypes.bool: ret = (r.src[0] > 0).where(ret, ret.const_like(identity_element(red.arg[0], red.dtype))) + else: ret = ret * r.src[0] if red.arg[0] is Ops.ADD else ret ** r.src[0] return ret pm_reduce_unparented = PatternMatcher([ diff --git a/tinygrad_repo/tinygrad/device.py b/tinygrad_repo/tinygrad/device.py index 36053a04c..a4a763251 100644 --- a/tinygrad_repo/tinygrad/device.py +++ b/tinygrad_repo/tinygrad/device.py @@ -2,11 +2,11 @@ from __future__ import annotations from dataclasses import dataclass, replace, field from collections import defaultdict from typing import Any, Callable, Generic, TypeVar, Iterator, Generator, Self, TYPE_CHECKING -import importlib, inspect, functools, pathlib, os, contextlib, re, atexit, pickle, decimal, subprocess, struct, mmap, time, statistics +import importlib, inspect, functools, itertools, pathlib, os, contextlib, re, atexit, pickle, decimal, subprocess, struct, mmap, time, statistics from tinygrad.helpers import mv_address, LRU, getenv, diskcache_get, diskcache_put, DEBUG, GlobalCounters, PROFILE, temp, colored from tinygrad.helpers import Context, CCACHE, ALLOW_DEVICE_USAGE, MAX_BUFFER_SIZE, cpu_events, ProfileEvent, ProfilePointEvent, suppress_finalizing from tinygrad.helpers import select_by_name, select_first_inited, DEV, TracingKey, size_to_str, pluralize, Target, unwrap, round_up, is_numpy_ndarray -from tinygrad.helpers import cpu_profile, perf_counter_us, to_name, HCQ_RUNTIME_DEV +from tinygrad.helpers import cpu_profile, perf_counter_us, to_name, HCQ_RUNTIME_DEV, to_tuple from tinygrad.dtype import dtypes, DType from tinygrad.runtime.support.memory import BumpAllocator, MMIOInterface if TYPE_CHECKING: @@ -64,7 +64,7 @@ def canonicalize_device(device:str|tuple|list|None) -> str|tuple[str, ...]: return canonical[0] if len(canonical:=tuple(Device.canonicalize(d) for d in device)) == 1 else canonical def is_disk_device(device:str|tuple[str, ...]) -> bool: - return any(d.split(":", 1)[0].upper() == "DISK" for d in ((device,) if isinstance(device, str) else device)) + return any(d.split(":", 1)[0].upper() == "DISK" for d in to_tuple(device)) # **************** Profile **************** @@ -108,7 +108,7 @@ class Buffer: def __init__(self, device:str, nbytes:int, opaque:Any=None, options:BufferSpec|None=None, initial_value:bytes|pickle.PickleBuffer|None=None, base:Buffer|None=None, offset:int=0, preallocate=False, allocator:Allocator|None=None): - self.device, self.nbytes, self.offset, self.allocated_views, self._base = Device.canonicalize(device), nbytes, offset, 0, base + self.device, self.nbytes, self.offset, self._base = Device.canonicalize(device), nbytes, offset, base if allocator is not None: self.allocator = allocator self.options = options if options is not None else BufferSpec() self._storage:BufferStorage|None = None @@ -185,7 +185,6 @@ class Buffer: GlobalCounters.mem_used += self.nbytes GlobalCounters.mem_used_per_device[self.device] += self.nbytes if PROFILE: Buffer.profile_events.append(ProfilePointEvent(self.device, "alloc", self.trace_num, {"nbytes":self.nbytes})) - elif self._storage is None: self.base.allocated_views += 1 self._storage, self._base_storage = storage, self.base._storage if self._base else None return self @@ -198,7 +197,6 @@ class Buffer: GlobalCounters.mem_used_per_device[self.device] -= self.nbytes if PROFILE: Buffer.profile_events.append(ProfilePointEvent(self.device, "free", self.trace_num)) self.allocator.free(self._storage, self.nbytes, self.options) - else: self.base.allocated_views -= 1 self._storage, self._base_storage = None, None def __reduce_ex__(self, protocol): @@ -250,11 +248,10 @@ DeviceType = TypeVar('DeviceType', bound='Compiled') class Allocator(Generic[DeviceType]): lru = True - def __init__(self, dev:DeviceType, supports_copy_from_disk:bool=True, supports_transfer:bool=True): + def __init__(self, dev:DeviceType): self.dev: DeviceType = dev self.default_buffer_spec: BufferSpec = BufferSpec() self.cache:dict[tuple[int, BufferSpec|None], list[BufferStorage]] = defaultdict(list) - self.supports_copy_from_disk, self.supports_transfer = supports_copy_from_disk, supports_transfer def alloc(self, size:int, options:BufferSpec|None=None) -> BufferStorage: assert size > 0, f"alloc size must be positive, getting {size}" @@ -291,10 +288,8 @@ class Allocator(Generic[DeviceType]): def _map(self, buf) -> BufferStorage: raise NotImplementedError("need map") def _unmap(self, mb): pass # default no-op; override if _map allocates iface-side state def _offset(self, buf, size:int, offset:int): raise NotImplementedError("need offset") - # def _transfer(self, dest, src, sz:int, src_dev, dest_dev): class HostAllocator(Allocator): - def __init__(self, dev): super().__init__(dev, supports_copy_from_disk=False, supports_transfer=False) def _alloc(self, size:int, options:BufferSpec) -> BufferStorage: if options.external_ptr is not None: view, meta = self._view(options.external_ptr, size), None elif (remote:=getattr(self.dev, "remote", None)) is not None: view, meta = remote.alloc_sysmem(round_up(size, mmap.PAGESIZE)) @@ -395,7 +390,7 @@ class Compiled: def has_copy_queue(self) -> bool: return True @property - def host(self) -> str: return f"CPU:{self.peer_group[7:]}" if self.peer_group.startswith("remote:") else HCQ_RUNTIME_DEV.value + def host(self) -> str: return f"CPU:{self.peer_group[7:]}" if self.peer_group.startswith("remote:") else HCQ_RUNTIME_DEV.device @property def renderer(self) -> Renderer: return self._select_renderer() @@ -463,10 +458,10 @@ class Compiled: def _renderer_name(self, r:type[Renderer]) -> str: return r.__name__.upper().removesuffix("RENDERER").removeprefix(devname:=self.device.split(':')[0].upper()) or devname - def _select_renderer(self) -> Renderer: + def _select_renderer(self, cv=DEV) -> Renderer: assert (rn:=next((self._renderer_name(r) for r in self.renderers if getenv(f"{self.device}_{self._renderer_name(r)}")), None)) is None, \ f"{self.device}_{rn}=1 is deprecated, use DEV={self.device}:{rn} instead" - t = DEV.target(self.device.split(':')[0], **({"arch":self.arch} if self.arch else {})) + t = cv.target(self.device.split(':')[0], **({"arch":self.arch} if self.arch else {})) return select_first_inited(select_by_name(self.renderers, self._renderer_name, t.renderer, f"{self.device} has no renderer {t.renderer!r}"), f"No renderer for {self.device} is available", self.cached_renderer, t) @@ -483,10 +478,9 @@ class Compiled: # profiling def collect_prof(self): - if PROFILE: - es = list(self.prof_ents.items()) - sigs = [buf.host.view(fmt='Q')[i]/decimal.Decimal(self.timestamp_divider) for (buf, _), e in es for i in (e.st_id, e.en_id)] - Compiled.profile_events.append(ProfileGraphEvent([replace(e, st_id=2*i, en_id=2*i+1) for i,(_, e) in enumerate(es)], [], sigs)) + div = decimal.Decimal(self.timestamp_divider) + for buf, g in itertools.groupby(self.prof_ents.items(), key=lambda x: x[0][0]) if PROFILE else []: + Compiled.profile_events.append(ProfileGraphEvent([e for _, e in g], [], [t/div for t in buf.host.view(fmt='Q')[:]])) self.prof_ents.clear() def _at_profile_finalize(self): diff --git a/tinygrad_repo/tinygrad/dtype.py b/tinygrad_repo/tinygrad/dtype.py index 27a67a9ff..1e5dd2cf7 100644 --- a/tinygrad_repo/tinygrad/dtype.py +++ b/tinygrad_repo/tinygrad/dtype.py @@ -202,19 +202,13 @@ def can_lossless_cast(dt0:DType, dt1:DType) -> bool: # return if dt1 preserves value of dt0 # similar to https://numpy.org/doc/stable/reference/generated/numpy.can_cast.html if dt0 == dt1 or dt0 == dtypes.bool: return True - match dt1: - case dtypes.weakint: return dt0 in dtypes.ints - case dtypes.double: return dt0 in (dtypes.float, dtypes.half, dtypes.bfloat16, *dtypes.fp8s, - dtypes.uint32, dtypes.uint16, dtypes.uint8, dtypes.int32, dtypes.int16, dtypes.int8) - case dtypes.float: return dt0 in (dtypes.half, dtypes.bfloat16, *dtypes.fp8s, dtypes.uint16, dtypes.uint8, dtypes.int16, dtypes.int8) - case dtypes.half: return dt0 in (*dtypes.fp8s, dtypes.uint8, dtypes.int8) - case dtypes.uint64: return dt0 in (dtypes.uint32, dtypes.uint16, dtypes.uint8) - case dtypes.uint32: return dt0 in (dtypes.uint16, dtypes.uint8) - case dtypes.uint16: return dt0 in (dtypes.uint8,) - case dtypes.int64: return dt0 in (dtypes.uint32, dtypes.uint16, dtypes.uint8, dtypes.int32, dtypes.int16, dtypes.int8) - case dtypes.int32: return dt0 in (dtypes.uint16, dtypes.uint8, dtypes.int16, dtypes.int8) - case dtypes.int16: return dt0 in (dtypes.uint8, dtypes.int8) - case _: return False + if dtypes.is_int(dt1): return dtypes.is_int(dt0) and dt1.min <= dt0.min and dt0.max <= dt1.max + if dt1 not in dtypes.floats: return False + e1, m1 = dtypes.finfo(dt1) + if dtypes.is_int(dt0): return dt0.bitsize - (dt0 in dtypes.sints) <= m1 + 1 + if dt0 not in dtypes.floats: return False + e0, m0 = dtypes.finfo(dt0) + return e0 <= e1 and m0 <= m1 and (e0, m0) != (e1, m1) def sum_acc_dtype(dt:DType): # default acc dtype for sum diff --git a/tinygrad_repo/tinygrad/engine/jit.py b/tinygrad_repo/tinygrad/engine/jit.py index 9d6b68c32..271d8b61c 100644 --- a/tinygrad_repo/tinygrad/engine/jit.py +++ b/tinygrad_repo/tinygrad/engine/jit.py @@ -61,7 +61,7 @@ class CapturedJit(Generic[ReturnType]): def __reduce__(self): return self.__class__, (self.ret, self._linear, self.expected_names, self.expected_input_info) @functools.cached_property - def _jit_bufs(self) -> set[UOp]: return {u for u in self._linear.toposort(enter_calls=False) if u.op is Ops.BUFFER} + def _jit_bufs(self) -> set[UOp]: return {u for u in self._linear.toposort() if u.op is Ops.BUFFER} @functools.cached_property def _symbolic_ret(self) -> list[tuple[Tensor, UOp, dict[Variable, int]]]: diff --git a/tinygrad_repo/tinygrad/engine/realize.py b/tinygrad_repo/tinygrad/engine/realize.py index 7bfc9df2a..d03c7be3a 100644 --- a/tinygrad_repo/tinygrad/engine/realize.py +++ b/tinygrad_repo/tinygrad/engine/realize.py @@ -2,10 +2,10 @@ from __future__ import annotations from typing import cast, Iterator, Any, Sequence import decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import CAPTURE_PROCESS_REPLAY, colored, DEBUG, GlobalCounters, ansipad, prod, flatten, Context, to_tuple, tqdm +from tinygrad.helpers import CAPTURE_PROCESS_REPLAY, colored, DEBUG, GlobalCounters, ansipad, prod, flatten, Context, to_tuple, tqdm, unwrap from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us, cpu_profile from tinygrad.uop.ops import get_process_replay_loc, Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo, KernelInfo -from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry +from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry, DEV, HCQ_RUNTIME_DEV from tinygrad.renderer import Estimates, Renderer from tinygrad.codegen import to_program, to_program_cache, to_program_key, to_program_context from tinygrad.engine.worker import get_worker_pool, terminate_worker_pool @@ -32,7 +32,7 @@ def get_call_kernels(call:UOp) -> list[tuple[str, UOp, tuple|None]]: return kernels + [(d, call, (name, estimates, key, bufs, io)) for devices,name,estimates,_,key,bufs,io in call.arg.aux.kernels for d in devices] ast = call.body if ast.op is Ops.CUSTOM_FUNCTION and ast.arg.name == "validate": return [] - return [(d, call, None) for d in to_tuple(call.src[1].device)] + return [(d, call, None) for d in to_tuple(call.device)] def get_call_name(call:UOp, bufs:Sequence[Buffer|UOp], var_vals:dict[str, int]|None=None) -> str: def _uop_sz_to_str(uop:UOp) -> str: return size_to_str(sym_infer(prod(uop.shape) * uop.dtype.itemsize, var_vals or {})) @@ -130,8 +130,10 @@ def unwrap_multi(call:UOp, resolved:list[UOp]) -> Iterator[tuple[list[Buffer], d if not any(isinstance(b, MultiBuffer) for b in bufs): yield cast(list[Buffer], bufs), {} else: # the DEVICE axis is bound per device at launch: it's a RANGE in the AST and the _device_num variable after codegen - has_dnum = any((x.op is Ops.RANGE and x.axis_type is AxisType.DEVICE) or (x.op is Ops.PARAM and x.arg.name == '_device_num') - for x in call.body.toposort()) + # on archs like x86, stack args (after the 6th) aren't PARAMs in the body, so also check the PROGRAM's vars + has_dnum = ((isinstance(call.body.arg, ProgramInfo) and any(v.arg.name == '_device_num' for v in call.body.arg.vars)) or + any((x.op is Ops.RANGE and x.axis_type is AxisType.DEVICE) or (x.op is Ops.PARAM and x.arg.name == '_device_num') + for x in call.body.toposort())) lanes = max(len(b.bufs) for b in bufs if isinstance(b, MultiBuffer)) # a single buffer is shared by every lane per_lane = [b.bufs if isinstance(b, MultiBuffer) else (b,)*lanes for b in bufs] for j, per_dev in enumerate(zip(*per_lane)): yield list(per_dev), {"_device_num": j} if has_dnum else {} @@ -139,12 +141,7 @@ def unwrap_multi(call:UOp, resolved:list[UOp]) -> Iterator[tuple[list[Buffer], d def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: for bufs, device_vars in unwrap_multi(call, resolve_params(call, ctx.input_uops)): dest, src = bufs[0].ensure_allocated(), bufs[1].ensure_allocated() - if hasattr(dest.allocator,'_transfer') and dest.allocator.supports_transfer and dest.device.split(":")[0] == src.device.split(":")[0]: - dest.allocator._transfer(dest._buf, src._buf, dest.nbytes, src_dev=src.allocator.dev, dest_dev=dest.allocator.dev) - elif src.device.startswith("DISK") and getattr(src.allocator.dev, 'fd', None) is not None \ - and hasattr(dest.allocator, 'copy_from_disk') and src.nbytes >= 4096 and dest.allocator.supports_copy_from_disk: - dest.allocator.copy_from_disk(dest._buf, src._buf, src.nbytes) - elif dest.get_storage().host is not None and src.get_storage().host is not None: + if dest.get_storage().host is not None and src.get_storage().host is not None: for b in (dest, src): b.allocator.dev.synchronize() with cpu_profile(f"{src.device} -> {dest.device}", f"{src.device}:COPY"): dest.host[:] = src.host[:] elif dest._host_mv() is not None: src.allocator._copyout(dest.as_memoryview(allow_zero_copy=True), src._buf) @@ -187,12 +184,13 @@ def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: info = call.arg.aux ctx = replace(ctx, wait=ctx.wait and not info.skip_wait, var_vals={**ctx.var_vals, **{k: v for d in info.device for k, v in cast(Any, Device[d]).var_vals.items()}}) + slots = {d: cast(Buffer, call.src[1 + i].buffer) for d, i in info.slots} + for d in {d for devs, _, _, prof, *_ in info.kernels for d in devs if prof and (slots[d], prof[0]) in Device[d].prof_ents}: Device[d].synchronize() ets = exec_kernel(ctx, call, ast, devices=(Device[info.device[0]].host,)) for d in info.device: if (err:=Device[d].error_state.host.view(fmt='q')[0]): raise RuntimeError(f"{d} failed with {err}") if not (ctx.wait or PROFILE): return ets - slots = {d: cast(Buffer, call.src[1 + i].buffer) for d, i in info.slots} for devs, name, _, prof, pkey, *_ in info.kernels: for d in (devs if prof else ()): cast(Any, Device[d]).prof_ents[(slots[d], prof[0])] = ProfileGraphEntry(d, name, prof[0], prof[1], pkey) if ctx.wait: @@ -231,12 +229,12 @@ def _get_call_to_compile(c:UOp) -> tuple[UOp, Renderer]|None: # a PROGRAM with a ProgramInfo and a BINARY is already compiled if (ast.op is Ops.SINK and isinstance(ast.arg, KernelInfo)) or \ (ast.op is Ops.PROGRAM and not (isinstance(ast.arg, ProgramInfo) and ast.src[-1].op is Ops.BINARY)): - return ast, Device[c.device if isinstance(c.device, str) else c.device[0]].renderer + return ast, Device[to_tuple(c.device)[0]]._select_renderer(HCQ_RUNTIME_DEV if isinstance(c.arg.aux, HCQInfo) else DEV) return None def lower_and_compile(linear:UOp, verbose=True) -> UOp: # collect the kernels to lower and compile, deduped by their compile cache key - if not len(ar:={c: a for c in linear.toposort(enter_calls=False) if c.op is Ops.CALL and (a:=_get_call_to_compile(c)) is not None}): return linear + if not len(ar:={c: a for c in linear.toposort() if c.op is Ops.CALL and (a:=_get_call_to_compile(c)) is not None}): return linear # lower and compile what's not cached, in parallel if there's a worker pool keys = {c: to_program_key(*a) for c, a in ar.items()} @@ -294,8 +292,8 @@ def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None linear = link_linear(compile_linear(UOp(Ops.LINEAR, src=(call,)), beam=0, profile=True, cache=False), allow_cache=ctx.cache) while True: if clear_l2: - if hasattr(dev:=Device[call.src[1].device], 'invalidate_caches'): dev.invalidate_caches() + if hasattr(dev:=Device[call.device], 'invalidate_caches'): dev.invalidate_caches() else: from tinygrad.tensor import Tensor with Context(DEBUG=0, BEAM=0, CAPTURING=0, TRACK_MATCH_STATS=0): Tensor.ones(1024, 1024).contiguous().realize(do_update_stats=False) - yield max(pm_exec.rewrite(linear.src[0].without_after, ctx) or [0.0]) + yield max(map(unwrap, pm_exec.rewrite(c:=linear.src[0].without_after, ctx)[isinstance(c.arg.aux, HCQInfo):])) diff --git a/tinygrad_repo/tinygrad/function.py b/tinygrad_repo/tinygrad/function.py index ec702cb9a..f94b69c74 100644 --- a/tinygrad_repo/tinygrad/function.py +++ b/tinygrad_repo/tinygrad/function.py @@ -21,7 +21,7 @@ pm_ctx = PatternMatcher([ (UPat(GroupOp.Defines, name="x"), lambda ctx,x: add_to_ctx(ctx,x) if is_implicit_storage(ctx, x) else None), (UPat((Ops.AFTER, Ops.STAGE), name="x"), lambda ctx,x: add_to_ctx(ctx,x) if not any(p.op is Ops.PARAM and p.arg.slot >= 0 for p in x.backward_slice) and - any(is_implicit_storage(ctx, b) for b in x.toposort(enter_calls=False)) else None), + any(is_implicit_storage(ctx, b) for b in x.toposort()) else None), ]) def invalid_outputs(uret:UOp) -> set[UOp]: @@ -33,7 +33,7 @@ def invalid_outputs(uret:UOp) -> set[UOp]: def renumber_invalid_outputs(uret:UOp) -> UOp: invalid = invalid_outputs(uret) return uret.substitute({b:b.replace(op=Ops.ALLOC, arg=replace(b.arg, slot=i, buffer=None, bind_on_realize=False)) - for i,b in enumerate(x for x in uret.toposort(enter_calls=False) if x in invalid)}) + for i,b in enumerate(x for x in uret.toposort() if x in invalid)}) ReturnType = TypeVar('ReturnType') class _function(Generic[ReturnType]): diff --git a/tinygrad_repo/tinygrad/helpers.py b/tinygrad_repo/tinygrad/helpers.py index 24788b8cd..3085bbc2e 100644 --- a/tinygrad_repo/tinygrad/helpers.py +++ b/tinygrad_repo/tinygrad/helpers.py @@ -249,7 +249,7 @@ RING, ALL2ALL, ALLREDUCE_CAST = ContextVar("RING", 1), ContextVar("ALL2ALL", 0), ALLREDUCE_NODE_NDEVS = ContextVar("ALLREDUCE_NODE_NDEVS", 0) # gpus per node, the nodes cabled gpu k to gpu k CACHELEVEL, IGNORE_BEAM_CACHE = ContextVar("CACHELEVEL", 2), ContextVar("IGNORE_BEAM_CACHE", 0) VALIDATE_WITH_CPU = ContextVar("VALIDATE_WITH_CPU", 0) -HCQ_RUNTIME_DEV = ContextVar("HCQ_RUNTIME_DEV", "CPU") +HCQ_RUNTIME_DEV = _DEV("HCQ_RUNTIME_DEV", "CPU:LLVM") # TODO: this is broken for some indexing DISABLE_FAST_IDIV = ContextVar("DISABLE_FAST_IDIV", 1) FUSE_OPTIM = ContextVar("FUSE_OPTIM", 0) diff --git a/tinygrad_repo/tinygrad/llm/cli.py b/tinygrad_repo/tinygrad/llm/cli.py index b92ef47bc..2a008564e 100644 --- a/tinygrad_repo/tinygrad/llm/cli.py +++ b/tinygrad_repo/tinygrad/llm/cli.py @@ -170,7 +170,7 @@ def main(): if not args.no_chat_template and (ct := kv.get('tokenizer.chat_template')) is not None: try: import jinja2 - env = jinja2.Environment() + env = jinja2.Environment(extensions=['jinja2.ext.loopcontrols']) # (for {% break %}) env.filters['tojson'] = lambda obj, **kwargs: json.dumps(obj, **kwargs) # jinja2's tojson escapes <>& for HTML safety env.globals['raise_exception'] = lambda msg: (_ for _ in ()).throw(RuntimeError(msg)) env.globals['strftime_now'] = lambda fmt: time.strftime(fmt) diff --git a/tinygrad_repo/tinygrad/llm/kernels/amd.py b/tinygrad_repo/tinygrad/llm/kernels/amd.py index 3d7228995..d60cd086f 100644 --- a/tinygrad_repo/tinygrad/llm/kernels/amd.py +++ b/tinygrad_repo/tinygrad/llm/kernels/amd.py @@ -667,7 +667,7 @@ def _amd_flash_attention(o:UOp, q:UOp, cache:UOp, valid_kv_len:int|UOp, q_start: Q_lds = QP_lds[:, :D] Q_store = Q_lds.after(n_tile).reshape(THREADS_PER_BLOCK, Q_ELEMS_PER_THREAD)[tid].store(q.reshape(THREADS_PER_BLOCK, Q_ELEMS_PER_THREAD)[tid]) load_k = UOp.range(KV_ELEMS_PER_THREAD, 90) - kval = k.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_k].float() + kval = k.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_k] K_store = KV_lds.reshape(THREADS_PER_BLOCK, KV_ELEMS_PER_THREAD)[tid, load_k].store(kval).end(load_k) Q_lds, KV_lds_k = Q_lds.after(Q_store, K_store), KV_lds.after(Q_store, K_store) S_reg = _reg((TM, TN), 0, n_tile) @@ -709,7 +709,7 @@ def _amd_flash_attention(o:UOp, q:UOp, cache:UOp, valid_kv_len:int|UOp, q_start: V_lds = UOp.alloc((D, BLOCK_N + LDS_PAD), dtypes.half, addrspace=AddrSpace.LOCAL)[:, :BLOCK_N] V_copy, load_v = V_lds.after(qk_done).permute(1, 0), UOp.range(KV_ELEMS_PER_THREAD, 390) v_pos = n_tile*BLOCK_N + (tid*KV_ELEMS_PER_THREAD + load_v)//D - vval = (v_pos < valid_kv_len).where(v.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_v].float(), 0) + vval = (v_pos < valid_kv_len).where(v.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_v], 0) V_store = V_copy.reshape(THREADS_PER_BLOCK, KV_ELEMS_PER_THREAD)[tid, load_v].store(vval).end(load_v) P_lds, V_lds = P_lds.after(P_store, V_store), V_lds.after(P_store, V_store) pv_acc = _reg((TM, TD), 0, n_tile) diff --git a/tinygrad_repo/tinygrad/llm/model.py b/tinygrad_repo/tinygrad/llm/model.py index 8d19736cc..05447993c 100644 --- a/tinygrad_repo/tinygrad/llm/model.py +++ b/tinygrad_repo/tinygrad/llm/model.py @@ -106,7 +106,7 @@ class TransformerConfig: expert_bias: bool = False expert_proj_bias: bool = False swiglu_alpha: float = 1.0 - swiglu_clamp_exp: float|None = None + swiglu_limit: float|None = None swiglu_up_bias: float = 0.0 sliding_window: int = 0 sliding_window_pattern: int = 0 @@ -157,16 +157,16 @@ class FFNBlock: if normalize_topk: probs = probs / probs.sum(axis=-1, keepdim=True) probs = probs * self.config.routed_scaling_factor gate, up = self.ffn_gate_exps(sel, h), self.ffn_up_exps(sel, h) - act = gated_activation(gate, up, alpha=self.config.swiglu_alpha, limit=self.config.swiglu_clamp_exp, up_bias=self.config.swiglu_up_bias) + act = gated_activation(gate, up, alpha=self.config.swiglu_alpha, limit=self.config.swiglu_limit, up_bias=self.config.swiglu_up_bias) x_down = self.ffn_down_exps(sel, act.contiguous()) # (B, T, k, D) out = (x_down * probs.unsqueeze(-1)).sum(axis=2) # (B, T, D) if hasattr(self, 'ffn_gate_shexp'): - shexp = self.ffn_down_shexp(self.ffn_gate_shexp(x).silu().contiguous() * self.ffn_up_shexp(x)) + shexp = self.ffn_down_shexp(gated_activation(self.ffn_gate_shexp(x), self.ffn_up_shexp(x), limit=self.config.swiglu_limit).contiguous()) if hasattr(self, 'ffn_gate_inp_shexp'): shexp = shexp * (x * self.ffn_gate_inp_shexp["weight"]).sum(axis=-1, keepdim=True).sigmoid() out = out + shexp return out # TODO: remove the need for this contiguous - return self.ffn_down(self.ffn_gate(x).silu().contiguous() * self.ffn_up(x)) + return self.ffn_down(gated_activation(self.ffn_gate(x), self.ffn_up(x), limit=self.config.swiglu_limit).contiguous()) # given the token-prefix match, return how much cached state this block can still reuse def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return prefix_len @@ -286,14 +286,16 @@ class MLATransformerBlock(FFNBlock): k = Tensor(self.cache_k.uop.after(self.cache_k[:, :, start_pos:start_pos+T, :].uop.store(k_store.uop)))[:, :, 0:start_pos+T, :] v = k[..., :self.config.kv_lora_rank] - mask = Tensor.full((1, 1, T, start_pos+T), float("-inf"), dtype=x.dtype, buffer=False).triu(start_pos+1) \ - if resolve(T != 1) else None attn = q @ k.transpose(-1, -2) * (1.0 / self.config.head_dim ** 0.5) - if mask is not None: attn = attn + mask + if (mask:=self._attn_mask(x, start_pos)) is not None: attn = attn + mask attn = attn.softmax(-1) attn = ((attn @ v) @ self.attn_v_b["weight"].transpose(-1, -2)).transpose(1, 2).reshape(B, T, -1) return self.attn_output(attn) + def _attn_mask(self, x:Tensor, start_pos:int|UOp) -> Tensor|None: # (1, 1, T, start_pos+T), None: everything is visible + T = x.shape[1] + return Tensor.full((1, 1, T, start_pos+T), float("-inf"), dtype=x.dtype, buffer=False).triu(start_pos+1) if resolve(T != 1) else None + def _init_state(self, x:Tensor): if not hasattr(self, "cache_k"): self.cache_k = Tensor.empty(x.shape[0], 1, self.config.max_context, self.config.kv_lora_rank + self.config.rope_dim, device=x.device) @@ -335,8 +337,8 @@ class GatedDeltaNetBlock(FFNBlock): out_gate = out_gate.reshape(B, T, self.num_v_heads, self.head_v_dim) beta = self.ssm_beta(x).sigmoid().reshape(B, T, self.num_v_heads) alpha = self.ssm_f_b(self.ssm_f_a(x)) if is_kda else self.ssm_alpha(x) - log_alpha = ((alpha.float() + self.ssm_dt["bias"]).softplus().reshape(B, T, self.num_v_heads, -1) * - self.ssm_a.reshape(self.num_v_heads, -1)) + g, a = (alpha.float() + self.ssm_dt["bias"]).reshape(B, T, self.num_v_heads, -1), self.ssm_a.reshape(self.num_v_heads, -1) + log_alpha = g.softplus() * a # qkv conv, conv_state is reset when starting from position 0 conv_state = initial.where(0, self.conv_state) @@ -353,9 +355,9 @@ class GatedDeltaNetBlock(FFNBlock): out_gate = out_gate.pad_to((B, T_pad, self.num_v_heads, self.head_v_dim)) beta, log_alpha = beta.pad_to((B, T_pad, self.num_v_heads)), log_alpha.pad_to((B, T_pad, *log_alpha.shape[2:])) q, k, v = conv_out.split([self.q_dim, self.q_dim, self.conv_channels - 2*self.q_dim], dim=-1) - qk_eps = 1e-12 if is_kda else 1e-6 - q, k = (z.reshape(B, T_pad, self.num_k_heads, self.head_k_dim).normalize(dim=-1, eps=qk_eps) - .repeat(1, 1, self.num_v_heads//self.num_k_heads, 1) for z in (q, k)) + q, k = (z.reshape(B, T_pad, self.num_k_heads, self.head_k_dim) for z in (q, k)) + # l2 norm x/sqrt(sum(x²)+1e-6) as in the FLA kernels, then the v heads share the k heads + q, k = ((z.rmsnorm(eps=1e-6/self.head_k_dim) / self.head_k_dim**0.5).repeat(1, 1, self.num_v_heads//self.num_k_heads, 1) for z in (q, k)) v = v.reshape(B, T_pad, self.num_v_heads, self.head_v_dim) # layout the per-step operands to broadcast against the (B, H, V, K) state q, k, v, beta = (z.transpose(1, 2).float() for z in (q, k, v, beta)) @@ -451,17 +453,19 @@ class Transformer: max_context = min(max_context, kv[f'{arch}.context_length']) if max_context is not None else kv[f'{arch}.context_length'] + num_blocks = kv[f'{arch}.block_count'] - kv.get(f'{arch}.nextn_predict_layers', 0) ssm = None ssm_layers: tuple[bool, ...] = () if arch in ('qwen35', 'qwen35moe'): ssm = SSMConfig(**{k: kv[f'{arch}.ssm.{k}'] for k in ('conv_kernel','state_size','group_count','time_step_rank','inner_size')}) - ssm_layers = tuple((i+1) % kv[f'{arch}.full_attention_interval'] != 0 for i in range(kv[f'{arch}.block_count'])) + ssm_layers = tuple((i+1) % kv[f'{arch}.full_attention_interval'] != 0 for i in range(num_blocks)) elif arch == 'kimi-linear': - ssm_layers = tuple(x == 0 for x in n_kv_heads) + ssm_layers = tuple(x == 0 for x in n_kv_heads)[:num_blocks] n_kv_heads = max(n_kv_heads) ssm = SSMConfig(kv[f'{arch}.ssm.conv_kernel'], kv[f'{arch}.kda.head_dim'], n_heads, n_heads, n_heads*kv[f'{arch}.kda.head_dim'], kda=True) for i, is_ssm in enumerate(ssm_layers): if not is_ssm: continue + state_dict[f"blk.{i}.ssm_a"] = state_dict[f"blk.{i}.ssm_a"].reshape(n_heads, 1) state_dict[f"blk.{i}.attn_qkv.weight"] = state_dict.pop(f"blk.{i}.attn_q.weight").cat( state_dict.pop(f"blk.{i}.attn_k.weight"), state_dict.pop(f"blk.{i}.attn_v.weight"), dim=0).contiguous() state_dict[f"blk.{i}.ssm_conv1d.weight"] = state_dict.pop(f"blk.{i}.ssm_conv1d_q.weight").cat( @@ -491,12 +495,12 @@ class Transformer: elif kv_lora_rank and 'attn_kv_a_mqa.weight' in name: state_dict[name] = state_dict[name][:kv_lora_rank].cat(state_dict[name][kv_lora_rank:].rearrange("(h two) d -> (two h) d", two=2), dim=0) config = TransformerConfig( - num_blocks=kv[f'{arch}.block_count'] - kv.get(f'{arch}.nextn_predict_layers', 0), dim=kv[f'{arch}.embedding_length'], + num_blocks=num_blocks, dim=kv[f'{arch}.embedding_length'], hidden_dim=kv.get(f'{arch}.expert_feed_forward_length', kv.get(f'{arch}.feed_forward_length', 0)), n_heads=n_heads, n_kv_heads=n_kv_heads, norm_eps=kv[f'{arch}.attention.layer_norm_rms_epsilon'], vocab_size=len(kv['tokenizer.ggml.tokens']), head_dim=head_dim, - rope_theta=kv[f'{arch}.rope.freq_base'], rope_dim=rope_dim, yarn=yarn, + rope_theta=kv.get(f'{arch}.rope.freq_base', 10000.0), rope_dim=rope_dim, yarn=yarn, v_head_dim=kv.get(f'{arch}.attention.value_length_mla', kv.get(f'{arch}.attention.value_length', head_dim)), max_context=max_context, qk_norm=int(state_dict['blk.0.attn_q_norm.weight'].shape[0]) if 'blk.0.attn_q_norm.weight' in state_dict else 0, @@ -516,7 +520,7 @@ class Transformer: qkv_bias='blk.0.attn_q.bias' in state_dict, expert_bias=f"blk.{kv.get(f'{arch}.leading_dense_block_count', 0)}.exp_probs_b.bias" in state_dict, expert_proj_bias='blk.0.ffn_gate_exps.bias' in state_dict, attn_output_bias='blk.0.attn_output.bias' in state_dict, - swiglu_alpha=1.702 if arch == 'gpt-oss' else 1.0, swiglu_clamp_exp=7.0 if arch == 'gpt-oss' else None, + swiglu_alpha=1.702 if arch == 'gpt-oss' else 1.0, swiglu_limit=7.0 if arch == 'gpt-oss' else kv.get(f'{arch}.swiglu_clamp_exp', [None])[0], swiglu_up_bias=1.0 if arch == 'gpt-oss' else 0.0, attn_sinks='blk.0.attn_sinks.weight' in state_dict, sliding_window=kv.get(f'{arch}.attention.sliding_window', 0), sliding_window_pattern=kv.get(f'{arch}.attention.sliding_window_pattern', 2 if arch == 'gpt-oss' else 0)) diff --git a/tinygrad_repo/tinygrad/mixin/creation.py b/tinygrad_repo/tinygrad/mixin/creation.py index 77df13c1b..ef5f2587c 100644 --- a/tinygrad_repo/tinygrad/mixin/creation.py +++ b/tinygrad_repo/tinygrad/mixin/creation.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Callable, Self from tinygrad.dtype import ConstType, DType, DTypeLike, Invalid, dtypes, to_dtype -from tinygrad.helpers import argfix, prod +from tinygrad.helpers import argfix from tinygrad.mixin.dtype import DTypeMixin from tinygrad.mixin.movement import MovementMixin @@ -32,12 +32,11 @@ class CreationMixin(DTypeMixin, MovementMixin): print(t.shape) ``` """ - from tinygrad.uop.ops import UOp, Ops, ParamArg, to_max_shape + from tinygrad.uop.ops import UOp from tinygrad.device import canonicalize_device dt = to_dtype(dtype) if dtype is not None else dtypes.default_float if dt in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {dt}") new_shape = argfix(*shape) - max_shape = to_max_shape(new_shape) dev = canonicalize_device(device) if axis is not None and isinstance(dev, tuple): axis = range(max(1, len(new_shape)))[axis] @@ -45,10 +44,7 @@ class CreationMixin(DTypeMixin, MovementMixin): if new_shape[axis] % len(dev) != 0: raise RuntimeError(f"can't split {new_shape} on axis {axis} over {dev}") local = tuple(s//len(dev) if i == axis else s for i,s in enumerate(new_shape)) return cls._wrap_uop(cls.empty(*local, device=dev, dtype=dt)._uop.unshard(axis)) - u = UOp(Ops.ALLOC, src=(UOp.const(prod(max_shape)),)+UOp.device_range_src(dev), - arg=ParamArg(next(UOp.unique_num), dt, device=dev, bind_on_realize=True)) - u = u.reshape(max_shape).shrink_to(new_shape) - return cls._wrap_uop(u) + return cls._wrap_uop(UOp.alloc(new_shape, dt, device=dev, bind_on_realize=True)) def empty_like(self, dtype: DTypeLike|None=None, device: str|tuple[str, ...]|None=None) -> Self: """ diff --git a/tinygrad_repo/tinygrad/mixin/gradient.py b/tinygrad_repo/tinygrad/mixin/gradient.py index b89159e95..338815b9a 100644 --- a/tinygrad_repo/tinygrad/mixin/gradient.py +++ b/tinygrad_repo/tinygrad/mixin/gradient.py @@ -27,7 +27,7 @@ def call_gradient(ctx:UOp, k:UOp, needed:set[int]) -> tuple[UOp|None, ...]: git = iter(k.arg.grad_fxn(*real, call=k) if len(real) > 1 else k.arg.grad_fxn(real[0], k)) return (None,) + tuple(None if i in outputs else next(git) for i in range(len(args))) assert outputs, f"expected a CALL with output STOREs or a grad_fxn, got {fxn.op}" - params = {p.arg.slot:p for p in fxn.toposort(enter_calls=False) if p.op is Ops.PARAM and p.arg.slot >= 0} + params = {p.arg.slot:p for p in fxn.toposort() if p.op is Ops.PARAM and p.arg.slot >= 0} grad_args = tuple(ctx.src[i] for i in outputs) root_grad = UOp.sink(*[g if g.device is None else g.param_like(len(args)+i) for i,g in enumerate(grad_args)]) grads = compute_gradient(UOp.sink(*[st.src[1] for st in outputs.values()]), root_grad, set(params.values())) @@ -41,7 +41,7 @@ def call_gradient(ctx:UOp, k:UOp, needed:set[int]) -> tuple[UOp|None, ...]: args += grad_args bwd_body = renumber_invalid_outputs(bwd_body) # Compact this scope's PARAMs, capturing free Variables as inputs. - used = sorted((p for p in bwd_body.toposort(enter_calls=False) if p.op is Ops.PARAM), key=lambda p:p.arg.slot) + used = sorted((p for p in bwd_body.toposort() if p.op is Ops.PARAM), key=lambda p:p.arg.slot) bwd_args = [p if p.arg.slot < 0 else args[p.arg.slot] for p in used] bwd_body = bwd_body.substitute({p:p.replace(arg=dataclasses.replace(p.arg, slot=i, name=None, val=None)) for i,p in enumerate(used)}, walk=True) @@ -52,14 +52,13 @@ def call_gradient(ctx:UOp, k:UOp, needed:set[int]) -> tuple[UOp|None, ...]: def partial_store_gradient(ctx:UOp, dest:UOp, view:UOp): # A write through a non-overlapping view replaces only that region of the returned state. path, base = [], view - while base is not dest and base.op in {Ops.RESHAPE, Ops.SHRINK, Ops.PERMUTE, Ops.FLIP, Ops.PAD}: + while base is not dest and base.op in {Ops.RESHAPE, Ops.SHRINK, Ops.PERMUTE, Ops.PAD}: path.append(base) base = base.src[0] if base is not dest: return None grad = ctx for mop in reversed(path): grad = mop.replace(src=(grad,)+mop.src[1:]) - mask = grad.const_like(1) - for mop in path: mask = pm_gradient.rewrite(mop, ctx=mask)[0] + mask = compute_gradient(view, grad.const_like(1), {dest})[dest] return mask.cast(dtypes.bool).where(0, ctx), grad # ctx is grad_output @@ -85,9 +84,8 @@ pm_gradient = PatternMatcher([ (UPat(Ops.RESHAPE, name="ret"), lambda ctx, ret: (ctx.reshape(ret.src[0].shape), None)), (UPat(Ops.EXPAND), lambda ctx: (ctx, None)), (UPat(Ops.PAD, name="ret"), lambda ctx, ret: (ctx.shrink(tuple([(p[0], s+p[0]) for s,p in zip(ret.src[0].shape, ret.marg)])), None, None)), - (UPat(Ops.SHRINK, name="ret"), lambda ctx, ret: (ctx.pad(tuple([(p[0], s-p[0]-p[1]) for s,p in zip(ret.src[0].shape, ret.marg)])), None, None)), + (UPat(Ops.SHRINK, name="ret"), lambda ctx, ret: (ctx._mop(Ops.PAD, tuple((p[0], s) for s,p in zip(ret.src[0].shape, ret.marg))), None, None)), (UPat(Ops.PERMUTE, name="ret"), lambda ctx, ret: (ctx.permute(argsort(ret.marg)),)), - (UPat(Ops.FLIP, name="ret"), lambda ctx, ret: (ctx.flip([i for i,x in enumerate(ret.marg) if x]),)), (UPat(Ops.STACK, name="ret"), lambda ctx, ret: tuple(ctx[i] for i in range(len(ret.src)))), (UPat(Ops.COPY, name="ret"), lambda ctx, ret: (ctx.copy_to_device(ret.src[0].device),) + (None,) * (len(ret.src)-1)), (UPat(Ops.UNSHARD, name="ret"), lambda ctx, ret: ctx.shard(ret.device, ret.axis).src), diff --git a/tinygrad_repo/tinygrad/mixin/movement.py b/tinygrad_repo/tinygrad/mixin/movement.py index 35ebe0eb7..092b031aa 100644 --- a/tinygrad_repo/tinygrad/mixin/movement.py +++ b/tinygrad_repo/tinygrad/mixin/movement.py @@ -250,8 +250,22 @@ class MovementMixin: assert all(not isinstance(x, bool) and x >= 0 and x < self.ndim for x in axis_arg), f"flip args must be axis ints {axis_arg}" if len(axis_arg) != len(dedup(axis_arg)): raise RuntimeError(f"dim can appear at most once, getting {axis_arg}") - flip_arg = tuple([i in axis_arg for i in range(len(self.shape))]) - return self._mop(Ops.FLIP, arg=flip_arg) if any(flip_arg) else self + ret = self + for axis in axis_arg: + if resolve((n:=ret.shape[axis]) <= 1, False): continue + x = ret.transpose(0, axis) + bounds, crop = (None,) * (x.ndim-1), tuple((0, s) for s in x.shape[1:]) + tail = tuple(s if resolve(s > 0, False) else s+1 for s in x.shape[1:]) + m, tail_pad = n if resolve(n >= 2, False) else n+2, tuple((0, s) for s in tail) + # Place values on an anti-diagonal, transpose it, then extract the anti-diagonal. + # Pad possibly empty dimensions before reshaping, then crop them back afterwards. + x = x._mop(Ops.PAD, ((0, m),)+tail_pad).reshape(m, 1, *tail) + x = x._mop(Ops.PAD, ((0, m), (0, m-1))+tail_pad).reshape(m*(m-1), *tail) + x = x._mop(Ops.PAD, ((m-1, m*m),)+tail_pad).reshape(m, m, *tail).transpose(0, 1).reshape(m*m, *tail) + x = x._mop(Ops.SHRINK, ((m-1, m*(m-1)),)+tail_pad).reshape(m, m-1, *tail) + x = x.shrink((None, (0, 1))+bounds).reshape(m, *tail) + ret = x.shrink(((m-n, m),)+crop).transpose(0, axis) + return ret def stack(self, *args: Self, dim: int = 0) -> Self: """ @@ -284,10 +298,10 @@ class MovementMixin: """`.view` is an alias for `.reshape`.""" return self.reshape(shape, *args) - def squeeze(self, dim: int | None = None) -> Self: + def squeeze(self, dim: int | Sequence[int] | None = None) -> Self: """ Returns a tensor with specified dimensions of input of size 1 removed. - If `dim` is not specified, all dimensions with size 1 are removed. + `dim` can be an int or a sequence of ints. If `dim` is not specified, all dimensions with size 1 are removed. ```python exec="true" source="above" session="tensor" result="python" t = Tensor.zeros(2, 1, 2, 1, 2) @@ -299,11 +313,12 @@ class MovementMixin: ```python exec="true" source="above" session="tensor" result="python" print(t.squeeze(1).shape) ``` + ```python exec="true" source="above" session="tensor" result="python" + print(t.squeeze((1, 3)).shape) + ``` """ - if dim is None: - return self.reshape(tuple(dim for dim in self.shape if dim != 1)) - dim = self._resolve_dim(dim) - return self if not self.ndim or self.shape[dim] != 1 else self.reshape(self.shape[:dim] + self.shape[dim + 1 :]) + dims = range(self.ndim) if dim is None else [self._resolve_dim(d) for d in make_tuple(dim, 1)] + return self.reshape(tuple(s for i,s in enumerate(self.shape) if i not in dims or s != 1)) def unsqueeze(self, dim: int) -> Self: """ @@ -587,6 +602,7 @@ class MovementMixin: ``` """ repeats = argfix(repeats, *args) + if any(resolve(r < 0, False) for r in repeats): raise RuntimeError(f"repeats should be non negative, got {repeats} instead") base_shape = _align_left(self.shape, repeats)[0] unsqueezed_shape = flatten([[s] if r == 1 else [1, s] for r, s in zip(repeats, base_shape)]) expanded_shape = flatten([[s] if r == 1 else [r, s] for r, s in zip(repeats, base_shape)]) diff --git a/tinygrad_repo/tinygrad/mixin/op.py b/tinygrad_repo/tinygrad/mixin/op.py index 04412b5e2..0c45d6302 100644 --- a/tinygrad_repo/tinygrad/mixin/op.py +++ b/tinygrad_repo/tinygrad/mixin/op.py @@ -1172,7 +1172,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): for dim in reversed(axes): mask, values = functools.reduce(lambda x,y: (x[0]|y[0], y[0].where(y[1], x[1])), zip(mask.split(1, dim), values.split(1, dim))) # remove extra dims from reduce - for dim in reversed(axes): mask, values = mask.squeeze(dim), values.squeeze(dim) + mask, values = mask.squeeze(axes), values.squeeze(axes) # select from values for each True element in mask else select from self return mask.where(values, self) @@ -1641,6 +1641,23 @@ class OpMixin(ElementwiseMixin, ReduceMixin): padding = flatten((((k-1)*d-pB,(k-1)*d-pA+op) for k,d,(pB,pA),op in reversed(list(zip(HW, dilation, padding, output_padding))))) return x.conv2d(w.flatten(end_dim=1), groups=groups, bias=bias, dilation=dilation, padding=padding) + def rmsnorm(self, axis:int|tuple[int,...]=-1, eps:float=1e-6) -> Self: + """ + Applies Root Mean Square Normalization over a mini-batch of inputs. + + - Paper: https://arxiv.org/abs/1910.07467 + + ```python exec="true" source="above" session="tensor" result="python" + t = Tensor.randn(8, 10, 16) * 2 + 8 + print(t.square().mean().sqrt().item()) + ``` + ```python exec="true" source="above" session="tensor" result="python" + t = t.rmsnorm() + print(t.square().mean().sqrt().item()) + ``` + """ + return self.mul(self.square().mean(axis, keepdim=True).add(eps).rsqrt()) + def layernorm(self, axis:int|tuple[int,...]=-1, eps:float=1e-5) -> Self: """ Applies Layer Normalization over a mini-batch of inputs. @@ -1656,8 +1673,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): print(t.mean().item(), t.std().item()) ``` """ - y = (self - self.mean(axis, keepdim=True)) - return y.mul((y*y).mean(axis, keepdim=True).add(eps).rsqrt()) + return (self - self.mean(axis, keepdim=True)).rmsnorm(axis, eps) def batchnorm(self, weight:Self|None, bias:Self|None, mean:Self, invstd:Self, axis:int|tuple[int, ...]=1) -> Self: """ diff --git a/tinygrad_repo/tinygrad/nn/__init__.py b/tinygrad_repo/tinygrad/nn/__init__.py index 15e4b1358..646e9064d 100644 --- a/tinygrad_repo/tinygrad/nn/__init__.py +++ b/tinygrad_repo/tinygrad/nn/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations import math, functools from tinygrad.tensor import Tensor from tinygrad.dtype import dtypes -from tinygrad.helpers import prod, make_tuple, flatten, USE_ATOMICS, TRAINING +from tinygrad.helpers import prod, make_tuple, flatten, USE_ATOMICS, TRAINING, to_tuple from tinygrad.nn import optim, state, datasets # noqa: F401 class BatchNorm: @@ -297,10 +297,8 @@ class RMSNorm: self.eps = eps self.weight = Tensor.ones(dim) if elementwise_affine else None - def _norm(self, x:Tensor) -> Tensor: return x * (x.square().mean(-1, keepdim=True) + self.eps).rsqrt() - def __call__(self, x:Tensor) -> Tensor: - x = self._norm(x.float()).cast(x.dtype) + x = x.float().rmsnorm(eps=self.eps).cast(x.dtype) return x if self.weight is None else x * self.weight from tinygrad.uop.ops import UOp, KernelInfo, Ops, AxisType @@ -326,8 +324,7 @@ def _embedding_bwd(grad_emb:UOp, call:UOp) -> tuple: return out.flatten()[i].store(0).end(i).sink(arg=KernelInfo(name="zero")) grad_weight_uop = grad_weight_uop.custom_kernel(fxn=_zero_kernel)[0] - # TODO: do we have a universal helper for this? - device = call.device.split(":")[0] if not isinstance(call.device, tuple) else call.device[0].split(":")[0] + device = to_tuple(call.device)[0].split(":")[0] # this is the real atomic kernel def _embedding_bwd_kernel(grad_weight:UOp, grad_emb:UOp, idx:UOp) -> UOp: diff --git a/tinygrad_repo/tinygrad/nn/onnx.py b/tinygrad_repo/tinygrad/nn/onnx.py index d61c97882..24a3ff4f6 100644 --- a/tinygrad_repo/tinygrad/nn/onnx.py +++ b/tinygrad_repo/tinygrad/nn/onnx.py @@ -683,9 +683,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT def Shrink(x:Tensor, bias:float=0.0, lambd:float=0.5): return (x < -lambd).where(x+bias, (x > lambd).where(x-bias, 0)).cast(x.dtype) def Transpose(x:Tensor, perm:tuple[int, ...]|None=None): return x.permute(order=perm or list(range(x.ndim)[::-1])) - def Squeeze(data:Tensor, axes:Sequence[int]|None=None): - if axes is None: return data.squeeze() - return functools.reduce(lambda d, dim: d.squeeze(dim), sorted(map(data._resolve_dim, axes), reverse=True), data) + def Squeeze(data:Tensor, axes:Sequence[int]|None=None): return data.squeeze(axes) def Unsqueeze(data:Tensor, axes:Sequence[int]): return functools.reduce(lambda d, dim: d.unsqueeze(dim), sorted(data._resolve_dim(a, extra=len(axes)) for a in axes), data) @@ -1094,8 +1092,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT def RMSNormalization(X:Tensor, scale:Tensor, axis:int=-1, epsilon:float=1e-5, stash_type:int=1): assert stash_type == 1, "only float32 is supported" - norm = X.cast(dtypes.float).square().mean(axis=tuple(range(axis + X.ndim if axis < 0 else axis, X.ndim)), keepdim=True).add(epsilon).rsqrt() - return X * norm * scale + return X.cast(dtypes.float).rmsnorm(axis=tuple(range(axis + X.ndim if axis < 0 else axis, X.ndim)), eps=epsilon) * scale def RotaryEmbedding(X:Tensor, cos_cache:Tensor, sin_cache:Tensor, position_ids:Tensor|None=None, interleaved:int=0, num_heads:int|None=None, rotary_embedding_dim:int=0): diff --git a/tinygrad_repo/tinygrad/renderer/amd/__init__.py b/tinygrad_repo/tinygrad/renderer/amd/__init__.py index 875005f57..96a763c23 100644 --- a/tinygrad_repo/tinygrad/renderer/amd/__init__.py +++ b/tinygrad_repo/tinygrad/renderer/amd/__init__.py @@ -6,7 +6,7 @@ from tinygrad.renderer.amd.dsl import Inst, FixedBitField, EnumBitField # 0xf9 (249) = SDWA, 0xfa (250) = DPP16 for CDNA (GFX9) _VARIANT_SRC0 = {"_SDWA_SDST": 0xf9, "_SDWA": 0xf9, "_DPP16": 0xfa} -def _matches(data: bytes, cls: type[Inst]) -> bool: +def _matches(data: bytes|memoryview, cls: type[Inst]) -> bool: """Check if data matches all FixedBitFields and op is in allowed.""" for _, field in cls._fields: dword_idx = field.lo // 32 @@ -63,7 +63,7 @@ def _load_formats() -> dict[str, list[type[Inst]]]: } return _FORMATS -def detect_format(data: bytes, arch: str = "rdna3") -> type[Inst]: +def detect_format(data: bytes|memoryview, arch: str = "rdna3") -> type[Inst]: """Detect instruction format from machine code bytes.""" assert len(data) >= 4, f"need at least 4 bytes, got {len(data)}" for cls in _load_formats()[arch]: diff --git a/tinygrad_repo/tinygrad/renderer/amd/dsl.py b/tinygrad_repo/tinygrad/renderer/amd/dsl.py index 6526706be..211696f04 100644 --- a/tinygrad_repo/tinygrad/renderer/amd/dsl.py +++ b/tinygrad_repo/tinygrad/renderer/amd/dsl.py @@ -437,7 +437,7 @@ class Inst: return None @classmethod - def from_bytes(cls, data: bytes): + def from_bytes(cls, data: bytes|memoryview): inst = object.__new__(cls) inst._raw = int.from_bytes(data[:cls._base_size], 'little') # Upgrade to variant class if needed (_LIT, _DPP8, _DPP16, _SDWA) diff --git a/tinygrad_repo/tinygrad/renderer/cstyle.py b/tinygrad_repo/tinygrad/renderer/cstyle.py index 012b25277..67ebc3da8 100644 --- a/tinygrad_repo/tinygrad/renderer/cstyle.py +++ b/tinygrad_repo/tinygrad/renderer/cstyle.py @@ -267,8 +267,9 @@ class CStyleLanguage(Renderer): def param_type(self, p:UOp): return "volatile "*p.arg.volatile + self._render_dtype(p.dtype, 1, p.addrspace, True, p.addrspace != AddrSpace.ALU) def render(self, uops:list[UOp]) -> str: prefix, call_bodies, self.fn_names = [], [], dict[UOp, str]() - prefix += [f"extern void {f}();" for f in dedup(u.arg.name for u in UOp.sink(*uops).toposort() if u.op is Ops.CUSTOM_FUNCTION)] # symbols to link - for body in (u for u in UOp.sink(*uops).toposort() if u.op is Ops.LINEAR): + topo = UOp.sink(*uops).toposort(enter_calls=True) + prefix += [f"extern void {f}();" for f in dedup(u.arg.name for u in topo if u.op is Ops.CUSTOM_FUNCTION)] # symbols to link + for body in (u for u in topo if u.op is Ops.LINEAR): self.fn_names[body] = body.arg + (f"_{n}" if (n:=sum(b.arg == body.arg for b in self.fn_names)) else "") # a name traced with other args _, call, bufs = self._render(body.src) params = ', '.join(f"{self.param_type(p)} {n}" for n,(p,_) in bufs) @@ -502,9 +503,9 @@ class HIPRenderer(CStyleLanguage): global_prod_max = (0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF) @staticmethod - def is_cdna(arch): return arch.split(":")[0] in {"gfx942", "gfx950"} + def is_cdna(arch): return arch in {"gfx942", "gfx950"} @staticmethod - def is_cdna4(arch): return arch.split(":")[0] == "gfx950" + def is_cdna4(arch): return arch == "gfx950" def __init__(self, target:Target, use_hipcc=False): # gfx942 => MI300, gfx1100 => RX 7900, gfx1201 => RX 9700 super().__init__(target) from tinygrad.runtime.support.compiler_amd import HIPCompiler, HIPCCCompiler @@ -588,14 +589,16 @@ class HIPRenderer(CStyleLanguage): elif (N, M, K) == (16, 16, 32): type_map = {**type_map, dtypes.bf16: "_bf16", dtypes.f16: "_f16"} elif (N, M, K) == (16, 16, 128): type_map = {**type_map, dtypes.fp8e4m3: "_f8f6f4", dtypes.fp8e5m2: "_f8f6f4"} prefix.append(f"#define __{name} __builtin_amdgcn_mfma_{'scale_' if K == 128 else ''}f32_{N}x{M}x{K}{type_map[dtype_in]}") + elif dtype_out == dtypes.int32: + # RDNA4 uses 8 int8 values per lane in 2 VGPRs, RDNA3 uses 16 in 4 VGPRs + num_regs = 2 if (is_rdna4:=self.tensor_cores == tc.amd_rdna4) else 4 + prefix.append(f"typedef int wmma_int{num_regs} __attribute__((ext_vector_type({num_regs})));\n"+ + f"static inline __attribute__((device)) int8 __{name}"+f"""(signed_char{num_regs*4} a, signed_char{num_regs*4} b, int8 c) {{ + return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32{'_gfx12' if is_rdna4 else ''}(true, __builtin_bit_cast(wmma_int{num_regs}, a), + true, __builtin_bit_cast(wmma_int{num_regs}, b), c, false);\n}}""") # #define __WMMA_16_16_16_f16_f16 __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12 elif self.tensor_cores == tc.amd_rdna4: prefix.append(f"#define __{name} __builtin_amdgcn_wmma_{type_map[dtype_out]}_16x16x16_{type_map[dtype_in]}_w32_gfx12") - elif dtype_out == dtypes.int32: - prefix.append("typedef int wmma_int4 __attribute__((ext_vector_type(4)));\n"+ - f"static inline __attribute__((device)) int8 __{name}"+"""(signed_char16 a, signed_char16 b, int8 c) { - return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, __builtin_bit_cast(wmma_int4, a), - true, __builtin_bit_cast(wmma_int4, b), c, false);\n}""") elif dtype_out == dtypes.float: prefix.append(f"#define __{name} __builtin_amdgcn_wmma_f32_16x16x16_{'f16' if dtype_in == dtypes.half else 'bf16'}_w32") else: prefix.append(f"static inline __attribute__((device)) half8 __{name}"+"""(half16 a, half16 b, half8 c) { diff --git a/tinygrad_repo/tinygrad/renderer/isa/x86.py b/tinygrad_repo/tinygrad/renderer/isa/x86.py index 56d4f0ebe..37e36c641 100644 --- a/tinygrad_repo/tinygrad/renderer/isa/x86.py +++ b/tinygrad_repo/tinygrad/renderer/isa/x86.py @@ -319,7 +319,7 @@ def copy_op(dt:DType) -> X86Ops: isel_matcher = PatternMatcher([ # **** Op -> Op **** # range is lowered to acc, cmp, jmp after regalloc - (UPat(Ops.RANGE, src=(UPat.cvar("c").cast(),), allow_any_len=True, name="x"), lambda c,x: x.replace(src=(imm(x.dtype, c.val),) + x.src[1:])), + (UPat(Ops.RANGE, src=(UPat.cvar("c").cast(),), name="x"), lambda c,x: x.replace(src=(imm(x.dtype, c.val),))), # BACKEDGE becomes a conditional jump referencing the RANGE start label (UPat(Ops.BACKEDGE, src=(UPat(), UPat(), UPat(GroupOp.Comparison, name="cond")), name="x"), lambda x,cond: cond.ins(X86Ops.LOOP_CMP, tag=cond.op, src=cond.src + x.src[:2])), @@ -497,7 +497,7 @@ def lower_range(ctx, x:UOp) -> tuple[UOp, list[UOp]]: if x.dtype is dtypes.void: return (label, [label]) else: # an AFTER-wrapped bound carries the loop's ordering deps, they keep the acc unique per loop (hash-consing) - bound, deps = ((b:=x.src[0]).src[0], b.src[1:]) if (b:=x.src[0]).op is Ops.AFTER else (x.src[0], x.src[1:]) + bound, deps = ((b:=x.src[0]).src[0], b.src[1:]) if (b:=x.src[0]).op is Ops.AFTER else (x.src[0], ()) acc = x.ins(X86Ops.MOVi, src=(imm(x.dtype, 0),) + tuple(deps)) cmp = UOp(Ops.INS, arg=(X86Ops.CMPi if bound.op is Ops.CAST else X86Ops.CMP, dtypes.void), src=(acc, bound)) jump_out = UOp(Ops.INS, arg=(X86Ops.JGE, dtypes.void), src=(cmp,), tag=f".LOOP_OUT_{loop_label}") diff --git a/tinygrad_repo/tinygrad/renderer/llvmir.py b/tinygrad_repo/tinygrad/renderer/llvmir.py index 940bfc1b5..530dce9b5 100644 --- a/tinygrad_repo/tinygrad/renderer/llvmir.py +++ b/tinygrad_repo/tinygrad/renderer/llvmir.py @@ -16,7 +16,7 @@ def ldt(dt:DType, count=1, ptr=False): dtypes.uint8: "i8", dtypes.uint16: "i16", dtypes.uint32: "i32", dtypes.uint64: "i64", **{d: "i8" for d in dtypes.fp8s}, dtypes.float16: "half", dtypes.bfloat16: "bfloat", dtypes.float32: "float", dtypes.float64: "double"}[dt] -def lparam(u:UOp) -> str: return ldt(u.dtype, ptr=u.op is Ops.INDEX) # an external function's arg, an index is its address +def lparam(u:UOp) -> str: return ldt(u.dtype, ptr=u.addrspace not in (None, AddrSpace.ALU)) # an external function's arg, an index is its address def lconst(x, dtype:DType): if dtype in dtypes.floats: @@ -79,7 +79,7 @@ lop = {**{x:unsigned_lop for x in (dtypes.bool,)+dtypes.uints}, **{x:signed_lop base_rewrite = PatternMatcher([ # memory load/store - (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat((Ops.BUFFER, Ops.PARAM, Ops.AFTER)),), allow_any_len=True, name="x"), lambda ctx,x: + (UPat((Ops.INDEX, Ops.SHRINK), name="x"), lambda ctx,x: None if x.src[0].addrspace in (None, AddrSpace.ALU) else f" {ctx[x]} = getelementptr inbounds {ldt(x.dtype)}, {ldt(x.dtype, ptr=True)} {ctx[x.src[0]]}, {ldt(x.src[1].dtype)} {ctx[x.src[1]]}"), # register index (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.cvar("c").cast()), name="x"), lambda ctx,buf,c,x: @@ -196,8 +196,9 @@ class LLVMRenderer(Renderer): kernel.append(f" {r[u]} = addrspacecast [{size} x {ldt(u.dtype)}] addrspace(3)* @{r[u][1:]} to [{size} x {ldt(u.dtype)}]*") else: kernel.append(f" {r[u]} = alloca [{size} x {ldt(u.dtype)}], align 16") + elif u.op is Ops.BINARY: r[u] = f"@bin_{u.key.hex()}" elif u.op is Ops.CAST and u.src[0].op is Ops.CONST: r[u] = lconst(u.src[0].val, u.dtype) - elif u.op is Ops.CAST and ldt(u.dtype) == ldt(u.src[0].dtype): + elif (u.op is Ops.CAST and ldt(u.dtype) == ldt(u.src[0].dtype)) or (u.op is Ops.BITCAST and u.addrspace not in (None, AddrSpace.ALU)): r[u] = r[u.src[0]] # cast from signed to unsigned of the same size is a noop, or pointer cast else: vc += 1 @@ -216,13 +217,15 @@ class CPULLVMRenderer(LLVMRenderer): abi = 'win64cc' if sys.platform == 'win32' else None string_rewrite = base_rewrite def render(self, uops: list[UOp]) -> str: # the kernel is first, it is the entry. its functions follow, a name traced with other args gets a suffix - fns = {b: f"{b.arg}_{i}" for i, b in enumerate(b for b in UOp.sink(*uops).toposort() if b.op is Ops.LINEAR)} + fns = {b: f"{b.arg}_{i}" for i, b in enumerate(b for b in UOp.sink(*uops).toposort(enter_calls=True) if b.op is Ops.LINEAR)} defs = [self._render_kernel(b.src, name=n, fns=fns)[1] for b, n in fns.items()] return "\n".join((k:=self._render_kernel(uops, fns=fns))[0] + (k[1], *defs, self._render_footer(uops))) def _render_footer(self, uops: list[UOp]) -> str: decls = {x.src[0].arg.name: f"declare {ldt(x.dtype)} @{x.src[0].arg.name}({', '.join(map(lparam, x.src[1:]))})" - for x in UOp.sink(*uops).toposort() if x.op is Ops.CALL and x.src[0].op is Ops.CUSTOM_FUNCTION} - return "\n".join([*decls.values(), 'attributes #0 = { alwaysinline nounwind "no-builtins" "no-trapping-math"="true" }']) + for x in UOp.sink(*uops).toposort(enter_calls=True) if x.op is Ops.CALL and x.src[0].op is Ops.CUSTOM_FUNCTION} + blobs = {x.key: f"@bin_{x.key.hex()} = private constant [{len(x.arg)} x i8] c\"" + "".join("\\%02X" % b for b in x.arg) + '", align 16' + for x in UOp.sink(*uops).toposort(enter_calls=True) if x.op is Ops.BINARY} + return "\n".join([*decls.values(), *blobs.values(), 'attributes #0 = { alwaysinline nounwind "no-builtins" "no-trapping-math"="true" }']) def __init__(self, target:Target): super().__init__(target) from tinygrad.runtime.support.compiler_llvm import CPULLVMCompiler @@ -285,7 +288,7 @@ exit: %packed = phi i32 [%packed_bf8, %do_bf8], [%packed_fp8, %do_fp8]\n %trunc f'"amdgpu-flat-work-group-size"="1,{requiredMaxThreadsPerBlock}"', '"no-trapping-math"="true"'] return 'attributes #0 = { ' + ' '.join(attributes) + ' }' @staticmethod - def is_rdna4(arch): return arch.split(':')[0] in {'gfx1200', 'gfx1201'} + def is_rdna4(arch): return arch in {'gfx1200', 'gfx1201'} def __init__(self, target:Target): super().__init__(target) from tinygrad.runtime.support.compiler_llvm import AMDLLVMCompiler diff --git a/tinygrad_repo/tinygrad/renderer/nir.py b/tinygrad_repo/tinygrad/renderer/nir.py index 4b23f6629..f6a07b32b 100644 --- a/tinygrad_repo/tinygrad/renderer/nir.py +++ b/tinygrad_repo/tinygrad/renderer/nir.py @@ -158,7 +158,7 @@ class NIRRenderer(Renderer): (UPat(Ops.STACK, name="x"), lambda ctx,x: nalu(ctx.b, f"vec{x.max_numel()}", *[ctx.r[src] for src in x.src])), (UPat(GroupOp.ALU, name="x"), lambda ctx,x: nalu(ctx.b, aop[x.src[0].dtype][x.op], *[ctx.r[src] for src in x.src])), (UPat(Ops.CAST, name="x"), lambda ctx,x: ncast(ctx.b, ctx.r[x.src[0]], x.src[0].dtype, x.dtype)), - (UPat(Ops.BITCAST, src=(UPat.var("a"),), allow_any_len=True), lambda ctx,a: ctx.r[a]), + (UPat(Ops.BITCAST, src=(UPat.var("a"),)), lambda ctx,a: ctx.r[a]), (UPat(Ops.BUFFER, name="x"), lambda ctx,x: mesa.nir_local_variable_create(ctx.b.impl, mesa.glsl_array_type(glsl_type(x.dtype), x.max_numel(), 0).contents, f"acc{x.arg.slot}".encode()).contents), (UPat(Ops.BARRIER), lambda ctx: nbarrier(ctx.b)), diff --git a/tinygrad_repo/tinygrad/renderer/ptx.py b/tinygrad_repo/tinygrad/renderer/ptx.py index 0b6db72c8..249cdc421 100644 --- a/tinygrad_repo/tinygrad/renderer/ptx.py +++ b/tinygrad_repo/tinygrad/renderer/ptx.py @@ -90,7 +90,7 @@ string_rewrite = PatternMatcher([ (UPat((Ops.CMPLT, Ops.CMPNE, Ops.CMPEQ), name="x", allow_any_len=True, src=(UPat.var("src0"),)), lambda ctx, x, src0: ctx.code_for_op[x.op](ctx.r[x], *[ctx.r[v] for v in x.src], src0.dtype, ctx.types[src0.dtype])), (UPat(GroupOp.ALU, name="x"), lambda ctx, x: ctx.code_for_op[x.op](ctx.r[x], *[ctx.r[v] for v in x.src], x.dtype, ctx.types[x.dtype])), - (UPat(Ops.BITCAST, name="x", src=(UPat.var("a"),), allow_any_len=True), lambda ctx, x, a: f"mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {ctx.r[a]};"), + (UPat(Ops.BITCAST, name="x", src=(UPat.var("a"),)), lambda ctx, x, a: f"mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {ctx.r[a]};"), (UPat(Ops.CAST, name="x", src=(UPat(dtype=dtypes.bool, name="a"),)), lambda ctx, x, a: f"selp.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {render_val(1, x.dtype)}, {render_val(0, x.dtype)}, {ctx.r[a]};"), (UPat(Ops.CAST, name="x", src=(UPat.var("a"),)), diff --git a/tinygrad_repo/tinygrad/renderer/tc.py b/tinygrad_repo/tinygrad/renderer/tc.py index fa59f50c0..52b590460 100644 --- a/tinygrad_repo/tinygrad/renderer/tc.py +++ b/tinygrad_repo/tinygrad/renderer/tc.py @@ -78,7 +78,7 @@ amd_rdna3 = [TensorCore(dtype_in=di, dtype_out=do, frag_a=(("m0", "m1", "m2", "m amd_rdna4 = [TensorCore(dtype_in=di, dtype_out=do, frag_a=(("m0", "m1", "m2", "m3", "k2"), ("k0", "k1", "k3")), frag_b=(("n0", "n1", "n2", "n3", "k2"), ("k0", "k1", "k3")), frag_c=(("n0", "n1", "n2", "n3", "m3"), ("m0", "m1", "m2"))) for di,do in [(dtypes.half,dtypes.float),(dtypes.half,dtypes.half),(dtypes.bfloat16,dtypes.float),(dtypes.bfloat16,dtypes.bfloat16), - (dtypes.fp8e4m3,dtypes.float),(dtypes.fp8e5m2,dtypes.float)]] + (dtypes.fp8e4m3,dtypes.float),(dtypes.fp8e5m2,dtypes.float),(dtypes.int8,dtypes.int32)]] # https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-cdna4-instruction-set-architecture.pdf def mfma(K:int, di:DType, do:DType) -> TensorCore: @@ -120,6 +120,9 @@ def pm_wmma_fp8(dtype:DType) -> PatternMatcher: ]) pm_validate_wmma_rdna4 = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.int32), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.int32), x.src[1].bitcast(dtypes.int32), x.src[2])) + if x.src[0].dtype == dtypes.int8 and x.src[0].max_numel() == 8 else None), (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), diff --git a/tinygrad_repo/tinygrad/renderer/wgsl.py b/tinygrad_repo/tinygrad/renderer/wgsl.py index bc34e7f63..ebf762e71 100644 --- a/tinygrad_repo/tinygrad/renderer/wgsl.py +++ b/tinygrad_repo/tinygrad/renderer/wgsl.py @@ -16,8 +16,9 @@ def packed_store(s:UOp): bidx, var, *gate = s.src idx, shift_am, mask = packed_field(bidx, var.dtype) new_v, wmask = (var.cast(dtypes.uint32) & mask) << shift_am, ((mask << shift_am) ^ 0xFFFFFFFF).cast(dtypes.uint32) - buf = idx.cast(dtypes.uint32).load(*((UOp.const(0, dtypes.uint32), *gate) if gate else ())) - return idx.store((buf & wmask) | new_v, *gate) + word = idx.cast(dtypes.uint32) + buf = word.load(*((UOp.const(0, dtypes.uint32), *gate) if gate else ())) + return word.store((buf & wmask) | new_v, *gate) # load for char: sign_extend(buf[idx/4] >> ((idx%4)*8)) def packed_load(root:UOp): @@ -85,7 +86,7 @@ class WGSLRenderer(CStyleLanguage): lambda ctx,b,v,gate: f"select({ctx[v]}, {ctx.render_load(ctx[b], b.src[0])}, {ctx[gate]})"), (UPat.load(UPat.var("b")), lambda ctx, b: ctx.render_load(ctx[b], b)), # packed_store writes (load & wmask) | new_v: atomicAnd clears the field, atomicAdd sets it. new_v is gone when it is 0 - (UPat.store(UPat.var("b"), UPat.any(packed_rmw, packed_rmw | UPat.var("nv"))), lambda ctx,b,wmask,nv=None: + (UPat.store(UPat(Ops.CAST, src=(UPat.var("b"),)), UPat.any(packed_rmw, packed_rmw | UPat.var("nv"))), lambda ctx,b,wmask,nv=None: f"atomicAnd(&{ctx[b]},{ctx[wmask]});"+(f"\n atomicAdd(&{ctx[b]},{ctx[nv]});" if nv is not None else "") if is_packed(b) else None), (UPat.store(UPat.var("b"), UPat.var("v")), lambda ctx,b,v: f"{ctx[b]} = {ctx[v]};"), (UPat(Ops.INDEX, src=(UPat.var("b"), UPat.var("idx"))), diff --git a/tinygrad_repo/tinygrad/runtime/ops_amd.py b/tinygrad_repo/tinygrad/runtime/ops_amd.py index a5df720ac..853c5e317 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_amd.py +++ b/tinygrad_repo/tinygrad/runtime/ops_amd.py @@ -164,7 +164,7 @@ class AMDComputeQueue(HWQueue): memsel_dw = self.pm4.DATA_SEL(data_sel) | self.pm4.INT_SEL(int_sel) ctxid = 0 addr_w = address if isinstance(address, UOp) else UOp.const(address, dtypes.uint64) - val_w = value.cast(dtypes.uint64) if isinstance(value, UOp) else UOp.const(value, dtypes.uint64) + val_w = UOp.const(value, dtypes.uint64) self.pkt3(self.pm4.PACKET3_RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, addr_w, val_w, ctxid) def memory_barrier(self): diff --git a/tinygrad_repo/tinygrad/runtime/ops_cuda.py b/tinygrad_repo/tinygrad/runtime/ops_cuda.py index 8f8a81d13..8d74851e2 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_cuda.py +++ b/tinygrad_repo/tinygrad/runtime/ops_cuda.py @@ -1,5 +1,6 @@ from __future__ import annotations import ctypes, functools, mmap, struct, time +from typing import Any from tinygrad.helpers import DEBUG, DEV, getenv, unwrap from tinygrad.device import Buffer, BufferStorage, BufferSpec, Allocator, Compiled, MMIOInterface, HCQ_RUNTIME_DEV from tinygrad.dtype import dtypes @@ -20,7 +21,7 @@ def check(status:int): def as_int(handle) -> int: return unwrap(ctypes.cast(handle, ctypes.c_void_p).value) def host_stamp(slot:int): ctypes.c_uint64.from_address(slot).value = time.perf_counter_ns() -def extern(ptr:int, meta=None) -> Buffer: return Buffer(HCQ_RUNTIME_DEV.value, 8, opaque=BufferStorage(ptr, meta)) +def extern(ptr:int, meta=None) -> Buffer: return Buffer(HCQ_RUNTIME_DEV.device, 8, opaque=BufferStorage(ptr, meta)) # ***************** # queue @@ -47,7 +48,7 @@ class CUDAQueue(HWQueue): addr = UOp(Ops.LINEAR, src=tuple(pack_args([(0, UOp.const(size, dtypes.uint64))] + rows, 8 + size)), arg="kernargs").getaddr(self.devs) # extra: [buffer pointer, &args, buffer size, &size, end] self.call(cuda.cuLaunchKernel, self.extern((self.dev.tag("function"), obj.lib, obj.name)), *prg.arg.global_size, *prg.arg.local_size, 0, None, 0, - (self.q(*[UOp.const(v, dtypes.uint64) if isinstance(v, int) else v for v in (1, addr + 8, 2, addr, 0)]) - 40,)) + (self.q(*[UOp.const(v, dtypes.uint64) if isinstance(v, int) else v for v in (1, addr + 8, 2, addr, self.dev.launch_end)]) - 40,)) def copy(self, dst:UOp, src:UOp, sz:int): self.call(cuda.cuMemcpyAsync, dst.getaddr(self.devs), src.getaddr(self.devs), sz, None) def wait(self, sig:UOp, val:UOp): self.call(cuda.cuStreamWaitValue64_v2, None, sig.getaddr(self.devs), val, cuda.CU_STREAM_WAIT_VALUE_GEQ) @@ -64,29 +65,32 @@ class CUDAQueue(HWQueue): class CUDAAllocator(Allocator['CUDADevice']): def _alloc(self, size:int, options:BufferSpec) -> BufferStorage: - check(cuda.cuCtxSetCurrent(self.dev.context)) + check((api:=self.dev.api).cuCtxSetCurrent(self.dev.context)) if options.external_ptr: return BufferStorage(options.external_ptr) if options.host or options.cpu_access: - ptr = init_c_var(ctypes.c_void_p, lambda x: check(cuda.cuMemHostAlloc(ctypes.byref(x), size, 0))).value + ptr = init_c_var(ctypes.c_void_p, lambda x: check(api.cuMemHostAlloc(ctypes.byref(x), size, 0))).value return BufferStorage(ptr, None, MMIOInterface(ptr, size)) - return BufferStorage(init_c_var(cuda.CUdeviceptr, lambda x: check(cuda.cuMemAlloc_v2(ctypes.byref(x), size))).value) + return BufferStorage(init_c_var(api.CUdeviceptr, lambda x: check(api.cuMemAlloc_v2(ctypes.byref(x), size))).value) def _free(self, storage:BufferStorage, options:BufferSpec): self.dev.synchronize() - check((cuda.cuMemFreeHost if options.host or options.cpu_access else cuda.cuMemFree_v2)(storage.buf)) + check((self.dev.api.cuMemFreeHost if options.host or options.cpu_access else self.dev.api.cuMemFree_v2)(storage.buf)) def _map(self, buf:Buffer) -> BufferStorage: - if buf.device.startswith("CUDA"): return BufferStorage(buf._buf) + if buf.device.split(":")[0] == self.dev.device.split(":")[0]: return BufferStorage(buf._buf) if (host:=buf.get_storage().host) is None or host.addr % mmap.PAGESIZE: raise RuntimeError(f"{buf.device} memory is not page aligned host memory") - check(cuda.cuCtxSetCurrent(self.dev.context)) - if (status:=cuda.cuMemHostRegister_v2(host.addr, buf.nbytes, 0)) != cuda.CUDA_ERROR_HOST_MEMORY_ALREADY_REGISTERED: + check((api:=self.dev.api).cuCtxSetCurrent(self.dev.context)) + if (status:=api.cuMemHostRegister_v2(host.addr, buf.nbytes, 0)) != cuda.CUDA_ERROR_HOST_MEMORY_ALREADY_REGISTERED: check(status) # another device pinned it already return BufferStorage(host.addr, status == cuda.CUDA_SUCCESS) def _unmap(self, mapping:BufferStorage): - if mapping.meta: check(cuda.cuMemHostUnregister(mapping.buf)) + if mapping.meta: check(self.dev.api.cuMemHostUnregister(mapping.buf)) def _offset(self, buf:int, size:int, offset:int) -> int: return buf + offset class CUDADevice(Compiled): + api:Any = cuda + launch_end = 0 # CU_LAUNCH_PARAM_END + renderers = [CUDARenderer, PTXRenderer, NVCCRenderer] pm_encode = PatternMatcher([ (UPat(Ops.CALL, src=(UPat.custom_function("submit_cuda_compute"), UPat()), name="s"), lambda s: CUDAQueue(s).encode()), (UPat(Ops.CALL, src=(UPat.custom_function("submit_cuda_copy"), UPat()), name="s"), lambda s: CUDAQueue(s).encode()), @@ -94,34 +98,37 @@ class CUDADevice(Compiled): def __init__(self, device:str=""): device_id = int(device.split(":")[1]) if ":" in device else 0 - check(cuda.cuInit(0)) - self.cu_device = init_c_var(cuda.CUdevice, lambda x: check(cuda.cuDeviceGet(ctypes.byref(x), device_id))) - self.context = init_c_var(cuda.CUcontext, lambda x: check(cuda.cuCtxCreate_v2(ctypes.byref(x), 0, self.cu_device))) - check(cuda.cuDeviceComputeCapability(ctypes.byref(major:=ctypes.c_int()), ctypes.byref(minor:=ctypes.c_int()), device_id)) - self.streams = [init_c_var(cuda.CUstream, lambda x: check(cuda.cuStreamCreate(ctypes.byref(x), cuda.CU_STREAM_NON_BLOCKING))) for _ in range(2)] - super().__init__(device, CUDAAllocator(self), [CUDARenderer, PTXRenderer, NVCCRenderer], None, arch=f"sm_{major.value}{minor.value}") + check((api:=self.api).cuInit(0)) + self.cu_device = init_c_var(api.CUdevice, lambda x: check(api.cuDeviceGet(ctypes.byref(x), device_id))) + self.context = init_c_var(api.CUcontext, lambda x: check(api.cuCtxCreate_v2(ctypes.byref(x), 0, self.cu_device))) + self.streams = [init_c_var(api.CUstream, lambda x: check(api.cuStreamCreate(ctypes.byref(x), cuda.CU_STREAM_NON_BLOCKING))) for _ in range(2)] + super().__init__(device, CUDAAllocator(self), self.renderers, None, arch=self.get_arch(device_id)) Compiled.pm_bufferize += PatternMatcher([ (UPat(Ops.ALLOC, tag=self.tag("cuda")), lambda d=self: d.handles), (UPat(Ops.ALLOC, tag=self.tag("stamp")), lambda d=self: d.stamp), (UPat(Ops.ALLOC, name="b"), lambda b, d=self: d.function(*b.tag[1:]) if isinstance(b.tag, tuple) and b.tag[0] == d.tag("function") else None)]) + def get_arch(self, device_id:int) -> str: + check(cuda.cuDeviceComputeCapability(ctypes.byref(major:=ctypes.c_int()), ctypes.byref(minor:=ctypes.c_int()), device_id)) + return f"sm_{major.value}{minor.value}" + @functools.cached_property def handles(self) -> Buffer: - return Buffer(HCQ_RUNTIME_DEV.value, 32, initial_value=struct.pack("4Q", *[as_int(h) for h in (self.context, *self.streams)], 0)) + return Buffer(HCQ_RUNTIME_DEV.device, 32, initial_value=struct.pack("4Q", *[as_int(h) for h in (self.context, *self.streams)], 0)) @functools.cached_property def stamp(self) -> Buffer: return extern(as_int(fn:=cuda.CUhostFn(host_stamp)), fn) @functools.cache def function(self, lib:bytes, name:str) -> Buffer: - if DEBUG >= 5: print("\n".join([f"{i+1:>3} {line}" for i, line in enumerate(pretty_ptx(lib.decode()).split("\n"))])) - check(cuda.cuCtxSetCurrent(self.context)) - check(cuda.cuModuleLoadData(ctypes.byref(module:=cuda.CUmodule()), lib)) - check(cuda.cuModuleGetFunction(ctypes.byref(func:=cuda.CUfunction()), module, name.encode())) + if DEBUG >= 5 and self.api is cuda: print("\n".join([f"{i+1:>3} {line}" for i, line in enumerate(pretty_ptx(lib.decode()).split("\n"))])) + check(self.api.cuCtxSetCurrent(self.context)) + check(self.api.cuModuleLoadData(ctypes.byref(module:=self.api.CUmodule()), lib)) + check(self.api.cuModuleGetFunction(ctypes.byref(func:=self.api.CUfunction()), module, name.encode())) return extern(as_int(func)) - def count(self) -> int: return init_c_var(ctypes.c_int, lambda x: check(cuda.cuDeviceGetCount(ctypes.byref(x)))).value + def count(self) -> int: return init_c_var(ctypes.c_int, lambda x: check(self.api.cuDeviceGetCount(ctypes.byref(x)))).value def _wait_signal(self, sig:MMIOInterface|memoryview, value:int, timeout:int|None=None): # a fault raises here - check(cuda.cuCtxSetCurrent(self.context)) - check(cuda.cuCtxSynchronize()) + check(self.api.cuCtxSetCurrent(self.context)) + check(self.api.cuCtxSynchronize()) diff --git a/tinygrad_repo/tinygrad/runtime/ops_disk.py b/tinygrad_repo/tinygrad/runtime/ops_disk.py index 424feaad9..fa2fe1c36 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_disk.py +++ b/tinygrad_repo/tinygrad/runtime/ops_disk.py @@ -1,22 +1,22 @@ -import os, sys, mmap, io, ctypes, contextlib, pathlib -from typing import Generator, Callable -from tinygrad.helpers import OSX, round_up, mv_address -from tinygrad.device import BufferStorage, MMIOInterface, Compiled, Allocator +import os, sys, mmap, io, ctypes, contextlib, pathlib, functools, collections, struct, itertools +from dataclasses import replace +from typing import cast +from tinygrad.helpers import OSX, mv_address, flatten, to_tuple, unwrap, ceildiv +from tinygrad.device import BufferStorage, MMIOInterface, Compiled, Allocator, Buffer, BufferSpec, Device, HCQ_RUNTIME_DEV +from tinygrad.dtype import dtypes +from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, uopfunc +from tinygrad.runtime.support.hcq2 import ccall, ins, patch with contextlib.suppress(ImportError): import _posixshmem from tinygrad.runtime.autogen import io_uring, libc class DiskDevice(Compiled): - _tried_io_uring_init = False - def synchronize(self, timeout:int|None=None): pass def __init__(self, device:str): - if not DiskDevice._tried_io_uring_init: self._iouring_setup() - self.size: int|None = None self.fd: int|None = None - self.refcount = 0 + self.refcount, self.info = 0, Buffer(HCQ_RUNTIME_DEV.device, 16, options=BufferSpec(nolru=True)) # info: [fd, mmap address] for batch reads super().__init__(device, DiskAllocator(self), [], None) def _might_open(self, size:int): assert self.size is None or size <= self.size, f"can't reopen Disk tensor with larger size, opened with {self.size}, tried to open with {size}" @@ -34,6 +34,7 @@ class DiskDevice(Compiled): except OSError: self.fd = os.open(filename, os.O_RDWR|os.O_CREAT) if not pathlib.Path(filename).is_block_device() and os.fstat(self.fd).st_size < size: os.ftruncate(self.fd, size) self.mem = mmap.mmap(self.fd, size) + self.info.host.view()[:16] = struct.pack("2Q", self.fd, mv_address(memoryview(self.mem))) self.size = size if hasattr(self.mem, 'madvise') and (hp := getattr(mmap, "MADV_HUGEPAGE", None)) is not None: with contextlib.suppress(OSError): self.mem.madvise(hp) # some systems have transparent_hugepage disabled @@ -41,34 +42,11 @@ class DiskDevice(Compiled): def _might_close(self): self.refcount -= 1 if self.refcount == 0: - if self.fd is not None: - os.close(self.fd) + if self.fd is not None: os.close(self.fd) if hasattr(self, "mem"): try: self.mem.close() except BufferError: pass self.size = None - def _iouring_setup(self): - DiskDevice._tried_io_uring_init = True - - if sys.platform == 'linux' and not hasattr(sys, "getandroidapilevel"): - fd = libc.syscall(io_uring.NR_io_uring_setup, 4096, ctypes.byref(p:=io_uring.struct_io_uring_params())) - if fd < 0: return - - sq_ptr = libc.mmap(0, p.sq_off.array + p.sq_entries * 4, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED | MAP_POPULATE, fd, 0) - cq_ptr = libc.mmap(0, p.cq_off.cqes + p.cq_entries * ctypes.sizeof(io_uring.struct_io_uring_cqe), - mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED | MAP_POPULATE, fd, io_uring.IORING_OFF_CQ_RING) - sqes = libc.mmap(0, p.sq_entries * ctypes.sizeof(io_uring.struct_io_uring_sqe), - mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED | MAP_POPULATE, fd, io_uring.IORING_OFF_SQES) - - def u32ptr(val): return ctypes.cast(val, ctypes.POINTER(ctypes.c_uint32)) - sqdesc = io_uring.struct_io_uring_sq(khead=u32ptr(sq_ptr+p.sq_off.head), ktail=u32ptr(sq_ptr+p.sq_off.tail), - array=u32ptr(sq_ptr+p.sq_off.array), - kring_mask=u32ptr(sq_ptr+p.sq_off.ring_mask), sqes=ctypes.cast(sqes, ctypes.POINTER(io_uring.struct_io_uring_sqe))) - - cqdesc = io_uring.struct_io_uring_cq(khead=u32ptr(cq_ptr+p.cq_off.head), ktail=u32ptr(cq_ptr+p.cq_off.tail), - kring_mask=u32ptr(sq_ptr+p.cq_off.ring_mask), cqes=ctypes.cast(cq_ptr+p.cq_off.cqes, ctypes.POINTER(io_uring.struct_io_uring_cqe))) - - DiskDevice.io_uring = io_uring.struct_io_uring(ring_fd=fd, sq=sqdesc, cq=cqdesc) # type: ignore class DiskBuffer: def __init__(self, device:DiskDevice, size:int, offset=0): @@ -98,47 +76,95 @@ class DiskAllocator(Allocator): else: dest[:] = src._buf() - def _copyout_sharded(self, src:DiskBuffer, size:int, _get_free_buf:Callable, seg_len:int, - use_ioring:bool=True) -> Generator[tuple[int, int, int, int], None, None]: - fd_offset = src.offset - (minor_offset := src.offset % mmap.PAGESIZE) - processed_reqs_cnt, copied_in, next_read_offset, total_copy_size = 0, 0, 0, round_up(size + minor_offset, mmap.PAGESIZE) - - if not hasattr(DiskDevice, 'io_uring') or not use_ioring: - local_buf = memoryview(bytearray(seg_len)) - for off in range(0, total_copy_size, seg_len): - while (copy_batch := _get_free_buf()) is None: pass - read_size = min(seg_len, total_copy_size - off, src.device.size - fd_offset - off) - self._copyout(local_buf[:read_size], DiskBuffer(src.device, read_size, fd_offset + off)) - copy_batch[0].view(size=read_size)[:] = local_buf[:read_size] - real_copy_size = min(read_size - minor_offset, size - copied_in) - yield (copy_batch, copied_in, minor_offset, real_copy_size) - copied_in, minor_offset = copied_in + real_copy_size, 0 - return - - reqs: list[tuple[int, int, int, int]] = [] - while next_read_offset < total_copy_size or len(reqs) != processed_reqs_cnt: - if next_read_offset < total_copy_size and (copy_batch := _get_free_buf()) is not None: - # Prepare sqe - sqe_index = (tail:=DiskDevice.io_uring.sq.ktail[0]) & DiskDevice.io_uring.sq.kring_mask[0] - sqe = DiskDevice.io_uring.sq.sqes[sqe_index] - sqe.opcode, sqe.fd, sqe.off = io_uring.IORING_OP_READ, self.dev.fd, fd_offset + next_read_offset - sqe.addr, sqe.len, sqe.user_data = copy_batch[0].addr, min(seg_len, total_copy_size - next_read_offset), len(reqs) - - # Send sqe - DiskDevice.io_uring.sq.array[sqe_index] = sqe_index - DiskDevice.io_uring.sq.ktail[0] = tail + 1 - libc.syscall(io_uring.NR_io_uring_enter, DiskDevice.io_uring.ring_fd, 1, 1, io_uring.IORING_ENTER_GETEVENTS) - - reqs.append((copy_batch, copied_in, minor_offset, real_copy_size:=min(sqe.len - minor_offset, size - copied_in))) - next_read_offset += sqe.len - copied_in += real_copy_size - minor_offset = 0 - - if (head:=DiskDevice.io_uring.cq.khead[0]) != DiskDevice.io_uring.cq.ktail[0]: - cqe = DiskDevice.io_uring.cq.cqes[head & DiskDevice.io_uring.cq.kring_mask[0]] - assert cqe.res >= 0, f"read from disk failed, err: {cqe.res}" - yield reqs[cqe.user_data] - DiskDevice.io_uring.cq.khead[0] = head + 1 # advance - processed_reqs_cnt += 1 - def _offset(self, buf:DiskBuffer, size:int, offset:int): return DiskBuffer(buf.device, size, offset) + +# ***************** +# UOps implementation + +CHUNK_SZ, READ_SZ, SLOTS, RING_ENTRIES = 64 << 20, 1 << 20, 2, 256 +SLOT_SZ, STAGE_SZ = CHUNK_SZ + 4096, 4096 + SLOTS * (CHUNK_SZ + 4096) # +4096: reads are page aligned. stage: a page of counts, then the slots + +@uopfunc +def disk_read(file:UOp, srcs:UOp, counts:UOp, rings:UOp, sqes:UOp) -> UOp: # srcs: [address in the file's mmap, bytes] per copy + fd, p = unwrap(uring())[:2] + file, srcs, counts, rings, sqes = (b.replace(arg=replace(b.arg, volatile=True, device=None)) for b in (file, srcs, counts, rings, sqes)) + + # for each copy + src_va, nbytes = srcs[(copy:=UOp.range(srcs.max_numel() // 2, 0, dtype=dtypes.uint64)) * 2], srcs[copy * 2 + 1] + + # for each chunk: the next chunk n waits for the gpu to free its slot + n = counts.after(chunk:=UOp.range(ceildiv(nbytes, CHUNK_SZ), 1, dtype=dtypes.uint64))[0] + free = (copied:=counts.after(n, loop:=UOp.loop(3))[1]).backedge(loop, copied + SLOTS <= n) + pos = src_va + chunk * CHUNK_SZ - file[1] + span = (pos % 4096 + (nbytes - chunk * CHUNK_SZ).minimum(CHUNK_SZ) + 4095) // 4096 * 4096 + + # for each read: queues READ_SZ of the chunk's pages. an sqe is 8 words: fd << 32 | flags << 8 | opcode, file offset, address, bytes + tail, read = rings.after(free)[p.sq_off.tail // 4].cast(dtypes.uint64), UOp.range(nreads:=ceildiv(span, READ_SZ), 2, dtype=dtypes.uint64) + sqe = [file[0] << 32 | io_uring.IOSQE_ASYNC << 8 | io_uring.IORING_OP_READ, pos - pos % 4096 + read * READ_SZ, + counts.getaddr(HCQ_RUNTIME_DEV.device) + 4096 + n % SLOTS * SLOT_SZ + read * READ_SZ, (span - read * READ_SZ).minimum(READ_SZ)] + queued = UOp.group(*[sqes[(tail + read) % RING_ENTRIES * 8 + j].store(v) for j, v in enumerate(sqe)]).end(read) + + # submits the reads, waits for all, marks the chunk read + to_submit = nreads.cast(dtypes.int).after(rings.after(queued)[p.sq_off.tail // 4].store((tail + nreads).cast(dtypes.uint32))) + done = ccall(libc.syscall, io_uring.NR_io_uring_enter, fd, to_submit, to_submit, io_uring.IORING_ENTER_GETEVENTS, 0, 0) + reaped = rings.after(done)[p.cq_off.head // 4].store(rings.after(done)[p.cq_off.tail // 4]) + return counts.after(reaped)[0].store(n + 1).end(chunk).end(copy).sink() + +# ***************** +# 2. rewriter + +def is_disk_read(c:UOp) -> bool: + dev, src = [to_tuple(b.device)[0] for b in c.src[1:3]] if c.op is Ops.CALL and c.body.op is Ops.STORE else ("", "") + return src.startswith("DISK:") and not src.startswith("DISK:shm:") and Device[dev].has_copy_queue and not dev.startswith("NULL") and bool(uring()) + +def disk_copy_rewriter(s:UOp) -> UOp|None: + lins = [submit.without_after.src[1].without_after for submit in s.src] + if not (copies:=[c for lin in lins for c in lin.src if is_disk_read(c)]): return None + rings = UOp.alloc((unwrap(uring())[2].nbytes // 4,), dtypes.uint32, 0, device=HCQ_RUNTIME_DEV.device).rtag("uring_rings") + sqes = UOp.alloc((8 * RING_ENTRIES,), dtypes.uint64, 0, device=HCQ_RUNTIME_DEV.device).rtag("uring_sqes") + + # [mmap address, bytes] per copy in a table: an arg each would hit ctypes' 1024 limit + words = flatten((c.src[2].getaddr(HCQ_RUNTIME_DEV.device), UOp.const(c.src[2].nbytes(), dtypes.uint64)) for c in copies) + table = UOp.alloc((8 * len(words),), dtypes.uint8, device=HCQ_RUNTIME_DEV.device) + srcs = patch(table, [(8 * i, w) for i, w in enumerate(words)]).bitcast(dtypes.uint64) + + # per gpu: [chunks read, chunks copied] at the start of its stage + devs = [to_tuple(c.src[1].device)[0] for c in copies] + counts = {d: UOp.alloc((STAGE_SZ,), dtypes.uint8, 0, device=d).rtag("disk_stage")[:16].bitcast(dtypes.uint64) for d in devs} + + # cpu part + runs = [(key, list(ks)) for key, ks in itertools.groupby(range(len(copies)), lambda k: (devs[k], to_tuple(copies[k].src[2].device)[0]))] + done, offs, gpu_ops = s.src[-1], collections.Counter[str](), collections.defaultdict[UOp, list[UOp]](list) + for (dev, disk), ks in runs: + file = UOp.alloc((2,), dtypes.uint64, 0, device=disk).rtag("disk_info") + done = disk_read(file, srcs[2 * ks[0]:2 * ks[-1] + 2], counts[dev].after(done), rings, sqes) + + # gpu part + for k, (c, dev) in enumerate(zip(copies, devs)): + first, slots = counts[dev].index(0).load() + offs[dev], counts[dev].getaddr(dev) + 4096 + srcs.index(2 * k).load() % 4096 + offs[dev] += ceildiv(c.src[2].nbytes(), CHUNK_SZ) + for chunk in range(ceildiv(c.src[2].nbytes(), CHUNK_SZ)): + n, dst, nb = first + chunk, c.src[1].getaddr() + chunk * CHUNK_SZ, min(CHUNK_SZ, c.src[2].nbytes() - chunk * CHUNK_SZ) + gpu_ops[c] += [ins("wait", counts[dev], n + 1), ins("copy", dst, slots + n % SLOTS * SLOT_SZ, nb), ins("store", counts[dev][1:], n + 1)] + return s.replace(src=(*s.src, done)).substitute({lin: lin.replace(src=tuple(flatten(gpu_ops.get(c, [c]) for c in lin.src))) for lin in lins}) +Compiled.pm_batch = Compiled.pm_batch + PatternMatcher([(UPat(Ops.SINK, name="s"), disk_copy_rewriter)]) + +# ***************** +# 3. bufferize + +def shared(fd:int, at:int, n:int) -> Buffer: # n bytes of memory the kernel shares at offset at + return Buffer(HCQ_RUNTIME_DEV.device, n, options=BufferSpec(external_ptr=libc.mmap(0, n, mmap.PROT_READ|mmap.PROT_WRITE, mmap.MAP_SHARED, fd, at))) + +@functools.cache +def uring() -> tuple[int, io_uring.struct_io_uring_params, Buffer, Buffer]|None: # fd, offsets, the rings (heads, tails, cqes), the sqes + if sys.platform != "linux" or hasattr(sys, "getandroidapilevel"): return None + p = io_uring.struct_io_uring_params(flags=io_uring.IORING_SETUP_NO_SQARRAY) # type: ignore[unreachable] + if (fd:=libc.syscall(io_uring.NR_io_uring_setup, RING_ENTRIES, ctypes.byref(p))) < 0: return None + return fd, p, shared(fd, io_uring.IORING_OFF_SQ_RING, p.cq_off.cqes + 16 * p.cq_entries), shared(fd, io_uring.IORING_OFF_SQES, 64 * RING_ENTRIES) + +@functools.cache +def _stage(dev) -> Buffer: return Buffer(Device[dev].host, STAGE_SZ, options=BufferSpec(cpu_access=True), initial_value=bytes(STAGE_SZ)) + +Compiled.pm_bufferize += PatternMatcher([(UPat(Ops.ALLOC, tag="disk_info", name="b"), lambda b: cast(DiskDevice, Device[b.device]).info), + (UPat(Ops.ALLOC, tag="disk_stage", name="b"), lambda b: _stage(b.device)), + (UPat(Ops.ALLOC, tag="uring_rings"), lambda: unwrap(uring())[2]), (UPat(Ops.ALLOC, tag="uring_sqes"), lambda: unwrap(uring())[3])]) diff --git a/tinygrad_repo/tinygrad/runtime/ops_hip.py b/tinygrad_repo/tinygrad/runtime/ops_hip.py index b5fe55502..495c68367 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_hip.py +++ b/tinygrad_repo/tinygrad/runtime/ops_hip.py @@ -1,71 +1,35 @@ -import ctypes -from tinygrad.helpers import mv_address, getenv, suppress_finalizing -from tinygrad.device import BufferStorage, Compiled, Allocator, BufferSpec, Program, TinyELF -from tinygrad.runtime.autogen import hip +from types import SimpleNamespace +from tinygrad.helpers import getenv +from tinygrad.dtype import dtypes +from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite from tinygrad.renderer.cstyle import HIPRenderer -from tinygrad.runtime.support.c import init_c_var, init_c_struct_t +from tinygrad.runtime.autogen import hip +from tinygrad.runtime.support.hcq2 import ccall +from tinygrad.runtime.support.c import init_c_var +from tinygrad.runtime.ops_cuda import CUDAQueue, CUDADevice, check if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl # noqa: F401 # pylint: disable=unused-import -def check(status): - if status != 0: raise RuntimeError(f"HIP Error {status}, {ctypes.string_at(hip.hipGetErrorString(status)).decode()}") +HIPIFY = dict(CUdevice=hip.hipDevice_t, CUcontext=hip.hipCtx_t, CUstream=hip.hipStream_t, CUdeviceptr=hip.hipDeviceptr_t, CUmodule=hip.hipModule_t, + CUfunction=hip.hipFunction_t, cuInit=hip.hipInit, cuDeviceGet=hip.hipDeviceGet, cuDeviceGetCount=hip.hipGetDeviceCount, + cuCtxCreate_v2=hip.hipCtxCreate, cuCtxSetCurrent=hip.hipCtxSetCurrent, cuCtxSynchronize=hip.hipDeviceSynchronize, + cuStreamCreate=hip.hipStreamCreateWithFlags, cuMemAlloc_v2=hip.hipMalloc, cuMemFree_v2=hip.hipFree, + cuMemHostAlloc=lambda p, sz, _: hip.hipHostMalloc(p, sz, hip.hipHostMallocCoherent), cuMemFreeHost=hip.hipHostFree, + cuMemHostRegister_v2=hip.hipHostRegister, cuMemHostUnregister=hip.hipHostUnregister, cuModuleLoadData=hip.hipModuleLoadData, + cuModuleGetFunction=hip.hipModuleGetFunction, cuLaunchKernel=hip.hipModuleLaunchKernel, cuMemcpyAsync=hip.hipMemcpyAsync, + cuStreamWaitValue64_v2=hip.hipStreamWaitValue64, cuStreamWriteValue64_v2=hip.hipStreamWriteValue64, cuLaunchHostFunc=hip.hipLaunchHostFunc) -class HIPDevice(Compiled): - def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - self.arch = init_c_var(hip.hipDeviceProp_t, lambda x: check(hip.hipGetDeviceProperties(x, self.device_id))).gcnArchName.decode() - self.time_event_st, self.time_event_en = [init_c_var(hip.hipEvent_t, lambda x: hip.hipEventCreate(ctypes.byref(x), 0)) for _ in range(2)] +def hipify(c:UOp, f:UOp) -> UOp|None: + if (fn:=HIPIFY.get(f.arg.name)) is None: return None + args = list(c.src[1:]) + if fn is hip.hipMemcpyAsync: args.insert(3, UOp.const(hip.hipMemcpyDefault, dtypes.int)) + if fn is hip.hipStreamWaitValue64: args.append(UOp.const((1<<64)-1, dtypes.uint64)) + return ccall(fn, *args) +pm_hipify = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, name="f"),), allow_any_len=True, name="c"), hipify)]) - super().__init__(device, HIPAllocator(self), [HIPRenderer], HIPProgram, arch=self.arch) +class HIPDevice(CUDADevice): + api, launch_end, renderers = SimpleNamespace(**HIPIFY), 3, [HIPRenderer] + pm_encode = PatternMatcher([(UPat(Ops.CALL, src=(UPat.custom_function(f"submit_hip_{q}"), UPat()), name="s"), + lambda s: graph_rewrite(CUDAQueue(s).encode(), pm_hipify, enter_calls=True, name="hipify")) for q in ("compute", "copy")]) - def count(self) -> int: return init_c_var(ctypes.c_int, lambda x: check(hip.hipGetDeviceCount(x))).value - - def synchronize(self, timeout:int|None=None): - check(hip.hipSetDevice(self.device_id)) - check(hip.hipDeviceSynchronize()) - -class HIPProgram(Program[HIPDevice]): - def __init__(self, dev:HIPDevice, obj:TinyELF): - self.dev, self.name, self.lib, self.signature = dev, obj.name, obj.lib, obj.signature - check(hip.hipSetDevice(self.dev.device_id)) - self.module = init_c_var(hip.hipModule_t, lambda x: check(hip.hipModuleLoadData(ctypes.byref(x), obj.lib))) - self.prg = init_c_var(hip.hipFunction_t, lambda x: check(hip.hipModuleGetFunction(ctypes.byref(x), self.module, obj.name.encode("utf-8")))) - - @suppress_finalizing - def __del__(self): - if hasattr(self, 'module'): check(hip.hipModuleUnload(self.module)) - - def __call__(self, *args, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw): - check(hip.hipSetDevice(self.dev.device_id)) - if not hasattr(self, "vargs"): - fields = ([(f'f{i}', hip.hipDeviceptr_t, i*8) for i in range(len(args))] + - [(f'v{i}', getattr(ctypes, f"c_int{dt.bitsize}"), o) for i,(o,dt) in enumerate(TinyELF.iter_sig(self.signature[len(args):], len(args)*8))]) - self.c_args = init_c_struct_t(fields[-1][2] + ctypes.sizeof(fields[-1][1]) if len(fields) else 0, tuple(fields))(*args, *vals) - self.vargs = (ctypes.c_void_p * 5)(1, ctypes.cast(ctypes.byref(self.c_args), ctypes.c_void_p), 2, - ctypes.cast(ctypes.pointer(ctypes.c_size_t(ctypes.sizeof(self.c_args))), ctypes.c_void_p), 3) - - for i in range(len(args)): self.c_args.__setattr__(f'f{i}', args[i]) - for i in range(len(vals)): self.c_args.__setattr__(f'v{i}', vals[i]) - - if wait: check(hip.hipEventRecord(self.dev.time_event_st, None)) - - check(hip.hipModuleLaunchKernel(self.prg, *global_size, *local_size, 0, None, None, self.vargs)) - - if wait: - check(hip.hipEventRecord(self.dev.time_event_en, None)) - check(hip.hipEventSynchronize(self.dev.time_event_en)) - check(hip.hipEventElapsedTime(ctypes.byref(ret := ctypes.c_float()), self.dev.time_event_st, self.dev.time_event_en)) - return ret.value * 1e-3 - -class HIPAllocator(Allocator[HIPDevice]): - def _alloc(self, size:int, options:BufferSpec) -> BufferStorage: - check(hip.hipSetDevice(self.dev.device_id)) - return BufferStorage(init_c_var(hip.hipDeviceptr_t, lambda x: check(hip.hipMalloc(ctypes.byref(x), size)))) - - def _free(self, storage:BufferStorage, options:BufferSpec): check(hip.hipFree(storage.buf)) - def _copyin(self, dest, src: memoryview): - check(hip.hipSetDevice(self.dev.device_id)) - check(hip.hipMemcpy(dest, mv_address(src), len(src), hip.hipMemcpyHostToDevice)) - def _copyout(self, dest:memoryview, src): - self.dev.synchronize() - check(hip.hipMemcpy(mv_address(dest), src, len(dest), hip.hipMemcpyDeviceToHost)) - def _offset(self, buf, size:int, offset:int): return hip.hipDeviceptr_t(buf.value + offset) + def get_arch(self, device_id:int) -> str: + return init_c_var(hip.hipDeviceProp_t, lambda x: check(hip.hipGetDeviceProperties(x, device_id))).gcnArchName.decode().split(":")[0] diff --git a/tinygrad_repo/tinygrad/runtime/ops_metal.py b/tinygrad_repo/tinygrad/runtime/ops_metal.py index e9b1be707..b76d0c684 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_metal.py +++ b/tinygrad_repo/tinygrad/runtime/ops_metal.py @@ -7,7 +7,7 @@ from tinygrad.dtype import dtypes, AddrSpace from tinygrad.renderer.cstyle import MetalRenderer from tinygrad.runtime.autogen import metal from tinygrad.runtime.support.c import DLL -from tinygrad.runtime.support.hcq2 import HWQueue, ccall, patch, layout_args, to_name +from tinygrad.runtime.support.hcq2 import HWQueue, ccall, patch, layout_args, to_name, bufferize_buf from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, uopfunc from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops @@ -141,7 +141,7 @@ class MetalQueue(HWQueue): dims = (*prg.arg.global_size, *prg.arg.local_size) if any(isinstance(d, UOp) for d in dims): self.sizes.append((len(self.cmds), at:=round_up(self.nbytes, 8))) - self.rows += layout_args([d.cast(dtypes.uint64) if isinstance(d, UOp) else UOp.const(d, dtypes.uint64) for d in dims], at) + self.rows += layout_args([UOp.const(d, dtypes.uint64) for d in dims], at) self.nbytes = at + 48 self.cmds.append((obj.lib, obj.name, tuple(1 if isinstance(d, UOp) else int(d) for d in dims), off)) @@ -217,7 +217,8 @@ class MetalDevice(Compiled): (UPat(Ops.ALLOC, tag=self.tag("handles")), lambda d=self: d.handles), (UPat(Ops.ALLOC, tag="slots", name="b"), # with stamps lambda b, d=self: d.new_slots(b.max_numel()) if b.device == d.device and b.max_numel() > 4 else None), - (UPat(Ops.ALLOC, name="b"), lambda b, d=self: d.new_icb(*b.tag[1:]) if isinstance(b.tag, tuple) and b.tag[0] == d.tag("mtl_icb") else None)]) + (UPat(Ops.ALLOC, name="b"), lambda ctx, b, d=self: + d.new_icb(bufferize_buf(ctx, b.rtag(None)).buffer, *b.tag[1:]) if isinstance(b.tag, tuple) and b.tag[0] == d.tag("mtl_icb") else None)]) @functools.cached_property def handles(self) -> Buffer: @@ -247,9 +248,8 @@ class MetalDevice(Compiled): descriptor.setSupportIndirectCommandBuffers(True) return checked(self.sysdevice.newComputePipelineStateWithDescriptor_options_reflection_error, descriptor, metal.MTLPipelineOptionNone, None) - def new_icb(self, cmds:tuple[tuple[bytes, str, tuple[int, ...], int], ...], header:int) -> Buffer: + def new_icb(self, buf:Buffer, cmds:tuple[tuple[bytes, str, tuple[int, ...], int], ...], header:int) -> Buffer: pipes = dedup(c[:2] for c in cmds) - buf = Buffer(self.device, header + 8 * (1 + len(cmds) + len(pipes)), options=BufferSpec(nolru=True), preallocate=True) desc = metal.MTLIndirectCommandBufferDescriptor.new() desc.setCommandTypes(metal.MTLIndirectCommandTypeConcurrentDispatch) desc.setMaxKernelBufferBindCount(1) @@ -259,7 +259,7 @@ class MetalDevice(Compiled): for cmd, (lib, name, dims, off) in zip(commands, cmds): cmd.setComputePipelineState(state:=self.pipeline(lib, name)) if prod(dims[3:]) > (mx:=state.maxTotalThreadsPerThreadgroup()): raise RuntimeError(f"local size {dims[3:]} bigger than {mx}") - cmd.setKernelBuffer_offset_atIndex(buf.get_storage().meta, off, 0) + cmd.setKernelBuffer_offset_atIndex(buf.meta, buf.offset + off, 0) cmd.concurrentDispatchThreadgroups_threadsPerThreadgroup(metal.MTLSize(*dims[:3]), metal.MTLSize(*dims[3:])) cmd.setBarrier() buf.host.view(fmt='Q')[header // 8:] = array.array('Q', [icb.value, *[c.value for c in commands], *[self.pipeline(*p).value for p in pipes]]) diff --git a/tinygrad_repo/tinygrad/runtime/ops_qcom.py b/tinygrad_repo/tinygrad/runtime/ops_qcom.py index b699dfb77..e778c1f8b 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_qcom.py +++ b/tinygrad_repo/tinygrad/runtime/ops_qcom.py @@ -201,7 +201,7 @@ class QCOMComputeQueue(HWQueue): ib, ib_off = unwrap_view(cmdbuf) fd, ctxid = [UOp.variable(n, 0, 2**31 - 1, dtypes.int32) for n in ("kgsl_fd", "kgsl_ctx")] obj = cstruct(kgsl.struct_kgsl_command_object, gpuaddr=ib.getaddr(self.devs) + ib_off, size=cmdbuf.max_numel(), flags=kgsl.KGSL_CMDLIST_IB) - req = cstruct(kgsl.struct_kgsl_gpu_command, cmdlist=obj.getaddr(HCQ_RUNTIME_DEV.value), cmdsize=ctypes.sizeof(kgsl.struct_kgsl_command_object), + req = cstruct(kgsl.struct_kgsl_gpu_command, cmdlist=obj.getaddr(HCQ_RUNTIME_DEV.device), cmdsize=ctypes.sizeof(kgsl.struct_kgsl_command_object), numcmds=1, context_id=ctxid) return qcom_submit(req.after(cmdbuf), UOp.alloc((1,), dtypes.int32, device=self.devs[0]).rtag("submit_ret"), fd) diff --git a/tinygrad_repo/tinygrad/runtime/support/hcq2.py b/tinygrad_repo/tinygrad/runtime/support/hcq2.py index 322eeabcd..652a19f01 100644 --- a/tinygrad_repo/tinygrad/runtime/support/hcq2.py +++ b/tinygrad_repo/tinygrad/runtime/support/hcq2.py @@ -4,11 +4,11 @@ import functools, itertools, weakref, ctypes, struct, time, hashlib, pathlib from dataclasses import replace, dataclass, field from collections import defaultdict from tinygrad.helpers import dedup, pluralize, unwrap, to_tuple, ContextVar, Context, panic, partition, getenv, to_name -from tinygrad.helpers import DEBUG, VIZ, DEV, ALL2ALL, PROFILE, diskcache_get, diskcache_put +from tinygrad.helpers import DEBUG, VIZ, ALL2ALL, PROFILE, diskcache_get, diskcache_put from tinygrad.device import Device, Buffer, BufferSpec, Compiled, TinyELF, HCQ_RUNTIME_DEV, ProfileProgramEvent from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo, GroupOp, graph_rewrite, rewrite_group, exec_alu, uopfunc, sym_infer from tinygrad.uop.ops import pm_renumber_slots -from tinygrad.dtype import dtypes, DType, DTYPES_DICT, AddrSpace +from tinygrad.dtype import dtypes, DType, AddrSpace, truncate, to_dtype from tinygrad.renderer import Estimates from tinygrad.engine.realize import get_call_arg_uops, get_call_name, get_call_outs_ins from tinygrad.engine.realize import estimate_uop, pm_flatten_linear, lower_and_compile @@ -17,8 +17,9 @@ from tinygrad.engine.realize import estimate_uop, pm_flatten_linear, lower_and_c # 0. helpers HCQ_CACHE_THRESH = ContextVar("HCQ_CACHE_THRESH", 64) -HCQ_DEVS = frozenset(("AMD", "NV", "QCOM", "CUDA", "NULL", "METAL")) +HCQ_DEVS = frozenset(("AMD", "NV", "QCOM", "CUDA", "HIP", "NULL", "METAL")) HOST_DEVS = frozenset(("CPU", "PYTHON")) +UVM_DEVS = frozenset(("QCOM", "METAL")) # the host maps their memory, so copies are host memcpys @dataclass(frozen=True) class HCQInfo: @@ -33,14 +34,12 @@ class HCQInfo: def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c -def get_enqueue_devs(call:UOp) -> Any|None: - if call.op is not Ops.CALL: return None # entries can be AFTER-wrapped calls - if call.body.op not in (Ops.PROGRAM, Ops.STORE) and not (call.body.op is Ops.CUSTOM_FUNCTION and call.body.arg.name == "encdec"): return None - if not (bufs:=get_call_arg_uops(call)): return None - if call.body.op is Ops.STORE: bufs = bufs[::-1] # copies push from the src device: p2p writes are faster than reads - devs = min(bufs, key=lambda b: not all_devices_in(b.device, HCQ_DEVS)).device - if not all_devices_in(devs, HCQ_DEVS): return None - if call.body.op is Ops.STORE and to_tuple(devs)[0].startswith(("QCOM", "METAL")): return None # unified memory uses host copies +def get_enqueue_devs(call:UOp) -> tuple[str, ...]: + if call.op is not Ops.CALL: return () # entries can be AFTER-wrapped calls + if call.body.op not in (Ops.PROGRAM, Ops.STORE) and not (call.body.op is Ops.CUSTOM_FUNCTION and call.body.arg.name == "encdec"): return () + runs_on = call.device if call.body.op is Ops.STORE and all_devices_in(call.device, HCQ_DEVS) else call.src[1].device + devs = tuple(Device.canonicalize(d) for d in to_tuple(runs_on)) + if not all_devices_in(devs, HCQ_DEVS) or (call.body.op is Ops.STORE and all_devices_in(devs, UVM_DEVS)): return () return devs def unwrap_view(v:UOp) -> tuple[UOp, int]: # look through views to (base, byte offset) @@ -57,13 +56,13 @@ def unwrap_lane(v:UOp) -> tuple[UOp, int|None, int]: # look through views and a def select_lane(u:UOp, lane:int) -> UOp: return u.src[lane] if u.op is Ops.MSTACK else u.mselect(lane) if len(to_tuple(u.device)) > 1 else u -def timeline(devs:tuple[str, ...]) -> UOp: return UOp.alloc((2,), dtypes.uint64, 0, device=devs[0]).rtag("timeline") +def timeline(devs:tuple[str, ...]) -> UOp: return UOp.alloc((2,), dtypes.u64, 0, device=devs[0]).rtag("timeline") def timeline_value(devs:tuple[str, ...]) -> UOp: return timeline(devs).index(1).load() def make_program(prg:UOp, size:int, device:str) -> UOp: slot = int.from_bytes(prg.key[:8], "little") if PROFILE: Compiled.profile_events.append(ProfileProgramEvent(device, prg.src[0].arg.function_name, prg.src[3].arg, None, slot, prg.key)) - return UOp.alloc((size,), dtypes.uint8, slot, device=device).rtag("program") + return UOp.alloc((size,), dtypes.u8, slot, device=device).rtag("program") def make_submit(*cmds, devs:str|tuple[str, ...], queue:str, fn:str|None=None, deps:tuple[UOp, ...]=()) -> UOp: # the order is on the arg lin = UOp(Ops.LINEAR, src=tuple(cmds), arg=(to_tuple(devs), queue)).after(*deps) @@ -71,14 +70,14 @@ def make_submit(*cmds, devs:str|tuple[str, ...], queue:str, fn:str|None=None, de def ins(name, *src) -> UOp: return UOp(Ops.INS, tuple(UOp.const(s, dtypes.u32) if isinstance(s, int) else s for s in src), (name, dtypes.void)) -def chunks(nbytes:int, chunk_sz:int, dtype:DType=dtypes.int) -> list[tuple[UOp|int, int]]: # (index, bytes): full chunks as one range, then the tail +def chunks(nbytes:int, chunk_sz:int, dtype:DType=dtypes.i32) -> list[tuple[UOp|int, int]]: # (index, bytes): full chunks as one range, then the tail full, tail = divmod(nbytes, chunk_sz) # no one-trip loops return ([(UOp.range(full, next(UOp.unique_num), dtype=dtype) if full > 1 else 0, chunk_sz)] if full else []) + ([(full, tail)] if tail else []) # C FFI def layout_args(args:Sequence[UOp|int], offset:int=0) -> list[tuple[int, UOp]]: - words = [a if isinstance(a, UOp) else UOp.const(a, dtypes.uint32) for a in args] + words = [a if isinstance(a, UOp) else UOp.const(a, dtypes.u32) for a in args] return [(offset + o, w) for (o, _), w in zip(TinyELF.iter_sig(tuple((None, i, w.dtype, ()) for i, w in enumerate(words))), words)] def pack_args(args:list[tuple[int, UOp]], size:int) -> list[UOp]: @@ -89,27 +88,22 @@ def pack_args(args:list[tuple[int, UOp]], size:int) -> list[UOp]: return words + [UOp(Ops.BINARY, arg=bytes(size - end))] def ccall(fn:Any, *args:UOp|int) -> UOp: - ret = dtypes.void if fn.restype is None else dtypes.uint64 if fn.restype is ctypes.c_void_p else \ - next(d for d in DTYPES_DICT.values() if d.fmt == fn.restype._type_) - return UOp.custom_function(fn.__name__, dtype=ret).call(*[UOp.const(a, dtypes.int) if isinstance(a, int) else a for a in args]) + ret = {None: dtypes.void, ctypes.c_void_p: dtypes.u64}.get(fn.restype) or to_dtype(fn.restype.__name__[2:]) # void, void*, c_int -> int + return UOp.custom_function(fn.__name__, dtype=ret).call(*[UOp.const(a, dtypes.i32) if isinstance(a, int) else a for a in args]) @uopfunc def do_get_time_ms(ms:UOp) -> UOp: - ts = UOp.placeholder((2,), dtypes.uint64, addrspace=AddrSpace.REG) - ts = ts.after(UOp.custom_function("clock_gettime", dtype=dtypes.int).call(UOp.const(time.CLOCK_MONOTONIC, dtypes.int), ts.index(0))) + ts = UOp.placeholder((2,), dtypes.u64, addrspace=AddrSpace.REG) + ts = ts.after(UOp.custom_function("clock_gettime", dtype=dtypes.i32).call(UOp.const(time.CLOCK_MONOTONIC, dtypes.i32), ts.index(0))) return ms.index(0).store(ts[0] * 1000 + ts[1] // 1000000).sink() -def get_time_ms(dep:UOp) -> UOp: return (ms:=UOp.placeholder((1,), dtypes.uint64, addrspace=AddrSpace.REG)).after(do_get_time_ms(ms.after(dep)))[0] - -CDTYPE = {1: dtypes.uchar, 2: dtypes.ushort, 4: dtypes.uint, 8: dtypes.ulong} # a C field as the unsigned int of its size +def get_time_ms(dep:UOp) -> UOp: return (ms:=UOp.placeholder((1,), dtypes.u64, addrspace=AddrSpace.REG)).after(do_get_time_ms(ms.after(dep)))[0] def cstruct(struct_t, **fields:UOp|int) -> UOp: - flds = {n: (o, CDTYPE[ctypes.sizeof(t)]) for n, t, o, *_ in struct_t._real_fields_ if ctypes.sizeof(t)} # skips zero length arrays - rows = [(flds[n][0], v.cast(flds[n][1]) if isinstance(v, UOp) else UOp.const(v, flds[n][1])) for n, v in fields.items()] - buf = UOp.alloc((ctypes.sizeof(struct_t),), dtypes.uint8, device=HCQ_RUNTIME_DEV.value).rtag(struct_t.__name__) + flds = {n: (o, {d.itemsize: d for d in dtypes.uints}[ctypes.sizeof(t)]) for n, t, o, *_ in struct_t._real_fields_ if ctypes.sizeof(t)} + rows = [(flds[n][0], UOp.const(v, flds[n][1])) for n, v in fields.items()] + buf = UOp.alloc((ctypes.sizeof(struct_t),), dtypes.u8, device=HCQ_RUNTIME_DEV.device).rtag(struct_t.__name__) return patch(buf, rows, bytes(ctypes.sizeof(struct_t))) -def cfield(buf:UOp, struct_t, name:str) -> UOp: return buf[(f:=getattr(struct_t, name)).offset:f.offset + f.size].bitcast(CDTYPE[f.size]).index(0) - # ***************** # 0.1. prep: eager buffers become tagged params @@ -124,8 +118,8 @@ pm_replace_buffers = PatternMatcher([(UPat(Ops.BUFFER, name="b"), lambda ctx, b: # 1.1. prep: unwrap multi def unwrap_call(call:UOp) -> UOp|None: - if get_enqueue_devs(call) is None or (n:=max(len(to_tuple(a.device)) for a in get_call_arg_uops(call))) == 1: return None - dnum = UOp.variable("_device_num", 0, n - 1, dtypes.int) + if not get_enqueue_devs(call) or (n:=max(len(to_tuple(a.device)) for a in get_call_arg_uops(call))) == 1: return None + dnum = UOp.variable("_device_num", 0, n - 1, dtypes.i32) return UOp(Ops.LINEAR, src=tuple(call.replace(src=(call.body, *[a if a.is_bound_var else select_lane(a, i) for a in call.src[1:]], dnum.bind(i))) for i in range(n))) pm_unwrap_multi = PatternMatcher([(UPat(Ops.CALL, name="call"), unwrap_call)]) @@ -133,11 +127,6 @@ pm_unwrap_multi = PatternMatcher([(UPat(Ops.CALL, name="call"), unwrap_call)]) # ***************** # 1.2. prep: staging copies -STAGING_SIZE, STAGING_SLOTS = (4 if DEV.interface.startswith("MOCK") else 128) << 20, 2 - -@functools.cache -def _staging(device:str) -> Buffer: return Buffer(device, STAGING_SIZE, preallocate=True) - def split_rdma(call:UOp, dst:UOp, src:UOp) -> UOp|None: devs = [to_tuple(b.device)[0] for b in (dst, src)] if not all(hasattr(Device[d], "iface") for d in devs) or Device[devs[0]].peer_group == Device[devs[1]].peer_group: return None # not 2 nodes @@ -153,21 +142,18 @@ def split_rdma(call:UOp, dst:UOp, src:UOp) -> UOp|None: def stage_copy(call:UOp, dst:UOp, src:UOp) -> UOp|None: devs = [to_tuple(b.device)[0] for b in (dst, src)] if any(d.startswith("RDMA") for d in devs): return None # over the nic + if not (enqueued:=get_enqueue_devs(call)): return None + device = enqueued[0] - if (device:=get_enqueue_devs(call)) is None: return None + from tinygrad.runtime.ops_disk import is_disk_read dev, host, usb_memcpys = Device[device], Device[device].host, getattr(Device[device], "is_usb", False) - mappable = {"CPU", "PYTHON"} | ({"NPY", "DISK"} if usb_memcpys else set()) + mappable = {"CPU", "PYTHON"} | ({"NPY", "DISK"} if usb_memcpys else {"DISK"} if is_disk_read(call) else set()) # ops_disk stages it in the batch if device != host and not all(Device[d].peer_group == dev.peer_group or (d.split(":")[0] in mappable and Device[d].host == host) for d in devs): - (staging:=_staging(host)).get_buf(device) - base, it, copies = UOp.from_buffer(staging, dtypes.uint8), src.dtype.itemsize, [] - chunk = (STAGING_SIZE // STAGING_SLOTS) // it - for i, off in enumerate(range(0, src.max_numel(), chunk)): - stage, part = base[(so:=(i % STAGING_SLOTS) * chunk * it):so + (n:=min(chunk, src.max_numel() - off)) * it], src[off:off+n] - copies += [stage.store_call(part), dst[off:off+n].store_call(stage)] - return UOp(Ops.LINEAR, src=tuple(copies)) + stage = UOp.alloc(src.max_shape, src.dtype, int.from_bytes(call.key[:8], "little"), device=host) + return UOp(Ops.LINEAR, src=(stage.store_call(src), dst.store_call(stage))) if Device[device].has_copy_queue: return None - out, inp = (UOp.param(i, dtypes.uint8, b.nbytes(), device=device) for i, b in enumerate((dst, src))) + out, inp = (UOp.param(i, dtypes.u8, b.nbytes(), device=device) for i, b in enumerate((dst, src))) ast = out.index(r:=UOp.range(src.nbytes(), 0)).store(inp.index(r).load()).end(r).sink(arg=KernelInfo()) return lower_and_compile(call.replace(src=(ast, *call.src[1:]))) @@ -181,36 +167,23 @@ pm_insert_copy_staging = PatternMatcher([ class DepsTracker: def __init__(self): - # tracks (offset, end, dep) ranges per base buffer/lane to handle suballocated buffers correctly. - self.w_dependency_map: dict[Any, list[tuple[int, int, Any]]] = defaultdict(list) - self.r_dependency_map: dict[Any, list[tuple[int, int, Any]]] = defaultdict(list) + self.writes:dict[Any, list[tuple[int, int, Any]]] = defaultdict(list) + self.reads:dict[Any, list[tuple[int, int, Any]]] = defaultdict(list) - def access_resources(self, bufs:Sequence[UOp|Buffer], write:list[int], new_dependency:Any): - ranges:list[tuple[Any, int, int]] = [] - for buf in bufs: - if isinstance(buf, Buffer): ranges.append((id(buf.base), buf.offset, buf.offset + buf.nbytes)) - else: - base, lane, off = unwrap_lane(buf) - ranges.append(((base, lane), off, off + buf.max_numel() * buf.dtype.itemsize)) - wait_nodes = [] - for i, (key, s, e) in enumerate(ranges): - wait_nodes += [dep for st,en,dep in self.w_dependency_map[key] if st < e and s < en] - if i in write: wait_nodes += [dep for st,en,dep in self.r_dependency_map[key] if st < e and s < en] - for i, (key, s, e) in enumerate(ranges): - if i in write: - for dmap in [self.w_dependency_map, self.r_dependency_map]: - kept = [] - for entry in dmap[key]: - st, en, dep = entry - if st == en: continue - if en <= s or e <= st: kept.append(entry) - else: - if st < s: kept.append((st, s, dep)) - if e < en: kept.append((e, en, dep)) - dmap[key] = kept - self.w_dependency_map[key].append((s, e, new_dependency)) - else: self.r_dependency_map[key].append((s, e, new_dependency)) - return list({id(x):x for x in wait_nodes}.values()) + def access_resources(self, bufs:Sequence[UOp], write:list[int], new_dependency:Any) -> list: + accesses = [(i in write, (base, lane), off, off + b.max_numel() * b.dtype.itemsize) + for i, b in enumerate(bufs) for base, lane, off in [unwrap_lane(b)]] + + waits = [dep for is_write, key, start, end in accesses for lo, hi, dep in (self.writes[key] + self.reads[key] if is_write else self.writes[key]) + if lo < end and start < hi] + + for is_write, key, start, end in accesses: + if is_write: + for ranges in (self.writes, self.reads): + before_and_after = [((lo, min(hi, start), dep), (max(lo, end), hi, dep)) for lo, hi, dep in ranges[key]] + ranges[key] = [(lo, hi, dep) for parts in before_and_after for lo, hi, dep in parts if lo < hi] + (self.writes if is_write else self.reads)[key].append((start, end, new_dependency)) + return dedup(waits) @dataclass class BatchCtx: @@ -236,7 +209,7 @@ class BatchCtx: self.queues.setdefault(d, []) self.signal_tags = {tag for (dev, q), tag in self.last.items() if q != self.epilogue_queue(dev) or (dev, q) in self.peers} # a slot is [signal][timestamp], 16 bytes: the queue signals, the timeline, then two per call if profiling - self.slots = {dev: UOp.alloc((2 * (len(qs) + 1 + (2 * len(self.batch) if self.profile else 0)),), dtypes.uint64, device=dev, + self.slots = {dev: UOp.alloc((2 * (len(qs) + 1 + (2 * len(self.batch) if self.profile else 0)),), dtypes.u64, device=dev, spec=BufferSpec(host=True, uncached=True, cpu_access=True)).rtag("slots") for dev, qs in self.queues.items()} def epilogue_queue(self, dev:str) -> str: return "COMPUTE:0" if len(self.queues[dev]) != 1 else self.queues[dev][0] # closes the device @@ -256,11 +229,10 @@ def _wait_ins(ctx:BatchCtx, call:UOp, device:str, queue:str, tag:int) -> list[UO if latest and device.split(":")[0] == "NV" and queue.startswith("COMPUTE") and (p:=ctx.prev[tag]) is not None: latest[(device, queue)] = p ctx.signal_tags |= set(latest.values()) - return [UOp(Ops.INS, arg=("wait", dtypes.void), src=(ctx.queue_signal((d,), q), UOp.const(t + 1, dtypes.uint64))) for (d, q), t in latest.items()] + return [ins("wait", ctx.queue_signal((d,), q), UOp.const(t + 1, dtypes.u64)) for (d, q), t in latest.items()] def _start_ins(ctx:BatchCtx, dev:str, queue:str) -> list[UOp]: # a queue first waits for prior work of its device and of the peers it touches - return [UOp(Ops.INS, arg=("barrier", dtypes.void), src=())] + \ - [UOp(Ops.INS, arg=("wait", dtypes.void), src=(timeline((d,)), timeline_value((d,)))) for d in [dev, *sorted(ctx.peers.get((dev, queue), ()))]] + return [ins("barrier")] + [ins("wait", timeline((d,)), timeline_value((d,))) for d in [dev, *sorted(ctx.peers.get((dev, queue), ()))]] def _build_queues(ctx:BatchCtx) -> dict[tuple[tuple[str, ...], str], list[UOp]]: # find all waits first to mark calls that must signal @@ -270,20 +242,19 @@ def _build_queues(ctx:BatchCtx) -> dict[tuple[tuple[str, ...], str], list[UOp]]: if not (q:=queues.setdefault((devices, queue), [])): q += _start_ins(ctx, devices[0], queue) # first use of a queue # dependency waits, then the call between its timestamps - ts_ins = [UOp(Ops.INS, arg=("timestamp", dtypes.void), src=(ctx.slot(devices, i),)) for i in ctx.stamps(devices, tag)] + ts_ins = [ins("timestamp", ctx.slot(devices, i)) for i in ctx.stamps(devices, tag)] q += waits + ts_ins[:1] + [call] + ts_ins[1:] # signal the queue if someone waits for us - if tag in ctx.signal_tags: - q += [UOp(Ops.INS, arg=("store", dtypes.void), src=(ctx.queue_signal(devices, queue), UOp.const(tag + 1, dtypes.uint64)))] + if tag in ctx.signal_tags: q += [ins("store", ctx.queue_signal(devices, queue), UOp.const(tag + 1, dtypes.u64))] # one queue advances the device timeline after all other queues finish, and after the queues of the peers that touched the device for dev in ctx.queues: touched = sorted(k for k, ds in ctx.peers.items() if dev in ds) qdev, queue = touched[0] if dev.split(":")[0] in HOST_DEVS else (dev, ctx.epilogue_queue(dev)) # the host has no queues, a toucher closes it - waits = [UOp(Ops.INS, arg=("wait", dtypes.void), src=(ctx.queue_signal((d,), q), UOp.const(ctx.last[(d, q)] + 1, dtypes.uint64))) + waits = [ins("wait", ctx.queue_signal((d,), q), UOp.const(ctx.last[(d, q)] + 1, dtypes.u64)) for d, q in [(dev, q) for q in ctx.queues[dev]] + touched if (d, q) != (qdev, queue)] - bump = UOp(Ops.INS, arg=("store", dtypes.void), src=(timeline((dev,)), timeline_value((dev,)) + UOp.const(1, dtypes.uint64))) + bump = ins("store", timeline((dev,)), timeline_value((dev,)) + 1) # multiple copy queues may need a new compute stream. a peer without calls of its own starts like any queue if not (q:=queues.setdefault(((qdev,), queue), [])) and qdev not in {d for d, _ in ctx.last}: q += _start_ins(ctx, qdev, queue) @@ -318,7 +289,7 @@ def _finalize_batch(ctx:BatchCtx, skip_wait:bool=False) -> UOp: @rewrite_group(new_ctx=False) def sched_batches(l:UOp, profile:bool) -> UOp: - devs = [() if (d:=get_enqueue_devs(c)) is None else tuple(Device.canonicalize(x) for x in to_tuple(d)) for c in l.src] + devs = [get_enqueue_devs(c) for c in l.src] # assign to queues peers = sorted({Device.canonicalize(d) for c in l.src if c.op is Ops.CALL and c.body.op is Ops.STORE @@ -326,7 +297,7 @@ def sched_batches(l:UOp, profile:bool) -> UOp: num_queues = max(1, getenv("HCQ_NUM_SDMA", min(len(peers), 8) if ALL2ALL >= 1 else 1)) queues = ["COMPUTE:0" if c.op is Ops.CALL and c.body.op is Ops.PROGRAM else "COPY:0" for c in l.src] for i, c in enumerate(l.src): - if c.op is Ops.CALL and c.body.op is Ops.CUSTOM_FUNCTION and c.body.arg.name == "encdec": queues[i] = "ENCDEC:0" + if c.op is Ops.CALL and c.body.op is Ops.CUSTOM_FUNCTION: queues[i] = "ENCDEC:0" if c.op is Ops.CALL and c.body.op is Ops.STORE and all(b.device in peers for b in get_call_arg_uops(c)): queues[i] = f"COPY:{(peers.index(c.src[1].device) - peers.index(c.src[2].device) - 1) % len(peers) % num_queues}" @@ -378,8 +349,8 @@ class HWQueue: self.patches.append((len(self.blob), w)) self.blob += bytes(w.dtype.itemsize) else: - v, n = (c.val, w.dtype.itemsize) if isinstance(w, UOp) else (c, 4) - self.blob += (v & (1 << 8 * n) - 1).to_bytes(n, 'little') + v, dt = (c.val, w.dtype) if isinstance(w, UOp) else (c, dtypes.u32) + self.blob += struct.pack(unwrap(dt.fmt), truncate[dt](v)) return len(self.blob) def loop(self, body:UOp, r:UOp): @@ -387,7 +358,7 @@ class HWQueue: for u in body.src: self.q_rewrite.rewrite(u, ctx=self) trip, words = len(self.blob) - start, self.patches[first:] # each trip gets its words as dwords: a trip need not be 8 bytes aligned - self.patches[first:] = [(o + 4 * k + r * trip, (w >> (32 * k)).cast(dtypes.uint32)) for o, w in words for k in range(w.dtype.itemsize // 4)] + self.patches[first:] = [(o + 4 * k + r * trip, (w >> (32 * k)).cast(dtypes.u32)) for o, w in words for k in range(w.dtype.itemsize // 4)] self.blob += self.blob[start:] * int(r.vmax) def memory_barrier(self): pass # a copy queue has nothing to flush @@ -424,7 +395,7 @@ def hcq_fence(slots:UOp, tl:UOp, tv:UOp, last:int) -> UOp: # wait for the previo loop = UOp.range(UOp(Ops.NOOP).after(start:=get_time_ms(target:=slots.index(last).load())), next(UOp.unique_num), dtype=dtypes.void) done = tl.after(target, loop).index(0).load() - bumped = tl.after(done.backedge(loop, (done < target) & (get_time_ms(done) - start < 30000))).index(1).store(nxt:=tv + UOp.const(1, dtypes.uint64)) + bumped = tl.after(done.backedge(loop, (done < target) & (get_time_ms(done) - start < 30000))).index(1).store(nxt:=tv + 1) return slots.after(bumped).index(last).store(nxt).sink() def encode_fence(f:UOp) -> UOp: @@ -453,7 +424,7 @@ def encode_cmdbuf(hq:HWQueue, lin:UOp|None=None, name:str="cmdbuf", device:str|t rt = [(o, w) for o, w in patches if isinstance(o, int) and o % 4 == 0 and w.dtype.itemsize in (4, 8) and not _is_link_patch(w)] uses = {w: [o for o, _ in grp] for w, grp in itertools.groupby(sorted(rt, key=lambda p: p[1].key), key=lambda p: p[1])} looped = {w: (at, [*w.ranges][0] if w.ranges else UOp.range(len(at), next(UOp.unique_num))) for w, at in uses.items() if len(at) > 1 or w.ranges} - dwords = [(UOp(Ops.BINARY, arg=struct.pack(f"<{len(at)}I", *at)).bitcast(dtypes.uint32).index(r).load() + 4 * k, (w >> 32 * k).cast(dtypes.uint32)) + dwords = [(UOp(Ops.BINARY, arg=struct.pack(f"<{len(at)}I", *at)).bitcast(dtypes.u32).index(r).load() + 4 * k, (w >> 32 * k).cast(dtypes.u32)) for w, (at, r) in looped.items() for k in range(w.dtype.itemsize // 4)] patches = [(o, w) for o, w in patches if w not in looped] + dwords nested = dedup([g.src[0] for _, w in patches for g in w.toposort() if g.op is Ops.GETADDR and g.src[0].op is Ops.LINEAR]) @@ -466,7 +437,7 @@ def encode_cmdbuf(hq:HWQueue, lin:UOp|None=None, name:str="cmdbuf", device:str|t bufs.append((offs, encode_cmdbuf(hq, name=lname))) views = {l: buf.without_after[o:e] for offs, buf in bufs for l, (o, e) in offs.items()} - buf = UOp.alloc((len(stream),), dtypes.uint8, device=device or hq.devs[0]).rtag(to_name(name, hq.queue)).after(*hq.deps) + buf = UOp.alloc((len(stream),), dtypes.u8, device=device or hq.devs[0]).rtag(to_name(name, hq.queue)).after(*hq.deps) words = UOp.sink(*[w for _, w in patches]).substitute(views).src return patch(buf, list(zip([o for o, _ in patches], words)), stream).after(*[b for _, b in bufs]) @@ -494,12 +465,12 @@ def _needs_arg(u:UOp, root:bool) -> bool: def _param_for(u:UOp, slot:int) -> UOp: if u.op is Ops.GETADDR or u.is_variable: - return UOp.param(slot, u.commit_dtype(dtypes.int), name=u.arg.name if u.is_variable else None, addrspace=AddrSpace.ALU).cast(u.dtype) - return UOp.param(slot, u.dtype, u.max_numel(), HCQ_RUNTIME_DEV.value, name=f"{u.tag}_{slot}" if isinstance(u.tag, str) else None) + return UOp.param(slot, u.commit_dtype(dtypes.i32), name=u.arg.name if u.is_variable else None, addrspace=AddrSpace.ALU).cast(u.dtype) + return UOp.param(slot, u.dtype, u.max_numel(), HCQ_RUNTIME_DEV.device, name=f"{u.tag}_{slot}" if isinstance(u.tag, str) else None) def lift(call:UOp, root:bool=False) -> UOp: # callees are lifted already body, args = graph_rewrite(call.body, pm_lift_deps + pm_renumber_slots, ctx=itertools.count(), walk=True, name="lift deps"), list(call.src[1:]) - nodes = body.toposort(gate=lambda u: u.op is not Ops.GETADDR, enter_calls=False) + nodes = body.toposort(gate=lambda u: u.op is not Ops.GETADDR) leaves = dedup([u for u in nodes if _needs_arg(u, root)] + [g for u in nodes for g in u.src if g.op is Ops.GETADDR]) slots = args + (new:=[u for u in leaves if u not in args]) body = body.substitute({u: _param_for(u, slots.index(u)) for u in leaves}, walk=True) @@ -543,7 +514,7 @@ def _source_stamp() -> str: return hashlib.sha256(str(files).encode()).hexdigest() def _disk_key(linear:UOp, profile:bool) -> str: - devs = sorted({d for c in linear.src if (ds:=get_enqueue_devs(c)) is not None for d in to_tuple(ds)} | {HCQ_RUNTIME_DEV.value}) + devs = sorted({d for c in linear.src if (ds:=get_enqueue_devs(c)) is not None for d in to_tuple(ds)} | {HCQ_RUNTIME_DEV.device}) return "|".join([_source_stamp(), linear.key.hex(), str(profile), str(ALL2ALL >= 1), *(f"{d}={Device[d].renderer.target!r}" for d in devs)]) @rewrite_group(lambda linear,input_uops,profile,cache=False,ret=None: f"HCQ Compile {pluralize('Kernel', len(ret.src))}") @@ -577,37 +548,32 @@ def bufferize_buf(ctx:LinkCtx, b:UOp) -> UOp: # ctx: a kept link (the jit's) own dev, spec = Device[b.device], b.arg.spec or BufferSpec(cpu_access=True) # data the device reads, unless the alloc says otherwise # a device owns the placeholders it names, the rest are allocated where they live - if (r:=cast(Buffer|None, Compiled.pm_bufferize.rewrite(b))) is not None: pass - elif not ctx.use_rt: r = Buffer(dev.device, max(b.max_numel(), 1) * b.dtype.itemsize, options=spec, preallocate=True) + if (r:=cast(Buffer|None, Compiled.pm_bufferize.rewrite(b, ctx))) is not None: pass + elif not ctx.use_rt or b.tag is None: r = Buffer(dev.device, max(b.max_numel(), 1) * b.dtype.itemsize, options=spec, preallocate=True) else: r = dev.rt_buffer(spec).view(b.nbytes(), dev.rt_allocator(spec).alloc(max(b.nbytes(), 1), alignment=256)).ensure_allocated() - return UOp.from_buffer(r, b.dtype, HCQ_RUNTIME_DEV.value) + return UOp.from_buffer(r, b.dtype, HCQ_RUNTIME_DEV.device) def resolve_getaddr(ctx:LinkCtx, g:UOp) -> UOp|None: if unwrap_lane(buf:=unwrap_view(g.src[0])[0])[0].op is not Ops.BUFFER: return None # input address, resolved per run ctx.refs.append(buf) # add to refs - return UOp.const(g.val, dtypes.uint64) + return UOp.const(g.val, dtypes.u64) -def fold_binary(buf:UOp, blob:UOp) -> UOp: - base, off = unwrap_view(buf) - cast(Buffer, base.buffer).ensure_allocated().host.view(fmt='B')[off:off + len(blob.arg)] = blob.arg - return UOp(Ops.NOOP) - -def write_words(buf:UOp, writes:list[tuple[int, int, int]]) -> UOp: # (word index, size, value) +def _write_host(buf:UOp, writes:list[tuple[int, bytes]]) -> UOp: base, off = unwrap_view(buf) mv = cast(Buffer, base.buffer).ensure_allocated().host.view(fmt='B') - for o, n, v in writes: mv[off + o * n:off + (o + 1) * n] = (v & (1 << 8 * n) - 1).to_bytes(n, 'little') + for o, data in writes: mv[off + o:off + o + len(data)] = data return UOp(Ops.NOOP) -def words(offs:UOp, ws:UOp): return zip(*[s.src if s.op is Ops.STACK else (s,) for s in (offs, ws)]) # a stack or one word - -def fold_words(buf:UOp, offs:UOp, ws:UOp) -> UOp: return write_words(buf, [(o.val, w.dtype.itemsize, w.val) for o, w in words(offs, ws)]) +def fold_words(buf:UOp, offs:UOp, ws:UOp) -> UOp: + return _write_host(buf, [(o.val * w.dtype.itemsize, struct.pack(w.dtype.fmt, truncate[w.dtype](w.val))) for o, w in zip(offs.src, ws.src)]) def fold_ranged(buf:UOp, offs:UOp, ws:UOp, e:UOp) -> UOp: # the words of every trip vs = {r: UOp.variable(f"r{i}", 0, r.vmax) for i, r in enumerate(e.src[1:])} - ows = list(words(offs.substitute(vs), ws.substitute(vs))) + ows = list(zip(offs.substitute(vs).src, ws.substitute(vs).src)) trips = [dict(zip([v.expr for v in vs.values()], t)) for t in itertools.product(*[range(int(r.vmax) + 1) for r in vs])] - return write_words(buf, [(sym_infer(o, t), w.dtype.itemsize, sym_infer(w, t)) for t in trips for o, w in ows]) + data = [(sym_infer(o, t) * w.dtype.itemsize, struct.pack(w.dtype.fmt, truncate[w.dtype](sym_infer(w, t)))) for t in trips for o, w in ows] + return _write_host(buf, data) pm_link = PatternMatcher([ # collapse committed const conversions @@ -620,10 +586,10 @@ pm_link = PatternMatcher([ # math on consts is a const (UPat(GroupOp.ALU, src=UPat.cvar().or_casted(), name="a"), lambda a: UOp.const(exec_alu(a.op, a.dtype, [s.val for s in a.src], False), a.dtype)), # fold rules - (UPat(name="buf").store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast())), fold_binary), + (UPat(name="buf").store(UPat(Ops.BINARY, name="blob").or_bitcasted()), lambda buf, blob: _write_host(buf, [(0, blob.arg)])), (UPat(name="buf").index(UPat(Ops.STACK, src=UPat.cvar(), name="offs")).store(UPat(Ops.STACK, src=UPat.cvar().or_casted(), name="ws")), fold_words), - (UPat(name="buf").index(UPat.cvar("offs")).store(UPat.cvar().or_casted("ws")), fold_words), - (UPat(name="buf").index(UPat(name="offs")).store(UPat(name="ws")).end(allow_any_len=True, name="e"), fold_ranged), + (UPat(name="buf").index(UPat.cvar("off")).store(UPat.cvar().or_casted("w")), lambda buf, off, w: fold_words(buf, UOp.stack(off), UOp.stack(w))), + (UPat(name="buf").index(UPat(Ops.STACK, name="offs")).store(UPat(Ops.STACK, name="ws")).end(allow_any_len=True, name="e"), fold_ranged), # a call keeps the deps that are not written yet (UPat(Ops.AFTER, src=(UPat(Ops.CALL),), allow_any_len=True, name="a"), lambda a: a.src[0].after(*(s for s in a.src[1:] if s.op is not Ops.NOOP))), (UPat(Ops.AFTER, name="a"), lambda a: None if a.without_after.op is Ops.CALL else @@ -637,7 +603,7 @@ def hcq_link(linear:UOp, input_uops:list[UOp]|None=None, allow_cache=True) -> UO if allow_cache and (linked:=link_linear_cache.get(linear)) is not None: return linked # if we have any link time buffers, do not cache this linear - cache = allow_cache and not any(u.tag == "lt_input" for u in linear.toposort(enter_calls=False) if u.op is Ops.PARAM) + cache = allow_cache and not any(u.tag == "lt_input" for u in linear.toposort() if u.op is Ops.PARAM) inputs = {UOp.param(i, b.dtype, b.max_numel(), b.device).replace(tag="lt_input"): b for i, b in enumerate(input_uops or ())} linked = graph_rewrite(linear, pm_link, ctx=(ctx:=LinkCtx(inputs, use_rt=allow_cache and not cache)), walk=True, name="link") diff --git a/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py b/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py index eafb18f76..e27af2fa1 100644 --- a/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py +++ b/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py @@ -95,9 +95,6 @@ class NVDev: def rreg(self, addr:int) -> int: return self.mmio[addr // 4] def _early_ip_init(self): - self.reg_names:set[str] = set() - self.reg_offsets:dict[str, tuple[int, int]] = {} - self.include("nv_ref", "") self.include("dev_fb", "tu102") self.include("dev_gc6_island", "ga102") diff --git a/tinygrad_repo/tinygrad/runtime/support/usb.py b/tinygrad_repo/tinygrad/runtime/support/usb.py index fe864c240..c85d93e8e 100644 --- a/tinygrad_repo/tinygrad/runtime/support/usb.py +++ b/tinygrad_repo/tinygrad/runtime/support/usb.py @@ -4,7 +4,7 @@ from tinygrad.helpers import DEBUG, DEV, HCQ_RUNTIME_DEV, to_mv, round_up, ceild from tinygrad.dtype import dtypes, DType, AddrSpace from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, uopfunc from tinygrad.device import Buffer, BufferSpec, Compiled -from tinygrad.runtime.support.hcq2 import HCQ_DEVS, CDTYPE, ccall, patch, unwrap_view, all_devices_in, to_name, get_time_ms, ins, chunks +from tinygrad.runtime.support.hcq2 import HCQ_DEVS, ccall, patch, unwrap_view, all_devices_in, to_name, get_time_ms, ins, chunks from tinygrad.runtime.support.memory import MMIOInterface from tinygrad.runtime.support import c @@ -207,7 +207,7 @@ class USBMMIOInterface(MMIOInterface): HALF, CHUNK, SLOT, STREAM = 0x40000, 0x40000 - 512, 0x4000, 1 << 20 # sram half, payload, slot, stream HOST_SIZE = 64 + 2 * HALF + STREAM # link, staging, zeros -def usb_host(dev:str) -> UOp: return UOp.alloc((HOST_SIZE,), dtypes.uint8, 0, device=HCQ_RUNTIME_DEV.value).rtag(to_name(dev, "usb_host")) +def usb_host(dev:str) -> UOp: return UOp.alloc((HOST_SIZE,), dtypes.uint8, 0, device=HCQ_RUNTIME_DEV.device).rtag(to_name(dev, "usb_host")) def usb_link(dev:str) -> UOp: return usb_host(dev)[:48].bitcast(dtypes.uint64) # handle, context, prev chunks, two transfers, error state def usb_stage(dev:str) -> UOp: return usb_host(dev)[64:64 + 2 * HALF] def usb_zeros(dev:str) -> UOp: return usb_host(dev)[64 + 2 * HALF:] @@ -222,9 +222,8 @@ def usb_sram(dev:str) -> UOp: return usb_asm24(dev)[0x5000:0x5000 + 2 * HALF] # ***************** # 2. transfers -def usb_word(v:UOp|int, dt:DType) -> UOp: return v.cast(dt) if isinstance(v, UOp) else UOp.const(v, dt) def usb_stack(dt:DType, *vals:UOp|int) -> UOp: - return (r:=UOp.placeholder((len(vals) or 1,), dt, addrspace=AddrSpace.REG)).after(*[r.index(i).store(usb_word(v, dt)) for i, v in enumerate(vals)]) + return (r:=UOp.placeholder((len(vals) or 1,), dt, addrspace=AddrSpace.REG)).after(*[r.index(i).store(UOp.const(v, dt)) for i, v in enumerate(vals)]) def usb_fail(link:UOp, code:UOp) -> UOp: return link.index(UOp.const(5).valid(link.after(code).index(5).load().eq(0))).store(code.cast(dtypes.uint64)) def usb_ctrl(link:UOp, rtype:int, req:int, val:UOp|int, idx:UOp|int, data:UOp, n:UOp|int, timeout:int=1000) -> UOp: @@ -283,9 +282,10 @@ def usb_store(b:UOp, idx:UOp, v:UOp) -> UOp: # kernargs: poke on change stored = usb_store(b, idx.src[0], v.src[0]) for i, w in zip(idx.src[1:], v.src[1:]): stored = usb_store(b.after(stored), i, w) return stored - link, addr, v = usb_link(b.device).after(*usb_deps(b)), usb_addr(b, idx, v.dtype), v.bitcast(CDTYPE[v.dtype.itemsize]) + link, addr = usb_link(b.device).after(*usb_deps(b)), usb_addr(b, idx, v.dtype) + v = v.bitcast({d.itemsize: d for d in dtypes.uints}[v.dtype.itemsize]) # bitcast to uint raw bits if str(unwrap_view(b)[0].tag).startswith("kernargs"): - cache = UOp.alloc((int(idx.vmax - idx.vmin) + 1,), v.dtype, device=HCQ_RUNTIME_DEV.value).rtag("usb_arg_cache") + cache = UOp.alloc((int(idx.vmax - idx.vmin) + 1,), v.dtype, device=HCQ_RUNTIME_DEV.device).rtag("usb_arg_cache") return usb_patch(link, addr, v, patch(cache, [], bytes(v.dtype.itemsize * cache.max_numel())).index(idx - idx.vmin)) return usb_poke_word(link, addr, v) @@ -306,7 +306,7 @@ pm_usb_lower = PatternMatcher([ def usb_wire(size:UOp|int) -> UOp|int: return (size + 512 + SLOT - 1) // SLOT * SLOT # payload + sentinel block, slot aligned def usb_sentinel(n:UOp) -> UOp: return ((n & 0xFFFFFF) | 0x51000000).cast(dtypes.uint32) def usb_put(link:UOp, ptr:UOp, dt:DType, *vals:UOp|int) -> UOp: # store through a pointer - return ccall(libc.memcpy, ptr, usb_stack(dt, *vals).after(link).index(0), dt.itemsize * len(vals)).cast(dtypes.void) + return ccall(libc.memcpy, ptr, usb_stack(dt, *vals).after(link).index(0), dt.itemsize * len(vals)) @uopfunc def usb_reap(link:UOp, xfer:UOp) -> UOp: # poll while pending (0xff), any other status but completed fails the link @@ -358,7 +358,7 @@ def usb_recv(link:UOp, table:UOp, i:UOp, run:UOp, go:UOp, stage:UOp) -> UOp: # c released = usb_poke(link.after(armed), go, (i + run + 1).cast(dtypes.uint32)) received = usb_bulk(link.after(released), 0x81, stage.index(0), wire) lower = ccall(libc.memcpy, addr, stage.after(received).index(0), first.cast(dtypes.uint64)) - return ccall(libc.memcpy, addr + CHUNK, stage.after(lower).index(HALF), second.cast(dtypes.uint64)).cast(dtypes.void).sink() + return ccall(libc.memcpy, addr + CHUNK, stage.after(lower).index(HALF), second.cast(dtypes.uint64)).sink() # ***************** # 5. batch and encode @@ -371,8 +371,8 @@ def usb_hostaddr(host:UOp, dev:str) -> UOp: def usb_table(hosts:list[tuple[UOp, int]], n:int, win:int, dev:str) -> UOp: # [address, bytes] per chunk rows = [(16 * (k + r) + o, w) for host, k in hosts for r, nb in chunks(host.nbytes(), win) - for o, w in ((0, usb_hostaddr(host, dev) + usb_word(r, dtypes.uint64) * win), (8, UOp.const(nb, dtypes.uint64)))] - return patch(UOp.alloc((2 * n,), dtypes.uint64, device=HCQ_RUNTIME_DEV.value).rtag("usb_table"), rows) + for o, w in ((0, usb_hostaddr(host, dev) + UOp.const(r, dtypes.uint64) * win), (8, UOp.const(nb, dtypes.uint64)))] + return patch(UOp.alloc((2 * n,), dtypes.uint64, device=HCQ_RUNTIME_DEV.device).rtag("usb_table"), rows) def usb_chunks(call:UOp, first:int, run:int) -> list[UOp]: # gpu side. first: chunk id of the copy and of its run dst, src = call.src[1:] @@ -380,7 +380,7 @@ def usb_chunks(call:UOp, first:int, run:int) -> list[UOp]: # gpu side. first: ch sram = usb_sram(dev:=vram.device).getaddr(dev) for r, nb in chunks(host.nbytes(), win): - n, va = (i:=usb_word(r, dtypes.uint64)) + first, vram.getaddr(dev) + i * win + n, va = (i:=UOp.const(r, dtypes.uint64)) + first, vram.getaddr(dev) + i * win if is_host(src): # copyin: wait sentinel, copy, release end = sram + (((n - run) & 1) + 1) * HALF cmds = [ins("wait_eq", end - 4, usb_sentinel(n)), ins("copy", va, end - usb_wire(nb), nb), ins("store", end - 4, 0)] diff --git a/tinygrad_repo/tinygrad/schedule/__init__.py b/tinygrad_repo/tinygrad/schedule/__init__.py index 4730e14ef..d8227b822 100644 --- a/tinygrad_repo/tinygrad/schedule/__init__.py +++ b/tinygrad_repo/tinygrad/schedule/__init__.py @@ -86,21 +86,19 @@ from tinygrad.schedule.rangeify import get_kernel_graph from tinygrad.helpers import CAPTURING from tinygrad.uop.ops import PatternMatcher, UPat -def create_new_buffer(ctx:tuple[dict[UOp, UOp], tuple[UOp, ...]], b:UOp): - if (ret:=ctx[0].get(b, None)) is None: - device = b.device if b.device is not None else next(a.device for a in ctx[1] if a.device is not None) - ctx[0][b] = ret = UOp.new_buffer(device, b.max_numel(), b.dtype) - return ret +def create_new_buffer(ctx:tuple[UOp, ...], b:UOp): + device = b.device if b.device is not None else next(a.device for a in ctx if a.device is not None) + return UOp.new_buffer(device, b.max_numel(), b.dtype) pm_post_sched_cache = PatternMatcher([ # Resolve positional arguments outside kernel bodies; free Variables have slot -1. - (UPat(Ops.PARAM, name="x"), lambda ctx,x: ctx[1][x.arg.slot] if x.arg.slot >= 0 else None), + (UPat(Ops.PARAM, name="x"), lambda ctx,x: ctx[x.arg.slot] if x.arg.slot >= 0 else None), # bind ALLOCs to fresh BUFFERs for this invocation (UPat(Ops.ALLOC, name="b"), create_new_buffer), ]) def resolve_linear_call(linear_call:UOp, outer_binds:dict[int, UOp]|None=None): - linear = graph_rewrite(linear_call.body, pm_post_sched_cache, ctx=({}, linear_call.src[1:]), walk=True, name="params to buffers") + linear = graph_rewrite(linear_call.body, pm_post_sched_cache, ctx=linear_call.src[1:], walk=True, name="params to buffers") # nested LINEAR calls are lexical scopes: their positional params shadow the enclosing scope, while calls without # scalar args (e.g. precompiled allreduce) inherit it binds = {**(outer_binds or {}), diff --git a/tinygrad_repo/tinygrad/schedule/allreduce.py b/tinygrad_repo/tinygrad/schedule/allreduce.py index aba5603e2..b92aa5d92 100644 --- a/tinygrad_repo/tinygrad/schedule/allreduce.py +++ b/tinygrad_repo/tinygrad/schedule/allreduce.py @@ -1,6 +1,6 @@ import functools, itertools from tinygrad.helpers import all_int, prod, DEBUG, RING, ALL2ALL, ALLREDUCE_NODE_NDEVS, getenv -from tinygrad.uop.ops import UOp, Ops, ParamArg +from tinygrad.uop.ops import UOp # *** allreduce implementation *** def handle_allreduce(buf:UOp, red:UOp) -> UOp|None: @@ -67,9 +67,7 @@ def handle_allreduce(buf:UOp, red:UOp) -> UOp|None: def create_allreduce_function(buf:UOp, red:UOp, output:UOp|None=None) -> UOp|None: if output is None: - output = UOp(Ops.ALLOC, src=(UOp.const(red.max_numel()),)+UOp.device_range_src(red.device), - arg=ParamArg(next(UOp.unique_num), red.dtype, device=red.device)) - output = output.reshape(red.max_shape).shrink_to(red.shape) + output = UOp.alloc(red.shape, red.dtype, device=red.device) to = red.param_like(0) src = buf.param_like(1) red = src.allreduce(*red.arg) diff --git a/tinygrad_repo/tinygrad/schedule/indexing.py b/tinygrad_repo/tinygrad/schedule/indexing.py index 3a705e6a5..2d50c5f05 100644 --- a/tinygrad_repo/tinygrad/schedule/indexing.py +++ b/tinygrad_repo/tinygrad/schedule/indexing.py @@ -33,7 +33,7 @@ def realize_srcs(ctx:IndexingContext, rb:UOp) -> None: def realize_store_after_src(ctx:IndexingContext, dest:UOp, src:UOp): # you don't usually have to do this for assign unless there's a WAR hazard like TestAssign.test_assign_double_diamond_reduce - if dest.base in src.toposort(enter_calls=False): ctx.realize_map[src] = None + if dest.base in src.toposort(): ctx.realize_map[src] = None def realize_custom_kernel_srcs(ctx:IndexingContext, c:UOp) -> None: for s in c.src[1:]: @@ -171,7 +171,6 @@ def apply_movement_op(op:Ops, in_shape:tuple[sint,...], arg:tuple, rngs:tuple[UO match op: case Ops.SHRINK: rngs = tuple(a if off == 0 else a+off for a,(off,_) in zip(rngs, arg)) case Ops.PERMUTE: rngs = tuple(rngs[p] for p in argsort(arg)) - case Ops.FLIP: rngs = tuple(((s-1)-a) if f else a for a,s,f in zip(rngs, in_shape, arg)) case Ops.EXPAND: rngs = rngs[len(arg):] case Ops.PAD: # NOTE: the .where(r-s, i) is not inside the graph_rewrite so that `convert_pad_to_where_to_keep_behavior_local` @@ -180,7 +179,7 @@ def apply_movement_op(op:Ops, in_shape:tuple[sint,...], arg:tuple, rngs:tuple[UO symbolic+pm_simplify_valid, name="pad")) for r,sh,(off,sz) in zip(rngs, in_shape, arg)) case Ops.RESHAPE: sink = UOp.sink(*rngs).simplify() # NOTE: this applies any commutative flips to the rngs early - sub_array = {r:r.replace(src=r.src[:1], arg=(AxisType.PLACEHOLDER, i)) for i,r in enumerate(sink.ranges)} + sub_array = {r:r.replace(arg=(AxisType.PLACEHOLDER, i)) for i,r in enumerate(sink.ranges)} rngs = _apply_reshape(in_shape, arg, sink.substitute(sub_array)).substitute({v:k for k,v in sub_array.items()}).src case _: raise RuntimeError(f"{op} is not a MovementOp") return rngs diff --git a/tinygrad_repo/tinygrad/schedule/memory.py b/tinygrad_repo/tinygrad/schedule/memory.py index 3a734727a..a13749ee3 100644 --- a/tinygrad_repo/tinygrad/schedule/memory.py +++ b/tinygrad_repo/tinygrad/schedule/memory.py @@ -1,5 +1,5 @@ from collections import defaultdict -from tinygrad.helpers import NO_MEMORY_PLANNER, DEBUG, round_up +from tinygrad.helpers import NO_MEMORY_PLANNER, DEBUG, round_up, to_tuple from tinygrad.uop.ops import UOp, Ops from tinygrad.dtype import dtypes from tinygrad.runtime.support.memory import TLSFAllocator @@ -11,9 +11,8 @@ def _collect_bufs(u:UOp) -> list[UOp]: def _can_plan(b:UOp, held_bufs:set[UOp]) -> bool: if b in held_bufs: return False - devs = (b.device,) if isinstance(b.device, str) else b.device # CL and WEBGPU do not support views, see explanation in contiguous_view_offset - return all(not d.startswith(("DISK", "CL", "WEBGPU")) for d in devs) + return all(not d.startswith(("DISK", "CL", "WEBGPU")) for d in to_tuple(b.device)) LaneKey = tuple[str, int] diff --git a/tinygrad_repo/tinygrad/schedule/multi.py b/tinygrad_repo/tinygrad/schedule/multi.py index 618ccb631..758753c7d 100644 --- a/tinygrad_repo/tinygrad/schedule/multi.py +++ b/tinygrad_repo/tinygrad/schedule/multi.py @@ -126,7 +126,7 @@ def reshape_multi(root:UOp, multi:UOp): # map every sharded axis through the reshape: the axis boundary must survive intact and stay divisible by its shard count arg_acc:list[sint] = [1] for s in new_shape: arg_acc.append(ssimplify(arg_acc[-1]*s)) - new_shardings = [] + new_shardings, new_counts = [], {} for ax, rng in multi.sharding: count = int(rng.vmax)+1 target = ssimplify(prod(multi.shape[:ax])) @@ -134,8 +134,8 @@ def reshape_multi(root:UOp, multi:UOp): new_ax = len(arg_acc) - arg_acc[::-1].index(target) - 1 if new_shape[new_ax] % count != 0: raise RuntimeError(f"reshape {multi.shape} -> {new_shape} moved items between shards") new_shardings.append((new_ax, rng)) - new_axs = {a for a, _ in new_shardings} - new_shape = tuple(s//(int(rng.vmax)+1) if a in new_axs else s for a,s in enumerate(new_shape)) + new_counts[new_ax] = count + new_shape = tuple(s//new_counts.get(a, 1) for a,s in enumerate(new_shape)) return multi.src[0].reshape(new_shape).unshard(tuple(a for a,_ in new_shardings), tuple(r for _,r in new_shardings)) def expand_multi(root:UOp, multi:UOp): @@ -179,11 +179,6 @@ def shrink_multi(root:UOp, multi:UOp): val = multi.src[0]._mop(Ops.SHRINK, tuple(local_marg)) return val if not remaining else val.unshard(tuple(a for a,_ in remaining), tuple(r for _,r in remaining)) -def flip_multi(root:UOp, multi:UOp): - for ax, _ in multi.sharding: - if root.marg[ax]: raise RuntimeError(f"flipping not supported on sharded axis {ax}") - return multi.src[0].flip([i for i,x in enumerate(root.marg) if x]).unshard(multi.arg, multi.src[1:]) - def stack_multi(root:UOp): # STACK adds a leading axis: srcs are sharded one axis below the output multis = [m for m in root.src if m.op is Ops.UNSHARD] @@ -284,7 +279,6 @@ multi_pm = PatternMatcher([ (UPat(Ops.PAD, src=(UPat(Ops.UNSHARD, name="multi"), UPat(), UPat()), name="root"), pad_multi), (UPat(Ops.SHRINK, src=(UPat(Ops.UNSHARD, name="multi"), UPat(), UPat()), name="root"), shrink_multi), (UPat(Ops.PERMUTE, src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), permute_multi), - (UPat(Ops.FLIP, src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), flip_multi), (UPat(Ops.STACK, name="root", custom_early_reject=set([Ops.UNSHARD])), stack_multi), (UPat(Ops.INDEX, src=(UPat(Ops.UNSHARD, name="multi"),), name="root", allow_any_len=True), index_multi), (UPat(Ops.AFTER, src=(UPat(Ops.UNSHARD), UPat(Ops.STORE, src=(UPat(Ops.UNSHARD, name="dest"), UPat(Ops.UNSHARD, name="src"))))), store_after_multi), diff --git a/tinygrad_repo/tinygrad/schedule/prepare.py b/tinygrad_repo/tinygrad/schedule/prepare.py index 4db6c5d8a..82c514681 100644 --- a/tinygrad_repo/tinygrad/schedule/prepare.py +++ b/tinygrad_repo/tinygrad/schedule/prepare.py @@ -1,6 +1,6 @@ from dataclasses import replace from tinygrad.dtype import dtypes, to_dtype -from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, GroupOp, ParamArg +from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, GroupOp from tinygrad.uop.ops import graph_rewrite, rewrite_group, identity_element, resolve_returned_after from tinygrad.uop.movement import mop_cleanup from tinygrad.helpers import prod, getenv, all_int, DEBUG, SPLIT_REDUCEOP, OPENPILOT_HACKS, FLOAT16, argsort @@ -25,7 +25,7 @@ def forward_call_outputs(sink:UOp) -> UOp: # Forward the allocation, not just one view of it, so saved values and other aliases follow the same placement. key = base if base.op is Ops.ALLOC else src if key not in placed and (src.op is Ops.STAGE or target.has_buffer_identity()) and \ - target.storage_base not in st.src[1].toposort(enter_calls=False): + target.storage_base not in st.src[1].toposort(): if base.op is Ops.ALLOC and src.has_buffer_identity() and base.max_numel() == target.storage_base.max_numel(): placed[key] = target.storage_base elif src.op is Ops.STAGE: placed[key] = target.after(target.store(src.src[0])) @@ -76,6 +76,11 @@ def _mop_index(r:UOp, idx:UOp): pm_mops = PatternMatcher([ # handle movement ops on INDEX (UPat(GroupOp.Movement, name="r").f(Ops.INDEX, allow_any_len=True, name="idx"), _mop_index), + # A store through a padded view gates the destination index, not its loaded value. + (UPat.var("a").where(UPat.var("b").where(UPat.var("dst"), 0), UPat.var("pad")).store(UPat.var("val")), + lambda a,b,dst,pad,val: (a&b).where(dst, 0).store(val) if pad.is_invalid else None), + (UPat.var("cond").where(UPat.var("buf").index(UPat.var("idx")), 0).store(UPat.var("val")), + lambda cond,buf,idx,val: buf.index(idx.valid(cond)).store(val)), # move movement ops and INDEX after AFTER (UPat(GroupOp.Movement|{Ops.INDEX}, name="r").after(name="a", allow_any_len=True), lambda r,a: UOp(r.op, src=(a.replace(src=(r.src[0],)+a.src[1:]),)+r.src[1:], arg=r.arg)), @@ -92,9 +97,9 @@ def store_hazard_boundary(s:UOp): return True def fix_store_hazard(target:UOp, src:UOp): - if (base:=target.base) not in src.toposort(enter_calls=False): return None - # PERMUTE and FLIP reorder indices, SHRINK can have overlapping regions when dest is also shrunk - unsafe = {Ops.PERMUTE, Ops.FLIP} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) + if (base:=target.base) not in src.toposort(): return None + # PERMUTE reorders indices, SHRINK can have overlapping regions when dest is also shrunk + unsafe = {Ops.PERMUTE} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) reaches_base: dict[UOp, bool] = {} for s in src.toposort(gate=store_hazard_boundary): reaches_base[s] = s is base or any(reaches_base.get(c) for c in s.src) @@ -126,7 +131,7 @@ def split_reduceop(reduce:UOp, x:UOp): def resolve_function(c:UOp) -> UOp|None: if not c.is_inline_call: return None - nodes = c.body.toposort(enter_calls=False) + nodes = c.body.toposort() # Input and output PARAMs both bind to explicit arguments by slot; unused arguments are allowed. args = c.src[1:] @@ -140,7 +145,7 @@ def resolve_function(c:UOp) -> UOp|None: for p, a in dict_map.items(): if p.shape: n, flat = flat_storage(a) - if p.src[0].val != n: raise TypeError(f"arg {p.arg.slot} shape mismatch: expected size {p.src[0].val}, got {a.shape}") + if p.max_numel() != n: raise TypeError(f"arg {p.arg.slot} shape mismatch: expected size {p.max_numel()}, got {a.shape}") dict_map[p] = flat elif a.shape != (): raise TypeError(f"arg {p.arg.slot} shape mismatch: expected scalar, got {a.shape}") @@ -165,14 +170,13 @@ def copy_to_anon_store(x:UOp, copy:UOp): # copies are always cross device: pad to the max shape so the copy reads a whole buffer (SDMA can't do offset copies) x = x.pad_to(x.max_shape) # the buffer takes the DEVICE range from the copy (no-op for single device copies) - buf = UOp(Ops.ALLOC, src=(UOp.const(prod(x.max_shape)),)+copy.src[1:], - arg=ParamArg(next(UOp.unique_num), copy.dtype, device=copy.device)).reshape(x.max_shape) + buf = UOp.alloc((x.max_numel(),), copy.dtype, device=copy.device) + buf = buf.replace(src=buf.src[:1]+copy.src[1:]).reshape(x.max_shape) return buf.after(buf.store(x)).shrink_to(copy.shape) def stage_to_anon_store(x:UOp, stg:UOp): # the buffer created here is inside the call and is not persisted, like the buffers created for copies - buf = UOp(Ops.ALLOC, src=(UOp.const(prod(x.max_shape)),)+UOp.device_range_src(x.device), - arg=ParamArg(next(UOp.unique_num), stg.dtype, device=x.device)).reshape(x.max_shape) + buf = UOp.alloc(x.max_shape, stg.dtype, device=x.device) view = buf.shrink_to(stg.shape) return view.after(view.store(x)) diff --git a/tinygrad_repo/tinygrad/schedule/rangeify.py b/tinygrad_repo/tinygrad/schedule/rangeify.py index 26e28fa1d..bb4d8308a 100644 --- a/tinygrad_repo/tinygrad/schedule/rangeify.py +++ b/tinygrad_repo/tinygrad/schedule/rangeify.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field, replace from typing import cast import itertools from tinygrad.dtype import dtypes, AddrSpace, Invalid -from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg +from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group from tinygrad.uop.symbolic import symbolic from tinygrad.helpers import prod, dedup, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC @@ -222,7 +222,7 @@ def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp): return buf.after(*ended_stores) if x.arg.addrspace == AddrSpace.GLOBAL: - buf = UOp(Ops.ALLOC, src=(UOp.const(size),)+UOp.device_range_src(x.arg.device), arg=ParamArg(next(ctx), dtype, device=x.arg.device)) + buf = UOp.alloc((size,), dtype, slot=next(ctx), device=x.arg.device) do_store = buf.index(idx).store(x.src[0].cast(dtype)).end(*rngs) return buf.after(do_store).cast(x.dtype) @@ -276,7 +276,7 @@ class LocalAddBufferContext: range:int = 0 def debuf(ctx:LocalAddBufferContext, buf:UOp): - param = UOp(Ops.PARAM, src=(UOp.const(prod(buf.max_shape)),), arg=ParamArg(ctx.dg, buf.dtype, addrspace=buf.addrspace, device=buf.device)) + param = UOp.param(ctx.dg, buf.dtype, buf.max_numel(), addrspace=buf.addrspace, device=buf.device) ret = param.reshape(buf.max_shape) # if the buffer has symbolic shape, shrink the max-sized view to the actual shape if buf.max_shape != buf.shape: ret = ret.shrink(tuple((0, s) for s in buf.shape)) @@ -367,5 +367,5 @@ def get_kernel_graph(tsink:UOp) -> UOp: if SPEC: # validate the kernel graph from tinygrad.uop.spec import type_verify, spec_kernel_graph - type_verify(tsink, spec_kernel_graph, enter_calls=False) + type_verify(tsink, spec_kernel_graph) return tsink diff --git a/tinygrad_repo/tinygrad/tensor.py b/tinygrad_repo/tinygrad/tensor.py index f8ac0f292..cdad2de29 100644 --- a/tinygrad_repo/tinygrad/tensor.py +++ b/tinygrad_repo/tinygrad/tensor.py @@ -180,7 +180,7 @@ class Tensor(RandMixin): # Rebuild in dependency order: replacement values already reference the other outputs' storage. tensor_map:dict[UOp, UOp] = {} - for x in sink.toposort(enter_calls=False): + for x in sink.toposort(): u = x.replace(src=tuple(tensor_map.get(s, s) for s in x.src)) if x.op is Ops.ALLOC and (x.arg.bind_on_realize or x in bases): u = UOp.new_buffer(x.device, x.max_numel(), x.dtype) elif x in bases and u.needs_storage(): @@ -202,7 +202,7 @@ class Tensor(RandMixin): # Realized outputs become the storage their AFTER sequenced a store into. Compose with tensor_map before updating # Tensors so map values reference final storage. becomes_map = {u:graph_rewrite(u.src[0], pm_drop_after, bottom_up=True, name="drop after").shrink_to(u.shape) - for u in sink.toposort(enter_calls=False) if is_store_after(u)} + for u in sink.toposort() if is_store_after(u)} assert not any(x in becomes_map for x in becomes_map.values()) tensor_map = dict(zip(tensor_map, UOp.sink(*tensor_map.values()).substitute(becomes_map, walk=True).src)) _apply_map_to_tensors(becomes_map | tensor_map, name="bufferize") diff --git a/tinygrad_repo/tinygrad/uop/__init__.py b/tinygrad_repo/tinygrad/uop/__init__.py index dd785ec64..7991eae74 100644 --- a/tinygrad_repo/tinygrad/uop/__init__.py +++ b/tinygrad_repo/tinygrad/uop/__init__.py @@ -89,8 +89,8 @@ class Ops(FastEnum): # buffer ops STAGE = auto(); COPY = auto(); MSELECT = auto(); MSTACK = auto(); CUSTOM_FUNCTION = auto() - # the core 6 movement ops! these only exist in the tensor graph - RESHAPE = auto(); PERMUTE = auto(); EXPAND = auto(); PAD = auto(); FLIP = auto() + # the core 5 movement ops! these only exist in the tensor graph + RESHAPE = auto(); PERMUTE = auto(); EXPAND = auto(); PAD = auto() UNSHARD = auto() # UNSHARD is really a movement op # reduce @@ -114,7 +114,7 @@ class GroupOp: Defines = {Ops.PARAM, Ops.BUFFER, Ops.ALLOC} Irreducible = {Ops.CONST, Ops.SPECIAL, Ops.RANGE, Ops.PARAM, Ops.GETADDR} - Movement = {Ops.RESHAPE, Ops.EXPAND, Ops.PERMUTE, Ops.PAD, Ops.SHRINK, Ops.FLIP} + Movement = {Ops.RESHAPE, Ops.EXPAND, Ops.PERMUTE, Ops.PAD, Ops.SHRINK} # BinaryOps that can be flipped Commutative = {Ops.ADD, Ops.MUL, Ops.MAX, Ops.CMPNE, Ops.CMPEQ, Ops.XOR, Ops.AND, Ops.OR} diff --git a/tinygrad_repo/tinygrad/uop/ops.py b/tinygrad_repo/tinygrad/uop/ops.py index 6c59f3f2b..f07de64a5 100644 --- a/tinygrad_repo/tinygrad/uop/ops.py +++ b/tinygrad_repo/tinygrad/uop/ops.py @@ -212,7 +212,7 @@ class recursive_property: self.__doc__ = fxn.__doc__ def __get__(self, x:UOp|None, owner=None): if x is None: return self - for node in x.toposort(gate=lambda node: self.nm not in node.__dict__): node.__dict__[self.nm] = self.fxn(node) + for node in x.toposort(gate=lambda node: self.nm not in node.__dict__, enter_calls=True): node.__dict__[self.nm] = self.fxn(node) return x.__dict__[self.nm] class _UOpTuple(tuple): @@ -267,7 +267,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @functools.cached_property def backward_slice(self:UOp) -> dict[UOp, None]: - res: dict[UOp, None] = self.toposort(enter_calls=False) + res: dict[UOp, None] = self.toposort() res.pop(self) return res @@ -283,7 +283,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @property def bool_slice(self) -> frozenset[UOp]: return self._bool_slice | {self} if self.dtype is dtypes.bool else self._bool_slice - def toposort(self, gate:Callable|None=None, enter_calls=True) -> dict[UOp, None]: + def toposort(self, gate:Callable|None=None, enter_calls=False) -> dict[UOp, None]: cache: dict[UOp, None] = {} stack: list[tuple[UOp, bool]] = [(self, False)] # each stack entry is (node, visited_flag) while stack: @@ -409,9 +409,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if len(ps) != len(self.marg) or not all(resolve(0<=o) and resolve(sz>=0) and resolve(o+sz<=s) for s,(o,sz) in zip(ps, self.marg)): raise ValueError(f"invalid shrink {self.marg} for {ps}") return tuple(sz for _,sz in self.marg) - case Ops.FLIP: - if len(ps) != len(self.marg) or not all(isinstance(x, bool) for x in self.marg): raise ValueError(f"bad flip on {ps}, {self.marg}") - return ps case Ops.UNSHARD: return tuple(s*(int(self.src[1:][self.arg.index(a)].vmax)+1) if a in self.arg else s for a,s in enumerate(ps)) case Ops.REDUCE: num_axes = self.arg[1] @@ -793,7 +790,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): match self.op: case Ops.RESHAPE | Ops.EXPAND: return self.src[1].as_shape case Ops.PAD | Ops.SHRINK: return tuple(zip(self.src[1].as_shape, self.src[2].as_shape)) - case Ops.PERMUTE | Ops.FLIP: return self.arg + case Ops.PERMUTE: return self.arg case _: raise RuntimeError(f"{self.op} is not a MovementOp") def _mop(self, op:Ops, arg) -> UOp: @@ -805,7 +802,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): match op: case Ops.RESHAPE | Ops.EXPAND: src_args = [arg] case Ops.PAD | Ops.SHRINK: src_args = list(zip(*arg)) - case Ops.PERMUTE | Ops.FLIP: src_args = [] + case Ops.PERMUTE: src_args = [] case Ops.STACK: srcs = (self,)+tuple(arg) dtype = dtype_from_uop(Ops.STACK, srcs, None) @@ -828,21 +825,24 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # BUFFER/ALLOC/COPY carry a DEVICE range when targeting multiple devices return (UOp.range(len(device), 0, AxisType.DEVICE),) if isinstance(device, tuple) else () @staticmethod + def _buffer(size:int, dtype:DType, slot:int, addrspace=AddrSpace.GLOBAL, device=None, buffer:Buffer|MultiBuffer|None=None) -> UOp: + return UOp(Ops.BUFFER, src=(shape_to_shape_arg((size,)),)+UOp.device_range_src(device), + arg=ParamArg(slot, dtype, addrspace=addrspace, device=device, buffer=buffer)) + @staticmethod def new_buffer(device:str|tuple[str, ...], size:int, dtype:DType, num=None): if dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {dtype}") assert dtype.itemsize > 0, f"not a storage dtype: {dtype}" assert isinstance(size, int), f"new_buffer size must be a concrete int, got {size}" slot = next(UOp.unique_num) if num is None else num buf = MultiBuffer(device, size * dtype.itemsize) if isinstance(device, tuple) else Buffer(device, size * dtype.itemsize) - return UOp(Ops.BUFFER, src=(UOp.const(size),)+UOp.device_range_src(device), arg=ParamArg(slot, dtype, device=device, buffer=buf)) + return UOp._buffer(size, dtype, slot, device=device, buffer=buf) @staticmethod def from_buffer(opaque:Buffer|MultiBuffer, dtype:DType, device:str|tuple[str, ...]|None=None): if dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {dtype}") assert dtype.itemsize > 0, f"not a storage dtype: {dtype}" assert opaque.nbytes % dtype.itemsize == 0, f"{opaque.nbytes} bytes is not divisible by {dtype.itemsize} for {dtype}" # Both storage identity and interpretation are part of the UOp key. - return UOp(Ops.BUFFER, src=(UOp.const(opaque.nbytes // dtype.itemsize),)+UOp.device_range_src(device or opaque.device), - arg=ParamArg(-id(opaque), dtype, device=device or opaque.device, buffer=opaque)) + return UOp._buffer(opaque.nbytes // dtype.itemsize, dtype, -id(opaque), device=device or opaque.device, buffer=opaque) def empty_like(self, dtype:DTypeLike|None=None, device:str|tuple[str, ...]|None=None) -> UOp: device = canonicalize_device(self.device if device is None else device) axis = self.axis if isinstance(device, tuple) else None @@ -880,6 +880,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op in {Ops.BUFFER, Ops.ALLOC}: return self.arg.device if self.op is Ops.COPY: return self.arg if self.op is Ops.ALLREDUCE: return self.arg[1] + if self.op is Ops.CALL and self.body.op is Ops.STORE: return self.src[2].device # a copy pushes from its src for x in self.src: if x.device is not None: return x.device return None @@ -922,7 +923,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # CL 1.1 provides the clCreateSubBuffer API, but at the time of writing, relevant CL runtimes (rusticl, adreno, nvidia, amd) do not provide # reasonable values for CL_DEVICE_MEM_BASE_ADDR_ALIGN. cl_ext_buffer_device_address could potentially help, but this extension is not provided # by relevant CL runtimes at time of writing. - if (dev:=self.device) is not None and any(d.startswith(("WEBGPU", "CL")) for d in ((dev,) if isinstance(dev, str) else dev)): return None + if (dev:=self.device) is not None and any(d.startswith(("WEBGPU", "CL")) for d in to_tuple(dev)): return None rng = UOp.range(self.numel(), 0) out = graph_rewrite(self.flatten().index(rng), pm_mops+symbolic, name="contiguous_view_offset") @@ -959,10 +960,10 @@ class UOp(RandMixin, metaclass=UOpMetaClass): def buffer(self) -> Buffer|MultiBuffer: # a bare STAGE (same-device materialization) keeps the source's buffer if self.op is Ops.STAGE and self.arg is None: return self.src[0].buffer - if self.op in {Ops.CONTIGUOUS_BACKWARD, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer + if self.op in {Ops.CONTIGUOUS_BACKWARD, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER, Ops.BITCAST}: return self.src[0].buffer # this buffer can process disk tensors and simple movement ops. # NOTE: the view Buffer returned here is transient (short-lived), it only wraps an offset into the base BUFFER's storage - if self is not self.base or self.op is Ops.BITCAST: + if self is not self.base: src, offset = self._buffer_view if isinstance(buf:=src.buffer, MultiBuffer): mbuf = MultiBuffer.__new__(MultiBuffer) @@ -1120,6 +1121,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if s1_vmax < 0: return (s1_vmin+1, 0) if self.op is Ops.XOR and s1_vmin == s1_vmax == -1 and isinstance(s0_vmin, int) and isinstance(s0_vmax, int): return ~int(s0_vmax), ~int(s0_vmin) + if self.op is Ops.XOR and dtypes.is_int(self.dtype) and s0_vmin >= 0 and s1_vmin >= 0: + return 0, (1 << int(max(s0_vmax, s1_vmax)).bit_length()) - 1 if self.op is Ops.MAX: return max(s0_vmin, s1_vmin), max(s0_vmax, s1_vmax) if self.op is Ops.CMPLT: return (s0_vmax 1: ret = ret.reshape(shape) return ret def placeholder_like(self, slot:int, addrspace=AddrSpace.GLOBAL): @@ -1225,13 +1228,12 @@ class UOp(RandMixin, metaclass=UOpMetaClass): return self.replace(arg=replace(self.arg, slot=slot, name=name, val=None)) # multi-device values become a per-shard sized param wrapped in UNSHARD: the sharding lives in the graph, not the arg if self.axis is not None and isinstance(self.device, tuple): - return UOp(Ops.PARAM, src=(UOp.const(prod(to_max_shape(self.shard_shape))),), - arg=ParamArg(slot, self.dtype, name=name, device=self.device)).view_as(self.shard_shape, self.axis) + return UOp.param(slot, self.dtype, prod(to_max_shape(self.shard_shape)), self.device, name=name).view_as(self.shard_shape, self.axis) return UOp.param(slot, self.dtype, self._shape, self.device, name=name) def view_as(self:UOp, shape:tuple[sint, ...], axis:int|None=None) -> UOp: """view flat storage as the given (possibly symbolic) shape, optionally sharded on axis, the UNSHARD gives back the multiplied shape""" max_shape = to_max_shape(shape) - ret = self.reshape(max_shape) if len(shape) > 1 else self + ret = self.reshape(max_shape) if shape else self if tuple(max_shape) != tuple(shape): ret = ret.shrink_to(shape) return ret if axis is None else ret.unshard(axis) @@ -1357,7 +1359,7 @@ class ProgramInfo: ins: list[int] = [] global_size: list[int] = [1, 1, 1] local_size: list[int] = [1, 1, 1] - for u in sink.toposort(enter_calls=False): + for u in sink.toposort(): if u.op is Ops.PARAM and u.addrspace == AddrSpace.ALU: _vars.append(u) if u.op is Ops.PARAM and u.addrspace != AddrSpace.ALU: _globals.append(u.arg.slot) if u.op in (Ops.STORE, Ops.LOAD): diff --git a/tinygrad_repo/tinygrad/uop/render.py b/tinygrad_repo/tinygrad/uop/render.py index 8817e91dc..03066faac 100644 --- a/tinygrad_repo/tinygrad/uop/render.py +++ b/tinygrad_repo/tinygrad/uop/render.py @@ -63,7 +63,7 @@ def _render_arg(x:UOp) -> str: def _inline(u:UOp) -> bool: return u.op is Ops.CONST or (u.op is Ops.STACK and all(s.op is Ops.CONST for s in u.src)) def render_uir(root:UOp|list[UOp]) -> str: - nodes = [u for u in (list(root.toposort()) if isinstance(root, UOp) else list(root)) if not _inline(u)] + nodes = [u for u in (list(root.toposort(enter_calls=True)) if isinstance(root, UOp) else list(root)) if not _inline(u)] table = {u:i for i,u in enumerate(nodes)} def src_str(u:UOp) -> str: if not _inline(u): return f"%{table[u]}" @@ -92,7 +92,6 @@ def marg_str(ctx, a:sint) -> str: return str(a) if not isinstance(a, UOp) else c def render_marg(ctx,x:UOp): if x.op is Ops.PERMUTE: return str(x.marg) - if x.op is Ops.FLIP: return str(tuple([i for i,x in enumerate(x.marg) if x])) pieces = [] if x.op in {Ops.RESHAPE, Ops.EXPAND}: pieces = [marg_str(ctx, a) for a in x.marg] if x.op in {Ops.PAD, Ops.SHRINK}: pieces = [f"({marg_str(ctx, a[0])}, {marg_str(ctx, a[1])})" for a in x.marg] @@ -148,7 +147,6 @@ def _render_with_splits(lst:list[UOp], pm:PatternMatcher, to_render:set[UOp], sp depth[u] = 0 if u in to_render else op_depth ren = pm.rewrite(u, ctx=r) assert isinstance(ren, str) - if u.tag is not None: ren += f".rtag({repr(u.tag)})" if u not in to_render: r[u] = ren else: r[u] = f"c{i}" if u is not lst[-1] else "ast" diff --git a/tinygrad_repo/tinygrad/uop/spec.py b/tinygrad_repo/tinygrad/uop/spec.py index cb2d3479d..66d63d6fc 100644 --- a/tinygrad_repo/tinygrad/uop/spec.py +++ b/tinygrad_repo/tinygrad/uop/spec.py @@ -1,7 +1,7 @@ from tinygrad.uop.ops import PatternMatcher, UPat, GroupOp, Ops, UOp, AxisType, ParamArg, CallInfo, OPAQUE_CALL_BODIES, \ CustomFunction from tinygrad.uop.render import render_uir -from tinygrad.dtype import DType, dtypes, AddrSpace, Invalid +from tinygrad.dtype import DType, dtypes, AddrSpace, Invalid, weak_dtype from tinygrad.helpers import DEBUG, Context, CHECK_OOB, all_same, is_image_shape from tinygrad.device import is_disk_device @@ -31,7 +31,7 @@ def valid_device_range(device:str|tuple[str, ...]|None, src:tuple[UOp, ...]) -> rng = src[0] return rng.op is Ops.RANGE and rng.axis_type is AxisType.DEVICE and int(rng.vmax)+1 == len(device) -def type_verify(ast:UOp|list[UOp], check_spec:PatternMatcher, enter_calls=True): +def type_verify(ast:UOp|list[UOp], check_spec:PatternMatcher, enter_calls=False): lst = list(ast.toposort(enter_calls=enter_calls)) if isinstance(ast, UOp) else ast with Context(TRACK_MATCH_STATS=0): @@ -58,20 +58,20 @@ spec_shared = PatternMatcher([ # STACK is everywhere too (UPat(Ops.STACK, dtype=dtypes.void, src=()), lambda: True), (UPat(Ops.STACK, src=(UPat(),), allow_any_len=True, name="s"), - lambda s: all_same([x.shape for x in s.src]) and all(matches_dtype(x, s.dtype) or x.dtype in dtypes.weaks for x in s.src)), + lambda s: all_same([x.shape for x in s.src]) and all(matches_dtype(x, s.dtype) or x.dtype == weak_dtype(s.dtype) for x in s.src)), # ALUs: operands match the result dtype, except comparisons/WHERE; renderer-lowered shifts may use a uint32 count - # a weak dtype matches any dtype until lowering commits its operand + # a weak dtype matches any dtype of its kind until lowering commits its operand (UPat(Ops.WHERE, name="w", src=(UPat(dtype=dtypes.bool), UPat(), UPat())), - lambda w: all(matches_dtype(s, w.dtype) or s.dtype in dtypes.weaks for s in w.src[1:])), + lambda w: all(matches_dtype(s, w.dtype) or s.dtype == weak_dtype(w.dtype) for s in w.src[1:])), (UPat(GroupOp.Comparison, dtype=dtypes.bool, src=(UPat.var("x"), UPat.var("y"))), - lambda x,y: matches_dtype(x, y.dtype) or matches_dtype(y, x.dtype) or x.dtype in dtypes.weaks or y.dtype in dtypes.weaks), + lambda x,y: matches_dtype(x, y.dtype) or matches_dtype(y, x.dtype) or x.dtype == weak_dtype(y.dtype) or y.dtype == weak_dtype(x.dtype)), (UPat((Ops.AND, Ops.OR, Ops.XOR, Ops.SHL, Ops.SHR), name="x"), lambda x: False if any(dtypes.is_float(s.dtype) for s in x.src) else None), (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat.var("c")), name="a"), lambda a,x,c: matches_dtype(c, a.dtype) or c.dtype in (dtypes.uint, dtypes.weakint) or x.base.is_invalid), (UPat((Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD), name="x"), lambda x: None if dtypes.is_int(x.dtype) or any(s.base.is_invalid for s in x.src) else False), - (UPat(GroupOp.ALU, name="x"), lambda x: all(matches_dtype(y, x.dtype) or y.dtype in dtypes.weaks for y in x.src)), + (UPat(GroupOp.ALU, name="x"), lambda x: all(matches_dtype(y, x.dtype) or y.dtype == weak_dtype(x.dtype) for y in x.src)), # CAST (UPat((Ops.BITCAST, Ops.CAST), src=(UPat(),), name="x"), lambda x: isinstance(x.arg, DType)), @@ -121,7 +121,9 @@ spec_shared = PatternMatcher([ # assembly instruction (UPat(Ops.INS, name="x"), lambda x: isinstance(x.arg, tuple) and len(x.arg) == 2 and isinstance(x.arg[1], DType)), - # LOAD(idx) / STORE(idx, val) with gates on the LOAD/STORE + # LOAD(idx) / STORE(idx, val) with gates on the LOAD/STORE; a STORE's value matches its target's dtype + (UPat(Ops.STORE, src=(UPat(name="t"), UPat(name="v")), allow_any_len=True), + lambda t,v: None if matches_dtype(v, t.dtype) or v.dtype == weak_dtype(t.dtype) else False), (UPat((Ops.INDEX, Ops.SHRINK), name="uidx").or_casted().load(), validate_index), (UPat((Ops.INDEX, Ops.SHRINK), name="uidx").or_casted().load(UPat.var("alt"), UPat.var("gate", dtype=dtypes.bool), name="load"), lambda uidx,gate,alt,load: validate_index(uidx, gate) if matches_dtype(alt, load.dtype) else False), @@ -161,9 +163,9 @@ spec_tensor = PatternMatcher([ (UPat(Ops.SPECIAL, src=(UPat(dtype=dtypes.weakint),), name="s"), lambda s: isinstance(s.arg, str)), # movement ops - (UPat((Ops.RESHAPE, Ops.EXPAND), src=(UPat(), UPat())), lambda: True), + (UPat((Ops.RESHAPE, Ops.EXPAND), src=(UPat(), UPat(dtype=dtypes.ints+(dtypes.weakint, dtypes.void)))), lambda: True), (UPat((Ops.PAD, Ops.SHRINK), src=(UPat(), UPat(), UPat()), name="x"), lambda x: x.src[1].shape == x.src[2].shape), - (UPat((Ops.PERMUTE, Ops.FLIP), name="mv", src=(UPat(),)), lambda mv: isinstance(mv.arg, tuple)), + (UPat(Ops.PERMUTE, name="mv", src=(UPat(),)), lambda mv: isinstance(mv.arg, tuple)), # REDUCE has arg=(op, num_axes), src[1:] are ranges after lowering (UPat(Ops.REDUCE, src=(UPat(),), allow_any_len=True, name="x"), @@ -179,7 +181,7 @@ spec_tensor = PatternMatcher([ # UNSHARD/MSELECT/MSTACK # an UNSHARD carries the value and one sharding range per sharded axis (usually a DEVICE RANGE, but can be a derived expression) (UPat(Ops.UNSHARD, name="multi"), lambda multi: len(multi.src) == 1+len(multi.arg) - and all(isinstance(a, int) for a in multi.arg) and all(r.dtype in dtypes.weaks for r in multi.src[1:])), + and all(isinstance(a, int) for a in multi.arg) and all(r.dtype == dtypes.weakint for r in multi.src[1:])), (UPat(Ops.MSELECT, name="x"), lambda x: isinstance(x.src[0].device, tuple) and x.arg < len(x.src[0].device)), (UPat(Ops.MSTACK, name="x"), lambda x: all(isinstance(s.device, str) for s in x.src) or (all_same(x.src) and x.src[0].device is None)), diff --git a/tinygrad_repo/tinygrad/uop/symbolic.py b/tinygrad_repo/tinygrad/uop/symbolic.py index 9c6d130b3..bd12e42fc 100644 --- a/tinygrad_repo/tinygrad/uop/symbolic.py +++ b/tinygrad_repo/tinygrad/uop/symbolic.py @@ -3,7 +3,7 @@ import math from collections import defaultdict from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu, promo_dtype from tinygrad.dtype import PyConst, ConstFloat, dtypes, can_lossless_cast, Invalid, bitcast, truncate -from tinygrad.helpers import partition, all_same, prod, flatten, unwrap, IMAGE, dedup +from tinygrad.helpers import partition, all_same, prod, flatten, IMAGE, dedup, ceildiv from tinygrad.uop.divandmod import div_and_mod_symbolic from tinygrad.uop.movement import mop_cleanup from tinygrad.uop.weak import pm_uncast_const @@ -204,9 +204,10 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ # ******** phase 2 builds on phase 1, it includes the old "symbolic", rules that match deeper ******** def lt_folding(x:UOp, c:int) -> UOp|None: - p, np = partition(x.split_uop(Ops.ADD), lambda u: u.const_factor() == 1) - if np and (d:=math.gcd(*[u.const_factor() for u in np], c)) > 1 and 0 <= sum(u.vmin for u in p) and sum(u.vmax for u in p) < d: - return unwrap(UOp.usum(*np).divides(d))<(c//d) + p, np = partition([u.pop_const(Ops.MUL) for u in x.split_uop(Ops.ADD)], lambda uf: uf[1] == 1) + # d*y+p < c <=> y < ceildiv(c-p, d), so it folds when that bound is the same for p.vmin and p.vmax + if np and (d:=math.gcd(*[int(f) for _,f in np])) > 1 and (t:=ceildiv(c-sum(u.vmin for u,_ in p), d)) == ceildiv(c-sum(u.vmax for u,_ in p), d): + return UOp.usum(*[u*(f//d) for u,f in np]) UOp|None: @@ -286,9 +287,6 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ # ** lt ** # c0+x x < c1-c0 ((UPat.cvar("c0") + UPat.var("x", dtype=dtypes.ints+(dtypes.weakint,))) < UPat.cvar("c1"), lambda x,c0,c1: x<(c1-c0)), - # c0*x sign(c0)*x < ceil(c1/abs(c0)) - ((UPat.cvar("c0")*UPat.var("x", dtype=dtypes.weakint)) 0 else -x)<-(-c1.val//abs(c0.val)) if abs(c0.val) > 1 else None), # x//d x0, and -> c*d 0 else (x>c.val*d.val) if d.val < 0 else None), @@ -297,7 +295,7 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ ((UPat.var("x") * UPat.cvar("c1")) * UPat.var("y"), lambda x,c1,y: (x*y)*c1 if y.op is not Ops.CONST else None), # *** rules from symbolic *** # generic lt folding - (UPat.var("x", dtypes.weakint) 0. NOTE: not x < 1 means x > 0 ((UPat.var("x", dtypes.weakint)<1).ne(True), lambda x: (newx<1).ne(True) if (newx:=canonicalize_simplex(x)) is not None else None), @@ -332,15 +330,11 @@ def parse_valid(v:UOp) -> tuple[UOp, bool, int]|None: # if it's X <= c, returns X, True, c # if it's X >= c, returns X, False, c - if v.op is Ops.CMPNE and v.src[1].op is Ops.CONST and v.src[1].val == 1 and (s0:=v.src[0]).op is Ops.CMPLT and dtypes.is_int(s0.src[0].dtype): - # (X < c).ne(True) -> X >= c - return s0.src[0], False, int(s0.src[1].vmin) - if v.op is Ops.CMPLT and dtypes.is_int(v.src[0].dtype): - # c < X -> X >= c+1 (a const on the left is a lower bound on the right) - if v.src[0].op is Ops.CONST: return v.src[1], False, int(v.src[0].val)+1 - # X < c -> X <= c-1 - return v.src[0], True, int((v.src[1]).vmax)-1 - return None + # a=a+1 for a const a), its negation a>=b.vmin (b<=a for a const a) + neg = v.op is Ops.CMPNE and v.src[1].op is Ops.CONST and v.src[1].val == 1 + if (lt:=v.src[0] if neg else v).op is not Ops.CMPLT or not dtypes.is_int(lt.src[0].dtype): return None + if (a:=lt.src[0]).op is Ops.CONST: return lt.src[1], neg, int(a.val) + (0 if neg else 1) + return a, not neg, int(lt.src[1].vmin) if neg else int(lt.src[1].vmax)-1 def uop_given_valid(valid:UOp, uop:UOp, try_simplex=True) -> UOp: # return simplified uop (might be the same as input) diff --git a/tinygrad_repo/tinygrad/viz/serve.py b/tinygrad_repo/tinygrad/viz/serve.py index 5223a62c8..e82a2b1d1 100755 --- a/tinygrad_repo/tinygrad/viz/serve.py +++ b/tinygrad_repo/tinygrad/viz/serve.py @@ -74,6 +74,7 @@ def load_rewrites(data:VizData) -> None: for j,s in enumerate(rewrites:=data.trace.rewrites[i]): steps.append(create_step(s.name, ("/graph-rewrites", i, j), loc=s.loc, match_count=len(s.matches), code_line=printable(s.loc), trace=k.tb if j==0 else None, depth=s.depth)) + if s.name == "View Base AST": data.ref_map[canonicalize_ast(_reconstruct(data, s.sink))] = i # get source and binary from Ops.PROGRAM if s.name == "linearize/render": lin_idx = j if lin_idx is not None and (j+1 == len(rewrites) or rewrites[j+1].depth <= rewrites[lin_idx].depth): @@ -82,7 +83,7 @@ def load_rewrites(data:VizData) -> None: steps.append(create_step("View Disassembly", ("/asm", i, len(steps)), (k.ret, lin_idx), depth=0)) lin_idx = None if s.name == "View Program": ki = _reconstruct(data, s.sink, depth=1).src[0].arg - for key in k.keys: data.ref_map[canonicalize_ast(key) if isinstance(key, UOp) else key] = i + for key in k.keys: data.ref_map[key] = i data.ctxs.append({"name":k.display_name, "steps":steps, "ki":ki}) # ** get the complete UOp graphs for one rewrite @@ -102,7 +103,7 @@ def fmt_colored(s:str) -> str: return ansistrip(s) if NO_COLOR else s def canonicalize_ast(u:UOp) -> UOp: return u.replace(arg=KernelInfo()) if u.op is Ops.SINK and isinstance(u.arg, KernelInfo) else u def tokenize_uir(data:VizData, root:UOp) -> list[dict]: - nodes = [u for u in root.toposort() if not _inline(u)] + nodes = [u for u in root.toposort(enter_calls=True) if not _inline(u)] refs = {f"%{i}":{"id":str(id(u))} for i,u in enumerate(nodes)} lines = [[{"st":s, **refs.get(s, {})} for s in re.split(r"( : [^\n]*|%\d+\b)", line) if s] for line in render_uir(root).split("\n")] for u,line in zip(nodes, lines): @@ -114,7 +115,7 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: assert isinstance(x, UOp) graph: dict[int, dict] = {} excluded: set[UOp] = set() - for u in (toposort:=x.toposort()): + for u in (toposort:=x.toposort(enter_calls=True)): # always exclude CONST if u.op is Ops.CONST and u is not x: excluded.add(u) if u.op is Ops.STACK and len(u.src) == 0: excluded.add(u) @@ -187,7 +188,8 @@ def get_full_rewrite(data:VizData, ctx:TrackedGraphRewrite, depth:int|None=None, match_repr = f"# {dur*1e6:.2f} us\n"+printable(upat_loc) diff = difflib.unified_diff(u0.render_uir().splitlines(), u1.render_uir().splitlines()) yield {"graph":(sink_json:=uop_to_json(data, new_sink)), "uop":tokenize_uir(data, new_sink), "upat":(upat_loc, match_repr), "_sink":new_sink, - "change":[id(x) for x in u1.toposort() if id(x) in sink_json], "diff":[ansistrip(x) for x in diff if not x.startswith(("---","+++","@@"))]} + "change":[id(x) for x in u1.toposort(enter_calls=True) if id(x) in sink_json], + "diff":[ansistrip(x) for x in diff if not x.startswith(("---","+++","@@"))]} if not ctx.bottom_up: next_sink = new_sink def get_sink_at(upats:tuple[str, ...], viz_data:VizData, kernel_idx:int, lin_idx:int, depth:int|None=None, alt:str|None=None) -> UOp|None: @@ -280,7 +282,7 @@ def graph_layout(k:str, dev_events:list[tuple[int, int, float, DevEvent]], start if mem > peak: peak = mem if e.name == "exec" and e.arg["bufs"]: for b in e.arg["bufs"]: buf_ei.setdefault(b, []).append(e) - if e.name == "free": + if e.name == "free" and e.key in temp: events.append(encode_mem_free(e.key, rel_ts(e.ts, start_ts, f"free on {e.device}"), buf_ei.pop(e.key, []), scache)) mem -= temp.pop(e.key) for t in temp: events.append(encode_mem_free(t, rel_ts(end_ts, start_ts, f"end_ts for {k}"), buf_ei.pop(t, []), scache)) @@ -481,7 +483,7 @@ def get_profile(data:VizData, profile:list[ProfileEvent], sort_fn:Callable[[str] for ts,en,e in flatten_events(profile, device_ts_diffs): if isinstance(e, ProfilePointEvent) and e.name == "JSON": ext_data[e.key] = e.arg else: - dev_events.setdefault(e.device,[]).append((st:=int(ts), et:=int(en), float(en-ts), e)) + dev_events.setdefault(e.device,[]).append((st:=int(ts), et:=int(en) if isinstance(e,ProfilePointEvent) else max(st+1,int(en)), float(et-st),e)) if start_ts is None or st < start_ts: start_ts = st if end_ts is None or et > end_ts: end_ts = et if isinstance(e, ProfilePointEvent) and e.name == "marker": markers.append(e) @@ -550,18 +552,18 @@ def amd_decode(buf:bytes, arch:str, off:int=0) -> dict[int, Inst]: from tinygrad.runtime.autogen.amd.rdna3.ins import s_code_end code_end = s_code_end().to_bytes()*5 if arch.startswith("rdna") else None addr_table:dict[int, Inst] = {} - offset = 0 + offset, view = 0, memoryview(buf) while offset < len(buf): - remaining = buf[offset:] - if code_end is not None and remaining.startswith(code_end): break + if code_end is not None and buf.startswith(code_end, offset): break + remaining = view[offset:] fmt = detect_format(remaining, arch) decoded = fmt.from_bytes(remaining) addr_table[off+offset] = decoded offset += decoded.size() return addr_table -def parse_branch(inst) -> int|None: - if "branch" in getattr(inst, "op_name", "").lower(): +def parse_branch(inst, op:str) -> int|None: + if "branch" in op: x = inst.simm16 & 0xffff return (x - 0x10000 if x & 0x8000 else x)*4 return None @@ -580,7 +582,9 @@ def get_cfg(pc_table:dict[int, Inst]) -> dict: # get leaders leaders:set[int] = {next(iter(pc_table))} for pc, inst in pc_table.items(): - if (offset:=parse_branch(inst)) is not None: leaders.update((pc+inst.size()+offset, pc+inst.size())) + op = getattr(inst, "op_name", "").lower() + if (offset:=parse_branch(inst, op)) is not None: leaders.update((pc+inst.size()+offset, pc+inst.size())) + elif op in {"s_endpgm", "s_barrier", "s_barrier_wait"}: leaders.add(pc+inst.size()) # build the cfg curr:int|None = None blocks:dict[int, list[int]] = {} @@ -593,10 +597,10 @@ def get_cfg(pc_table:dict[int, Inst]) -> dict: blocks[curr].append(pc) # otherwise a basic block can have exactly one or two paths nx = pc+inst.size() - if (offset:=parse_branch(inst)) is not None: - if inst.op_name == "S_BRANCH": paths[curr][nx+offset] = UNCOND + if (offset:=parse_branch(inst, op:=getattr(inst, "op_name", "").lower())) is not None: + if op == "s_branch": paths[curr][nx+offset] = UNCOND else: paths[curr].update([(nx+offset, COND_TAKEN), (nx, COND_NOT_TAKEN)]) - elif nx in leaders: paths[curr][nx] = UNCOND + elif nx in leaders and op != "s_endpgm": paths[curr][nx] = UNCOND pc_tokens:dict[int, list[dict]] = {} for pc, inst in pc_table.items(): pc_tokens[pc] = tokens = [] @@ -626,7 +630,7 @@ def get_render(viz_data:VizData, query:str, **kwargs) -> dict: if (sink:=get_sink_at(("do_linearize",), viz_data, i, data, alt="View Program")) is None: return {"src":"No linear found"} if sink.op is Ops.REWRITE_ERROR: return {"src":sink.arg} ret:dict = {} - with soft_err(lambda err: ret.update(err)): ret["src"] = render_uir(list(sink.src[1].toposort())[:-1]) + with soft_err(lambda err: ret.update(err)): ret["src"] = render_uir(list(sink.src[1].toposort(enter_calls=True))[:-1]) return ret if fmt == "code": if (sink:=get_sink_at(("do_render",), viz_data, i, data, depth=1, alt="View Program")) is None: return {"src":"No source found"}