From 449cb119382c72190696efe199445bf55fe61500 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:59:09 -0500 Subject: [PATCH] Pin --- docs/MODEL_REBUILD.md | 24 +- scripts/model_compiler.py | 5 +- scripts/model_release.py | 2 +- scripts/tests/test_model_release.py | 4 +- selfdrive/modeld/modeld.py | 4 + starpilot/assets/model_manager.py | 2 +- starpilot/assets/tests/test_model_pipeline.py | 14 +- .../.github/actions/setup-tinygrad/action.yml | 63 +- tinygrad_repo/.github/workflows/autogen.yml | 47 +- tinygrad_repo/.github/workflows/benchmark.yml | 275 +- tinygrad_repo/.github/workflows/docs.yml | 2 +- tinygrad_repo/.github/workflows/platform.yml | 34 +- .../.github/workflows/python-publish.yml | 2 +- tinygrad_repo/.github/workflows/szdiff.yml | 6 +- tinygrad_repo/.github/workflows/test.yml | 119 +- tinygrad_repo/.gitignore | 1 + tinygrad_repo/.pre-commit-config.yaml | 2 +- tinygrad_repo/AGENTS.md | 1 + tinygrad_repo/README.md | 2 +- tinygrad_repo/TINYGRAD_COMMIT | 2 +- tinygrad_repo/conftest.py | 2 +- tinygrad_repo/docs/abstractions4.py | 2 +- tinygrad_repo/docs/developer/speed.md | 2 +- tinygrad_repo/examples/mlperf/dataloader.py | 10 +- tinygrad_repo/examples/mlperf/model_train.py | 72 +- .../examples/mlperf/models/gpt_oss.py | 70 +- tinygrad_repo/examples/mlperf/optim.py | 7 +- .../tinybox_8xMI350X/buid_dataset_cache.py | 13 + .../tinybox_8xMI350X/dev_beam.sh | 6 +- .../tinybox_8xMI350X/dev_run.sh | 10 +- .../tinybox_8xMI350X/run_and_time.sh | 6 +- .../tinybox_8xMI350X/dev_beam.sh | 1 + .../tinybox_8xMI350X/dev_run.sh | 1 + tinygrad_repo/examples/openpilot/compile3.py | 22 +- tinygrad_repo/extra/amdpci/am_smi.py | 6 +- tinygrad_repo/extra/bench_log.py | 18 +- tinygrad_repo/extra/bnxt_driver/bnxtdev.py | 238 + tinygrad_repo/extra/bnxt_driver/connect.py | 118 + tinygrad_repo/extra/bnxt_driver/loopback.py | 46 + tinygrad_repo/extra/export_model.py | 5 +- tinygrad_repo/extra/gemm/amd_asm_matmul.py | 2 +- tinygrad_repo/extra/gemm/cdna_asm_gemm.py | 7 +- tinygrad_repo/extra/gemm/gemm_mxfp4.py | 862 +-- tinygrad_repo/extra/gemm/max_matmul.py | 1 - tinygrad_repo/extra/gemm/moe_gemm.py | 27 +- tinygrad_repo/extra/gemm/moe_routing.py | 2 +- tinygrad_repo/extra/gemm/rdna4_asm_matmul.py | 2 +- .../extra/gemm/tinygrad_nv_matmul.py | 15 +- tinygrad_repo/extra/gemm/triton_nv_matmul.py | 2 +- .../extra/gptoss_kernels/rmsnorm/__init__.py | 29 + tinygrad_repo/extra/hcq/hcq_smi.py | 19 +- tinygrad_repo/extra/hcq2/ops_amd2.py | 335 +- tinygrad_repo/extra/llama_kernels/__init__.py | 6 +- .../quantize_fp8_delayed/__init__.py | 2 +- .../llama_kernels/quantize_mxfp4/__init__.py | 2 +- tinygrad_repo/extra/mmapeak/mmapeak.py | 4 +- tinygrad_repo/extra/models/llama.py | 6 +- tinygrad_repo/extra/onnx_helpers.py | 2 + .../extra/optimization/test_beam_search.py | 7 +- .../extra/runbook_digitalocean_mi350x.md | 197 + .../extra/sqtt/examples/generate_examples.py | 5 +- .../examples/gfx1100/profile_empty_run_0.pkl | Bin 66725 -> 37390 bytes .../examples/gfx1100/profile_empty_run_1.pkl | Bin 66740 -> 37390 bytes .../examples/gfx1100/profile_gemm_run_0.pkl | Bin 70989 -> 47881 bytes .../examples/gfx1100/profile_gemm_run_1.pkl | Bin 71008 -> 47881 bytes .../examples/gfx1100/profile_plus_run_0.pkl | Bin 83106 -> 37640 bytes .../examples/gfx1100/profile_plus_run_1.pkl | Bin 82917 -> 37448 bytes .../examples/gfx1100/profile_sync_run_0.pkl | Bin 137026 -> 70555 bytes .../examples/gfx1100/profile_sync_run_1.pkl | Bin 137010 -> 70555 bytes .../examples/gfx1200/profile_empty_run_0.pkl | Bin 45072 -> 18100 bytes .../examples/gfx1200/profile_empty_run_1.pkl | Bin 45065 -> 18100 bytes .../examples/gfx1200/profile_gemm_run_0.pkl | Bin 51709 -> 30159 bytes .../examples/gfx1200/profile_gemm_run_1.pkl | Bin 51739 -> 30191 bytes .../gfx1200/profile_handwritten_run_0.pkl | Bin 56301 -> 24148 bytes .../gfx1200/profile_handwritten_run_1.pkl | Bin 56173 -> 24020 bytes .../examples/gfx1200/profile_plus_run_0.pkl | Bin 60241 -> 18166 bytes .../examples/gfx1200/profile_plus_run_1.pkl | Bin 60227 -> 18166 bytes .../examples/gfx1200/profile_sync_run_0.pkl | Bin 97578 -> 37030 bytes .../examples/gfx1200/profile_sync_run_1.pkl | Bin 97696 -> 37158 bytes .../examples/gfx950/profile_empty_run_0.pkl | Bin 99408 -> 37448 bytes .../examples/gfx950/profile_empty_run_1.pkl | Bin 99294 -> 37320 bytes .../examples/gfx950/profile_gemm_run_0.pkl | Bin 130767 -> 85196 bytes .../examples/gfx950/profile_gemm_run_1.pkl | Bin 130773 -> 85196 bytes .../examples/gfx950/profile_plus_run_0.pkl | Bin 130351 -> 38626 bytes .../examples/gfx950/profile_plus_run_1.pkl | Bin 130345 -> 38626 bytes tinygrad_repo/extra/sqtt/roc.py | 119 +- tinygrad_repo/extra/thunder/amd/fa.py | 56 +- .../extra/thunder/amd/fa_fwd_causal.cpp | 20 + .../extra/thunder/amd/fused_qkv_rope.cpp | 69 - tinygrad_repo/extra/thunder/tiny/tk/tiles.py | 2 +- tinygrad_repo/extra/tinyfs/fetch_file.py | 38 - tinygrad_repo/extra/tinyfs/fetch_raid.py | 42 - tinygrad_repo/extra/tinyfs/upload_raid.py | 32 - tinygrad_repo/extra/torch_backend/backend.py | 212 +- tinygrad_repo/extra/torch_backend/test.py | 120 + tinygrad_repo/pyproject.toml | 8 +- tinygrad_repo/spec/tinyspec.pdf | Bin 98812 -> 98975 bytes tinygrad_repo/spec/tinyspec.tex | 66 +- tinygrad_repo/test/amd/hw/test_sop.py | 33 + tinygrad_repo/test/amd/hw/test_vop1.py | 61 + tinygrad_repo/test/amd/hw/test_vop2.py | 47 + tinygrad_repo/test/amd/hw/test_vop3.py | 92 + tinygrad_repo/test/amd/hw/test_vopc.py | 65 + tinygrad_repo/test/amd/test_asm_kernel.py | 12 +- tinygrad_repo/test/amd/test_emu2_pcode.py | 67 +- tinygrad_repo/test/amd/test_sqtt_examples.py | 1 - tinygrad_repo/test/amd/test_sqtt_profiler.py | 12 +- tinygrad_repo/test/backend/test_asm_gemm.py | 4 +- .../test/{unit => backend}/test_assign.py | 83 +- .../test/backend/test_const_folding.py | 9 +- .../test/backend/test_custom_kernel.py | 13 +- tinygrad_repo/test/backend/test_dtype.py | 18 +- tinygrad_repo/test/backend/test_dtype_alu.py | 4 + tinygrad_repo/test/backend/test_encodings.py | 11 +- tinygrad_repo/test/backend/test_isel.py | 14 +- tinygrad_repo/test/backend/test_jit.py | 26 +- tinygrad_repo/test/backend/test_linearizer.py | 67 +- .../test/backend/test_linearizer_dumb.py | 8 +- .../test/backend/test_llama_kernels.py | 21 +- .../test/backend/test_multitensor.py | 30 +- tinygrad_repo/test/backend/test_nn.py | 2 +- tinygrad_repo/test/backend/test_ops.py | 60 +- tinygrad_repo/test/backend/test_opt_gemm.py | 11 +- tinygrad_repo/test/backend/test_optim.py | 3 +- tinygrad_repo/test/backend/test_pickle.py | 10 +- .../test/backend/test_quantize_onnx.py | 21 +- tinygrad_repo/test/backend/test_rangeify.py | 131 +- .../test/backend/test_renderer_failures.py | 24 +- tinygrad_repo/test/backend/test_schedule.py | 59 +- tinygrad_repo/test/backend/test_setitem.py | 21 +- tinygrad_repo/test/backend/test_tensor.py | 10 + .../test/backend/test_tensor_variable.py | 13 +- tinygrad_repo/test/backend/test_uops.py | 64 +- tinygrad_repo/test/backend/test_wait_loop.py | 2 +- tinygrad_repo/test/device/test_hcq.py | 3 +- tinygrad_repo/test/device/test_hcq2.py | 64 + .../external/external_benchmark_all2all.py | 17 + .../external/external_benchmark_op_conv.py | 11 +- .../external/external_benchmark_pyrender.py | 4 +- .../test/external/external_test_gpu_crash.py | 6 +- .../test/external/external_test_onnx_ops.py | 15 + .../external/external_test_onnx_runner.py | 23 +- .../external_test_schedule_scaling.py | 17 +- .../test/external/external_test_usb_asm24.py | 2 +- .../test/external/external_uop_gc.py | 2 +- .../test/external/speed_v_theoretical.py | 4 +- tinygrad_repo/test/helpers.py | 11 +- tinygrad_repo/test/mockgpu/amd/amddriver.py | 47 +- tinygrad_repo/test/mockgpu/amd/emu.py | 1223 ++-- tinygrad_repo/test/mockgpu/amd/pcode.py | 168 +- tinygrad_repo/test/mockgpu/amd/sqtt_enc.py | 100 + tinygrad_repo/test/mockgpu/usb.py | 24 +- tinygrad_repo/test/models/test_whisper.py | 2 + tinygrad_repo/test/null/test_const_folding.py | 142 +- tinygrad_repo/test/null/test_dtype_spec.py | 9 +- tinygrad_repo/test/null/test_elf.py | 5 +- tinygrad_repo/test/null/test_gpudims.py | 2 +- tinygrad_repo/test/null/test_graph_rewrite.py | 193 +- .../test/null/test_linearizer_failures.py | 6 +- .../test/null/test_linearizer_rewrite.py | 8 +- tinygrad_repo/test/null/test_llm_tokenizer.py | 20 + .../test/null/test_memory_planner.py | 11 +- tinygrad_repo/test/null/test_mnist_dataset.py | 3 +- .../test/null/test_pattern_matcher.py | 2 +- .../test/null/test_process_replay.py | 3 +- .../test/null/test_simplify_valid_idx.py | 52 +- tinygrad_repo/test/null/test_tensor.py | 7 +- .../test/null/test_transcendental_helpers.py | 2 +- tinygrad_repo/test/null/test_uop_graph.py | 245 +- tinygrad_repo/test/null/test_uop_repr.py | 18 +- tinygrad_repo/test/null/test_uop_symbolic.py | 92 +- tinygrad_repo/test/null/test_uop_vmin_vmax.py | 4 +- tinygrad_repo/test/null/test_uops.py | 106 +- tinygrad_repo/test/null/test_uops_stats.py | 30 +- tinygrad_repo/test/null/test_validate_oob.py | 120 +- tinygrad_repo/test/null/test_viz.py | 63 +- tinygrad_repo/test/opt/test_gen_float4.py | 38 +- tinygrad_repo/test/opt/test_kernel_opts.py | 318 +- tinygrad_repo/test/opt/test_tensor_cores.py | 87 +- .../test/testextra/test_bench_log.py | 24 +- tinygrad_repo/test/unit/test_allreduce.py | 31 +- tinygrad_repo/test/unit/test_attention.py | 111 +- tinygrad_repo/test/unit/test_bnxt.py | 115 + .../test/unit/test_bnxt_transport.py | 40 + tinygrad_repo/test/unit/test_call.py | 80 +- tinygrad_repo/test/unit/test_dtype_weak.py | 116 +- tinygrad_repo/test/unit/test_function.py | 29 +- tinygrad_repo/test/unit/test_gguf.py | 99 +- tinygrad_repo/test/unit/test_gradient.py | 10 +- tinygrad_repo/test/unit/test_hcq_graph.py | 6 +- .../test/unit/test_invalid_tensor.py | 6 + tinygrad_repo/test/unit/test_llm_amd.py | 122 + tinygrad_repo/test/unit/test_llm_moe.py | 29 +- tinygrad_repo/test/unit/test_llm_server.py | 46 +- tinygrad_repo/test/unit/test_metal_graph.py | 40 +- tinygrad_repo/test/unit/test_multitensor.py | 82 +- .../test/unit/test_schedule_cache.py | 55 +- tinygrad_repo/test/unit/test_tinyfs.py | 83 - tinygrad_repo/tinygrad/codegen/__init__.py | 84 +- .../tinygrad/codegen/decomp/dtype.py | 80 +- tinygrad_repo/tinygrad/codegen/decomp/op.py | 12 +- .../tinygrad/codegen/decomp/transcendental.py | 4 +- tinygrad_repo/tinygrad/codegen/gpudims.py | 32 +- .../tinygrad/codegen/late/coalesce.py | 26 +- tinygrad_repo/tinygrad/codegen/late/gater.py | 6 +- .../tinygrad/codegen/late/regalloc.py | 8 +- .../tinygrad/codegen/opt/__init__.py | 3 +- .../tinygrad/codegen/opt/heuristic.py | 89 +- .../tinygrad/codegen/opt/postrange.py | 91 +- tinygrad_repo/tinygrad/codegen/opt/search.py | 43 +- tinygrad_repo/tinygrad/codegen/simplify.py | 18 +- tinygrad_repo/tinygrad/device.py | 56 +- tinygrad_repo/tinygrad/dtype.py | 15 +- tinygrad_repo/tinygrad/engine/jit.py | 31 +- tinygrad_repo/tinygrad/engine/realize.py | 305 +- tinygrad_repo/tinygrad/engine/worker.py | 49 + tinygrad_repo/tinygrad/function.py | 19 +- tinygrad_repo/tinygrad/helpers.py | 100 +- tinygrad_repo/tinygrad/llm/cli.py | 27 +- tinygrad_repo/tinygrad/llm/gguf.py | 71 +- tinygrad_repo/tinygrad/llm/kernels/amd.py | 654 ++ tinygrad_repo/tinygrad/llm/model.py | 151 +- tinygrad_repo/tinygrad/mixin/creation.py | 5 +- tinygrad_repo/tinygrad/mixin/dtype.py | 5 +- tinygrad_repo/tinygrad/mixin/elementwise.py | 38 +- tinygrad_repo/tinygrad/mixin/gradient.py | 76 +- tinygrad_repo/tinygrad/mixin/movement.py | 11 + tinygrad_repo/tinygrad/mixin/op.py | 42 +- tinygrad_repo/tinygrad/mixin/reduce.py | 6 +- tinygrad_repo/tinygrad/nn/__init__.py | 6 +- tinygrad_repo/tinygrad/nn/onnx.py | 95 +- tinygrad_repo/tinygrad/nn/optim.py | 4 +- tinygrad_repo/tinygrad/nn/state.py | 60 +- tinygrad_repo/tinygrad/renderer/__init__.py | 19 +- tinygrad_repo/tinygrad/renderer/amd/elf.py | 2 +- tinygrad_repo/tinygrad/renderer/amd/sqtt.py | 17 +- tinygrad_repo/tinygrad/renderer/cstyle.py | 123 +- .../tinygrad/renderer/isa/__init__.py | 2 +- tinygrad_repo/tinygrad/renderer/isa/x86.py | 351 +- tinygrad_repo/tinygrad/renderer/llvmir.py | 61 +- tinygrad_repo/tinygrad/renderer/nir.py | 33 +- tinygrad_repo/tinygrad/renderer/ptx.py | 54 +- .../tinygrad/{codegen/opt => renderer}/tc.py | 36 + tinygrad_repo/tinygrad/renderer/wgsl.py | 78 +- .../tinygrad/runtime/autogen/__init__.py | 42 +- .../tinygrad/runtime/autogen/am/__init__.py | 9 +- .../tinygrad/runtime/autogen/am/am.py | 149 +- .../tinygrad/runtime/autogen/am/fw.py | 145 +- .../tinygrad/runtime/autogen/am/regs.py | 23 + .../tinygrad/runtime/autogen/bnxt.py | 5407 +++++++++++++++++ .../tinygrad/runtime/autogen/mesa.py | 2 +- tinygrad_repo/tinygrad/runtime/graph/hcq.py | 10 +- tinygrad_repo/tinygrad/runtime/graph/metal.py | 5 +- tinygrad_repo/tinygrad/runtime/ops_amd.py | 82 +- tinygrad_repo/tinygrad/runtime/ops_cl.py | 32 +- tinygrad_repo/tinygrad/runtime/ops_cpu.py | 225 +- tinygrad_repo/tinygrad/runtime/ops_dsp.py | 14 +- tinygrad_repo/tinygrad/runtime/ops_metal.py | 6 +- tinygrad_repo/tinygrad/runtime/ops_npy.py | 2 + tinygrad_repo/tinygrad/runtime/ops_null.py | 11 +- tinygrad_repo/tinygrad/runtime/ops_nv.py | 28 +- tinygrad_repo/tinygrad/runtime/ops_python.py | 26 +- tinygrad_repo/tinygrad/runtime/ops_qcom.py | 9 +- tinygrad_repo/tinygrad/runtime/ops_rdma.py | 3 +- tinygrad_repo/tinygrad/runtime/ops_tinyfs.py | 128 - tinygrad_repo/tinygrad/runtime/ops_webgpu.py | 2 +- .../tinygrad/runtime/support/am/amdev.py | 127 +- .../tinygrad/runtime/support/am/ip.py | 83 +- tinygrad_repo/tinygrad/runtime/support/c.py | 2 +- .../tinygrad/runtime/support/compiler_cpu.py | 11 +- .../tinygrad/runtime/support/compiler_cuda.py | 17 +- .../tinygrad/runtime/support/compiler_llvm.py | 14 +- .../tinygrad/runtime/support/compiler_qcom.py | 17 +- .../tinygrad/runtime/support/compileserver.py | 13 + tinygrad_repo/tinygrad/runtime/support/elf.py | 10 +- tinygrad_repo/tinygrad/runtime/support/hcq.py | 51 +- .../tinygrad/runtime/support/hcq2.py | 802 +-- .../tinygrad/runtime/support/memory.py | 17 +- .../tinygrad/runtime/support/mlx/mlxdev.py | 6 +- .../tinygrad/runtime/support/nv/ip.py | 23 +- .../tinygrad/runtime/support/system.py | 12 +- tinygrad_repo/tinygrad/runtime/support/usb.py | 131 +- tinygrad_repo/tinygrad/schedule/__init__.py | 35 +- tinygrad_repo/tinygrad/schedule/indexing.py | 41 +- tinygrad_repo/tinygrad/schedule/memory.py | 8 +- tinygrad_repo/tinygrad/schedule/multi.py | 51 +- tinygrad_repo/tinygrad/schedule/prepare.py | 221 + tinygrad_repo/tinygrad/schedule/rangeify.py | 315 +- tinygrad_repo/tinygrad/tensor.py | 290 +- tinygrad_repo/tinygrad/uop/__init__.py | 15 +- tinygrad_repo/tinygrad/uop/ops.py | 507 +- tinygrad_repo/tinygrad/uop/render.py | 45 +- tinygrad_repo/tinygrad/uop/spec.py | 139 +- tinygrad_repo/tinygrad/uop/symbolic.py | 103 +- tinygrad_repo/tinygrad/uop/upat.py | 46 +- tinygrad_repo/tinygrad/uop/validate.py | 7 +- tinygrad_repo/tinygrad/uop/weak.py | 134 +- tinygrad_repo/tinygrad/viz/index.html | 3 - tinygrad_repo/tinygrad/viz/js/index.js | 24 +- tinygrad_repo/tinygrad/viz/js/worker.js | 2 +- tinygrad_repo/tinygrad/viz/serve.py | 52 +- 301 files changed, 14621 insertions(+), 7377 deletions(-) create mode 100644 tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py create mode 100644 tinygrad_repo/extra/bnxt_driver/bnxtdev.py create mode 100644 tinygrad_repo/extra/bnxt_driver/connect.py create mode 100644 tinygrad_repo/extra/bnxt_driver/loopback.py create mode 100644 tinygrad_repo/extra/gptoss_kernels/rmsnorm/__init__.py create mode 100644 tinygrad_repo/extra/runbook_digitalocean_mi350x.md delete mode 100644 tinygrad_repo/extra/thunder/amd/fused_qkv_rope.cpp delete mode 100644 tinygrad_repo/extra/tinyfs/fetch_file.py delete mode 100644 tinygrad_repo/extra/tinyfs/fetch_raid.py delete mode 100644 tinygrad_repo/extra/tinyfs/upload_raid.py rename tinygrad_repo/test/{unit => backend}/test_assign.py (91%) create mode 100644 tinygrad_repo/test/device/test_hcq2.py create mode 100644 tinygrad_repo/test/external/external_benchmark_all2all.py create mode 100644 tinygrad_repo/test/mockgpu/amd/sqtt_enc.py create mode 100644 tinygrad_repo/test/unit/test_bnxt.py create mode 100644 tinygrad_repo/test/unit/test_bnxt_transport.py create mode 100644 tinygrad_repo/test/unit/test_llm_amd.py delete mode 100644 tinygrad_repo/test/unit/test_tinyfs.py create mode 100644 tinygrad_repo/tinygrad/engine/worker.py create mode 100644 tinygrad_repo/tinygrad/llm/kernels/amd.py rename tinygrad_repo/tinygrad/{codegen/opt => renderer}/tc.py (81%) create mode 100644 tinygrad_repo/tinygrad/runtime/autogen/bnxt.py delete mode 100644 tinygrad_repo/tinygrad/runtime/ops_tinyfs.py create mode 100644 tinygrad_repo/tinygrad/runtime/support/compileserver.py create mode 100644 tinygrad_repo/tinygrad/schedule/prepare.py diff --git a/docs/MODEL_REBUILD.md b/docs/MODEL_REBUILD.md index b803bc75c..e44150f5f 100644 --- a/docs/MODEL_REBUILD.md +++ b/docs/MODEL_REBUILD.md @@ -159,7 +159,21 @@ All four files must be updated together. ## Manifest -Generate the base manifest after compilation, then namespace the release artifacts as v23: +The current test branch uses manifest v25 and requests v25 only. Seed the new +manifest from the previous catalog, then replace entries as artifacts are +rebuilt with the pinned runtime: + +```bash +cp /path/to/model_names_v24.json /path/to/model_names_v25.json +``` + +The current tinygrad pin is `f6fc4e3f2c3db5fae1e19cbfbc3ad9fc579a12ae`, from +`openpilot` `origin/master` (`bump tg + TC_MIN_GLOBALS`). StarPilot's +multi-model `modeld` remains in place; do not replace it with upstream's +single-model `modeld`. + +For the older namespace migration workflow, generate the base manifest after +compilation and namespace the release artifacts as v23: ```bash python3 scripts/model_rebuild_pipeline.py manifest \ @@ -175,9 +189,9 @@ python3 scripts/namespace_model_artifacts.py \ The namespace command changes IDs such as `tr1422` to `tr14223`, renames the compiled and upload-ready files, and writes an ID map. It preserves display -names and behavioral versions. The current model manager requests v23 only; -the manifest is fetched from `Models/model_names_v23.json`, while v22 remains -available for devices that have not updated yet. +names and behavioral versions. The current model manager requests v25 only; the +manifest is fetched from `Models/model_names_v25.json`. Devices still running +the prior branch continue to request their existing manifest version. After importing newly compiled sources, normalize the release namespace before copying files into either resource repository: @@ -205,4 +219,4 @@ Compilation validates JIT capture/replay, pickle round-trip, finite outputs, met 4. Confirm `driverStateV2` on both supported camera resolutions. 5. Test download, selection, deletion, randomization, migration, and fallback in both device UIs and Galaxy. -The built-in RDF artifact is `selfdrive/modeld/models/driving_tinygrad.pkl`. If migration cannot download the selected v23 artifact, StarPilot switches to that built-in model. +The built-in RDF artifact is `selfdrive/modeld/models/driving_tinygrad.pkl`. If migration cannot download the selected v25 artifact, StarPilot switches to that built-in model. diff --git a/scripts/model_compiler.py b/scripts/model_compiler.py index 34475d722..89827cfcb 100644 --- a/scripts/model_compiler.py +++ b/scripts/model_compiler.py @@ -56,14 +56,13 @@ def build_compile_env(*, supercombo: bool = False) -> dict[str, str]: existing_pythonpath = env.get("PYTHONPATH", "") env["PYTHONPATH"] = f"{REPO_ROOT}{os.pathsep}{existing_pythonpath}" if existing_pythonpath else str(REPO_ROOT) defaults = { + "DEBUG": "0", "FLOAT16": "1", "IMAGE": "1" if supercombo else "2", "JIT_BATCH_SIZE": "0", "NOLOCALS": "1", "OPENPILOT_HACKS": "1", - } | ({} if supercombo else { - "DEBUG": "0", - }) + } for key, default in defaults.items(): try: int(str(env.get(key)), 0) diff --git a/scripts/model_release.py b/scripts/model_release.py index e6c6aabe3..82e56b5fe 100755 --- a/scripts/model_release.py +++ b/scripts/model_release.py @@ -31,7 +31,7 @@ OPENPILOT_REPO = "commaai/openpilot" RESOURCES_REPO = os.environ.get("STARPILOT_RESOURCES_REPO", "firestar5683/StarPilot-Resources") HF_BUCKET = os.environ.get("STARPILOT_HF_BUCKET", "StarPilot-Driving/StarPilot-Resources") RESOURCE_BRANCH = "Models" -MANIFEST_VERSION = "v24" +MANIFEST_VERSION = "v25" DEFAULT_BEHAVIOR_VERSION = "v16" DEVICE_ROOT = "/data/openpilot" REPOSITORY_FILE_LIMIT = 100_000_000 diff --git a/scripts/tests/test_model_release.py b/scripts/tests/test_model_release.py index 9a6d0e228..db4ac95ee 100644 --- a/scripts/tests/test_model_release.py +++ b/scripts/tests/test_model_release.py @@ -79,14 +79,14 @@ def test_runtime_scan_excludes_model_weights_but_flags_runtime_code(): def test_update_manifest_replaces_one_entry(tmp_path: Path): - manifest = tmp_path / "model_names_v24.json" + manifest = tmp_path / "model_names_v25.json" manifest.write_text(json.dumps({"models": [{"id": "old"}]}) + "\n") info = parse_pasted_release(RELEASE_TEXT, "bmrlnapv4", "v16") path = update_manifest( tmp_path, info, {"size": 123, "sha256": "a" * 64}, - "v24", + "v25", ) payload = json.loads(path.read_text()) assert len(payload["models"]) == 2 diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 38feb28bf..f3ed47690 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -7,6 +7,10 @@ import struct from openpilot.system.hardware import HARDWARE, TICI os.environ['GMMU'] = '0' os.environ['DEV'] = 'QCOM' if TICI else 'LLVM' +try: + int(os.getenv('DEBUG', '0'), 0) +except ValueError: + os.environ['DEBUG'] = '0' from tinygrad.device import Device from tinygrad.tensor import Tensor import time diff --git a/starpilot/assets/model_manager.py b/starpilot/assets/model_manager.py index a6540b789..51fe07887 100644 --- a/starpilot/assets/model_manager.py +++ b/starpilot/assets/model_manager.py @@ -24,7 +24,7 @@ from openpilot.starpilot.common.starpilot_utilities import delete_file from openpilot.starpilot.common.starpilot_variables import MODELS_PATH from openpilot.system.hardware.usb import chestnut_firmware_ready -MANIFEST_CANDIDATES = ("v24",) +MANIFEST_CANDIDATES = ("v25",) MODEL_NAMESPACE_SUFFIX = "3" DEFAULT_MODEL_KEY = "rdf43" LOCAL_MODEL_PREFIX = "local-" diff --git a/starpilot/assets/tests/test_model_pipeline.py b/starpilot/assets/tests/test_model_pipeline.py index 863e41a4d..fe868656b 100644 --- a/starpilot/assets/tests/test_model_pipeline.py +++ b/starpilot/assets/tests/test_model_pipeline.py @@ -15,12 +15,12 @@ from openpilot.starpilot.assets.model_manager import MANIFEST_CANDIDATES, ModelM from openpilot.starpilot.common.model_versions import UNIFIED_ARTIFACT_FORMAT -def test_v24_is_the_only_manifest_candidate(): - assert MANIFEST_CANDIDATES == ("v24",) +def test_v25_is_the_only_manifest_candidate(): + assert MANIFEST_CANDIDATES == ("v25",) -def test_v24_manifest_is_loaded_from_models_checkout(): - assert ModelManager._manifest_paths("v24") == ("Models/model_names_v24.json",) +def test_v25_manifest_is_loaded_from_models_checkout(): + assert ModelManager._manifest_paths("v25") == ("Models/model_names_v25.json",) def test_resource_sources_prefer_huggingface_then_github(monkeypatch): @@ -33,9 +33,9 @@ def test_resource_sources_prefer_huggingface_then_github(monkeypatch): def test_huggingface_manifest_has_root_and_manifests_fallbacks(): - assert ModelManager._hf_manifest_paths("v24") == ( - "model_names_v24.json", - "manifests/model_names_v24.json", + assert ModelManager._hf_manifest_paths("v25") == ( + "model_names_v25.json", + "manifests/model_names_v25.json", ) diff --git a/tinygrad_repo/.github/actions/setup-tinygrad/action.yml b/tinygrad_repo/.github/actions/setup-tinygrad/action.yml index 6f164a500..372f227e3 100644 --- a/tinygrad_repo/.github/actions/setup-tinygrad/action.yml +++ b/tinygrad_repo/.github/actions/setup-tinygrad/action.yml @@ -4,7 +4,7 @@ inputs: python-version: description: 'Python version to use' required: false - default: '' # if you don't set a version, the native python version will be used + default: '3.14' key: description: 'Key for the python cache' required: false @@ -41,12 +41,12 @@ inputs: description: "Install LLVM?" required: false default: 'false' - tinydreno: - description: "Install tinydreno" + qemu: + description: "Install qemu?" required: false default: 'false' - qemu: - description: "Install qemu" + ninja: + description: "Install ninja?" required: false default: 'false' runs: @@ -59,18 +59,18 @@ runs: echo "OMP_NUM_THREADS=1" >> "$GITHUB_ENV" # no buffers should be over 300MB in CI echo "MAX_BUFFER_SIZE=300000000" >> "$GITHUB_ENV" + if [[ "$RUNNER_OS" == "Linux" ]]; then + echo "VIRTUAL_ENV=/opt/venv/${{ inputs.python-version }}" >> "$GITHUB_ENV" + echo "UV_PYTHON_INSTALL_DIR=/opt/python" >> "$GITHUB_ENV" + else + echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> "$GITHUB_ENV" + fi - name: Set up uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b with: enable-cache: 'false' # see below for manual caching - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v6 - if: inputs.python-version != '' - with: - python-version: ${{ inputs.python-version }} - # **** Caching packages **** - name: Cache Python packages (PR) @@ -109,15 +109,15 @@ runs: if: inputs.deps != '' shell: bash run: | - uv venv .venv + uv venv --allow-existing --python ${{ inputs.python-version }} "$VIRTUAL_ENV" DEPS="${{ inputs.deps }}" - uv pip install --python .venv -e ".[${DEPS// /,}]" ${{ inputs.pydeps }} --torch-backend cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/ + uv pip install --python "$VIRTUAL_ENV" -e ".[${DEPS// /,}]" ${{ inputs.pydeps }} --torch-backend cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/ - name: Install dependencies in venv (without extra) if: inputs.deps == '' shell: bash run: | - uv venv .venv - uv pip install --python .venv -e . ${{ inputs.pydeps }} + uv venv --allow-existing --python ${{ inputs.python-version }} "$VIRTUAL_ENV" + uv pip install --python "$VIRTUAL_ENV" -e . ${{ inputs.pydeps }} - name: Prune uv cache if: github.event_name != 'pull_request' shell: bash @@ -125,16 +125,15 @@ runs: - name: Configure venv shell: bash run: | - echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> "$GITHUB_ENV" if [[ "$RUNNER_OS" == "Windows" ]]; then - echo "${{ github.workspace }}/.venv/Scripts" >> "$GITHUB_PATH" + echo "$VIRTUAL_ENV/Scripts" >> "$GITHUB_PATH" else - echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH" + echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH" fi # ******************* apt ******************* - name: Setup apt - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') shell: bash run: | sudo mkdir -p /var/cache/apt/archives @@ -162,7 +161,7 @@ runs: echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list - name: Compute Package List + Hash - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') id: apt-pkgs shell: bash run: | @@ -187,25 +186,37 @@ runs: if [[ "${{ inputs.qemu }}" == "true" ]]; then pkgs+=" qemu-user-static" fi + # **** ninja **** + if [[ "${{ inputs.ninja }}" == "true" ]]; then + pkgs+=" ninja-build" + fi echo "pkgs=$pkgs" >> "$GITHUB_OUTPUT" echo "hash=$(echo -n "$pkgs" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + installed=true + for pkg in $pkgs; do + info=$(dpkg-query -W -f='${db:Status-Abbrev} ${Version}' "$pkg" 2> /dev/null || true) + echo "${pkg}: ${info:-not in dpkg database}" + [[ "$info" == ii* ]] || installed=false + done + echo "installed=$installed" >> "$GITHUB_OUTPUT" + - name: Cache apt (PR) - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') && github.event_name == 'pull_request' + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name == 'pull_request' && steps.apt-pkgs.outputs.installed == 'false' uses: actions/cache/restore@v5 with: path: /var/cache/apt/archives/ key: ${{ runner.os }}-${{ runner.arch }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.CACHE_VERSION }} - name: Cache apt - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') && github.event_name != 'pull_request' + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name != 'pull_request' && steps.apt-pkgs.outputs.installed == 'false' uses: actions/cache@v5 with: path: /var/cache/apt/archives/ key: ${{ runner.os }}-${{ runner.arch }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.CACHE_VERSION }} - name: Run apt Update + Install - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && steps.apt-pkgs.outputs.installed == 'false' shell: bash run: | sudo apt -qq update || true @@ -277,12 +288,6 @@ runs: shell: bash run: brew install llvm@20 - # *** tinydreno *** - - name: Install tinydreno (linux) - if: inputs.tinydreno == 'true' && runner.os == 'Linux' - shell: bash - run: sudo curl -fL https://github.com/sirhcm/tinydreno/raw/refs/heads/master/libllvm-qcom.so -o /usr/lib/libllvm-qcom.so - # *** OpenCL *** - name: Install rusticl if: inputs.opencl == 'true' diff --git a/tinygrad_repo/.github/workflows/autogen.yml b/tinygrad_repo/.github/workflows/autogen.yml index 73174a6cd..df891d268 100644 --- a/tinygrad_repo/.github/workflows/autogen.yml +++ b/tinygrad_repo/.github/workflows/autogen.yml @@ -35,15 +35,15 @@ jobs: key: 'autogen' amd: 'true' llvm: 'true' - pydeps: 'pyyaml mako' + deps: 'autogen' - name: Install autogen support packages run: sudo apt-get install -y --no-install-recommends libclang-20-dev llvm-20-dev hip-dev libusb-1.0-0-dev libdrm-dev liburing-dev - name: Regenerate autogen files run: | - find tinygrad/runtime/autogen -type f -name "*.py" -not -path "*/amd/*" -not -name "__init__.py" -not -name "comgr.py" -not -name "metal.py" -not -name "iokit.py" -not -name "corefoundation.py" -not -name "libclang.py" -delete + find tinygrad/runtime/autogen -type f -name "*.py" -not -path "*/amd/*" -not -name "__init__.py" -not -name "metal.py" -not -name "iokit.py" -not -name "corefoundation.py" -not -name "libclang.py" -delete python3 -c "from tinygrad.runtime.autogen import opencl" python3 -c "from tinygrad.runtime.autogen import cuda, nvrtc, nvjitlink, nv_570, nv_580, nv_610, nv" - python3 -c "from tinygrad.runtime.autogen import comgr_3, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd, amdgpu_drm" + python3 -c "from tinygrad.runtime.autogen import comgr, comgr_3, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd, amdgpu_drm" python3 -c "from tinygrad.runtime.autogen.am import *" python3 -c "from tinygrad.runtime.autogen.nv_regs import *" python3 -c "from tinygrad.runtime.autogen import libc, kfd, io_uring, pci, vfio" @@ -54,7 +54,7 @@ jobs: python3 -c "from tinygrad.runtime.autogen import mesa" python3 -c "from tinygrad.runtime.autogen import avcodec" python3 -c "from tinygrad.runtime.autogen import llvm_qcom" - python3 -c "from tinygrad.runtime.autogen import mlx5" + python3 -c "from tinygrad.runtime.autogen import mlx5, bnxt" python3 -c "from tinygrad.runtime.autogen import ggml_common" REGEN=1 python3 -c "from tinygrad.runtime.autogen import libclang" - name: Check for differences @@ -102,42 +102,3 @@ jobs: with: name: autogen-macos-patch path: autogen-macos.patch - - autogen-comgr-2: - name: In-tree Autogen (comgr 2) - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: Setup Environment - uses: ./.github/actions/setup-tinygrad - with: - key: 'autogen-comgr' - - name: Install autogen support packages - run: | - wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - sudo tee /etc/apt/sources.list.d/rocm.list < autogen-comgr2.patch - echo "Autogen mismatch detected. Patch available at: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts" - exit 1 - fi - - name: Upload patch artifact - if: failure() - uses: actions/upload-artifact@v7 - with: - name: autogen-comgr2-patch - path: autogen-comgr2.patch diff --git a/tinygrad_repo/.github/workflows/benchmark.yml b/tinygrad_repo/.github/workflows/benchmark.yml index 739f7242c..41d78c9b2 100644 --- a/tinygrad_repo/.github/workflows/benchmark.yml +++ b/tinygrad_repo/.github/workflows/benchmark.yml @@ -88,13 +88,13 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 30 defaults: run: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -102,16 +102,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } - - name: Symlink models and datasets - run: | - mkdir -p weights - ln -s /raid/weights/LLaMA-3 weights/LLaMA-3 + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -121,18 +116,14 @@ jobs: run: python3 test/external/process_replay/reset.py - name: Run llama3.2 run: BENCHMARK_LOG=llama32_3b-f16 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m llama3.2:3b-f16 --benchmark --warmup - - name: Run qwen3.6 - # qwen3.6:35b-a3b doesn't fit on mac + - name: Run qwen3.8 + # qwen3.8:27b doesn't fit on mac if: ${{ matrix.dev != 'METAL' }} - run: BENCHMARK_LOG=qwen36_35b-a3b JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m qwen3.6:35b-a3b --benchmark --warmup + run: BENCHMARK_LOG=qwen38_27b JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m qwen3.8:27b --benchmark --warmup - name: Run olmoe # just metal for now if: ${{ matrix.dev == 'METAL' }} run: BENCHMARK_LOG=olmoe JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m olmoe --benchmark --warmup - - name: Run LLaMA-3 8B on 4 GPUs with BEAM - # only run on machines with multiple gpus - if: ${{ matrix.dev != 'METAL' }} - run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 - name: Run process replay tests uses: ./.github/actions/process-replay @@ -143,13 +134,13 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 10 defaults: run: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -157,12 +148,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -182,10 +172,6 @@ jobs: # slow on metal if: ${{ matrix.dev != 'METAL' }} run: time BENCHMARK_LOG=cifar DEFAULT_FLOAT=HALF STEPS=1000 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py - - name: Run full CIFAR training steps w 6 GPUS - # only run on machines with multiple gpus - if: ${{ matrix.dev != 'METAL' }} - run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py - name: Run process replay tests uses: ./.github/actions/process-replay @@ -196,13 +182,13 @@ jobs: fail-fast: false matrix: dev: ['AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 5 defaults: run: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -210,12 +196,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: Symlink models and datasets run: | mkdir -p extra/datasets @@ -227,15 +212,8 @@ jobs: rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: reset process replay run: test/external/process_replay/reset.py - - name: Run MLPerf resnet eval on training data - run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py - name: Run 10 MLPerf ResNet50 training steps (1 gpu) run: BENCHMARK_LOG=resnet_10steps DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py - - name: Run 10 MLPerf ResNet50 training steps (6 gpu) - run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py - - name: Run 10 MLPerf Bert training steps (6 gpu) - # TODO: remove BERT_LAYERS once scheduler is fast - run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py - name: Run process replay tests uses: ./.github/actions/process-replay @@ -246,13 +224,13 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 15 defaults: run: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -260,12 +238,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -285,6 +262,59 @@ jobs: - name: Run process replay tests uses: ./.github/actions/process-replay + multigpubenchmark: + name: Multi-GPU Benchmarks (DEV=${{ matrix.dev }}) + runs-on: [self-hosted, "${{ matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + strategy: + fail-fast: false + matrix: + dev: ['AMD', 'NV'] + timeout-minutes: 20 + defaults: + run: + shell: bash -e -o pipefail {0} + env: + DEV: ${{ matrix.dev }} + HCQ2: '0' + if: github.repository_owner == 'tinygrad' + steps: + - name: Checkout Code + uses: actions/checkout@v6 + - name: Setup (AMD) + if: ${{ matrix.dev == 'AMD' }} + run: | + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless + - name: Setup (NV) + if: ${{ matrix.dev == 'NV' }} + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } + - name: Symlink models and datasets + run: | + mkdir -p weights + mkdir -p extra/datasets + ln -s /raid/weights/LLaMA-3 weights/LLaMA-3 + ln -s /raid/datasets/imagenet extra/datasets/imagenet + - name: setup staging db + if: github.ref == 'refs/heads/update_benchmark_staging' + run: | + echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV + rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal + - name: reset process replay + run: python3 test/external/process_replay/reset.py + - name: Run LLaMA-3 8B on 4 GPUs with BEAM + run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 + - name: Run full CIFAR training steps w 6 GPUS + run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py + - name: Run MLPerf resnet eval on training data + run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py + - name: Run 10 MLPerf ResNet50 training steps (6 gpu) + run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py + - name: Run 10 MLPerf Bert training steps (6 gpu) + # TODO: remove BERT_LAYERS once scheduler is fast + run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py + - name: Run process replay tests + uses: ./.github/actions/process-replay + tests: name: Tests (DEV=${{ matrix.dev }}) runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] @@ -292,7 +322,7 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 10 defaults: run: shell: bash -e -o pipefail {0} @@ -305,12 +335,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -374,7 +403,7 @@ jobs: run: python test/external/external_benchmark_multitensor_allreduce.py - name: HEVC Decode Benchmark if: ${{ matrix.dev == 'NV' }} - run: VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py + run: IGNORE_BEAM_CACHE=1 VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py - uses: actions/upload-artifact@v7 if: ${{ matrix.dev != 'AMD' }} with: @@ -387,7 +416,7 @@ jobs: testusbgpu: name: UsbGPU Benchmark runs-on: [self-hosted, macOS] - timeout-minutes: 10 + timeout-minutes: 3 defaults: run: shell: bash -e -o pipefail {0} @@ -402,32 +431,66 @@ jobs: rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: Kill stale pids run: | - PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids - PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids - # since sudo is required for usbgpu on macos, do not write bytecode, as some of the files are owned by root + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless + ./extra/hcq/hcq_smi.py nv kill_pids --sudoless - name: UsbGPU boot time - run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus + run: GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus - name: UsbGPU tiny tests - run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py + run: GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py - name: UsbGPU copy speeds - run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds - #- name: UsbGPU openpilot test - # run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx + run: SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds - name: UsbGPU (USB4/TB) install script - run: PYTHONPATH=. sh extra/setup_tinygpu_osx.sh + run: sh extra/setup_tinygpu_osx.sh - name: UsbGPU (USB4/TB) boot time - run: PYTHONPATH=. DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus + run: DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus - name: UsbGPU (USB4/TB) tiny tests - run: PYTHONPATH=. DEV=PCI+NV:NAK python3.11 test/test_tiny.py + run: DEV=PCI+NV:NAK python3.11 test/test_tiny.py - testcommalatest: - name: comma Benchmark (0.11.2) - runs-on: [self-hosted, Linux, comma] - timeout-minutes: 12 + testcomma: + strategy: + matrix: + dev: ['QCOM', 'QCOM:IR3'] + version: ['0.11.0', '0.11.2'] + model: ['vision', 'policy', 'supercombo', 'dmonitoring'] + # exclude non-existent models + exclude: [{ version: '0.11.0', model: supercombo }, { version: '0.11.2', model: vision }, { version: '0.11.2', model: policy }] + include: + - version: '0.11.0' + model: vision + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx + timing: 18 + - version: '0.11.0' + model: policy + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx + timing: 3.4 + - version: '0.11.0' + model: dmonitoring + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx + timing: 13 + - version: '0.11.2' + model: supercombo + url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b + timing: 28 + - dev: QCOM:IR3 + version: '0.11.2' + model: supercombo + timing: 29 + - version: '0.11.2' + model: dmonitoring + url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316 + timing: 12.5 + fail-fast: false + name: openpilot ${{ matrix.version }} compile3 ${{ matrix.model }} (DEV=${{ matrix.dev }}) + runs-on: [self-hosted, Linux, comma4] + timeout-minutes: 5 defaults: run: shell: bash -e -o pipefail {0} if: github.repository_owner == 'tinygrad' + env: + DEV: ${{ matrix.dev }} + ASSERT_MIN_STEP_TIME: ${{ matrix.timing }} + BENCHMARK_LOG: ${{ matrix.dev == 'QCOM:IR3' && 'ir3_' || '' }}openpilot_${{ matrix.version }}_${{ matrix.model }} steps: - name: Checkout Code uses: actions/checkout@v6 @@ -438,45 +501,10 @@ jobs: rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: reset process replay run: test/external/process_replay/reset.py - - name: openpilot compile3 0.11.2 supercombo - run: BENCHMARK_LOG=openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b - - name: openpilot compile3 0.11.2 supercombo (from pickle) - run: BENCHMARK_LOG=openpilot_0_11_2_supercombo_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py - - name: IR3 openpilot compile3 0.11.2 supercombo - run: BENCHMARK_LOG=ir3_openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=41 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b - - name: openpilot compile3 0.11.2 dmonitoring - run: BENCHMARK_LOG=openpilot_0_11_2_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316 - - name: Run process replay tests - uses: ./.github/actions/process-replay - - testcommaold: - name: comma Benchmark (0.11.0) - runs-on: [self-hosted, Linux, comma] - timeout-minutes: 12 - defaults: - run: - shell: bash -e -o pipefail {0} - if: github.repository_owner == 'tinygrad' - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: setup staging db - if: github.ref == 'refs/heads/update_benchmark_staging' - run: | - echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV - rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - - name: reset process replay - run: test/external/process_replay/reset.py - - name: openpilot compile3 0.11.0 driving_vision - run: BENCHMARK_LOG=openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx - - name: openpilot compile3 0.11.0 driving_vision (from pickle) - run: BENCHMARK_LOG=openpilot_0_11_0_vision_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py - - name: IR3 openpilot compile3 0.11.0 driving_vision - run: BENCHMARK_LOG=ir3_openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=18 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx - - name: openpilot compile3 0.11.0 driving_policy - run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3.2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx - - name: openpilot compile3 0.11.0 dmonitoring - run: BENCHMARK_LOG=openpilot_0_11_0_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx + - name: compile + run: FLOAT16=1 IMAGE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py ${{ matrix.url }} openpilot.pkl + - name: run pickle + run: BENCHMARK_LOG="${BENCHMARK_LOG}_run_pickle" RUN_PICKLE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py - openpilot.pkl - name: Run process replay tests uses: ./.github/actions/process-replay @@ -489,15 +517,6 @@ jobs: shell: bash -e -o pipefail {0} if: github.repository_owner == 'tinygrad' steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: setup staging db - if: github.ref == 'refs/heads/update_benchmark_staging' - run: | - echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV - rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - - name: reset process replay - run: test/external/process_replay/reset.py - name: Checkout Code uses: actions/checkout@v6 - name: setup staging db @@ -510,8 +529,8 @@ jobs: - name: benchmark MobileNetV2 on DSP run: | # generate quantized weights - ln -s /data/home/tiny/tinygrad/extra/datasets/imagenet extra/datasets/imagenet - ln -s /data/home/tiny/tinygrad/testsig-*.so . + ln -s ~/tinygrad/extra/datasets/imagenet extra/datasets/imagenet + ln -s ~/tinygrad/testsig-*.so . PYTHONPATH=. DEV=CPU QUANT=1 CNT=0 python3 examples/test_onnx_imagenet.py https://github.com/xamcat/mobcat-samples/raw/refs/heads/master/onnx_runtime/InferencingSample/InferencingSample/mobilenetv2-7.onnx /tmp/model.quant.onnx # benchmark on DSP with NOOPT=1, the devectorizer has issues PYTHONPATH=. DEV=DSP NOOPT=1 CNT=2 DEBUG=2 python3 examples/test_onnx_imagenet.py /tmp/model.quant.onnx @@ -521,7 +540,7 @@ jobs: testcommausbgpubenchmark: name: UsbGPU Benchmark (comma) runs-on: [self-hosted, Linux, comma4] - timeout-minutes: 20 + timeout-minutes: 10 defaults: run: shell: bash -e -o pipefail {0} @@ -541,7 +560,7 @@ jobs: - name: openpilot run_pickle big_driving_supercombo run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo_run_pickle RUN_PICKLE=1 PICKLE_OOB=1 PYTHONPATH="." GMMU=0 DEV=USB+AMD ASSERT_MIN_STEP_TIME=50 python3 examples/openpilot/compile3.py - openpilot.pkl - name: Test copy speeds - run: SIZE=64e6 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds + run: SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds driverbenchmarks: name: PCI Driver Benchmark (DEV=${{ matrix.dev }}) @@ -550,7 +569,7 @@ jobs: fail-fast: false matrix: dev: ['AMD', 'NV'] - timeout-minutes: 20 + timeout-minutes: 5 defaults: run: shell: bash -e -o pipefail {0} @@ -562,9 +581,8 @@ jobs: uses: actions/checkout@v6 - name: Setup run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} rmmod - ./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} kill_pids + ./extra/hcq/hcq_smi.py ${{ matrix.dev }} rmmod --expect + ./extra/hcq/hcq_smi.py ${{ matrix.dev }} kill_pids --sudoless mkdir -p extra/datasets ln -s /raid/datasets/imagenet extra/datasets/imagenet - name: setup staging db @@ -599,6 +617,9 @@ jobs: run: | GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyDefaulttoCPUJit GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyCPUtoDefaultJit + - name: HEVC Decode Benchmark + if: ${{ matrix.dev == 'NV' }} + run: IGNORE_BEAM_CACHE=1 VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py - name: Run 10 MLPerf ResNet50 training steps (1 gpu) if: ${{ matrix.dev == 'NV' }} run: BENCHMARK_LOG=resnet_10steps MNISTMOCK=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py @@ -621,12 +642,12 @@ jobs: llvmspeed: name: LLVM Speed runs-on: [self-hosted, Linux, tinyboxrandom] - timeout-minutes: 20 + timeout-minutes: 10 if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code uses: actions/checkout@v6 - name: Speed Test - run: DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py + run: DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py - name: Speed Test (BEAM=2) - run: BEAM=2 DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py + run: IGNORE_BEAM_CACHE=1 BEAM=2 DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py diff --git a/tinygrad_repo/.github/workflows/docs.yml b/tinygrad_repo/.github/workflows/docs.yml index c32c1466b..b9149d9dc 100644 --- a/tinygrad_repo/.github/workflows/docs.yml +++ b/tinygrad_repo/.github/workflows/docs.yml @@ -8,7 +8,7 @@ permissions: contents: write jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 - name: Configure Git Credentials diff --git a/tinygrad_repo/.github/workflows/platform.yml b/tinygrad_repo/.github/workflows/platform.yml index bb27f446d..cdd00999e 100644 --- a/tinygrad_repo/.github/workflows/platform.yml +++ b/tinygrad_repo/.github/workflows/platform.yml @@ -166,7 +166,7 @@ jobs: uses: ./.github/actions/setup-tinygrad with: key: windows-${{ matrix.dev }}-minimal - deps: testing_unit + deps: testing_minimal pydeps: ${{ matrix.dev == 'WEBGPU' && 'dawn-python' || '' }} - name: Set env shell: bash @@ -179,35 +179,3 @@ jobs: - name: Run test_tiny shell: bash run: python -m pytest -n=auto test/test_tiny.py --durations=20 - - - qcomclcompiletests: - name: Compile-only (QCOM CL) - runs-on: ubuntu-24.04-arm - timeout-minutes: 15 - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: Setup Environment - uses: ./.github/actions/setup-tinygrad - with: - key: compile-qcomcl - deps: testing_unit - tinydreno: 'true' - - name: Set env - shell: bash - run: printf "DEV=NULL:QCOMCL:a630\nNULL_ALLOW_COPYOUT=1" >> $GITHUB_ENV - - name: Run test_ops - shell: bash - run: | - python -c "from tinygrad import Device; assert Device.DEFAULT == 'NULL'" - DEBUG=4 python3 test/backend/test_ops.py TestOps.test_add - python -m pytest -n=auto test/backend/test_ops.py --durations=20 - - name: Run test_ops (IMAGE) - shell: bash - env: - IMAGE: 1 - DEV: "NULL:QCOMCL:a630,IMAGE_PITCH_ALIGNMENT=64" - run: | - DEBUG=4 python test/backend/test_ops.py TestOps.test_gemm | grep read_imagef - python -m pytest -n=auto test/backend/test_ops.py --durations=20 diff --git a/tinygrad_repo/.github/workflows/python-publish.yml b/tinygrad_repo/.github/workflows/python-publish.yml index 397467a86..23aef5aca 100644 --- a/tinygrad_repo/.github/workflows/python-publish.yml +++ b/tinygrad_repo/.github/workflows/python-publish.yml @@ -10,7 +10,7 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 - name: Set up Python diff --git a/tinygrad_repo/.github/workflows/szdiff.yml b/tinygrad_repo/.github/workflows/szdiff.yml index 37a985f0c..900282151 100644 --- a/tinygrad_repo/.github/workflows/szdiff.yml +++ b/tinygrad_repo/.github/workflows/szdiff.yml @@ -10,7 +10,7 @@ concurrency: jobs: checkbranch: name: Check PR Branch status - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: branchstat: ${{ steps.brstat.outputs.stat}} steps: @@ -44,7 +44,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: checkbranch if: needs.checkbranch.outputs.branchstat == 'false' steps: @@ -87,7 +87,7 @@ jobs: name: Core Library Line Difference permissions: pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: checkbranch if: needs.checkbranch.outputs.branchstat == 'true' steps: diff --git a/tinygrad_repo/.github/workflows/test.yml b/tinygrad_repo/.github/workflows/test.yml index 45dfa5649..3b2e2bd58 100644 --- a/tinygrad_repo/.github/workflows/test.yml +++ b/tinygrad_repo/.github/workflows/test.yml @@ -21,7 +21,7 @@ concurrency: jobs: docs: name: Docs - runs-on: &linux ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 10 env: CHECK_OOB: 0 @@ -31,8 +31,7 @@ jobs: - name: Setup Environment uses: ./.github/actions/setup-tinygrad with: - deps: docs - pydeps: "capstone torch" + deps: "docs testing_minimal" - name: Build wheel and show size run: | uv build --wheel @@ -61,7 +60,7 @@ jobs: torchbackend: name: Torch Backend Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -73,10 +72,7 @@ jobs: deps: testing_unit pydeps: "pillow torchvision expecttest" llvm: 'true' - - name: Install ninja - run: | - sudo apt update || true - sudo apt install -y --no-install-recommends ninja-build + ninja: 'true' - name: Test ResNet-18 run: DEBUG=2 python3 extra/torch_backend/example.py - name: Test one op in torch tests @@ -86,9 +82,26 @@ jobs: - name: Custom tests run: DEV=CPU:LLVM GPUS=4 TINY_BACKEND=1 python3 -m pytest -nauto extra/torch_backend/test.py extra/torch_backend/test_inplace.py extra/torch_backend/test_multigpu.py extra/torch_backend/test_kernel_fusion.py --durations=20 + torchbackendtrain: + name: Torch Backend Training + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} + timeout-minutes: 15 + steps: + - name: Checkout Code + uses: actions/checkout@v6 + - name: Setup Environment + uses: ./.github/actions/setup-tinygrad + with: + key: torch-backend-pillow-torchvision-et-pt + deps: testing_unit + llvm: 'true' + ninja: 'true' + - name: Test beautiful_mnist in torch with TINY_BACKEND + run: STEPS=20 DEV=CPU TARGET_EVAL_ACC_PCT=90.0 MAX_BUFFER_SIZE=0 TINY_BACKEND=1 python3 examples/other_mnist/beautiful_mnist_torch.py + bepython: name: Python Backend - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -126,7 +139,7 @@ jobs: linter: name: Linters - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 10 steps: @@ -157,7 +170,7 @@ jobs: nulltest: name: Null Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: @@ -191,7 +204,7 @@ jobs: unittest: name: Unit Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: @@ -220,7 +233,7 @@ jobs: - name: Run process replay tests uses: ./.github/actions/process-replay - name: Repo line count <= 26000 lines - run: MAX_LINE_COUNT=26000 python sz.py + run: MAX_LINE_COUNT=26500 python sz.py spec: strategy: @@ -228,7 +241,7 @@ jobs: matrix: group: [1, 2] name: SPEC=2 (${{ matrix.group }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -244,7 +257,7 @@ jobs: fuzzing: name: Fuzzing - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 10 steps: - name: Checkout Code @@ -260,7 +273,7 @@ jobs: testopenclimage: name: CL IMAGE Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -280,7 +293,7 @@ jobs: testopenpilot: name: openpilot Compile Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -309,7 +322,7 @@ jobs: testonnxcpu: name: ONNX (CPU) Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 steps: @@ -328,7 +341,7 @@ jobs: testoptim: name: Optimization Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 steps: - name: Checkout Code @@ -360,7 +373,7 @@ jobs: testllm: name: Test LLM - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 env: CHECK_OOB: 0 @@ -377,17 +390,17 @@ jobs: run: | parallel --link --tagstring '[{1}]' '{2}' \ ::: llama 'llama q4' qwen3.5 qwen \ - ::: $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model llama3.2:1b | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model llama3.2:1b-q4 | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model qwen3.5:0.8b | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a female chicken called? Answer with only one word." | python3 -m tinygrad.llm --model qwen3:0.6b | tee /dev/stderr | grep -i hen' + ::: $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model llama3.2:1b | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model llama3.2:1b-q4 | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model qwen3.5:0.8b | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a female chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model qwen3:0.6b | tee /dev/stderr | grep -i hen' # NOTE: qwen is dumb and only knows about female chickens # ****** Models Tests ****** testmodels: name: Models - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -407,7 +420,7 @@ jobs: testdsp: name: Linux (DSP) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -435,7 +448,7 @@ jobs: - 'WEBGPU' name: Linux (DEV=${{ matrix.dev }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 steps: - name: Checkout Code @@ -461,7 +474,7 @@ jobs: testamdasm: name: AMD ASM IDE - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 env: DEV: MOCKKFD+AMD @@ -491,7 +504,7 @@ jobs: - name: Run AMD renderer tests (AMD:LLVM) run: DEV=MOCKKFD+AMD:LLVM python -m pytest -n=auto test/amd/ --durations 20 - name: Run SQTT profiling tests - run: PROFILE=1 SQTT=1 python3 -m pytest -n=auto test/amd/test_sqtt_profiler.py + run: VIZ=-2 python3 -m pytest -n=auto test/amd/test_sqtt_profiler.py - name: Run AMD emulated tests on NULL backend env: AMD: 0 @@ -507,7 +520,7 @@ jobs: hcq2: name: hcq2 - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 5 steps: - name: Checkout Code @@ -521,16 +534,15 @@ jobs: - name: Run HCQ2 tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/test_tiny.py - name: Run HCQ2 multi-device tests - run: | - HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_multitensor.py \ - TestMultiTensor.test_simple_add TestMultiTensor.test_shard_reduce \ - TestMultiTensor.test_backward_sum TestMultiTensor.test_matmul_shard_0_0 + run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python -m pytest -n=auto test/backend/test_multitensor.py - name: Run HCQ2 JIT tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_jit.py + - name: Run HCQ2 unit tests + run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python -m pytest test/device/test_hcq2.py testmockam: name: Linux (am) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 env: DEV: MOCKPCI+AMD @@ -566,7 +578,7 @@ jobs: arch: [gfx1100, gfx1201, gfx950] name: Linux (${{ matrix.backend }} ${{ matrix.arch }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 env: DEV: MOCKKFD+AMD:${{ matrix.backend == 'amdllvm' && 'LLVM' || '' }}:${{ matrix.arch }} @@ -589,7 +601,7 @@ jobs: if: ${{ matrix.backend == 'amd' && matrix.arch == 'gfx950' }} run: PYTHONPATH=. DEV=NULL:HIP:gfx950 MXFP4=1 LLAMA_LAYERS=2 BENCHMARK=3 NULL_ALLOW_COPYOUT=1 NO_HIPCC=1 ROCM_PATH=/opt/rocm JITBEAM=0 examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/profile.sh - name: Run pytest (amd) - run: python -m pytest -n=auto test/backend/test_ops.py test/backend/test_dtype.py test/backend/test_dtype_alu.py test/backend/test_linearizer.py test/backend/test_randomness.py test/backend/test_jit.py test/backend/test_graph.py test/backend/test_multitensor.py test/device/test_hcq.py test/external/external_test_am.py test/backend/test_asm_gemm.py::TestAsmGEMM test/opt/test_tensor_cores.py --durations=20 + run: python -m pytest -n=auto test/backend/test_ops.py test/backend/test_dtype.py test/backend/test_dtype_alu.py test/backend/test_linearizer.py test/backend/test_randomness.py test/backend/test_jit.py test/backend/test_graph.py test/backend/test_multitensor.py test/device/test_hcq.py test/external/external_test_am.py test/backend/test_asm_gemm.py::TestAsmGEMM --durations=20 - name: Run disk copy tests run: python -m pytest test/unit/test_disk_tensor.py -k test_copy_from_disk - name: Run TRANSCENDENTAL math @@ -604,7 +616,7 @@ jobs: backend: [ptx, nv] name: Linux (${{ matrix.backend }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 env: FORWARD_ONLY: 1 @@ -638,10 +650,17 @@ jobs: strategy: fail-fast: false matrix: - backend: [ir3, nak] - name: Compile-only (${{ matrix.backend }}) - runs-on: *linux + dev: + - 'NULL:IR3:a630' + - 'NULL:QCOMCL:a630' + - 'NULL:NAK:sm_120' + name: Compile-only (DEV=${{ matrix.dev }}) + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 + env: + NULL_ALLOW_COPYOUT: 1 + DEV: ${{ matrix.dev }}${{ contains(matrix.dev, 'a630') && ',IMAGE_PITCH_ALIGNMENT=64' || '' }} + IMAGE: ${{ contains(matrix.dev, 'a630') && '1' || '0' }} steps: - name: Checkout Code uses: actions/checkout@v6 @@ -650,21 +669,15 @@ jobs: with: key: compile-${{ matrix.backend }} deps: "testing_unit mesa" - - name: Set env + qemu: ${{ contains(matrix.dev, 'QCOMCL') }} + - name: Test IMAGE shell: bash - run: printf "NULL_ALLOW_COPYOUT=1\n${{ matrix.backend == 'ir3' && 'DEV=NULL:IR3:a630' || matrix.backend == 'nak' && 'DEV=NULL:NAK:sm_120' }}" >> $GITHUB_ENV + if: contains(matrix.dev, 'a630') + run: DEBUG=7 python3 test/backend/test_ops.py TestOps.test_gemm | grep isam - name: Run test_ops shell: bash run: | python -c "from tinygrad import Device; assert Device.DEFAULT == 'NULL'" DEBUG=4 python3 test/backend/test_ops.py TestOps.test_add - python -m pytest -n=auto test/backend/test_ops.py --durations=20 - - name: Run test_ops (IMAGE) - if: matrix.backend == 'ir3' - shell: bash - env: - IMAGE: 1 - DEV: "NULL:IR3:a630,IMAGE_PITCH_ALIGNMENT=64" - run: | - DEBUG=4 python3 test/backend/test_ops.py TestOps.test_gemm | grep image_load - python -m pytest -n=auto test/backend/test_ops.py --durations=20 + # QCOMCL compiles in qemu, too slow for parallel workers + ${{ contains(matrix.dev, 'QCOMCL') && 'PARALLEL=0' || '' }} python -m pytest -n=auto test/backend/test_ops.py --durations=20 diff --git a/tinygrad_repo/.gitignore b/tinygrad_repo/.gitignore index 12333d702..97d110257 100644 --- a/tinygrad_repo/.gitignore +++ b/tinygrad_repo/.gitignore @@ -69,3 +69,4 @@ mutants dagre/ graphlib/ uv.lock +pi_session_window0.jsonl diff --git a/tinygrad_repo/.pre-commit-config.yaml b/tinygrad_repo/.pre-commit-config.yaml index f5fdd5aae..380ddca53 100644 --- a/tinygrad_repo/.pre-commit-config.yaml +++ b/tinygrad_repo/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: pass_filenames: false - id: tests name: comprehensive test suite - entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/backend/test_ops.py test/backend/test_schedule.py test/unit/test_assign.py test/backend/test_tensor.py test/backend/test_jit.py test/unit/test_schedule_cache.py test/null/test_pattern_matcher.py test/null/test_uop_symbolic.py test/unit/test_helpers.py + entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/backend/test_ops.py test/backend/test_schedule.py test/backend/test_assign.py test/backend/test_tensor.py test/backend/test_jit.py test/unit/test_schedule_cache.py test/null/test_pattern_matcher.py test/null/test_uop_symbolic.py test/unit/test_helpers.py language: system always_run: true pass_filenames: false diff --git a/tinygrad_repo/AGENTS.md b/tinygrad_repo/AGENTS.md index 962039e68..00c02b4ab 100644 --- a/tinygrad_repo/AGENTS.md +++ b/tinygrad_repo/AGENTS.md @@ -4,3 +4,4 @@ - Run `python -m mypy tinygrad/` to typecheck - Run `python -m ruff check .` to lint - Read `./tinygrad/viz/README.md` for profiling and debugging rewrite rules +- Do not do amend commits. Always do a new commit if a force push to origin would be required. diff --git a/tinygrad_repo/README.md b/tinygrad_repo/README.md index 4e05df80e..0e7375ce0 100644 --- a/tinygrad_repo/README.md +++ b/tinygrad_repo/README.md @@ -140,7 +140,7 @@ Documentation along with a quick start guide can be found on the [docs website]( ```python from tinygrad import Tensor -x = Tensor.eye(3) +x = Tensor.eye(3).clone() # clone to make it a buffer y = Tensor([[2.0,0,-2.0]]) z = y.matmul(x).sum() z.backward() diff --git a/tinygrad_repo/TINYGRAD_COMMIT b/tinygrad_repo/TINYGRAD_COMMIT index 3c89d0c79..f09a3c1ee 100644 --- a/tinygrad_repo/TINYGRAD_COMMIT +++ b/tinygrad_repo/TINYGRAD_COMMIT @@ -1 +1 @@ -8611fe22a7fcc7d1928bbde19ded66277cb12f3e +f6fc4e3f2c3db5fae1e19cbfbc3ad9fc579a12ae diff --git a/tinygrad_repo/conftest.py b/tinygrad_repo/conftest.py index 8c84ede27..61d5ea1ac 100644 --- a/tinygrad_repo/conftest.py +++ b/tinygrad_repo/conftest.py @@ -2,7 +2,7 @@ import os, pytest, signal, threading @pytest.hookimpl(wrapper=True) def pytest_runtest_call(item): - t = threading.Timer(int(os.getenv("TEST_TIMEOUT", 300)), os.kill, args=(os.getpid(), signal.SIGABRT)) + t = threading.Timer(int(os.getenv("TEST_TIMEOUT", 90)), os.kill, args=(os.getpid(), signal.SIGABRT)) t.start() try: yield finally: diff --git a/tinygrad_repo/docs/abstractions4.py b/tinygrad_repo/docs/abstractions4.py index d28a3e3a3..9e00f732f 100644 --- a/tinygrad_repo/docs/abstractions4.py +++ b/tinygrad_repo/docs/abstractions4.py @@ -122,7 +122,7 @@ def example_5_custom_assembly(a:Tensor, correct): offset_dwords = (self.labels[inst._target] - inst._pos - inst.size()) // 4 if not -32768 <= offset_dwords <= 32767: raise ValueError(f"branch to '{inst._target}' offset {offset_dwords} exceeds simm16 range") inst.simm16 = offset_dwords - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in self.instructions])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in self.instructions])))) CU_COUNT = 32 LANES = 64 diff --git a/tinygrad_repo/docs/developer/speed.md b/tinygrad_repo/docs/developer/speed.md index 460e093b9..4cf4f3c66 100644 --- a/tinygrad_repo/docs/developer/speed.md +++ b/tinygrad_repo/docs/developer/speed.md @@ -52,7 +52,7 @@ In `kernel.py` we have a set of `OptOps`, these control the parameters of the sp The main bottleneck in most kernels is accessing memory. In a freshman algorithms class, you'll learn about cache aware matrix multiplication, and this is all forms of that. While the same math is run, the order in which you run it can have large impacts on the speed depending on if the data you are loading. OptOps will change this order. -Memory, even cache, is often much slower than accessing the register file. The amount of times data is used in math is called the "arithmetic intensity". For operations like BS=1 GEMV, the arithmetic intensity is 1, but for GEMMs and convs it can be much higher. OptOps like UPCAST and UNROLL can increase this, but be careful of making them too large, as if there's too much register pressure on the GPU the warp scheduler may not be able to fit many warps, or even worse, it could be spilling to local memory. +Memory, even cache, is often much slower than accessing the register file. The amount of times data is used in math is called the "arithmetic intensity". For operations like BS=1 GEMV, the arithmetic intensity is 1, but for GEMMs and convs it can be much higher. Splitting an axis into UPCAST can increase this, but be careful of making them too large, as if there's too much register pressure on the GPU the warp scheduler may not be able to fit many warps, or even worse, it could be spilling to local memory. 4090s have 1 TB/s of ram bandwidth and ~160 TFLOPS of compute, so you need to use each loaded value ~100 times. The L1 cache has around 40 TB/s of bandwidth, so in order to get full compute utilization you need to use each value ~4 times. diff --git a/tinygrad_repo/examples/mlperf/dataloader.py b/tinygrad_repo/examples/mlperf/dataloader.py index 275bcd4ae..3f5db0c74 100644 --- a/tinygrad_repo/examples/mlperf/dataloader.py +++ b/tinygrad_repo/examples/mlperf/dataloader.py @@ -5,7 +5,7 @@ from multiprocessing import Queue, Process, shared_memory, connection, Lock import numpy as np from tinygrad import dtypes, Tensor -from tinygrad.helpers import getenv, prod, Context, round_up, tqdm, OSX, NUM_CPU_THREADS +from tinygrad.helpers import getenv, prod, Context, round_up, tqdm, OSX, CPU_COUNT from tinygrad.nn.state import TensorIO ### ResNet @@ -131,7 +131,7 @@ def batch_load_resnet(batch_size=64, val=False, shuffle=True, seed=None, pad_fir else: X = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/{shm_name}") Y = [None] * (batch_size*BATCH_COUNT) - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): p = Process(target=loader_process, args=(q_in, q_out, X, seed)) p.daemon = True p.start() @@ -212,7 +212,7 @@ def batch_load_train_bert(BS:int, seed:int|None=None): rng.shuffle(fs) train_files.append(fs.pop(0)) - cycle_length = min(NUM_CPU_THREADS.value, len(train_files)) + cycle_length = min(CPU_COUNT, len(train_files)) assert cycle_length > 0, "cycle_length must be greater than 0" dataset = InterleavedDataset(train_files, cycle_length) @@ -301,7 +301,7 @@ def batch_load_unet3d(preprocessed_dataset_dir:Path, batch_size:int=6, val:bool= X = Tensor.empty(*sz, dtype=dtypes.float32, device=f"disk:/dev/shm/{shm_name_x}") Y = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/{shm_name_y}") - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): proc = Process(target=load_unet3d_data, args=(preprocessed_dataset_dir, seed, queue_in, queue_out, X, Y)) proc.daemon = True proc.start() @@ -437,7 +437,7 @@ def batch_load_retinanet(dataset, val:bool, base_dir:Path, batch_size:int=32, sh dataset_iter = iter(image_ids) try: - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): proc = Process( target=load_retinanet_data, args=(base_dir, val, queue_in, queue_out, imgs, boxes, labels), diff --git a/tinygrad_repo/examples/mlperf/model_train.py b/tinygrad_repo/examples/mlperf/model_train.py index 47c4977e3..d72399d83 100644 --- a/tinygrad_repo/examples/mlperf/model_train.py +++ b/tinygrad_repo/examples/mlperf/model_train.py @@ -1667,15 +1667,14 @@ def train_llama3(): def train_gptoss(): from examples.mlperf.models.gpt_oss import GPTOSS, GPT_OSS_20B, apply_grad, FP8_DTYPE from examples.mlperf.lr_schedulers import CosineAnnealingLRWithWarmup - from examples.mlperf.optim import GradAccClipAdamW, GradAccClipAdamWGroup, clip_grads + from examples.mlperf.optim import GradAccClipAdamW, GradAccClipAdamWGroup, fclip_grads BENCHMARK = getenv("BENCHMARK") config = {} BASEDIR = config["BASEDIR"] = Path(getenv("BASEDIR", "/raid/datasets/c4-8b/")) BS = config["BS"] = getenv("BS", 16) - grad_acc = config["GRADIENT_ACC_STEPS"] = getenv("GRADIENT_ACC_STEPS", 1) - GBS = config["GLOBAL_BATCH_SIZE"] = BS * grad_acc + GBS = config["GLOBAL_BATCH_SIZE"] = BS SEED = config["SEED"] = getenv("SEED", 5760) DATA_SEED = config["DATA_SEED"] = getenv("DATA_SEED", SEED) SEQLEN = config["SEQLEN"] = getenv("SEQLEN", 8192) @@ -1737,13 +1736,13 @@ def train_gptoss(): params_wd = [p for p in params if p.ndim >= 3] params_no_wd = [p for p in params if p.ndim < 3] optim = GradAccClipAdamWGroup( - GradAccClipAdamW(params_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=opt_adamw_weight_decay, grad_acc=grad_acc, device=optim_device), - GradAccClipAdamW(params_no_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=0.0, grad_acc=grad_acc, device=optim_device), + GradAccClipAdamW(params_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=opt_adamw_weight_decay, grad_acc=1, device=optim_device), + GradAccClipAdamW(params_no_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=0.0, grad_acc=1, device=optim_device), ) for p in optim.params: - grad_dtype = dtypes.bfloat16 if p.dtype == FP8_DTYPE else p.dtype - p.grad = p.zeros_like(dtype=grad_dtype).contiguous() + p.grad = p.zeros_like(dtype=dtypes.bfloat16 if p.dtype == FP8_DTYPE else p.dtype).contiguous() + if getattr(p, "_zero2", False): p.grad = optim.optimizers[0]._zero_shard(p.grad) grads = [p.grad for p in optim.params] from extra.gemm.cdna_asm_gemm import _mx_block_scale @@ -1770,31 +1769,32 @@ def train_gptoss(): @TinyJit @Context(TRAINING=1) - def minibatch(tokens:Tensor): + def step(tokens:Tensor): if is_dp: tokens = tokens.to(None).shard(device, 0) if not is_sharding: tokens = tokens.to(None) + logits:Tensor = model(tokens[:, :-1], save=True) - loss = logits.sparse_categorical_crossentropy(tokens[:, 1:]) + if getenv("FUSED_CE", 0): + from extra.llama_kernels.fused_ce import fused_ce_loss + loss = fused_ce_loss(logits.cast(dtypes.bfloat16), tokens[:, 1:], label_smoothing=0.0) + else: + loss = logits.sparse_categorical_crossentropy(tokens[:, 1:]) for g, new_g in zip(grads, loss.gradient(*optim.params)): apply_grad(g, new_g.uop) - loss_cpu = loss.flatten().float().to("CPU") - return loss_cpu.realize(*grads) + Tensor.realize(loss, *grads) - @TinyJit - def optim_step(): - grad_norm = clip_grads(grads, grad_acc, 1.0) - optim.fstep(grads, grad_norm) + clipped_grads, grad_norm = fclip_grads(grads, 1.0) + optim.fstep(clipped_grads, grad_norm) scheduler.step() - for g in grads: g.assign(0) - + loss_cpu = loss.flatten().float().to("CPU") lr_cpu = optim.lr.float().to("CPU") grad_norm_cpu = grad_norm.float().to("CPU") - Tensor.realize(lr_cpu, grad_norm_cpu, *grads, *fp8_inv_scales) + Tensor.realize(loss_cpu, lr_cpu, grad_norm_cpu, *grads, *fp8_inv_scales) - return lr_cpu, grad_norm_cpu + return loss_cpu, lr_cpu, grad_norm_cpu @TinyJit @Context(TRAINING=0) @@ -1843,30 +1843,20 @@ def train_gptoss(): profile_marker(f"train @ {i}") st = time.perf_counter() - stopped = False - losses, data_time, dev_time = [], 0, 0 - for _ in range(grad_acc if i >= 2 else 1): - ist = time.perf_counter() - try: tokens = next(train_iter) - except StopIteration: - stopped = True - break - mst = time.perf_counter() - data_time += mst - ist - losses.append(minibatch(tokens).item()) - dev_time += time.perf_counter() - mst - if stopped: break + ist = time.perf_counter() - gt = time.perf_counter() - ret = optim_step() - lr, grad_norm = ret[0].item(), ret[1].item() + try: tokens = next(train_iter) + except StopIteration: break + mst = time.perf_counter() + data_time = mst - ist + + ret = step(tokens) + dev_time = time.perf_counter() - mst + + loss, lr, grad_norm = ret[0].item(), ret[1].item(), ret[2].item() et = time.perf_counter() - loss = sum(losses) / len(losses) - optim_time = et - gt - dev_time += optim_time step_time = et - st - gbs_time = gt - st if BENCHMARK: step_times.append(step_time) i += 1 @@ -1876,7 +1866,7 @@ def train_gptoss(): gflops = GlobalCounters.global_ops / 1e9 / dev_time mfu = ((6 * num_params * SEQLEN * GBS) / (dev_time * device_count * 4.6e15)) * 100 tqdm.write( - f"{i:5} {step_time:.3f} s step, {gbs_time:.3f} s gbs, {optim_time:.3f} s optim, {data_time:.3f} s data, {loss:.4f} loss, " \ + f"{i:5} {step_time:.3f} s step, {dev_time:.3f} s dev, {data_time:.3f} s data, {loss:.4f} loss, " \ f"{lr:.12f} LR, {grad_norm:.6f} grad_norm, {mem_gb:.2f} GB used, {gflops:9.2f} GFLOPS, {mfu:5.2f}% MFU") if DEBUG >= 1: tqdm.write(" mem per device: " + ', '.join(f"{dev}: {mem/1e9:.2f} GB" for dev, mem in sorted(GlobalCounters.mem_used_per_device.items()))) @@ -1886,8 +1876,6 @@ def train_gptoss(): "train/lr": lr, "train/grad_norm": grad_norm, "train/step_time": step_time, - "train/gbs_time": gbs_time, - "train/optim_time": optim_time, "train/dev_time": dev_time, "train/data_time": data_time, "train/mem": mem_gb, diff --git a/tinygrad_repo/examples/mlperf/models/gpt_oss.py b/tinygrad_repo/examples/mlperf/models/gpt_oss.py index 1e157a94d..23d41650d 100644 --- a/tinygrad_repo/examples/mlperf/models/gpt_oss.py +++ b/tinygrad_repo/examples/mlperf/models/gpt_oss.py @@ -12,7 +12,7 @@ from tinygrad.helpers import Timing, colored, GlobalCounters, profile_marker from tinygrad.uop.ops import Ops, UOp from extra.models.llama import apply_rotary_emb from extra.llama_kernels.rmsnorm import rmsnorm -from extra.gemm.cdna_asm_gemm import _mx_block_scale, _mx_block_scale_3d, quantize_mxfp8 +from extra.gemm.cdna_asm_gemm import _mx_block_scale, _mx_block_scale_3d, quantize_mxfp8, asm_gemm, can_use_asm_gemm from extra.gemm.moe_gemm import grouped_mx_gemm from extra.gemm.moe_routing import route, dispatch, combine @@ -61,7 +61,25 @@ def dequant_weight(w_q:Tensor, w_scale:Tensor) -> Tensor: call = UOp.maketuple(fxn.uop).call(w_q.uop, w_scale.uop, grad_fxn=_dequant_bwd) return Tensor(call.gettuple(0)) -def matmul_mx(x:Tensor, w_q:Tensor, w_scale:Tensor) -> Tensor: +def matmul_mx(x:Tensor|tuple[Tensor, Tensor], w_q:Tensor, w_scale:Tensor) -> Tensor: + if isinstance(x, tuple): + assert ASM_GEMM, "pre-quantized MXFP8 input requires ASM_GEMM" + from extra.gemm.cdna_asm_gemm import asm_gemm, can_use_asm_gemm, mx_pack + x_q, x_e8 = x + l_shape, padded = x_q.shape[:-1], x_q.shape[-1] + x_q, x_e8 = x_q.reshape(-1, padded), x_e8.reshape(-1, padded // 32) + K, N = w_q.shape[1], w_q.shape[0] + assert padded >= K and (padded - K) % 32 == 0 and x_e8.shape[-1] == padded // 32 + wq, ws = w_q, w_scale + if (pad := padded - K): + wq = wq.pad(((0, 0), (0, pad))) + ws = ws.pad(((0, 0), (0, pad // 32)), value=127).cast(dtypes.uint8) + if (npad := (-N) % 256): + wq = wq.pad(((0, npad), (0, 0))) + ws = ws.pad(((0, npad), (0, 0)), value=127).cast(dtypes.uint8) + assert can_use_asm_gemm(x_q, wq.T) + out = asm_gemm(x_q, wq.T, mx=True, mx_scales=(mx_pack(x_e8), x_e8, mx_pack(ws), ws), mx_w_stored=True) + return (out[:, :N] if npad else out).reshape(*l_shape, N).cast(dtypes.bfloat16) l_shape = x.shape[:-1] if ASM_GEMM: from extra.gemm.cdna_asm_gemm import asm_gemm, can_use_asm_gemm, mx_pack @@ -146,6 +164,7 @@ class GPTOSS: return w_q, w_e8.is_param_(False) if moe: qs = [_one(*shape[1:]) for _ in range(shape[0])] + for q in qs: q[0]._zero2 = True # grad arrives sharded on the expert axis under ZeRO-2 (moe_gemm) return [q[0] for q in qs], [q[1] for q in qs] return _one(*shape) @@ -174,20 +193,30 @@ class GPTOSS: def attention(self, x:Tensor, freqs_cis:Tensor, mask:Tensor, sliding:bool, *, attention_norm:Tensor, wqkv:Tensor, wqkv_scale:Tensor, wqkv_bias:Tensor, wo:Tensor, wo_scale:Tensor, wo_bias:Tensor, sinks:Tensor): bsz, seqlen, _ = x.shape - x_normed, rrms = rmsnorm(x, self.norm_eps) - qkv = matmul_mx(x_normed * attention_norm, wqkv, wqkv_scale) + wqkv_bias + + if getenv("FUSED_RMSNORM_MUL", 0): + from extra.gptoss_kernels.rmsnorm import rmsnorm_mul + x_normed, rrms = rmsnorm_mul(x, attention_norm, self.norm_eps) + norm_saves = [x_normed, rrms] + else: + x_normed, rrms = rmsnorm(x, self.norm_eps) + qkv = matmul_mx(x_normed * attention_norm, wqkv, wqkv_scale) + wqkv_bias + norm_saves = [x_normed, rrms] + qkv = qkv.reshape(bsz, seqlen, self.n_kv_heads, self.n_rep + 2, self.head_dim) xq = qkv[:, :, :, :self.n_rep].reshape(bsz, seqlen, self.n_heads, self.head_dim) xk, xv = qkv[:, :, :, self.n_rep], qkv[:, :, :, self.n_rep + 1] xq, xk = apply_rotary_emb(xq, xk, freqs_cis) xq, xk, xv = xq.cast(dtypes.bfloat16), xk.cast(dtypes.bfloat16), xv.cast(dtypes.bfloat16) # (B,N,H,D)/(B,N,KV,D) - if sliding: - attn = self._sliding_attention(xq, xk, xv, sinks) - elif getenv("HK_FLASH_ATTENTION"): + fa_saves = [] + if getenv("HK_FLASH_ATTENTION"): from extra.thunder.amd.fa import flash_attention - attn, *_ = flash_attention(xq, xk, xv, is_causal=True, write_flat=True, sinks=sinks) + attn, _, l_vec = flash_attention(xq, xk, xv, is_causal=True, write_flat=True, sinks=sinks, window=self.sliding_window if sliding else 0) attn = attn.reshape(bsz, seqlen, self.n_heads * self.head_dim) + fa_saves = [xq, xk, xv, l_vec] + elif sliding: + attn = self._sliding_attention(xq, xk, xv, sinks) else: xqm = xq.reshape(bsz, seqlen, self.n_kv_heads, self.n_rep, self.head_dim).permute(0, 2, 3, 1, 4) xkm, xvm = xk.permute(0, 2, 1, 3).unsqueeze(2), xv.permute(0, 2, 1, 3).unsqueeze(2) @@ -199,13 +228,19 @@ class GPTOSS: attn = (w @ xvm).permute(0, 3, 1, 2, 4).reshape(bsz, seqlen, self.n_heads * self.head_dim) out = matmul_mx(attn, wo, wo_scale) + wo_bias - return out, [x_normed, rrms, attn] + return out, [attn] + norm_saves + fa_saves def feed_forward(self, x:Tensor, *, ffn_norm:Tensor, gate:Tensor, gate_bias:Tensor, w_gate_up:Tensor, w_gate_up_scale:Tensor, w_gate_up_bias:Tensor, w_down:Tensor, w_down_scale:Tensor, w_down_bias:Tensor): - x_normed, rrms = rmsnorm(x, self.norm_eps) - inp = x_normed * ffn_norm + if getenv("FUSED_RMSNORM_MUL", 0): + from extra.gptoss_kernels.rmsnorm import rmsnorm_mul + x_normed, rrms = rmsnorm_mul(x, ffn_norm, self.norm_eps) + inp = x_normed + else: + x_normed, rrms = rmsnorm(x, self.norm_eps) + inp = x_normed * ffn_norm + logits = inp.float() @ gate.float().T + gate_bias.float() dim, inter = self.dim, self.intermediate_size @@ -220,6 +255,7 @@ class GPTOSS: z = grouped_mx_gemm(_pad_cols(y.cast(dtypes.bfloat16)), (w_down, w_down_scale), r.off)[:, :dim] \ + (onehot @ w_down_bias.float()).cast(dtypes.bfloat16) out = combine(z, r, inp.shape[0], self.experts_per_tok).reshape(bsz, seqlen, dim) + return out, [x_normed, rrms, xg, h, y, z, r.weights, r.dest_row, r.off] else: thresh = logits.topk(self.experts_per_tok)[0][..., -1:] weights = (logits >= thresh).where(logits, -float("inf")).softmax(-1) @@ -263,7 +299,11 @@ class GPTOSS: w_down=self.w_down[i], w_down_scale=self.w_down_scale[i], w_down_bias=self.w_down_bias[i]) h, *_ = self.run_layer(h, freqs_cis, mask_full, i % 2 == 0, attn_kwargs, ffn_kwargs, save=save) - logits = self.norm(h) @ self.output.T + h_normed = self.norm(h) + pad = (-self.dim) % 256 + h_padded, w_padded = h_normed.pad((None, None, (0, pad))), self.output.pad(((0, 0), (0, pad))) + if ASM_GEMM and can_use_asm_gemm(h_padded, w_padded.T): logits = asm_gemm(h_padded, w_padded.T) + else: logits = h_normed @ self.output.T return logits def _get_pads(uop:UOp) -> list[UOp]: @@ -274,14 +314,14 @@ def apply_grad(grad_buf:Tensor, new_grad:UOp): pads = _get_pads(new_grad) if len(pads) <= 1: new_grad = new_grad.cast(grad_buf.dtype) - grad_buf.uop = grad_buf.uop.after(grad_buf.uop.store(grad_buf.uop + new_grad)) + grad_buf.uop = grad_buf.uop.after(grad_buf.uop.store(new_grad)) return cur = grad_buf.uop for pad in sorted(pads, key=lambda p: p.marg[0][0] if p.op == Ops.PAD else 0, reverse=True): if pad.op == Ops.PAD: - grad_shrink = tuple([(p[0], s+p[0]) for s,p in zip(pad.src[0].shape, pad.marg)]) + grad_shrink = tuple((p[0], s+p[0]) for s,p in zip(pad.src[0].shape, pad.marg)) buf_slice = cur.shrink(grad_shrink) - cur = cur.after(buf_slice.store(buf_slice + pad.src[0].cast(cur.dtype))) + cur = cur.after(buf_slice.store(pad.src[0].cast(cur.dtype))) else: cur = cur.after(cur.store(cur + pad.cast(cur.dtype))) grad_buf.uop = cur diff --git a/tinygrad_repo/examples/mlperf/optim.py b/tinygrad_repo/examples/mlperf/optim.py index f93ccef11..d7e3d861b 100644 --- a/tinygrad_repo/examples/mlperf/optim.py +++ b/tinygrad_repo/examples/mlperf/optim.py @@ -15,7 +15,7 @@ def stochastic_round_bf16(x:Tensor) -> Tensor: bits = x.bitcast(dtypes.uint32) if isinstance(x.device, tuple): shape = x.uop.shard_shape if x.uop.axis is not None else x.shape - noise = Tensor(UOp(Ops.MSTACK, dtypes.default_float, tuple(Tensor.rand(*shape, device=d).uop for d in x.device))) + noise = Tensor(UOp(Ops.MSTACK, src=tuple(Tensor.rand(*shape, device=d).uop for d in x.device))) else: noise = x.rand_like() noise = (noise * 0xFFFF).cast(dtypes.uint32) @@ -27,6 +27,11 @@ def clip_grads(grads:list[Tensor], grad_acc, clip_norm) -> Tensor: for g in grads: g.assign((g * (clip_norm / (total_norm + 1e-6)).clamp(max_=1.0)).cast(g.dtype)) return total_norm +def fclip_grads(grads:list[Tensor], clip_norm) -> Tensor: + total_norm = Tensor.stack(*[g.float().square().sum() for g in grads]).sum().sqrt().contiguous() + scale = (clip_norm / (total_norm + 1e-6)).clamp(max_=1.0) + return [(g * scale).cast(g.dtype) for g in grads], total_norm + class GradAccClipAdamW(Optimizer): def __init__(self, params:list[Tensor], lr=0.001, b1=0.9, b2=0.999, eps=1e-6, weight_decay=0.0, grad_acc=1, clip_norm=1.0, device=None, fused=FUSE_OPTIM): super().__init__(params, lr, device, fused) diff --git a/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py new file mode 100644 index 000000000..53510dd58 --- /dev/null +++ b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py @@ -0,0 +1,13 @@ +from pathlib import Path + +from examples.mlperf.dataloader import get_llama3_dataset +from tinygrad.helpers import getenv + +BASEDIR = Path(getenv("BASEDIR", "/raid/datasets/c4-8b/")) +SAMPLES = getenv("SAMPLES", 1_200_000 * 32) +EVAL_SAMPLES = getenv("EVAL_SAMPLES", 1024) +SEQLEN = getenv("SEQLEN", 8192) +DATA_SEED = getenv("DATA_SEED", 5760) + +get_llama3_dataset(SAMPLES, SEQLEN, BASEDIR, seed=DATA_SEED, val=False, small=True) +get_llama3_dataset(EVAL_SAMPLES, SEQLEN, BASEDIR, seed=0, val=True, small=True) diff --git a/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh index db5daa5f2..0da8692fa 100755 --- a/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh +++ b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh @@ -26,7 +26,7 @@ export FUSED_SILU_W13=${FUSED_SILU_W13:-1} export SPLIT_W13=${SPLIT_W13:-0} export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-0} -export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16" +export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="float32" export DP=${DP:-8} MP=${MP:-1} BS=${BS:-16} EVAL_BS=${EVAL_BS:-8} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2} export GBS=$((BS * GRADIENT_ACC_STEPS)) @@ -35,7 +35,7 @@ export BASEDIR="/raid/datasets/c4-8b/" export SMALL=1 export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"} export EVAL_TARGET=3.3 EVAL_FREQ=12288 -export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=4096 MAX_STEPS=1200000 +export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=2048 MAX_STEPS=1200000 export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS)) export SAMPLES=$((MAX_STEPS * GBS)) export SEQLEN=${SEQLEN:-8192} @@ -44,7 +44,7 @@ export SEED=${SEED:-5760} export DATA_SEED=${DATA_SEED:-5760} export JITBEAM=${JITBEAM:-3} -export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 +export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0 export FAKEDATA=${FAKEDATA:-1} BENCHMARK=${BENCHMARK:-10} if [ -z "$FULL_LAYERS" ]; then diff --git a/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh index 0bfe3ae82..4579bc645 100755 --- a/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +++ b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash export PYTHONPATH="." -export PATH="/opt/rocm-7.1.1/bin:$PATH" -export ROCM_PATH="/opt/rocm-7.1.1" +export ROCM_PATH=${ROCM_PATH:-/opt/rocm-7.1.1} +export PATH="$ROCM_PATH/bin:$PATH" export DEV=${DEV:-AMD} export CHECK_OOB=0 export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000 @@ -26,7 +26,7 @@ export FUSED_SILU_W13=${FUSED_SILU_W13:-1} export SPLIT_W13=${SPLIT_W13:-0} export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-0} -export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16" +export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="float32" export DP=${DP:-8} MP=${MP:-1} BS=${BS:-16} EVAL_BS=${EVAL_BS:-8} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2} export GBS=$((BS * GRADIENT_ACC_STEPS)) @@ -35,7 +35,7 @@ export BASEDIR="/raid/datasets/c4-8b/" export SMALL=1 export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"} export EVAL_TARGET=3.3 EVAL_FREQ=12288 -export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=4096 MAX_STEPS=1200000 +export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=2048 MAX_STEPS=1200000 export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS)) export SAMPLES=$((MAX_STEPS * GBS)) export SEQLEN=${SEQLEN:-8192} @@ -44,6 +44,6 @@ export SEED=${SEED:-$RANDOM} export DATA_SEED=${DATA_SEED:-5760} export JITBEAM=${JITBEAM:-3} -export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 +export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0 python3 examples/mlperf/model_train.py diff --git a/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh index e1d5a8673..8815f4484 100755 --- a/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh +++ b/tinygrad_repo/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh @@ -17,7 +17,7 @@ export USE_ATOMICS=1 export ASM_GEMM=1 export WQKV=1 export MASTER_WEIGHTS=1 -export FP8=1 +export MXFP4=1 export ALLREDUCE_CAST=1 export FAST_CE=1 export FUSED_INPUT_QUANTIZE=1 @@ -26,7 +26,7 @@ export FUSED_ADD_NORM_MUL_QUANTIZE=1 export FUSED_SILU_W13=1 export SPLIT_W13=0 -export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16" +export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="float32" export DP=8 MP=1 BS=16 EVAL_BS=8 GRADIENT_ACC_STEPS=2 export GBS=$((BS * GRADIENT_ACC_STEPS)) @@ -44,7 +44,7 @@ export SEED=$RANDOM export DATA_SEED=$SEED export JITBEAM=3 -export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 +export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0 export LOGMLPERF=1 diff --git a/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh b/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh index 35216e667..a6348b34d 100755 --- a/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh +++ b/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh @@ -11,6 +11,7 @@ export DEVICE_IN_FUNCTION_BUG=1 export DEBUG=${DEBUG:-2} export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1} export ASM_GEMM=${ASM_GEMM:-1} +export GROUPED_MOE=${GROUPED_MOE:-1} export ALL2ALL=${ALL2ALL:-1} export LATE_ALLREDUCE=${LATE_ALLREDUCE:-0} export ALLREDUCE_CAST=${ALLREDUCE_CAST:-1} diff --git a/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh b/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh index 9e071ea63..d7ae4cdcc 100755 --- a/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh +++ b/tinygrad_repo/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh @@ -11,6 +11,7 @@ export DEVICE_IN_FUNCTION_BUG=1 export DEBUG=${DEBUG:-0} export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1} export ASM_GEMM=${ASM_GEMM:-1} +export GROUPED_MOE=${GROUPED_MOE:-1} export ALL2ALL=${ALL2ALL:-1} export LATE_ALLREDUCE=${LATE_ALLREDUCE:-0} export ALLREDUCE_CAST=${ALLREDUCE_CAST:-1} diff --git a/tinygrad_repo/examples/openpilot/compile3.py b/tinygrad_repo/examples/openpilot/compile3.py index 7bf253fd8..20c98cd46 100644 --- a/tinygrad_repo/examples/openpilot/compile3.py +++ b/tinygrad_repo/examples/openpilot/compile3.py @@ -107,14 +107,21 @@ def compile(onnx_file): return inputs, test_val def test_vs_compile(run, inputs, test_val=None): + if (log:=bool(getenv("BENCHMARK_LOG", ""))): from extra.bench_log import WallTimeEvent, BenchEvent # run 20 times step_times = [] for _ in range(20): st = time.perf_counter() - out = run(**inputs) - mt = time.perf_counter() - val = out.numpy() + if log: + with WallTimeEvent(BenchEvent.STEP): + out = run(**inputs) + mt = time.perf_counter() + val = out.numpy() + else: + out = run(**inputs) + mt = time.perf_counter() + val = out.numpy() et = time.perf_counter() step_times.append((et-st)*1e3) print(f"enqueue {(mt-st)*1e3:6.2f} ms -- total run {step_times[-1]:6.2f} ms") @@ -160,12 +167,6 @@ def test_vs_onnx(new_inputs, test_val, onnx_file, tol): print("test vs onnx passed") return timings -def bench(run, inputs): - from extra.bench_log import WallTimeEvent, BenchEvent - for _ in range(10): - with WallTimeEvent(BenchEvent.STEP): - run(**inputs).numpy() - if __name__ == "__main__": if getenv("RUN_PICKLE"): with open(OUTPUT, "rb") as f: pickle_loaded = load_pickle(f) @@ -181,6 +182,3 @@ if __name__ == "__main__": test_vs_compile(pickle_loaded, inputs, outputs) if getenv("SELFTEST"): test_vs_onnx(inputs, outputs, onnx_file, 1e-4) - - if getenv("BENCHMARK_LOG", ""): - bench(pickle_loaded, inputs) diff --git a/tinygrad_repo/extra/amdpci/am_smi.py b/tinygrad_repo/extra/amdpci/am_smi.py index ed43f1075..888e77ccb 100755 --- a/tinygrad_repo/extra/amdpci/am_smi.py +++ b/tinygrad_repo/extra/amdpci/am_smi.py @@ -84,7 +84,8 @@ class AMSMI(AMDev): with open(f"/sys/bus/pci/devices/{self.pcibus}/power_state", "r") as f: return f.read().strip().rstrip() class SMICtx: - def __init__(self): + def __init__(self, dev_filter=None): + self.dev_filter = dev_filter self.devs = [] self.opened_pcidevs = [] self.opened_pci_resources = {} @@ -135,6 +136,7 @@ class SMICtx: pattern = os.path.join('/tmp', 'am_*.lock') for d in [f[8:-5] for f in glob.glob(pattern)]: if d.startswith("usb"): continue + if self.dev_filter is not None and d != self.dev_filter: continue if d not in self.opened_pcidevs: self._open_am_device(d) @@ -406,7 +408,7 @@ if __name__ == "__main__": try: if not args.list: os.system('clear') - smi_ctx = SMICtx() + smi_ctx = SMICtx(args.dev) while True: smi_ctx.rescan_devs() smi_ctx.draw(args.list) diff --git a/tinygrad_repo/extra/bench_log.py b/tinygrad_repo/extra/bench_log.py index 62695c693..26bc55d8b 100644 --- a/tinygrad_repo/extra/bench_log.py +++ b/tinygrad_repo/extra/bench_log.py @@ -35,7 +35,7 @@ class WallTimeEvent: return self def __exit__(self, *_): self.time = time.monotonic() - self.start - _events[self.event]["wall"].append(self.time) + _events[self.event]["wall"].append((self.time, BENCHMARK_LOG.value)) return False class KernelTimeEvent: @@ -47,19 +47,19 @@ class KernelTimeEvent: self.start = GlobalCounters.time_sum_s return self def __exit__(self, *_): - _events[self.event]["kernel"].append(GlobalCounters.time_sum_s - self.start) + _events[self.event]["kernel"].append((GlobalCounters.time_sum_s - self.start, BENCHMARK_LOG.value)) return False def log_event_instant(event:InstantBenchEvent, value:float): - _events[event].append(value) + _events[event].append((value, BENCHMARK_LOG.value)) if BENCHMARK_LOG: INFLUXDB_HOST = getenv("INFLUXDB_HOST", "") INFLUXDB_ORG = getenv("INFLUXDB_ORG", "tiny") INFLUXDB_TOKEN = getenv("INFLUXDB_TOKEN", "") - def _create_point(run_id, i, attempt, ref, commit, name, value, run): - point = Point(BENCHMARK_LOG.value).tag("id", run_id).tag("index", i) + def _create_point(run_id, i, attempt, ref, commit, name, value, log_name, run): + point = Point(log_name.replace(':', '_').replace('.', '_')).tag("id", run_id).tag("index", i) point = point.tag("device", Device.DEFAULT) point = point.tag("attempt", attempt).tag("ref", ref).tag("commit", commit) point = point.field(name, value).field("x", run) @@ -91,12 +91,12 @@ if BENCHMARK_LOG: run_id = str(uuid.uuid4()) if isinstance(event, BenchEvent): for event_type, values in _events[event].items(): - for i, value in enumerate(values): - point = _create_point(run_id, i, attempt, ref, commit, f"{event.value}_{event_type}", value, run) + for i, (value, log_name) in enumerate(values): + point = _create_point(run_id, i, attempt, ref, commit, f"{event.value}_{event_type}", value, log_name, run) points.append(point) else: - for i, value in enumerate(_events[event]): - point = _create_point(run_id, i, attempt, ref, commit, event.value, value, run) + for i, (value, log_name) in enumerate(_events[event]): + point = _create_point(run_id, i, attempt, ref, commit, event.value, value, log_name, run) points.append(point) write_options = WriteOptions(write_type=WriteType.synchronous, retry_interval=5000, max_retries=5, max_retry_delay=30000, exponential_base=2) diff --git a/tinygrad_repo/extra/bnxt_driver/bnxtdev.py b/tinygrad_repo/extra/bnxt_driver/bnxtdev.py new file mode 100644 index 000000000..bf898625d --- /dev/null +++ b/tinygrad_repo/extra/bnxt_driver/bnxtdev.py @@ -0,0 +1,238 @@ +import ctypes, struct +from tinygrad.helpers import ceildiv, getenv, wait_cond, DEBUG +from tinygrad.runtime.autogen import bnxt, pci +from tinygrad.runtime.support.system import PCIDevice, System, ipv4_to_gid + +BNXT_DEBUG = getenv("BNXT_DEBUG", 0) +BNXT_ACCESS, BNXT_INIT_MASK, BNXT_RTR_MASK, BNXT_RTS_MASK = 3, 0xd, 0x41515ad, 0xae005 +BNXT_CHIMP_COMM, BNXT_CHIMP_COMM_TRIGGER = 0x0, 0x100 +BNXT_BACKING_STORE = ((0, 2), (1, 0), (2, 2), (3, 0), (4, 2), (5, 0), (6, 0), (14, 2), (15, 0)) + +def db_value(xid, typ, index, epoch): + return (xid & bnxt.DBC_DBC_XID_MASK | bnxt.DBC_DBC_PATH_ROCE | typ | bnxt.BNXT_QPLIB_DBR_VALID) << 32 | \ + index & bnxt.DBC_DBC_INDEX_MASK | epoch << bnxt.BNXT_QPLIB_DBR_EPOCH_SHIFT + +def _pbl(dev, paddrs, queue=False): + if len(paddrs) == 1: return 0, paddrs[0] + values = [p | bnxt.PTU_PTE_VALID for p in paddrs] + if queue: + values[-1] |= bnxt.PTU_PTE_LAST + if len(values) > 1: values[-2] |= bnxt.PTU_PTE_NEXT_TO_LAST + table, table_paddrs = dev.pci_dev.alloc_sysmem(ceildiv(len(values), 512) * 0x1000) + table[:len(values) * 8] = struct.pack(f"<{len(values)}Q", *values) + if len(table_paddrs) == 1: return 1, table_paddrs[0] + top, top_paddrs = dev.pci_dev.alloc_sysmem(0x1000) + top[:len(table_paddrs) * 8] = struct.pack(f"<{len(table_paddrs)}Q", *(p | bnxt.PTU_PTE_VALID for p in table_paddrs)) + return 2, top_paddrs[0] + +def _queue(dev, stride:int=16, aux=False): + mem, paddrs = dev.pci_dev.alloc_sysmem(0x1000 + aux * 0x400) + level, base = _pbl(dev, paddrs, queue=True) + return {"mem":mem, "paddrs":paddrs, "stride":stride, "prod":0, "cons":0, "level":level, "base":base} + +def _qread(q, i): + off = (i & 15) * q["stride"] + return q["mem"][off:off + q["stride"]] + +def _qwrite(q, i, data, aux=False): + off = 0x1000 + i % 128 * 8 if aux else (i & 15) * q["stride"] + q["mem"][off:off + len(data)] = data + +class BNXTDev: + def __init__(self, pci_dev:PCIDevice, ip:str=getenv("BNXT_IP", "10.0.0.1")): + self.pci_dev, self.devfmt = pci_dev, pci_dev.pcibus + self.bar0, self.db = pci_dev.map_bar(0, fmt='I'), pci_dev.map_bar(2, fmt='Q') + pci_dev.write_config(pci.PCI_COMMAND, pci_dev.read_config(pci.PCI_COMMAND, 2) | pci.PCI_COMMAND_MASTER, 2) + self.resp, self.resp_pa = pci_dev.alloc_sysmem(0x1000) + self.seq = 0 + + ver = self.hwrm("ver_get") + if DEBUG >= 2: print(f"bnxt {self.devfmt}: firmware {ver.hwrm_fw_maj_8b}.{ver.hwrm_fw_min_8b}.{ver.hwrm_fw_bld_8b}") + self.hwrm("func_reset", timeout_ms=40000) + caps = self.hwrm("func_qcaps", fid=0xffff) + self.mac, self.port_id = int.from_bytes(bytes(caps.mac_address), 'big'), caps.port_id + self.hwrm("func_drv_rgtr") + self.db_off = self.hwrm("func_qcfg", fid=0xffff).legacy_l2_db_size_kb * 1024 + + self.setup_backing_store() + self._open_rcfw() + self._open_l2() + self.local_gid = ipv4_to_gid(ip) + gids, mac = (ctypes.c_uint32 * 4)(*(int.from_bytes(self.local_gid[i:i + 4], 'big') for i in (12, 8, 4, 0))), self.mac.to_bytes(6, 'big') + smac = (ctypes.c_uint16 * 3)(*(int.from_bytes(mac[i:i + 2], 'big') for i in (0, 2, 4))) + self.gid_id = self.rcfw("add_gid", gid=gids, src_mac=smac).xid + + if DEBUG >= 2: print(f"bnxt {self.devfmt}: booted mac={self.mac.to_bytes(6, 'big').hex(':')} gid={self.local_gid.hex()}") + + def hwrm(self, name, timeout_ms=10000, **fields): + inp, out = getattr(bnxt, f"struct_hwrm_{name}_input"), getattr(bnxt, f"struct_hwrm_{name}_output") + opcode = getattr(bnxt, f"HWRM_{name.upper()}") + self.seq = (self.seq + 1) & 0xffff + data = bytes(inp(req_type=opcode, cmpl_ring=bnxt.BNXT_HWRM_NO_CMPL_RING, seq_id=self.seq, target_id=bnxt.BNXT_HWRM_TARGET, + resp_addr=self.resp_pa[0], **fields)) + self.resp[:] = bytes(len(self.resp)) + System.memory_barrier() + for i, w in enumerate(memoryview(bytearray(data.ljust(bnxt.HWRM_MAX_REQ_LEN, b'\0'))).cast('I')): + self.bar0[BNXT_CHIMP_COMM // 4 + i] = w + self.bar0[BNXT_CHIMP_COMM_TRIGGER // 4] = 1 + def hdr(): return bnxt.struct_hwrm_resp_hdr.from_buffer_copy(bytes(self.resp[:8])) + wait_cond(lambda: (n := hdr().resp_len) and hdr().seq_id == self.seq and self.resp[n - 1], timeout_ms=timeout_ms, msg=f"HWRM {name}") + ret = out.from_buffer_copy(bytes(self.resp[:ctypes.sizeof(out)])) + assert ret.error_code == 0, f"HWRM {name}: {ret.error_code}" + return ret + + def setup_backing_store(self): + counts: dict[int, int] = {} + for typ, extra in BNXT_BACKING_STORE: + caps = self.hwrm("func_backing_store_qcaps_v2", type=typ) + size, splits = caps.entry_size, tuple(getattr(caps, f"split_entry_{j}") for j in range(caps.subtype_valid_cnt)) + counts[typ] = n = counts[0] if typ == 15 else max(caps.min_num_entries, sum(splits) + extra) + # a zero bitmap means the type has a single instance 0 + for instance in [i for i in range(8) if caps.instance_bit_map >> i & 1] or [0]: + mem, paddrs = self.pci_dev.alloc_sysmem(ceildiv(n * size, 0x1000) * 0x1000) + if caps.ctx_init_value: + for off in range(caps.ctx_init_offset, len(mem), size): mem[off] = caps.ctx_init_value + lvl, base = _pbl(self, paddrs) + self.hwrm("func_backing_store_cfg_v2", type=typ, instance=instance, entry_size=size, num_entries=n, page_dir=base, + page_size_pbl_level=lvl, subtype_valid_cnt=len(splits), + flags=bnxt.FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE if typ == 15 else 0, + **{f"split_entry_{j}": v for j, v in enumerate(splits)}) + + def _open_rcfw(self): + self.rcfw_first = True + + self.creq = _queue(self) + self.creq_id = self.hwrm("ring_alloc", ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_NQ, page_tbl_addr=self.creq["base"], + page_size=12, page_tbl_depth=self.creq["level"], length=16, int_mode=bnxt.RING_ALLOC_REQ_INT_MODE_MSIX).ring_id + + self.cmdq = _queue(self) + self.doorbell(self.creq_id, bnxt.DBC_DBC_TYPE_NQ_ARM, 0, 0) + init = bnxt.struct_cmdq_init(cmdq_pbl=self.cmdq["base"], creq_ring_id=self.creq_id, + cmdq_size_cmdq_lvl=16 << bnxt.CMDQ_INIT_CMDQ_SIZE_SFT) + + System.memory_barrier() + for i, w in enumerate(memoryview(bytearray(bytes(init))).cast('I')): self.bar0[bnxt.RCFW_COMM_BASE_OFFSET // 4 + i] = w + + _, p = self.pci_dev.alloc_sysmem(0x1000) + self.rcfw("initialize_fw", stat_ctx_id=self.hwrm("stat_ctx_alloc", stats_dma_addr=p[0], stats_dma_length=176).stat_ctx_id, + flags=bnxt.CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED) + + # RoCE notification ring: never armed or serviced, but CQ and L2 ring allocation require one + nq = _queue(self) + self.nq_id = self.hwrm("ring_alloc", ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_NQ, page_tbl_addr=nq["base"], + page_size=12, page_tbl_depth=nq["level"], length=16, logical_id=1, int_mode=bnxt.RING_ALLOC_REQ_INT_MODE_MSIX).ring_id + + def rcfw(self, name, timeout_ms=20000, **fields): + req_t, resp_t = getattr(bnxt, f"struct_cmdq_{name}"), getattr(bnxt, f"struct_creq_{name}_resp") + op = getattr(bnxt, f"CMDQ_BASE_OPCODE_{name.upper()}") + data = bytes(req_t(opcode=op, cmd_size=(slots := ceildiv(ctypes.sizeof(req_t), 16)), **fields)).ljust(slots * 16, b'\0') + for i in range(slots): _qwrite(self.cmdq, self.cmdq["prod"] + i, data[i * 16:(i + 1) * 16]) + + self.cmdq["prod"] += slots + prod = self.cmdq["prod"] & 0xffff + if self.rcfw_first: prod, self.rcfw_first = prod | 1 << bnxt.FIRMWARE_FIRST_FLAG, False + + System.memory_barrier() + + self.bar0[(bnxt.RCFW_COMM_BASE_OFFSET + bnxt.RCFW_PF_VF_COMM_PROD_OFFSET) // 4] = prod + self.bar0[(bnxt.RCFW_COMM_BASE_OFFSET + bnxt.RCFW_COMM_TRIG_OFFSET) // 4] = bnxt.RCFW_CMDQ_TRIG_VAL + + def poll(): + h = bnxt.struct_creq_base.from_buffer_copy(bytes(_qread(self.creq, self.creq["cons"]))) + return bool(h.v & bnxt.CREQ_BASE_V) != bool((self.creq["cons"] // 16) & 1) + wait_cond(poll, timeout_ms=timeout_ms, msg=f"RCFW {name}") + + ret = resp_t.from_buffer_copy(bytes(_qread(self.creq, self.creq["cons"]))) + self.creq["cons"] += 1 + + # NQ_ARM also publishes the CREQ consumer index, which is what frees ring space for the next command + self.doorbell(self.creq_id, bnxt.DBC_DBC_TYPE_NQ_ARM, self.creq["cons"] & 15, (self.creq["cons"] // 16) & 1) + assert ret.status == 0, f"RCFW {name}: {ret.status}" + + if BNXT_DEBUG >= 1: print(f"bnxt {self.devfmt}: rcfw {name} xid={getattr(ret, 'xid', 0):#x}") + return ret + + def doorbell(self, xid, typ, index, epoch): + System.memory_barrier() + self.db[self.db_off // 8] = db_value(xid, typ, index, epoch) + + # L2 receive path, required for RoCE ingress even though no ethernet receive buffers are posted + def _open_l2(self): + cq = _queue(self) + ci = self.hwrm("ring_alloc", enables=bnxt.RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID, ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_L2_CMPL, + page_tbl_addr=cq["base"], page_size=12, page_tbl_depth=cq["level"], length=16, nq_ring_id=self.nq_id).ring_id + rx = _queue(self) + ri = self.hwrm("ring_alloc", enables=bnxt.RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID | + bnxt.RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID, ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_RX, page_tbl_addr=rx["base"], + page_size=12, page_tbl_depth=rx["level"], length=16, rx_buf_size=640, nq_ring_id=self.nq_id).ring_id + vi = self.hwrm("vnic_alloc").vnic_id + self.hwrm("vnic_cfg", enables=bnxt.VNIC_CFG_REQ_ENABLES_MRU | bnxt.VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | + bnxt.VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID, vnic_id=vi, mru=9018, + default_rx_ring_id=ri, default_cmpl_ring_id=ci) + self.hwrm("cfa_l2_filter_alloc", flags=bnxt.CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX, + enables=bnxt.CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | bnxt.CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK | + bnxt.CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID, l2_addr=tuple(self.mac.to_bytes(6, 'big')), l2_addr_mask=(0xff,) * 6, dst_id=vi) + + def register_mem(self, paddrs:list[int], size:int, log_page_size:int=12) -> int: + level, base = _pbl(self, paddrs[:ceildiv(size, 1 << log_page_size)]) + return self.rcfw("register_mr", flags=bnxt.CMDQ_REGISTER_MR_FLAGS_ALLOC_MR, + log2_pg_size_lvl=level << bnxt.CMDQ_REGISTER_MR_LVL_SFT | log_page_size << bnxt.CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT, + access=bnxt.CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE | bnxt.CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE, + log2_pbl_pg_size=12, pbl=base, va=paddrs[0], mr_size=size).xid + +class BNXTQP: + def __init__(self, dev:BNXTDev): + self.dev, self.sq_psn, self.msn = dev, 0, 0 + + self.cqq = _queue(dev, ctypes.sizeof(bnxt.struct_cq_base)) + self.cq_id = dev.rcfw("create_cq", cq_size=16, pbl=self.cqq["base"], + pg_size_lvl=self.cqq["level"], cq_fco_cnq_id=dev.nq_id).xid + + self.sq = _queue(dev, aux=True) + self.qpn = dev.rcfw("create_qp", type=bnxt.CMDQ_CREATE_QP_TYPE_RC, + sq_size=16, sq_fwo_sq_sge=1, scq_cid=self.cq_id, rcq_cid=self.cq_id, + sq_pbl=self.sq["base"], sq_pg_size_sq_lvl=self.sq["level"]).xid + self.qp_op(1, BNXT_INIT_MASK, access=BNXT_ACCESS, pkey=0xffff) + + def qp_op(self, state, mask, network_type=0, **fields): + self.dev.rcfw("modify_qp", qp_cid=self.qpn, modify_mask=mask, + network_type_en_sqd_async_notify_new_state=state | network_type, **fields) + + def connect(self, qpn:int, gid:bytes, mac:int): + network_type = bnxt.CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 + dgid = (ctypes.c_uint32 * 4)(*(int.from_bytes(gid[i:i + 4], 'little') for i in (0, 4, 8, 12))) + dmac = (ctypes.c_uint16 * 3)(*(int.from_bytes(mac.to_bytes(6, 'big')[i:i + 2], 'little') for i in (0, 2, 4))) + + self.qp_op(2, BNXT_RTR_MASK, network_type=network_type, qp_type=bnxt.CMDQ_MODIFY_QP_QP_TYPE_RC, access=BNXT_ACCESS, + pkey=0xffff, dgid=dgid, sgid_index=self.dev.gid_id, hop_limit=64, dest_mac=dmac, + path_mtu_pingpong_push_enable=bnxt.CMDQ_MODIFY_QP_PATH_MTU_MTU_1024, max_dest_rd_atomic=4, + dest_qp_id=qpn) + self.qp_op(3, BNXT_RTS_MASK, network_type=network_type, qp_type=bnxt.CMDQ_MODIFY_QP_QP_TYPE_RC, access=BNXT_ACCESS, + max_rd_atomic=1) + + if BNXT_DEBUG >= 1: print(f"bnxt: QP {self.qpn:#x} connected (remote={qpn:#x})") + + def _poll(self, timeout): + def poll(): + base = bnxt.struct_cq_base.from_buffer_copy(bytes(_qread(self.cqq, self.cqq["cons"]))) + return bool(base.cqe_type_toggle & bnxt.CQ_BASE_TOGGLE) == (not bool((self.cqq["cons"] // 16) & 1)) + wait_cond(poll, timeout_ms=timeout, msg="BNXT CQ") + raw = bytes(_qread(self.cqq, self.cqq["cons"])) + self.cqq["cons"] += 1 + self.dev.doorbell(self.cq_id, bnxt.DBC_DBC_TYPE_CQ, self.cqq["cons"] & 15, (self.cqq["cons"] // 16) & 1) + return raw + + def rdma_write(self, rva, rkey, lva, lkey, size, timeout_ms=20000): + start = self.sq["prod"] & 15 + hdr = bytes(bnxt.struct_sq_rdma_hdr(wqe_type=bnxt.SQ_RDMA_HDR_WQE_TYPE_WRITE_WQE, + flags=bnxt.SQ_SEND_FLAGS_SIGNAL_COMP, wqe_size=3, length=size, remote_va=rva, remote_key=rkey)) + for i, data in enumerate((hdr[:16], hdr[16:32], bytes(bnxt.struct_sq_sge(va_or_pa=lva, l_key=lkey, size=size)))): + _qwrite(self.sq, start + i, data) + nxt = (self.sq_psn + max(1, ceildiv(size, 1024))) & 0xffffff + value = start << bnxt.SQ_MSN_SEARCH_START_IDX_SFT | nxt << bnxt.SQ_MSN_SEARCH_NEXT_PSN_SFT | self.sq_psn + _qwrite(self.sq, self.msn, struct.pack(" dict[str, Any]: + for line in iter(stream.readline, ""): + print(f" [remote] {line}", end="") + try: value = json.loads(line) + except json.JSONDecodeError: continue + if isinstance(value, dict): return value + raise RuntimeError(f"remote exited before publishing {what}") + +def wait_line(stream:IO[str], text:str) -> str: + for line in iter(stream.readline, ""): + print(f" [remote] {line}", end="") + if text in line: return line + raise RuntimeError(f"remote exited before reporting {text!r}") + +def send_line(stream:IO[str], value:str|dict[str, Any]): + stream.write((json.dumps(value) if isinstance(value, dict) else value) + "\n") + stream.flush() + +def qp_info(dev:BNXTDev, qp:BNXTQP) -> dict[str, Any]: + return {"qpn":qp.qpn, "mac":dev.mac.to_bytes(6, "big").hex(), "gid":dev.local_gid.hex()} + +def server(): + dev = BNXTDev(PCIDevice("bnxt", os.getenv("BNXT_PCI", "0000:41:00.0")), ip=os.getenv("BNXT_IP", REMOTE_IP)) + qp = BNXTQP(dev) + print(json.dumps(qp_info(dev, qp)), flush=True) + + peer = json.loads(sys.stdin.readline()) + qp.connect(peer["qpn"], bytes.fromhex(peer["gid"]), int(peer["mac"], 16)) + print("connected", flush=True) + + target, target_paddrs = dev.pci_dev.alloc_sysmem(0x1000) + target[:0x1000] = bytes(0x1000) + rkey = dev.register_mem(target_paddrs, 0x1000) + print(json.dumps({"target_addr":target_paddrs[0], "rkey":rkey}), flush=True) + + assert sys.stdin.readline().strip() == "done" + received = bytes(target).rstrip(b"\0") + print(f"AS TEXT: {received.decode(errors='replace')!r}", flush=True) + print(json.dumps({"data":received.hex()}), flush=True) + +def sync_remote(): + if os.getenv("SYNC", "1") == "0": return + print("syncing BNXT driver to remote") + subprocess.run(["rsync", "-azR", *SYNC_FILES, f"{REMOTE}:~/tinygrad/"], cwd=TINYGRAD, check=True) + +def start_remote() -> subprocess.Popen[str]: + print("booting remote") + command = (f"cd ~/tinygrad && sudo env PYTHONPATH=. PYTHONUNBUFFERED=1 BNXT_DEBUG={os.getenv('BNXT_DEBUG', '0')} " + f"BNXT_PCI={REMOTE_PCI} BNXT_IP={REMOTE_IP} python3 extra/bnxt_driver/connect.py --server") + return subprocess.Popen(SSH + [command], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stderr, text=True) + +def client(): + assert 0 < len(MESSAGE) <= 0x1000 + sync_remote() + remote = start_remote() + assert remote.stdin is not None and remote.stdout is not None + remote_info = read_json(remote.stdout, "QP information") + print("booting local") + dev = BNXTDev(PCIDevice("bnxt", LOCAL_PCI), ip=LOCAL_IP) + qp = BNXTQP(dev) + + send_line(remote.stdin, qp_info(dev, qp)) + wait_line(remote.stdout, "connected") + qp.connect(remote_info["qpn"], bytes.fromhex(remote_info["gid"]), int(remote_info["mac"], 16)) + print("both QPs in RTS") + + remote_target = read_json(remote.stdout, "MR information") + source, source_paddrs = dev.pci_dev.alloc_sysmem(0x1000) + source[:len(MESSAGE)] = MESSAGE + lkey = dev.register_mem(source_paddrs, 0x1000) + print(f"RDMA WRITE {len(MESSAGE)}B to remote phys 0x{remote_target['target_addr']:x}") + qp.rdma_write(remote_target["target_addr"], remote_target["rkey"], source_paddrs[0], lkey, len(MESSAGE)) + + send_line(remote.stdin, "done") + wait_line(remote.stdout, "AS TEXT") + result = read_json(remote.stdout, "RDMA result") + assert bytes.fromhex(result["data"]) == MESSAGE + print("RDMA WRITE data verified") + + remote.stdin.close() + assert remote.wait() == 0 + print("RDMA WRITE test complete") + +if __name__ == "__main__": + server() if "--server" in sys.argv else client() diff --git a/tinygrad_repo/extra/bnxt_driver/loopback.py b/tinygrad_repo/extra/bnxt_driver/loopback.py new file mode 100644 index 000000000..42b25a44a --- /dev/null +++ b/tinygrad_repo/extra/bnxt_driver/loopback.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Local BNXT RoCEv2 RDMA WRITE loopback using the firmware's PHY loopback mode. + +The kernel bnxt_en/bnxt_re modules must be unloaded first. + + sudo PYTHONPATH=. BNXT_PCI=0000:41:00.0 BNXT_IP=10.0.200.5 python3 extra/bnxt_driver/loopback.py +""" +import os +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")) + +from extra.bnxt_driver.bnxtdev import BNXTDev, BNXTQP +from tinygrad.runtime.autogen import bnxt +from tinygrad.runtime.support.system import PCIDevice + +BUF_SIZE = 0x1000 +BNXT_PCI = os.getenv("BNXT_PCI", "0000:41:00.0") +BNXT_IP = os.getenv("BNXT_IP", "10.0.200.5") + +if __name__ == "__main__": + print(f"[init] BNXT at {BNXT_PCI}") + dev = BNXTDev(PCIDevice("bnxt", BNXT_PCI), ip=BNXT_IP) + tx_qp, rx_qp = BNXTQP(dev), BNXTQP(dev) + print(f"[init] loopback-connect TX QP 0x{tx_qp.qpn:x} <-> RX QP 0x{rx_qp.qpn:x}") + tx_qp.connect(rx_qp.qpn, dev.local_gid, dev.mac) + rx_qp.connect(tx_qp.qpn, dev.local_gid, dev.mac) + + src, src_paddrs = dev.pci_dev.alloc_sysmem(BUF_SIZE) + dst, dst_paddrs = dev.pci_dev.alloc_sysmem(BUF_SIZE) + message = b"Hello from BNXT RoCE PHY loopback!" + src[:BUF_SIZE], dst[:BUF_SIZE] = bytes(BUF_SIZE), bytes(BUF_SIZE) + src[:len(message)] = message + lkey = dev.register_mem(src_paddrs, BUF_SIZE) + rkey = dev.register_mem(dst_paddrs, BUF_SIZE) + + print("[loopback] enabling local PHY loopback") + dev.hwrm("port_phy_cfg", port_id=dev.port_id, enables=bnxt.PORT_PHY_CFG_REQ_ENABLES_LPBK, lpbk=bnxt.PORT_PHY_CFG_REQ_LPBK_LOCAL) + time.sleep(1) + tx_qp.rdma_write(dst_paddrs[0], rkey, src_paddrs[0], lkey, len(message)) + got = bytes(dst[:len(message)]) + print(f"[result] {got!r}") + assert got == message + print("BNXT RoCE PHY loopback RDMA WRITE passed") + dev.hwrm("port_phy_cfg", port_id=dev.port_id, enables=bnxt.PORT_PHY_CFG_REQ_ENABLES_LPBK, lpbk=bnxt.PORT_PHY_CFG_REQ_LPBK_NONE) diff --git a/tinygrad_repo/extra/export_model.py b/tinygrad_repo/extra/export_model.py index a0024494b..a6baf20bc 100644 --- a/tinygrad_repo/extra/export_model.py +++ b/tinygrad_repo/extra/export_model.py @@ -35,7 +35,7 @@ def compile_net(linear:UOp, output_bufs:List[Buffer]) -> Tuple[Dict[str,str], Li return name for call in iter_kernel_calls(linear): - arg_uops = [b for b in call.src[1:] if b.op is not Ops.BIND] + arg_uops = [b for b in call.src[1:] if not b.is_bound_var] prg = to_program(call.src[0], Device[arg_uops[0].device].renderer) info = prg.arg functions[info.function_name] = prg.src[2].arg @@ -241,8 +241,7 @@ export default {model_name}; def export_model(model, target:str, *inputs, model_name: Optional[str] = "model", stream_weights=False): assert Device.DEFAULT in EXPORT_SUPPORTED_DEVICE, f"only {', '.join(EXPORT_SUPPORTED_DEVICE)} are supported" - # NOTE: NUM_CPU_THREADS=1, since export does not support threading - with Context(JIT=2, NUM_CPU_THREADS=1): linear, output_bufs = jit_model(model, *inputs) + with Context(JIT=2): linear, output_bufs = jit_model(model, *inputs) functions, statements, bufs, bufs_to_save = compile_net(linear, output_bufs) state = get_state_dict(model) weight_names = {(id(b), b.offset, b.size, b.dtype): name for name, x in state.items() if (b:=x.uop.base.realized) is not None} diff --git a/tinygrad_repo/extra/gemm/amd_asm_matmul.py b/tinygrad_repo/extra/gemm/amd_asm_matmul.py index a6c79df74..2feabab25 100644 --- a/tinygrad_repo/extra/gemm/amd_asm_matmul.py +++ b/tinygrad_repo/extra/gemm/amd_asm_matmul.py @@ -462,7 +462,7 @@ def test_matmul(): lds = UOp.placeholder((lds_size,), dtypes.uint8, 0, AddrSpace.LOCAL) sink = UOp.sink(A.base, B.base, C.base, lds, *gidxs, *lidxs, arg=KernelInfo(name=colored("kernel", "cyan"), estimates=Estimates(ops=N*N*N*2, mem=N*N*4*3))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) c = Tensor.custom_kernel(a, b, c, fxn=asm_kernel)[2] linear = c.schedule_linear() diff --git a/tinygrad_repo/extra/gemm/cdna_asm_gemm.py b/tinygrad_repo/extra/gemm/cdna_asm_gemm.py index e9cb78e98..801f40779 100644 --- a/tinygrad_repo/extra/gemm/cdna_asm_gemm.py +++ b/tinygrad_repo/extra/gemm/cdna_asm_gemm.py @@ -122,9 +122,10 @@ def custom_mxfp4_gemm(C:UOp, A:UOp, B:UOp, scale_a:UOp, scale_b:UOp, *extra:UOp, groups_x, groups_y = UOp.special(ceildiv(N, tile_n), "gidx0"), UOp.special(ceildiv(M, tile_m), "gidx1") lds = UOp.placeholder((163840,), dtypes.uint8, 0, AddrSpace.LOCAL) sink = UOp.sink(C.base, A.base, B.base, scale_a.base, scale_b.base, *(x.base for x in extra), lds, threads, groups_x, groups_y, - arg=KernelInfo(f"custom_mxfp4_gemm_{M}_{N}_{K}", estimates=Estimates(ops=2*M*N*K))) + arg=KernelInfo(f"mxfp4_gemm_{M}_{N}_{K}", + estimates=Estimates(ops=2*M*N*K, mem=(M*half_k+N*half_k)*A.dtype.itemsize+M*N*C.dtype.itemsize))) insts = build_kernel(M, N, K, tile_m, tile_n) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=x) for x in insts)))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts)))) def _mxfp4_gemm_quantized(a_q:Tensor, b_q:Tensor, scale_a:Tensor, scale_b:Tensor) -> Tensor: M, half_k = a_q.shape @@ -214,7 +215,7 @@ def custom_uop_gemm(C:UOp, A:UOp, B:UOp) -> UOp: k = UOp.range(K, 0, AxisType.REDUCE) mul = (A.flatten().index((m*UOp.const(K)+k))* B.flatten().index((k*UOp.const(N)+n))).cast(dtypes.float32) - red = mul.reduce(k, arg=Ops.ADD, dtype=dtypes.float32).cast(C.dtype) + red = mul.reduce(k, arg=Ops.ADD).cast(C.dtype) store = C.flatten().index((m*UOp.const(N)+n)).store(red).end(m, n) return store.sink(arg=KernelInfo(name=f'uop_gemm_{M}_{N}_{K}')) diff --git a/tinygrad_repo/extra/gemm/gemm_mxfp4.py b/tinygrad_repo/extra/gemm/gemm_mxfp4.py index 04d8195ea..4edbf1779 100644 --- a/tinygrad_repo/extra/gemm/gemm_mxfp4.py +++ b/tinygrad_repo/extra/gemm/gemm_mxfp4.py @@ -20,38 +20,39 @@ def v_mfma_fp4(dst, a, b, opsel, opsel_hi, scale_a, scale_b): def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k = Kernel() scale_k = K // 32 + k.emit(s_and_b32(s[1], s[1], LIT, 65535)) if (tile_m, tile_n) == (128, 512): - k.emit(s_and_b32(s[1], s[1], LIT, 65535)) k.emit(s_mov_b32(s[47], s[2])) k.emit(s_mov_b32(s[48], s[3])) - k.emit(s_mov_b32(s[64], s[4])) - k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[8], 0)) - k.emit(s_mov_b32(s[9], 0)) - k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[41], 1.0)) - k.emit(s_mov_b32(s[42], 0)) - k.emit(s_mov_b32(s[36], N)) - k.emit(s_mov_b32(s[37], K)) - k.emit(s_mov_b32(s[38], K)) - k.emit(s_mov_b32(s[43], M)) - k.emit(s_mov_b32(s[44], N)) - k.emit(s_mov_b32(s[45], K)) - k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[39], scale_k)) - k.emit(s_mov_b32(s[40], scale_k)) - k.emit(s_mov_b32(s[65], 0)) - k.emit(v_lshrrev_b32_e32(v[1], 10)) - k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) - k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) - k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) - k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) - k.emit(v_lshrrev_b32_e32(v[3], 6)) - k.emit(v_and_b32_e32(v[0], 63)) - k.emit(v_readfirstlane_b32_e32(v[46], v[3])) - k.emit(s_waitcnt(49279)) + k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) + k.emit(s_mov_b32(s[8], 0)) + k.emit(s_mov_b32(s[9], 0)) + k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) + k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) + k.emit(s_mov_b32(s[36], N)) + k.emit(s_mov_b32(s[37], K)) + k.emit(s_mov_b32(s[38], K)) + k.emit(s_mov_b32(s[43], M)) + k.emit(s_mov_b32(s[44], N)) + k.emit(s_mov_b32(s[45], K)) + k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) + k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) + k.emit(s_mov_b32(s[39], scale_k)) + k.emit(s_mov_b32(s[40], scale_k)) + k.emit(v_lshrrev_b32_e32(v[1], 10)) + k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) + k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) + k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) + k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) + k.emit(v_lshrrev_b32_e32(v[3], 6)) + k.emit(v_and_b32_e32(v[0], 63)) + if (tile_m, tile_n) == (256, 256): + k.emit(s_mov_b32(s[49], s[2])) + k.emit(s_mov_b32(s[47], s[3])) + k.emit(v_readfirstlane_b32_e32(v[46], v[3])) + k.emit(s_waitcnt(49279)) + + if (tile_m, tile_n) == (128, 512): for i in range(2): k.emit(s_mov_b32(s[6 + i * 8], -16)) k.emit(s_mov_b32(s[10 + i * 12], -16)) @@ -68,17 +69,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_or_b32(s[5 + i * 8], s[5 + i * 8], LIT, 262144)) k.emit(s_or_b32(s[9 + i * 12], s[9 + i * 12], LIT, 262144)) k.emit(s_or_b32(s[17 + i * 8], s[17 + i * 8], LIT, 262144)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(9), target='L0_0194') - k.emit(s_lshr_b32(s[66], s[45], s[65])) - k.emit(s_add_u32(s[66], s[66], LIT, 255)) - k.emit(s_lshr_b32(s[66], s[66], 8)) - k.emit(s_lshl_b32(s[66], s[66], 8)) - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_sub_i32(s[62], s[45], s[63])) - k.emit(s_cmp_lt_i32(s[62], s[66])) - k.emit(s_cselect_b32(s[45], s[62], s[66])) - k.label('L0_0194') k.emit(s_lshr_b32(s[37], s[37], 1)) k.emit(s_mul_i32(s[62], s[48], LIT, 128)) k.emit(s_mul_hi_u32(s[63], s[37], s[62])) @@ -92,14 +82,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[37], s[62])) k.emit(s_mov_b32(s[14], s[63])) k.emit(s_mov_b32(s[15], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(5), target='L0_01F4') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_lshr_b32(s[62], s[63], 1)) - k.emit(s_add_u32(s[12], s[12], s[62])) - k.emit(s_addc_u32(s[13], s[13], 0)) - k.emit(s_sub_u32(s[14], s[14], s[62])) - k.label('L0_01F4') k.emit(v_lshrrev_b32_e32(v[4], 3)) k.emit(v_lshrrev_b32_e32(v[5], 2, v[4])) k.emit(v_lshlrev_b32_e32(v[5], 4, v[5])) @@ -161,13 +143,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[39], s[62])) k.emit(s_mov_b32(s[22], s[63])) k.emit(s_mov_b32(s[23], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(4), target='L0_0334') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_add_u32(s[20], s[20], s[63])) - k.emit(s_addc_u32(s[21], s[21], 0)) - k.emit(s_sub_u32(s[22], s[22], s[63])) - k.label('L0_0334') k.emit(v_lshlrev_b32_e32(v[218], 2)) k.emit(s_mul_i32(s[63], s[46], 32)) k.emit(s_mul_i32(s[63], s[63], s[39])) @@ -189,15 +164,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[38], s[62])) k.emit(s_mov_b32(s[18], s[63])) k.emit(s_mov_b32(s[19], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(6), target='L0_03BC') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_lshr_b32(s[62], s[63], 1)) - k.emit(s_mul_i32(s[62], s[62], 16)) - k.emit(s_add_u32(s[16], s[16], s[62])) - k.emit(s_addc_u32(s[17], s[17], 0)) - k.emit(s_sub_u32(s[18], s[18], s[62])) - k.label('L0_03BC') k.emit(v_lshlrev_b32_e32(v[220], 4)) k.emit(s_mul_i32(s[63], s[46], LIT, 128)) k.emit(s_mul_i32(s[62], s[63], s[38])) @@ -223,13 +189,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[40], s[62])) k.emit(s_mov_b32(s[26], s[63])) k.emit(s_mov_b32(s[27], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(4), target='L0_047C') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_add_u32(s[24], s[24], s[63])) - k.emit(s_addc_u32(s[25], s[25], 0)) - k.emit(s_sub_u32(s[26], s[26], s[63])) - k.label('L0_047C') k.emit(v_lshlrev_b32_e32(v[236], 2)) k.emit(s_mul_i32(s[63], s[46], LIT, 128)) k.emit(s_mul_i32(s[63], s[63], s[40])) @@ -342,17 +301,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_sub_i32(s[62], s[62], s[63])) k.emit(s_mov_b32(s[6], s[62])) k.emit(s_mov_b32(s[7], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(11), target='L0_0F54') - k.emit(v_mul_i32_i24_e64(v[4], v[0], 4)) - k.emit(s_mul_i32(s[62], s[46], LIT, 256)) - k.emit(v_add_u32_e32(v[240], s[62], v[4])) - k.emit(v_add_u32_e32(v[241], LIT, v[240], 128)) - k.emit(s_mul_i32(s[62], s[36], 64)) - k.emit(v_add_u32_e32(v[242], s[62], v[240])) - k.emit(v_add_u32_e32(v[243], s[62], v[241])) - k.emit(s_branch(22), target='L0_0FAC') - k.label('L0_0F54') k.emit(v_and_b32_e64(v[4], v[0], 15)) k.emit(v_mul_lo_u32(v[240], s[36], v[4])) k.emit(v_lshrrev_b32_e32(v[4], 5)) @@ -1018,242 +966,14 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.label('L0_37CC') k.emit(s_waitcnt()) k.emit(s_barrier()) - k.emit(s_cmp_eq_u32(s[65], 0)) - k.emit(s_cbranch_scc1(2149), target='L0_5970') - k.emit(v_lshrrev_b32_e32(v[4], 4)) - k.emit(v_mul_i32_i24_e64(v[4], v[4], 8)) - k.emit(v_and_b32_e64(v[5], v[0], 15)) - k.emit(v_lshlrev_b32_e32(v[5], 8, v[5])) - k.emit(v_add_i32(v[4], v[4], v[5])) - k.emit(s_mul_i32(s[62], s[46], LIT, 16384)) - k.emit(s_add_i32(s[62], s[62], 0)) - k.emit(v_add_i32(v[4], v[4], s[62])) - for i in range(2): - for j9 in range(4): - k.emit(v_accvgpr_read(v[8 + j9 * 1 + i * 4], v[0 + j9 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j9 * 1 + i * 4], s[41], v[8 + j9 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17])) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32)) - for i in range(2): - for j10 in range(4): - k.emit(v_accvgpr_read(v[8 + j10 * 1 + i * 4], v[32 + j10 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j10 * 1 + i * 4], s[41], v[8 + j10 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 96)) - for i in range(3): - for j11 in range(2): - k.emit(v_accvgpr_read(v[8], v[4 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[5 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[6 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[7 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[20 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[21 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[22 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[23 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 0 + j11 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32 + j11 * 64, 16 + i * 16)) - for i in range(2): - for j12 in range(2): - k.emit(v_accvgpr_read(v[8 + j12 * 4], v[64 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j12 * 4], s[41], v[8 + j12 * 4])) - k.emit(v_accvgpr_read(v[9 + j12 * 4], v[65 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j12 * 4], s[41], v[9 + j12 * 4])) - k.emit(v_accvgpr_read(v[10 + j12 * 4], v[66 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j12 * 4], s[41], v[10 + j12 * 4])) - k.emit(v_accvgpr_read(v[11 + j12 * 4], v[67 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j12 * 4], s[41], v[11 + j12 * 4])) - for j13 in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + j13 * 1], v[8 + j13 * 2], v[9 + j13 * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + i * 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + i * 64)) - for i in range(3): - for j14 in range(2): - k.emit(v_accvgpr_read(v[8], v[68 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[69 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[70 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[71 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[84 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[85 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[86 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[87 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + j14 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + j14 * 64, 16 + i * 16)) - k.emit(s_waitcnt(49279)) - k.emit(v_mul_i32_i24_e64(v[4], v[0], 4)) - k.emit(v_add_i32(v[4], v[4], s[62])) - k.emit(s_mul_i32(s[63], s[36], 0)) - k.emit(v_add_u32_e32(v[244], s[63], v[240])) - k.emit(ds_read_b32(v[16], v[4])) - for i in range(15): - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 1 + i * 4)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 2 + i * 4)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 3 + i * 4)) - for j15 in range(4): - k.emit(s_waitcnt(50047 + j15 * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + j15 * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(s_mul_i32(s[63], s[36], 4 + i * 4)) - k.emit(v_add_u32_e32(v[244], s[63], v[240])) - k.emit(ds_read_b32(v[16], v[4], v[0], v[0], 0, 0, 4 + i * 4)) - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 61)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 62)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 63)) - for i in range(4): - k.emit(s_waitcnt(50047 + i * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(v_lshrrev_b32_e32(v[4], 4)) - k.emit(v_mul_i32_i24_e64(v[4], v[4], 8)) - k.emit(v_and_b32_e64(v[5], v[0], 15)) - k.emit(v_lshlrev_b32_e32(v[5], 8, v[5])) - k.emit(v_add_i32(v[4], v[4], v[5])) - k.emit(s_mul_i32(s[62], s[46], LIT, 16384)) - k.emit(s_add_i32(s[62], s[62], 0)) - k.emit(v_add_i32(v[4], v[4], s[62])) - for i in range(2): - for j16 in range(4): - k.emit(v_accvgpr_read(v[8 + j16 * 1 + i * 4], v[128 + j16 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j16 * 1 + i * 4], s[41], v[8 + j16 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17])) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32)) - for i in range(2): - for j17 in range(4): - k.emit(v_accvgpr_read(v[8 + j17 * 1 + i * 4], v[160 + j17 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j17 * 1 + i * 4], s[41], v[8 + j17 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 96)) - for i in range(3): - for j18 in range(2): - k.emit(v_accvgpr_read(v[8], v[132 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[133 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[134 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[135 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[148 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[149 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[150 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[151 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 0 + j18 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32 + j18 * 64, 16 + i * 16)) - for i in range(2): - for j19 in range(2): - k.emit(v_accvgpr_read(v[8 + j19 * 4], v[192 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j19 * 4], s[41], v[8 + j19 * 4])) - k.emit(v_accvgpr_read(v[9 + j19 * 4], v[193 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j19 * 4], s[41], v[9 + j19 * 4])) - k.emit(v_accvgpr_read(v[10 + j19 * 4], v[194 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j19 * 4], s[41], v[10 + j19 * 4])) - k.emit(v_accvgpr_read(v[11 + j19 * 4], v[195 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j19 * 4], s[41], v[11 + j19 * 4])) - for j20 in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + j20 * 1], v[8 + j20 * 2], v[9 + j20 * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + i * 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + i * 64)) - for i in range(3): - for j21 in range(2): - k.emit(v_accvgpr_read(v[8], v[196 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[197 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[198 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[199 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[212 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[213 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[214 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[215 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + j21 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + j21 * 64, 16 + i * 16)) - k.emit(s_waitcnt(49279)) - k.emit(v_mul_i32_i24_e64(v[4], v[0], 4)) - k.emit(v_add_i32(v[4], v[4], s[62])) - k.emit(s_mul_i32(s[63], s[36], 0)) - k.emit(v_add_u32_e32(v[244], s[63], v[242])) - k.emit(ds_read_b32(v[16], v[4])) - for i in range(15): - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 1 + i * 4)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 2 + i * 4)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 3 + i * 4)) - for j22 in range(4): - k.emit(s_waitcnt(50047 + j22 * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + j22 * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(s_mul_i32(s[63], s[36], 4 + i * 4)) - k.emit(v_add_u32_e32(v[244], s[63], v[242])) - k.emit(ds_read_b32(v[16], v[4], v[0], v[0], 0, 0, 4 + i * 4)) - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 61)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 62)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 63)) - for i in range(4): - k.emit(s_waitcnt(50047 + i * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(s_branch(1344), target='L0_6E70') - k.label('L0_5970') k.emit(s_mul_i32(s[62], s[36], 0)) k.emit(v_add_u32_e32(v[244], s[62], v[240])) for i in range(2): for j23 in range(2): k.emit(v_accvgpr_read(v[8 + j23 * 4], v[0 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j23 * 4], s[41], v[8 + j23 * 4])) k.emit(v_accvgpr_read(v[9 + j23 * 4], v[1 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j23 * 4], s[41], v[9 + j23 * 4])) k.emit(v_accvgpr_read(v[10 + j23 * 4], v[2 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j23 * 4], s[41], v[10 + j23 * 4])) k.emit(v_accvgpr_read(v[11 + j23 * 4], v[3 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j23 * 4], s[41], v[11 + j23 * 4])) for j24 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j24 * 1], v[8 + j24 * 2], v[9 + j24 * 2])) k.emit(s_nop(1)) @@ -1268,13 +988,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j25 in range(2): k.emit(v_accvgpr_read(v[8 + j25 * 4], v[4 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j25 * 4], s[41], v[8 + j25 * 4])) k.emit(v_accvgpr_read(v[9 + j25 * 4], v[5 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j25 * 4], s[41], v[9 + j25 * 4])) k.emit(v_accvgpr_read(v[10 + j25 * 4], v[6 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j25 * 4], s[41], v[10 + j25 * 4])) k.emit(v_accvgpr_read(v[11 + j25 * 4], v[7 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j25 * 4], s[41], v[11 + j25 * 4])) for j26 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j26 * 1], v[8 + j26 * 2], v[9 + j26 * 2])) k.emit(s_nop(1)) @@ -1289,13 +1005,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j27 in range(2): k.emit(v_accvgpr_read(v[8 + j27 * 4], v[8 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j27 * 4], s[41], v[8 + j27 * 4])) k.emit(v_accvgpr_read(v[9 + j27 * 4], v[9 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j27 * 4], s[41], v[9 + j27 * 4])) k.emit(v_accvgpr_read(v[10 + j27 * 4], v[10 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j27 * 4], s[41], v[10 + j27 * 4])) k.emit(v_accvgpr_read(v[11 + j27 * 4], v[11 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j27 * 4], s[41], v[11 + j27 * 4])) for j28 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j28 * 1], v[8 + j28 * 2], v[9 + j28 * 2])) k.emit(s_nop(1)) @@ -1310,13 +1022,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j29 in range(2): k.emit(v_accvgpr_read(v[8 + j29 * 4], v[12 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j29 * 4], s[41], v[8 + j29 * 4])) k.emit(v_accvgpr_read(v[9 + j29 * 4], v[13 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j29 * 4], s[41], v[9 + j29 * 4])) k.emit(v_accvgpr_read(v[10 + j29 * 4], v[14 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j29 * 4], s[41], v[10 + j29 * 4])) k.emit(v_accvgpr_read(v[11 + j29 * 4], v[15 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j29 * 4], s[41], v[11 + j29 * 4])) for j30 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j30 * 1], v[8 + j30 * 2], v[9 + j30 * 2])) k.emit(s_nop(1)) @@ -1331,13 +1039,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j31 in range(2): k.emit(v_accvgpr_read(v[8 + j31 * 4], v[64 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j31 * 4], s[41], v[8 + j31 * 4])) k.emit(v_accvgpr_read(v[9 + j31 * 4], v[65 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j31 * 4], s[41], v[9 + j31 * 4])) k.emit(v_accvgpr_read(v[10 + j31 * 4], v[66 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j31 * 4], s[41], v[10 + j31 * 4])) k.emit(v_accvgpr_read(v[11 + j31 * 4], v[67 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j31 * 4], s[41], v[11 + j31 * 4])) for j32 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j32 * 1], v[8 + j32 * 2], v[9 + j32 * 2])) k.emit(s_nop(1)) @@ -1352,13 +1056,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j33 in range(2): k.emit(v_accvgpr_read(v[8 + j33 * 4], v[68 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j33 * 4], s[41], v[8 + j33 * 4])) k.emit(v_accvgpr_read(v[9 + j33 * 4], v[69 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j33 * 4], s[41], v[9 + j33 * 4])) k.emit(v_accvgpr_read(v[10 + j33 * 4], v[70 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j33 * 4], s[41], v[10 + j33 * 4])) k.emit(v_accvgpr_read(v[11 + j33 * 4], v[71 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j33 * 4], s[41], v[11 + j33 * 4])) for j34 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j34 * 1], v[8 + j34 * 2], v[9 + j34 * 2])) k.emit(s_nop(1)) @@ -1373,13 +1073,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j35 in range(2): k.emit(v_accvgpr_read(v[8 + j35 * 4], v[72 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j35 * 4], s[41], v[8 + j35 * 4])) k.emit(v_accvgpr_read(v[9 + j35 * 4], v[73 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j35 * 4], s[41], v[9 + j35 * 4])) k.emit(v_accvgpr_read(v[10 + j35 * 4], v[74 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j35 * 4], s[41], v[10 + j35 * 4])) k.emit(v_accvgpr_read(v[11 + j35 * 4], v[75 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j35 * 4], s[41], v[11 + j35 * 4])) for j36 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j36 * 1], v[8 + j36 * 2], v[9 + j36 * 2])) k.emit(s_nop(1)) @@ -1394,13 +1090,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j37 in range(2): k.emit(v_accvgpr_read(v[8 + j37 * 4], v[76 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j37 * 4], s[41], v[8 + j37 * 4])) k.emit(v_accvgpr_read(v[9 + j37 * 4], v[77 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j37 * 4], s[41], v[9 + j37 * 4])) k.emit(v_accvgpr_read(v[10 + j37 * 4], v[78 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j37 * 4], s[41], v[10 + j37 * 4])) k.emit(v_accvgpr_read(v[11 + j37 * 4], v[79 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j37 * 4], s[41], v[11 + j37 * 4])) for j38 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j38 * 1], v[8 + j38 * 2], v[9 + j38 * 2])) k.emit(s_nop(1)) @@ -1415,13 +1107,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j39 in range(2): k.emit(v_accvgpr_read(v[8 + j39 * 4], v[128 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j39 * 4], s[41], v[8 + j39 * 4])) k.emit(v_accvgpr_read(v[9 + j39 * 4], v[129 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j39 * 4], s[41], v[9 + j39 * 4])) k.emit(v_accvgpr_read(v[10 + j39 * 4], v[130 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j39 * 4], s[41], v[10 + j39 * 4])) k.emit(v_accvgpr_read(v[11 + j39 * 4], v[131 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j39 * 4], s[41], v[11 + j39 * 4])) for j40 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j40 * 1], v[8 + j40 * 2], v[9 + j40 * 2])) k.emit(s_nop(1)) @@ -1436,13 +1124,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j41 in range(2): k.emit(v_accvgpr_read(v[8 + j41 * 4], v[132 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j41 * 4], s[41], v[8 + j41 * 4])) k.emit(v_accvgpr_read(v[9 + j41 * 4], v[133 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j41 * 4], s[41], v[9 + j41 * 4])) k.emit(v_accvgpr_read(v[10 + j41 * 4], v[134 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j41 * 4], s[41], v[10 + j41 * 4])) k.emit(v_accvgpr_read(v[11 + j41 * 4], v[135 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j41 * 4], s[41], v[11 + j41 * 4])) for j42 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j42 * 1], v[8 + j42 * 2], v[9 + j42 * 2])) k.emit(s_nop(1)) @@ -1457,21 +1141,13 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[244], s[62], v[242 + i * 1])) for j43 in range(2): k.emit(v_accvgpr_read(v[8], v[136 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) k.emit(v_accvgpr_read(v[9], v[137 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) k.emit(v_accvgpr_read(v[10], v[138 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) k.emit(v_accvgpr_read(v[11], v[139 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) k.emit(v_accvgpr_read(v[12], v[152 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) k.emit(v_accvgpr_read(v[13], v[153 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) k.emit(v_accvgpr_read(v[14], v[154 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) k.emit(v_accvgpr_read(v[15], v[155 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) @@ -1487,21 +1163,13 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[244], s[62], v[242 + i * 1])) for j44 in range(2): k.emit(v_accvgpr_read(v[8], v[140 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) k.emit(v_accvgpr_read(v[9], v[141 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) k.emit(v_accvgpr_read(v[10], v[142 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) k.emit(v_accvgpr_read(v[11], v[143 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) k.emit(v_accvgpr_read(v[12], v[156 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) k.emit(v_accvgpr_read(v[13], v[157 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) k.emit(v_accvgpr_read(v[14], v[158 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) k.emit(v_accvgpr_read(v[15], v[159 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) @@ -1518,13 +1186,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j45 in range(2): k.emit(v_accvgpr_read(v[8 + j45 * 4], v[200 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j45 * 4], s[41], v[8 + j45 * 4])) k.emit(v_accvgpr_read(v[9 + j45 * 4], v[201 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j45 * 4], s[41], v[9 + j45 * 4])) k.emit(v_accvgpr_read(v[10 + j45 * 4], v[202 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j45 * 4], s[41], v[10 + j45 * 4])) k.emit(v_accvgpr_read(v[11 + j45 * 4], v[203 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j45 * 4], s[41], v[11 + j45 * 4])) for j46 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j46 * 1], v[8 + j46 * 2], v[9 + j46 * 2])) k.emit(s_nop(1)) @@ -1539,13 +1203,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j47 in range(2): k.emit(v_accvgpr_read(v[8 + j47 * 4], v[204 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j47 * 4], s[41], v[8 + j47 * 4])) k.emit(v_accvgpr_read(v[9 + j47 * 4], v[205 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j47 * 4], s[41], v[9 + j47 * 4])) k.emit(v_accvgpr_read(v[10 + j47 * 4], v[206 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j47 * 4], s[41], v[10 + j47 * 4])) k.emit(v_accvgpr_read(v[11 + j47 * 4], v[207 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j47 * 4], s[41], v[11 + j47 * 4])) for j48 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j48 * 1], v[8 + j48 * 2], v[9 + j48 * 2])) k.emit(s_nop(1)) @@ -1555,37 +1215,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_nop(1)) k.emit(buffer_store_dwordx4(v[16:19], v[244], s[4:7], 0, 0, 1)) k.emit(v_add_i32(v[244], v[244], 64)) - k.label('L0_6E70') k.emit(s_waitcnt()) k.emit(s_endpgm()) elif (tile_m, tile_n) == (192, 256): - k.emit(s_and_b32(s[1], s[1], LIT, 65535)) - k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[8], 0)) - k.emit(s_mov_b32(s[9], 0)) - k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[41], 1.0)) - k.emit(s_mov_b32(s[42], 0)) - k.emit(s_mov_b32(s[36], N)) - k.emit(s_mov_b32(s[37], K)) - k.emit(s_mov_b32(s[38], K)) - k.emit(s_mov_b32(s[43], M)) - k.emit(s_mov_b32(s[44], N)) - k.emit(s_mov_b32(s[45], K)) - k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[39], scale_k)) - k.emit(s_mov_b32(s[40], scale_k)) - k.emit(v_lshrrev_b32_e32(v[1], 10)) - k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) - k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) - k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) - k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) - k.emit(v_lshrrev_b32_e32(v[3], 6)) - k.emit(v_and_b32_e32(v[0], 63)) - k.emit(v_readfirstlane_b32_e32(v[46], v[3])) - k.emit(s_waitcnt(49279)) k.emit(s_mul_i32(s[63], LIT, 8, 192)) k.emit(v_cvt_f32_u32_e32(v[4], s[63])) k.emit(s_sub_i32(s[62], 0, s[63])) @@ -2582,53 +2214,22 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_waitcnt()) k.emit(s_endpgm()) elif (tile_m, tile_n) == (256, 256): - k.emit(s_and_b32(s[1], s[1], LIT, 65535)) - k.emit(s_mov_b32(s[56], s[4])) - k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[8], 0)) - k.emit(s_mov_b32(s[9], 0)) - k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[38], 1.0)) - k.emit(s_mov_b32(s[39], 0)) - k.emit(s_mov_b32(s[40], N)) - k.emit(s_mov_b32(s[41], K)) - k.emit(s_mov_b32(s[42], K)) - k.emit(s_mov_b32(s[43], M)) - k.emit(s_mov_b32(s[44], N)) - k.emit(s_mov_b32(s[45], K)) - k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[36], scale_k)) - k.emit(s_mov_b32(s[37], scale_k)) - k.emit(s_mov_b32(s[57], 0)) - k.emit(v_lshrrev_b32_e32(v[1], 10)) - k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) - k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) - k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) - k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) - k.emit(v_lshrrev_b32_e32(v[3], 6)) - k.emit(v_and_b32_e32(v[0], 63)) - k.emit(s_mov_b32(s[46], s[2])) - k.emit(s_mov_b32(s[47], s[3])) - k.emit(v_readfirstlane_b32_e32(v[49], v[3])) - k.emit(s_waitcnt(49279)) k.emit(s_add_u32(s[55], s[44], LIT, 255)) k.emit(s_lshr_b32(s[54], s[55], 8)) k.emit(s_mul_i32(s[48], s[54], s[47])) - k.emit(s_add_i32(s[48], s[48], s[46])) + k.emit(s_add_i32(s[48], s[48], s[49])) k.emit(s_add_u32(s[55], s[43], LIT, 255)) k.emit(s_lshr_b32(s[52], s[55], 8)) k.emit(s_lshl_b32(s[52], s[52], 5)) - k.emit(s_mov_b32(s[46], 0)) + k.emit(s_mov_b32(s[49], 0)) k.label('L2_00E8') k.emit(s_cmp_lt_i32(s[48], s[52])) k.emit(s_cbranch_scc1(3), target='L2_00FC') k.emit(s_sub_i32(s[48], s[48], s[52])) - k.emit(s_add_i32(s[46], s[46], 32)) + k.emit(s_add_i32(s[49], s[49], 32)) k.emit(s_branch(65531), target='L2_00E8') k.label('L2_00FC') - k.emit(s_sub_i32(s[54], s[54], s[46])) + k.emit(s_sub_i32(s[54], s[54], s[49])) k.emit(s_cmp_lt_i32(s[54], 32)) k.emit(s_cbranch_scc1(3), target='L2_0114') k.emit(s_lshr_b32(s[47], s[48], 5)) @@ -2663,7 +2264,7 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[52], s[54], s[47])) k.emit(s_sub_i32(s[52], s[48], s[52])) k.label('L2_0194') - k.emit(s_add_i32(s[46], s[52], s[46])) + k.emit(s_add_i32(s[49], s[52], s[49])) k.emit(s_mov_b32(s[6], -16)) k.emit(s_mov_b32(s[10], -16)) k.emit(s_mov_b32(s[18], -16)) @@ -2680,37 +2281,18 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_or_b32(s[9], s[9], LIT, 262144)) k.emit(s_or_b32(s[17], s[17], LIT, 262144)) k.emit(s_or_b32(s[13], s[13], LIT, 262144)) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(9), target='L2_0234') - k.emit(s_lshr_b32(s[58], s[45], s[57])) - k.emit(s_add_u32(s[58], s[58], LIT, 255)) - k.emit(s_lshr_b32(s[58], s[58], 8)) - k.emit(s_lshl_b32(s[58], s[58], 8)) - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_sub_i32(s[52], s[45], s[53])) - k.emit(s_cmp_lt_i32(s[52], s[58])) - k.emit(s_cselect_b32(s[45], s[52], s[58])) - k.label('L2_0234') - k.emit(s_lshr_b32(s[41], s[41], 1)) - k.emit(s_mul_i32(s[52], s[41], s[43])) + k.emit(s_lshr_b32(s[37], s[37], 1)) + k.emit(s_mul_i32(s[52], s[37], s[43])) k.emit(s_mov_b32(s[14], s[52])) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(5), target='L2_025C') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_lshr_b32(s[52], s[53], 1)) - k.emit(s_add_u32(s[12], s[12], s[52])) - k.emit(s_addc_u32(s[13], s[13], 0)) - k.emit(s_sub_u32(s[14], s[14], s[52])) - k.label('L2_025C') - k.emit(s_lshr_b32(s[42], s[42], 1)) - k.emit(s_mul_i32(s[52], s[42], s[44])) + k.emit(s_lshr_b32(s[38], s[38], 1)) + k.emit(s_mul_i32(s[52], s[38], s[44])) k.emit(s_mov_b32(s[18], s[52])) k.emit(s_add_u32(s[52], s[43], 31)) k.emit(s_lshr_b32(s[52], s[52], 5)) k.emit(s_lshl_b32(s[52], s[52], 5)) - k.emit(s_mul_i32(s[53], s[52], s[36])) + k.emit(s_mul_i32(s[53], s[52], s[39])) k.emit(s_mov_b32(s[22], s[53])) - k.emit(s_mul_i32(s[53], s[44], s[37])) + k.emit(s_mul_i32(s[53], s[44], s[40])) k.emit(s_mov_b32(s[26], s[53])) k.emit(s_mov_b32(s[23], LIT, 131072)) k.emit(s_mov_b32(s[27], LIT, 131072)) @@ -2727,23 +2309,23 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[5], v[5], v[6])) k.emit(v_and_b32_e32(v[4], 1, v[4])) k.emit(v_add_u32_e32(v[5], v[5], v[4])) - k.emit(v_mul_lo_u32(v[212], s[41], v[5])) + k.emit(v_mul_lo_u32(v[212], s[37], v[5])) k.emit(v_and_b32_e32(v[4], 7)) k.emit(v_lshlrev_b32_e32(v[4], 4, v[4])) k.emit(v_add_u32_e32(v[212], v[212], v[4])) - k.emit(s_lshr_b32(s[52], s[49], 1)) + k.emit(s_lshr_b32(s[52], s[46], 1)) k.emit(s_mul_i32(s[52], s[52], 8)) - k.emit(s_and_b32(s[53], s[49], 1)) + k.emit(s_and_b32(s[53], s[46], 1)) k.emit(s_mul_i32(s[53], s[53], 2)) k.emit(s_add_u32(s[52], s[52], s[53])) k.emit(s_mul_i32(s[53], s[47], LIT, 256)) k.emit(s_add_u32(s[52], s[52], s[53])) - k.emit(s_mul_i32(s[52], s[41], s[52])) + k.emit(s_mul_i32(s[52], s[37], s[52])) k.emit(v_add_u32_e32(v[212], s[52], v[212])) - k.emit(s_mul_i32(s[52], s[41], 32)) + k.emit(s_mul_i32(s[52], s[37], 32)) for i in range(7): k.emit(v_add_u32_e32(v[213 + i * 1], s[52], v[212 + i * 1])) - k.emit(s_mul_i32(s[59], LIT, s[49], 1056)) + k.emit(s_mul_i32(s[59], LIT, s[46], 1056)) k.emit(s_add_u32(s[59], LIT, s[59], 4096)) k.emit(v_and_b32_e32(v[4], 15)) k.emit(v_lshrrev_b32_e32(v[5], 3, v[4])) @@ -2765,60 +2347,37 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mov_b32(s[52], LIT, 4096)) k.emit(v_add_u32_e64(v[220], v[220], s[52])) k.emit(v_add_u32_e32(v[221], LIT, v[220], 33792)) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(4), target='L2_03C4') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_add_u32(s[20], s[20], s[53])) - k.emit(s_addc_u32(s[21], s[21], 0)) - k.emit(s_sub_u32(s[22], s[22], s[53])) - k.label('L2_03C4') k.emit(v_lshlrev_b32_e32(v[222], 2)) k.emit(s_mul_i32(s[52], s[47], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 32)) + k.emit(s_mul_i32(s[53], s[46], 32)) k.emit(s_add_i32(s[52], s[53], s[52])) - k.emit(s_mul_i32(s[53], s[52], s[36])) + k.emit(s_mul_i32(s[53], s[52], s[39])) k.emit(v_add_u32_e32(v[222], s[53], v[222])) - k.emit(s_mul_i32(s[53], LIT, s[36], 128)) + k.emit(s_mul_i32(s[53], LIT, s[39], 128)) k.emit(v_add_u32_e32(v[223], s[53], v[222])) - k.emit(s_mul_i32(s[60], s[49], LIT, 256)) + k.emit(s_mul_i32(s[60], s[46], LIT, 256)) k.emit(s_add_i32(s[60], s[60], 0)) k.emit(v_lshlrev_b32_e32(v[224], 2)) k.emit(v_add_u32_e32(v[224], 0, v[224])) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(6), target='L2_0420') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_lshr_b32(s[52], s[53], 1)) - k.emit(s_mul_i32(s[52], s[52], 16)) - k.emit(s_add_u32(s[16], s[16], s[52])) - k.emit(s_addc_u32(s[17], s[17], 0)) - k.emit(s_sub_u32(s[18], s[18], s[52])) - k.label('L2_0420') k.emit(v_lshlrev_b32_e32(v[225], 4)) - k.emit(s_mul_i32(s[52], s[46], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 64)) + k.emit(s_mul_i32(s[52], s[49], LIT, 256)) + k.emit(s_mul_i32(s[53], s[46], 64)) k.emit(s_add_u32(s[52], s[52], s[53])) - k.emit(s_mul_i32(s[52], s[52], s[42])) + k.emit(s_mul_i32(s[52], s[52], s[38])) k.emit(v_add_u32_e32(v[225], s[52], v[225])) - k.emit(s_mul_i32(s[52], 16, s[42])) + k.emit(s_mul_i32(s[52], 16, s[38])) k.emit(v_add_u32_e32(v[226], s[52], v[225])) k.emit(v_add_u32_e32(v[227], s[52], v[226])) k.emit(v_add_u32_e32(v[228], s[52], v[227])) for i in range(4): k.emit(v_add_u32_e32(v[229 + i * 1], LIT, v[225 + i * 1], 1024)) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(4), target='L2_0484') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_add_u32(s[24], s[24], s[53])) - k.emit(s_addc_u32(s[25], s[25], 0)) - k.emit(s_sub_u32(s[26], s[26], s[53])) - k.label('L2_0484') k.emit(v_lshlrev_b32_e32(v[233], 2)) - k.emit(s_mul_i32(s[52], s[46], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 64)) + k.emit(s_mul_i32(s[52], s[49], LIT, 256)) + k.emit(s_mul_i32(s[53], s[46], 64)) k.emit(s_add_i32(s[52], s[53], s[52])) - k.emit(s_mul_i32(s[53], s[52], s[37])) + k.emit(s_mul_i32(s[53], s[52], s[40])) k.emit(v_add_u32_e32(v[233], s[53], v[233])) - k.emit(s_mul_i32(s[52], 32, s[37])) + k.emit(s_mul_i32(s[52], 32, s[40])) k.emit(v_add_u32_e32(v[234], s[52], v[233])) k.emit(s_mov_b32(s[61], LIT, 128)) k.emit(s_mov_b32(s[62], LIT, 2048)) @@ -2904,18 +2463,18 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(ds_read_b32(v[201], v[224], v[0], v[0], 0, 0, 1)) k.emit(ds_read_b32(v[202], v[224], v[0], v[0], 0, 0, 2)) k.emit(ds_read_b32(v[203], v[224], v[0], v[0], 0, 0, 3)) - k.emit(s_lshl_b32(s[40], s[40], 1)) + k.emit(s_lshl_b32(s[36], s[36], 1)) k.emit(s_mul_i32(s[52], s[47], LIT, 256)) - k.emit(s_mul_hi_u32(s[53], s[52], s[40])) + k.emit(s_mul_hi_u32(s[53], s[52], s[36])) k.emit(s_add_u32(s[5], s[5], s[53])) - k.emit(s_mul_i32(s[53], s[52], s[40])) + k.emit(s_mul_i32(s[53], s[52], s[36])) k.emit(s_add_u32(s[4], s[4], s[53])) k.emit(s_addc_u32(s[5], 0, s[5])) k.emit(s_sub_i32(s[52], s[43], s[52])) - k.emit(s_mul_i32(s[52], s[52], s[40])) + k.emit(s_mul_i32(s[52], s[52], s[36])) k.emit(s_mov_b32(s[6], s[52])) k.emit(v_and_b32_e64(v[235], v[0], 15)) - k.emit(v_mul_lo_u32(v[235], v[235], s[40])) + k.emit(v_mul_lo_u32(v[235], v[235], s[36])) k.emit(v_lshrrev_b32_e32(v[4], 5)) k.emit(v_mul_i32_i24_e32(v[4], 16, v[4])) k.emit(v_add_u32_e32(v[235], v[4], v[235])) @@ -2923,12 +2482,12 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_and_b32_e32(v[4], 1, v[4])) k.emit(v_mul_i32_i24_e32(v[4], 32, v[4])) k.emit(v_add_u32_e32(v[235], v[4], v[235])) - k.emit(s_mul_i32(s[52], s[46], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 64)) + k.emit(s_mul_i32(s[52], s[49], LIT, 256)) + k.emit(s_mul_i32(s[53], s[46], 64)) k.emit(s_add_i32(s[52], s[52], s[53])) k.emit(s_lshl_b32(s[52], s[52], 1)) k.emit(v_add_u32_e32(v[235], s[52], v[235])) - k.emit(s_mul_i32(s[53], s[40], 16)) + k.emit(s_mul_i32(s[53], s[36], 16)) for i in range(15): k.emit(v_add_u32_e64(v[236 + i * 1], v[235 + i * 1], s[53])) k.emit(s_mov_b32(s[50], 0)) @@ -2937,7 +2496,7 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_cmp_lt_u32(LIT, s[51], 512 + i * -256)) k.emit(s_cselect_b32(s[61 + i * 1], s[61 + i * 1], 0)) k.emit(s_cselect_b32(s[63 + i * 1], s[63 + i * 1], 0)) - k.emit(s_cmp_lt_i32(s[49], 2)) + k.emit(s_cmp_lt_i32(s[46], 2)) k.emit(s_cbranch_scc0(1367), target='L2_25B8') k.label('L2_105C') k.emit(s_waitcnt(122)) @@ -3619,270 +3178,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.label('L2_3B10') k.emit(s_waitcnt(112)) k.emit(s_barrier()) - k.emit(s_cmp_eq_u32(s[57], 0)) - k.emit(s_cbranch_scc1(1505), target='L2_52A4') - for i in range(2): - for j77 in range(4): - k.emit(v_accvgpr_read(v[8 + j77 * 1 + i * 4], v[0 + j77 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j77 * 1 + i * 4], s[38], v[8 + j77 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[235], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[235], v[235], 64)) - for i in range(2): - for j78 in range(4): - k.emit(v_accvgpr_read(v[8 + j78 * 1 + i * 4], v[64 + j78 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j78 * 1 + i * 4], s[38], v[8 + j78 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[235], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[235], v[235], 64)) - for i in range(2): - for j79 in range(4): - k.emit(v_accvgpr_read(v[8 + j79 * 1 + i * 4], v[4 + j79 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j79 * 1 + i * 4], s[38], v[8 + j79 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[236], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[236], v[236], 64)) - for i in range(2): - for j80 in range(4): - k.emit(v_accvgpr_read(v[8 + j80 * 1 + i * 4], v[68 + j80 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j80 * 1 + i * 4], s[38], v[8 + j80 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[236], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[236], v[236], 64)) - for i in range(2): - for j81 in range(4): - k.emit(v_accvgpr_read(v[8 + j81 * 1 + i * 4], v[8 + j81 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j81 * 1 + i * 4], s[38], v[8 + j81 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[237], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[237], v[237], 64)) - for i in range(2): - for j82 in range(4): - k.emit(v_accvgpr_read(v[8 + j82 * 1 + i * 4], v[72 + j82 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j82 * 1 + i * 4], s[38], v[8 + j82 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[237], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[237], v[237], 64)) - for i in range(2): - for j83 in range(4): - k.emit(v_accvgpr_read(v[8 + j83 * 1 + i * 4], v[12 + j83 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j83 * 1 + i * 4], s[38], v[8 + j83 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[238], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[238], v[238], 64)) - for i in range(2): - for j84 in range(4): - k.emit(v_accvgpr_read(v[8 + j84 * 1 + i * 4], v[76 + j84 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j84 * 1 + i * 4], s[38], v[8 + j84 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[238], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[238], v[238], 64)) - for i in range(2): - for j85 in range(4): - k.emit(v_accvgpr_read(v[8 + j85 * 1 + i * 4], v[16 + j85 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j85 * 1 + i * 4], s[38], v[8 + j85 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[239], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[239], v[239], 64)) - for i in range(2): - for j86 in range(4): - k.emit(v_accvgpr_read(v[8 + j86 * 1 + i * 4], v[80 + j86 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j86 * 1 + i * 4], s[38], v[8 + j86 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[239], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[239], v[239], 64)) - for i in range(2): - for j87 in range(4): - k.emit(v_accvgpr_read(v[8 + j87 * 1 + i * 4], v[20 + j87 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j87 * 1 + i * 4], s[38], v[8 + j87 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[240], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[240], v[240], 64)) - for i in range(2): - for j88 in range(4): - k.emit(v_accvgpr_read(v[8 + j88 * 1 + i * 4], v[84 + j88 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j88 * 1 + i * 4], s[38], v[8 + j88 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[240], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[240], v[240], 64)) - for i in range(2): - for j89 in range(4): - k.emit(v_accvgpr_read(v[8 + j89 * 1 + i * 4], v[24 + j89 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j89 * 1 + i * 4], s[38], v[8 + j89 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[241], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[241], v[241], 64)) - for i in range(2): - for j90 in range(4): - k.emit(v_accvgpr_read(v[8 + j90 * 1 + i * 4], v[88 + j90 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j90 * 1 + i * 4], s[38], v[8 + j90 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[241], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[241], v[241], 64)) - for i in range(2): - for j91 in range(4): - k.emit(v_accvgpr_read(v[8 + j91 * 1 + i * 4], v[28 + j91 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j91 * 1 + i * 4], s[38], v[8 + j91 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[242], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[242], v[242], 64)) - for i in range(2): - for j92 in range(4): - k.emit(v_accvgpr_read(v[8 + j92 * 1 + i * 4], v[92 + j92 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j92 * 1 + i * 4], s[38], v[8 + j92 * 1 + i * 4])) - for i in range(8): - for j93 in range(2): - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - k.emit(buffer_atomic_pk_add_bf16(v[16], v[242 + j93 * 1 + i * 1], s[4:7], 0, 0, 1)) - k.emit(buffer_atomic_pk_add_bf16(v[17], v[242 + j93 * 1 + i * 1], s[4:7], 0, 4, 1)) - k.emit(buffer_atomic_pk_add_bf16(v[18], v[242 + j93 * 1 + i * 1], s[4:7], 0, 8, 1)) - k.emit(buffer_atomic_pk_add_bf16(v[19], v[242 + j93 * 1 + i * 1], s[4:7], 0, 12, 1)) - k.emit(v_add_i32(v[242 + j93 * 1 + i * 1], v[242 + j93 * 1 + i * 1], 64)) - k.emit(v_accvgpr_read(v[8], v[128 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[38], v[8])) - k.emit(v_accvgpr_read(v[9], v[129 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[38], v[9])) - k.emit(v_accvgpr_read(v[10], v[130 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[38], v[10])) - k.emit(v_accvgpr_read(v[11], v[131 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[38], v[11])) - k.emit(v_accvgpr_read(v[12], v[160 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[38], v[12])) - k.emit(v_accvgpr_read(v[13], v[161 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[38], v[13])) - k.emit(v_accvgpr_read(v[14], v[162 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[38], v[14])) - k.emit(v_accvgpr_read(v[15], v[163 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[38], v[15])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[250], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[250], v[250], 64)) - k.emit(s_branch(1312), target='L2_6724') - k.label('L2_52A4') for i in range(2): for j94 in range(4): k.emit(v_accvgpr_read(v[8 + j94 * 1 + i * 4], v[0 + j94 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j94 * 1 + i * 4], s[38], v[8 + j94 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3895,7 +3193,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j95 in range(4): k.emit(v_accvgpr_read(v[8 + j95 * 1 + i * 4], v[64 + j95 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j95 * 1 + i * 4], s[38], v[8 + j95 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3908,7 +3205,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j96 in range(4): k.emit(v_accvgpr_read(v[8 + j96 * 1 + i * 4], v[4 + j96 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j96 * 1 + i * 4], s[38], v[8 + j96 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3921,7 +3217,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j97 in range(4): k.emit(v_accvgpr_read(v[8 + j97 * 1 + i * 4], v[68 + j97 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j97 * 1 + i * 4], s[38], v[8 + j97 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3934,7 +3229,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j98 in range(4): k.emit(v_accvgpr_read(v[8 + j98 * 1 + i * 4], v[8 + j98 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j98 * 1 + i * 4], s[38], v[8 + j98 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3947,7 +3241,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j99 in range(4): k.emit(v_accvgpr_read(v[8 + j99 * 1 + i * 4], v[72 + j99 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j99 * 1 + i * 4], s[38], v[8 + j99 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3960,7 +3253,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j100 in range(4): k.emit(v_accvgpr_read(v[8 + j100 * 1 + i * 4], v[12 + j100 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j100 * 1 + i * 4], s[38], v[8 + j100 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3973,7 +3265,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j101 in range(4): k.emit(v_accvgpr_read(v[8 + j101 * 1 + i * 4], v[76 + j101 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j101 * 1 + i * 4], s[38], v[8 + j101 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3986,7 +3277,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j102 in range(4): k.emit(v_accvgpr_read(v[8 + j102 * 1 + i * 4], v[16 + j102 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j102 * 1 + i * 4], s[38], v[8 + j102 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3999,7 +3289,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j103 in range(4): k.emit(v_accvgpr_read(v[8 + j103 * 1 + i * 4], v[80 + j103 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j103 * 1 + i * 4], s[38], v[8 + j103 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4012,7 +3301,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j104 in range(4): k.emit(v_accvgpr_read(v[8 + j104 * 1 + i * 4], v[20 + j104 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j104 * 1 + i * 4], s[38], v[8 + j104 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4025,7 +3313,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j105 in range(4): k.emit(v_accvgpr_read(v[8 + j105 * 1 + i * 4], v[84 + j105 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j105 * 1 + i * 4], s[38], v[8 + j105 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4038,7 +3325,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j106 in range(4): k.emit(v_accvgpr_read(v[8 + j106 * 1 + i * 4], v[24 + j106 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j106 * 1 + i * 4], s[38], v[8 + j106 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4051,7 +3337,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j107 in range(4): k.emit(v_accvgpr_read(v[8 + j107 * 1 + i * 4], v[88 + j107 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j107 * 1 + i * 4], s[38], v[8 + j107 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4064,7 +3349,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j108 in range(4): k.emit(v_accvgpr_read(v[8 + j108 * 1 + i * 4], v[28 + j108 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j108 * 1 + i * 4], s[38], v[8 + j108 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4077,7 +3361,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j109 in range(4): k.emit(v_accvgpr_read(v[8 + j109 * 1 + i * 4], v[92 + j109 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j109 * 1 + i * 4], s[38], v[8 + j109 * 1 + i * 4])) for i in range(8): for j110 in range(2): k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) @@ -4092,21 +3375,13 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(buffer_store_dwordx4(v[16:19], v[242 + j110 * 1 + i * 1], s[4:7], 0, 0, 1)) k.emit(v_add_i32(v[242 + j110 * 1 + i * 1], v[242 + j110 * 1 + i * 1], 64)) k.emit(v_accvgpr_read(v[8], v[128 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[38], v[8])) k.emit(v_accvgpr_read(v[9], v[129 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[38], v[9])) k.emit(v_accvgpr_read(v[10], v[130 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[38], v[10])) k.emit(v_accvgpr_read(v[11], v[131 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[38], v[11])) k.emit(v_accvgpr_read(v[12], v[160 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[38], v[12])) k.emit(v_accvgpr_read(v[13], v[161 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[38], v[13])) k.emit(v_accvgpr_read(v[14], v[162 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[38], v[14])) k.emit(v_accvgpr_read(v[15], v[163 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[38], v[15])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4116,7 +3391,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_nop(1)) k.emit(buffer_store_dwordx4(v[16:19], v[250], s[4:7], 0, 0, 1)) k.emit(v_add_i32(v[250], v[250], 64)) - k.label('L2_6724') k.emit(s_waitcnt()) k.emit(s_endpgm()) else: diff --git a/tinygrad_repo/extra/gemm/max_matmul.py b/tinygrad_repo/extra/gemm/max_matmul.py index 5a41fe17f..9e3be02a6 100644 --- a/tinygrad_repo/extra/gemm/max_matmul.py +++ b/tinygrad_repo/extra/gemm/max_matmul.py @@ -113,7 +113,6 @@ if __name__ == "__main__": } elif GEMM_VARIATION == "hcopt" and M == N == K == 4096 and DTYPE_IN == dtypes.half and DTYPE_OUT == dtypes.half and DTYPE_ACC == dtypes.float: print("Using CUDA and generated hcopt") - # [Opt(op=OptOps.TC, axis=0, amt=0), Opt(op=OptOps.UPCAST, axis=0, amt=4), Opt(op=OptOps.UPCAST, axis=1, amt=4), Opt(op=OptOps.LOCAL, axis=1, amt=4)] prog = CUDAProgram(device, "wmma_example", compiler.compile(open(os.path.join(script_dir, 'max_kernels/nv.fp16_fp32_fp16.hcopt.cu')).read())) args = (c, a, b) kwargs = { diff --git a/tinygrad_repo/extra/gemm/moe_gemm.py b/tinygrad_repo/extra/gemm/moe_gemm.py index 2b89f752e..4fba527d1 100644 --- a/tinygrad_repo/extra/gemm/moe_gemm.py +++ b/tinygrad_repo/extra/gemm/moe_gemm.py @@ -1,10 +1,32 @@ import functools, pathlib from tinygrad import Tensor, dtypes -from tinygrad.uop.ops import UOp, Ops, KernelInfo +from tinygrad.uop.ops import UOp, Ops, KernelInfo, AxisType +from tinygrad.helpers import getenv from tinygrad.renderer import Estimates from tinygrad.runtime.support.compiler_amd import HIPCCCompiler from extra.gemm.cdna_asm_gemm import quantize_mxfp8, _mx_block_scale, _mx_block_scale_3d +ZERO_OPTIM = getenv("ZERO_OPTIM", 0) + +def reduce_scatter_devaxis(out:Tensor, shard_axis:int=0) -> Tensor: + # out: sharded on the device axis, shape (ndev, *rest); return the device-axis sum left sharded on shard_axis. + u = out.uop + devs, rest = u.device, u.shape[1:] + assert rest[shard_axis] % len(devs) == 0, f"reduce_scatter needs even shards: {rest[shard_axis]} % {len(devs)}" + # reach the raw per-device buffer below the UNSHARD, keeping the AFTERs so reads stay ordered after the kernel writes + node, barriers = u, [] + while node.op is not Ops.UNSHARD: + if node.op is Ops.AFTER: barriers += node.src[1:] + node = node.src[0] + mbuf = node.src[0].after(*barriers) if barriers else node.src[0] + sz = rest[shard_axis] // len(devs) + shards = [] + for i in range(len(devs)): + bounds = tuple((0,s) if a != shard_axis else (i*sz,(i+1)*sz) for a,s in enumerate(rest)) + contribs = [mbuf.mselect(j).reshape(rest).shrink(bounds).copy_to_device(devs[i]) for j in range(len(devs))] + shards.append(functools.reduce(lambda a,b: a.alu(Ops.ADD, b), contribs)) + return Tensor(UOp.mstack(*shards).unshard(shard_axis, UOp.range(len(devs), -1, AxisType.DEVICE)), device=devs) + @functools.cache def custom_hk_grouped_mxfp8_gemm(C:UOp, A:UOp, B:UOp, scale_A:UOp, scale_B:UOp, *extra:UOp, dname:str, n_experts:int) -> UOp: M, K = A.shape @@ -58,7 +80,8 @@ def grouped_mx_wgrad(g:Tensor, xg:Tensor, expert_off:Tensor, n_experts:int) -> T out = Tensor(inv.uop.unshard(0), device=g.device) if is_multi else inv out = Tensor.custom_kernel(out, gT, xT, g_si, x_si, expert_off, fxn=functools.partial(custom_hk_grouped_mxfp8_wgrad, dname=dname, n_experts=n_experts))[0] - out = out.sum(0) if is_multi else out.squeeze(0) + if is_multi and ZERO_OPTIM: out = reduce_scatter_devaxis(out, 0) + else: out = out.sum(0) if is_multi else out.squeeze(0) return out.reshape(n_experts, N, K) def mx_pack_3d(e8:Tensor) -> Tensor: diff --git a/tinygrad_repo/extra/gemm/moe_routing.py b/tinygrad_repo/extra/gemm/moe_routing.py index 30ba001c3..321f241aa 100644 --- a/tinygrad_repo/extra/gemm/moe_routing.py +++ b/tinygrad_repo/extra/gemm/moe_routing.py @@ -53,7 +53,7 @@ def _ggather_bwd(gradient:UOp, kernel:UOp) -> tuple: g, m, j, jo, ji = _kv_ranges(Gk, M, Dk, _blk_for(Dk)) row = idx.index(g, m).cast(dtypes.weakint) val = gout.index(g, m, j).load().cast(dtypes.float32) - atomic = UOp(Ops.CUSTOM, dtypes.void, (gtab.index(g, row, j), val), arg=atomic_str) + atomic = UOp(Ops.CUSTOM, src=(gtab.index(g, row, j), val), arg=(atomic_str, dtypes.void)) return atomic.end(g, m, jo, ji).sink(arg=KernelInfo(name=f"ggather_bwd_{M}_{Dk}", opts_to_apply=())) grad_table = Tensor.custom_kernel(gt, go, Tensor(idx_u, device=dev), fxn=_bwd_kernel)[0] return (None, grad_table.cast(table_u.dtype).uop, None) diff --git a/tinygrad_repo/extra/gemm/rdna4_asm_matmul.py b/tinygrad_repo/extra/gemm/rdna4_asm_matmul.py index 4f655bcef..5b1817fd7 100644 --- a/tinygrad_repo/extra/gemm/rdna4_asm_matmul.py +++ b/tinygrad_repo/extra/gemm/rdna4_asm_matmul.py @@ -223,7 +223,7 @@ def test_matmul(): lds = UOp.placeholder((lds_size,), dtypes.uint8, 0, AddrSpace.LOCAL) sink = UOp.sink(A.base, B.base, C.base, lds, *gidxs, *lidxs, arg=KernelInfo(name=colored("kernel","cyan"), estimates=Estimates(ops=N*N*N*2, mem=N*N*2*3))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) c = Tensor.custom_kernel(a, b, c, fxn=asm_kernel)[2] linear = c.schedule_linear() diff --git a/tinygrad_repo/extra/gemm/tinygrad_nv_matmul.py b/tinygrad_repo/extra/gemm/tinygrad_nv_matmul.py index adc9a747e..a3826ee76 100644 --- a/tinygrad_repo/extra/gemm/tinygrad_nv_matmul.py +++ b/tinygrad_repo/extra/gemm/tinygrad_nv_matmul.py @@ -1,6 +1,7 @@ from tinygrad import Tensor, dtypes, Context from tinygrad.helpers import getenv from tinygrad.codegen.opt import Opt, OptOps +from tinygrad.uop.ops import AxisType from tinygrad.engine.realize import run_linear from dataclasses import replace @@ -13,17 +14,17 @@ if __name__ == "__main__": C = A.matmul(B) if getenv("GEMV"): opts = [ - Opt(op=OptOps.UNROLL, axis=0, amt=8), - Opt(op=OptOps.GROUP, axis=0, amt=32), + Opt(op=OptOps.SPLIT, axis=1, arg=(8, AxisType.UNROLL)), + Opt(op=OptOps.SPLIT, axis=1, arg=(32, AxisType.GROUP_REDUCE)), ] else: opts = [ Opt(op=OptOps.TC, axis=0, amt=0), - Opt(op=OptOps.UPCAST, axis=0, amt=4), - Opt(op=OptOps.UPCAST, axis=1, amt=8), - Opt(op=OptOps.LOCAL, axis=0, amt=2), - Opt(op=OptOps.LOCAL, axis=1, amt=2), - Opt(op=OptOps.LOCAL, axis=0, amt=2), + Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=1, arg=(8, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=0, arg=(2, AxisType.LOCAL)), + Opt(op=OptOps.SPLIT, axis=1, arg=(2, AxisType.LOCAL)), + Opt(op=OptOps.SPLIT, axis=0, arg=(2, AxisType.LOCAL)), ] linear = C.schedule_linear() call = linear.src[-1] diff --git a/tinygrad_repo/extra/gemm/triton_nv_matmul.py b/tinygrad_repo/extra/gemm/triton_nv_matmul.py index 133f09227..8477b5cba 100644 --- a/tinygrad_repo/extra/gemm/triton_nv_matmul.py +++ b/tinygrad_repo/extra/gemm/triton_nv_matmul.py @@ -79,7 +79,7 @@ if __name__ == "__main__": linear, var_vals = C.linear_with_vars() last_call = linear.src[-1] ast = last_call.src[0] - bufs = [s.buffer for s in last_call.src[1:] if s.op is not Ops.BIND] + bufs = [s.buffer for s in last_call.src[1:] if not s.is_bound_var] src = compiled.asm["ptx"] # specify the shared memory here so we don't need to do it dynamically diff --git a/tinygrad_repo/extra/gptoss_kernels/rmsnorm/__init__.py b/tinygrad_repo/extra/gptoss_kernels/rmsnorm/__init__.py new file mode 100644 index 000000000..260e03539 --- /dev/null +++ b/tinygrad_repo/extra/gptoss_kernels/rmsnorm/__init__.py @@ -0,0 +1,29 @@ +from __future__ import annotations +import functools +from tinygrad import Tensor +from tinygrad.uop.ops import UOp + +def rmsnorm_mul_fwd(x_in:Tensor, weight:Tensor, eps:float) -> tuple[Tensor, Tensor]: + x = x_in.float() + rrms = (x.square().mean(-1, keepdim=True) + eps).rsqrt() + return ((x * rrms) * weight.float()).cast(x_in.dtype), rrms + +@functools.cache +def _rmsnorm_mul_fwd_fxn(x_in_p, w_p, eps, device): + return rmsnorm_mul_fwd(Tensor(x_in_p, device=device), Tensor(w_p, device=device), eps) + +def _rmsnorm_mul_bwd(grad:UOp, call:UOp) -> tuple: + x = Tensor(call.src[1]).float(); weight = Tensor(call.src[2]).float() + rrms = Tensor(call.gettuple(1)) + x_normed = x * rrms # recompute unweighted normed (x is call.src[1]) + d_y = Tensor(grad).float() + dxn = d_y * weight # d/d(x_normed) + d_x = rrms * (dxn - x_normed * (dxn * x_normed).mean(-1, keepdim=True)) + dw = d_y * x_normed + d_weight = dw.sum(axis=tuple(range(dw.ndim - 1))) # reduce batch/seq -> [dim] + return (d_x.cast(call.src[1].dtype).uop, d_weight.cast(call.src[2].dtype).uop) + +def rmsnorm_mul(x_in:Tensor, weight:Tensor, eps:float) -> tuple[Tensor, Tensor]: + fxn = _rmsnorm_mul_fwd_fxn(x_in.as_param(0).uop, weight.as_param(1).uop, eps, x_in.device) + call = UOp.maketuple(fxn[0].uop, fxn[1].uop).call(x_in.uop, weight.uop, grad_fxn=_rmsnorm_mul_bwd) + return Tensor(call.gettuple(0)), Tensor(call.gettuple(1)) diff --git a/tinygrad_repo/extra/hcq/hcq_smi.py b/tinygrad_repo/extra/hcq/hcq_smi.py index edb465235..814c8f430 100755 --- a/tinygrad_repo/extra/hcq/hcq_smi.py +++ b/tinygrad_repo/extra/hcq/hcq_smi.py @@ -16,9 +16,12 @@ def _do_reset_device(pci_bus): os.system(f"sudo sh -c 'echo 1 > /sys/bus/pci/dev def _is_module_loaded(name: str) -> bool: return os.path.isdir(f"/sys/module/{name}") def cmd_remove_module(args): - modules = ["nvidia_drm", "nvidia_modeset", "nvidia_uvm", "nvidia", "ast"] if args.backend == "nv" else ["amdgpu"] + modules = ["nvidia_drm", "nvidia_modeset", "nvidia_uvm", "nvidia"] if args.backend == "nv" else ["amdgpu"] to_unload = [m for m in modules if _is_module_loaded(m)] if not to_unload: print("Kernel modules are not loaded") + elif getattr(args, "expect", False): + print(f"Kernel modules are loaded: {to_unload}") + sys.exit(1) else: print("Removing kernel modules:", ", ".join(to_unload)) try: subprocess.run(["sudo", "modprobe", "-r", *to_unload], check=True) @@ -60,17 +63,19 @@ def cmd_show_pids(args): def cmd_kill_pids(args): devs = scan_devs_based_on_lock(prefix:={"amd":"am", "nv":"nv"}[args.backend], args) + use_sudo = not getattr(args, "sudoless", False) for dev in devs: for i in range(128): if i > 0: time.sleep(0.2) try: - try: pid = subprocess.check_output(['sudo', 'lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1] + try: pid = subprocess.check_output((['sudo'] if use_sudo else []) + + ['lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1] except subprocess.CalledProcessError: break print(f"Killing process {pid} (which uses {dev})") - subprocess.run(['sudo', 'kill', '-9', pid], check=True) + subprocess.run((['sudo'] if use_sudo else []) + ['kill', '-9', pid], check=True) except subprocess.CalledProcessError as e: print(f"Failed to kill process for device {dev}: {e}", file=sys.stderr) @@ -79,6 +84,7 @@ def add_common_commands(parent_subparsers): p_insmod.set_defaults(func=cmd_insert_module) p_rmmod = parent_subparsers.add_parser("rmmod", help="Remove a kernel module") + p_rmmod.add_argument("--expect", action="store_true", help="Just assert that module is already unloaded") p_rmmod.set_defaults(func=cmd_remove_module) p_reset = parent_subparsers.add_parser("reset", help="Reset a device") @@ -91,17 +97,20 @@ def add_common_commands(parent_subparsers): p_reset = parent_subparsers.add_parser("kill_pids", help="Kill pids of processes using the device") p_reset.add_argument("--pci_bus", default="", help="PCI bus ID of the device") + p_reset.add_argument("--sudoless", action="store_true", help="Do not use sudo when detecting or killing pids") p_reset.set_defaults(func=cmd_kill_pids) if __name__ == "__main__": parser = argparse.ArgumentParser() backend_subparsers = parser.add_subparsers(dest="backend", required=True, metavar="{nv,amd}", help="Hardware backend to target") - nv_parser = backend_subparsers.add_parser("nv", help="NVIDIA GPUs") + nv_parser = backend_subparsers.add_parser("nv", aliases=["NV"], help="NVIDIA GPUs") + nv_parser.set_defaults(backend="nv") nv_commands = nv_parser.add_subparsers(dest="command", required=True) add_common_commands(nv_commands) - amd_parser = backend_subparsers.add_parser("amd", help="AMD GPUs") + amd_parser = backend_subparsers.add_parser("amd", aliases=["AMD"], help="AMD GPUs") + amd_parser.set_defaults(backend="amd") amd_commands = amd_parser.add_subparsers(dest="command", required=True) add_common_commands(amd_commands) diff --git a/tinygrad_repo/extra/hcq2/ops_amd2.py b/tinygrad_repo/extra/hcq2/ops_amd2.py index 0c8e99d69..a6f47142a 100644 --- a/tinygrad_repo/extra/hcq2/ops_amd2.py +++ b/tinygrad_repo/extra/hcq2/ops_amd2.py @@ -1,33 +1,32 @@ from __future__ import annotations -from typing import cast, Any, Callable -import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit +from typing import cast +import os, ctypes, struct, functools, importlib, mmap, errno, contextlib, sys, itertools, atexit assert sys.platform != 'win32' from dataclasses import dataclass -from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, HCQ2Buffer, encode_kernargs_clike, make_cmdbuf -from tinygrad.runtime.support.hcq2 import make_binary_patch +from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_buf, hcq_size_var from tinygrad.uop.ops import sint, UOp -from tinygrad.device import Compiled, BufferSpec, Buffer, Device +from tinygrad.device import BufferSpec, Buffer, Device from tinygrad.dtype import dtypes -from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, ProfileEvent, lo32, hi32, colored, prod, ContextVar, TracingKey -from tinygrad.helpers import VIZ, ceildiv, unwrap, pluralize, to_tuple +from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, lo32, hi32 +from tinygrad.helpers import ceildiv, unwrap, pluralize, to_tuple from tinygrad.renderer.cstyle import HIPRenderer, HIPCCRenderer from tinygrad.renderer.llvmir import AMDLLVMRenderer -from tinygrad.runtime.autogen import kfd, hsa, sqtt, amdgpu_kd, amdgpu_drm +from tinygrad.runtime.autogen import kfd, hsa, amdgpu_kd, amdgpu_drm from tinygrad.runtime.autogen.am import am from tinygrad.runtime.support.elf import elf_loader from tinygrad.runtime.support.hcq import FileIOInterface, HCQBuffer, MMIOInterface, hcq_filter_visible_devices from tinygrad.runtime.support.am.amdev import AMDev, AMMemoryManager from tinygrad.runtime.support.amd import AMDReg, AMDIP, import_module, import_soc, import_pmc from tinygrad.runtime.support.system import PCIIfaceBase, PCIAllocationMeta, USBPCIDevice, MAP_FIXED, MAP_NORESERVE -from tinygrad.runtime.support.usb import USB3 +from tinygrad.runtime.support.usb import USB3, pm_usb_bufferize from tinygrad.runtime.support.memory import AddrSpace, BumpAllocator -from tinygrad.runtime.ops_amd import SQTT, SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE, PMC -from tinygrad.runtime.ops_amd import EVENT_INDEX_PARTIAL_FLUSH, WAIT_REG_MEM_FUNCTION_EQ, WAIT_REG_MEM_FUNCTION_NEQ, WAIT_REG_MEM_FUNCTION_GEQ +from tinygrad.runtime.ops_amd import SQTT, PMC +from tinygrad.runtime.ops_amd import EVENT_INDEX_PARTIAL_FLUSH, WAIT_REG_MEM_FUNCTION_GEQ if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl # noqa: F401 # pylint: disable=unused-import -from tinygrad.engine.realize import get_runtime, pm_flatten_linear +from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops from tinygrad.uop import FastEnum, auto -from tinygrad.uop.ops import Ops, UPat, PatternMatcher, graph_rewrite +from tinygrad.uop.ops import Ops, UPat, PatternMatcher # ***************** # PM4 @@ -36,9 +35,11 @@ class PM4Ops(FastEnum): SET_SH_REG = auto(); SET_UCONFIG_REG = auto(); WAIT_REG_MEM = auto(); ACQUIRE_MEM = auto() # noqa: E702 RELEASE_MEM = auto(); DISPATCH_DIRECT = auto(); EVENT_WRITE = auto() # noqa: E702 +def _dw(vals) -> int: return sum(2 if isinstance(x, UOp) and x.dtype.itemsize == 8 else 1 for x in vals) + def pkt3(ctx, op:PM4Ops, *vals): - return UOp(Ops.INS, arg=op, src=tuple(UOp.const(x, dtypes.uint32) - for x in (ctx.pm4.PACKET3(getattr(ctx.pm4, f"PACKET3_{op.name}"), len(vals) - 1), *vals))) + return UOp(Ops.LINEAR, src=tuple(x if isinstance(x, UOp) else UOp.const(x, dtypes.uint32) + for x in (ctx.pm4.PACKET3(getattr(ctx.pm4, f"PACKET3_{op.name}"), _dw(vals) - 1), *vals))) def wreg(ctx, reg:AMDReg, *args:sint, **kwargs:int): if bool(args) == bool(kwargs): raise RuntimeError('One (and only one) of *args or **kwargs must be specified') @@ -52,7 +53,7 @@ def wreg(ctx, reg:AMDReg, *args:sint, **kwargs:int): def wait_reg_mem(ctx, value, mask=0xffffffff, mem=None, reg=None, reg_done=0, op=WAIT_REG_MEM_FUNCTION_GEQ): wrm_info_dw = ctx.pm4.WAIT_REG_MEM_MEM_SPACE(int(mem is not None)) | ctx.pm4.WAIT_REG_MEM_OPERATION(int(mem is None and reg_done > 0)) \ | ctx.pm4.WAIT_REG_MEM_FUNCTION(op) | ctx.pm4.WAIT_REG_MEM_ENGINE(0) - return pkt3(ctx, PM4Ops.WAIT_REG_MEM, wrm_info_dw, *(data64_le(mem) if mem is not None else (reg, reg_done)), value, mask, 4) + return pkt3(ctx, PM4Ops.WAIT_REG_MEM, wrm_info_dw, *((mem,) if mem is not None else (reg, reg_done)), value, mask, 4) def acquire_mem(ctx, addr=0x0, sz=(1 << 64)-1, gli=1, glm=1, glk=1, glv=1, gl1=1, gl2=1): if ctx.target[0] != 9: @@ -83,16 +84,18 @@ def release_mem(ctx, address=0x0, value=0, data_sel=0, int_sel=2, ctxid=0, cache event_dw = ctx.pm4.EVENT_TYPE(ctx.pm4.CACHE_FLUSH_AND_INV_TS_EVENT) | ctx.pm4.EVENT_INDEX(ctx.pm4.event_index__mec_release_mem__end_of_pipe) memsel_dw = ctx.pm4.DATA_SEL(data_sel) | ctx.pm4.INT_SEL(int_sel) ctxid = 0 - return pkt3(ctx, PM4Ops.RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, *data64_le(address), *data64_le(value), ctxid) + addr_w = address if isinstance(address, UOp) else UOp.const(address, dtypes.uint64) + val_w = value.cast(dtypes.uint64) if isinstance(value, UOp) else UOp.const(value, dtypes.uint64) + return pkt3(ctx, PM4Ops.RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, addr_w, val_w, ctxid) def memory_barrier(ctx): pf = '' if ctx.nbio.version[0] == 2 else '0' if ctx.nbio.version[:2] != (7, 11) else '1' - return UOp(Ops.LINEAR, dtypes.void, ( + return UOp(Ops.LINEAR, src=( wait_reg_mem(ctx, reg=getattr(ctx.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_REQ').addr[0], reg_done=getattr(ctx.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_DONE').addr[0], value=0xffffffff), acquire_mem(ctx))) -def pm4_wait(ctx, dst, val): return wait_reg_mem(ctx, val, mem=dst.getaddr(ctx.devs)) +def pm4_wait(ctx, dst, val): return wait_reg_mem(ctx, val.cast(dtypes.uint32), mem=dst.getaddr(ctx.devs)) def pm4_barrier(ctx): return memory_barrier(ctx) @@ -106,155 +109,130 @@ def pm4_timestamp(ctx, dst): ctx.pm4.int_sel__mec_release_mem__none) def pm4_program(ctx, call, prg): - data, info = prg.arg - lib_gpu = prg.src[0] - args = encode_kernargs_clike(call, prg, ctx.devs) - prog_addr = lib_gpu.getaddr(ctx.devs) + data.entry_point_offset - scratch_addr = UOp.placeholder((data.private_segment_size,), dtypes.uint8, 0, device=ctx.devs).rtag("scratch").getaddr(ctx.devs) - args_addr = args.getaddr(ctx.devs) + data, lib = amd_build_program(ctx.dev, prg) + info = prg.arg - user_regs = [] - if data.enable_private_segment_sgpr: - scratch_hilo = data64_le(scratch_addr) - user_regs = [scratch_hilo[0], scratch_hilo[1] | 1 << 31, 0xffffffff, 0x20c14000] - if data.enable_dispatch_ptr: user_regs += [*data64_le(args_addr + data.kernargs_segment_size)] - user_regs += [*data64_le(args_addr)] + # kernargs: a nested blob linear inside a getaddr, input addresses and variable values are filled per call through the input table + ka_words = [get_call_arg_uops(call)[gi].getaddr(ctx.devs) for gi in info.globals] + list(get_call_var_uops(call, prg)) + pad = data.kernargs_alloc_size - sum(w.dtype.itemsize for w in ka_words) + assert pad >= 0 and pad % 4 == 0, f"bad kernargs padding {pad}" + ka = UOp(Ops.LINEAR, src=tuple(ka_words) + (UOp.const(0, dtypes.uint32),) * (pad // 4)).rtag("kernargs") + + prog_addr = lib.getaddr(ctx.devs) + data.entry_point_offset + scratch_addr = UOp.placeholder((data.private_segment_size,), dtypes.uint8, 0, device=ctx.devs).rtag("scratch").getaddr(ctx.devs) + args_addr = ka.getaddr(ctx.devs) + + user_regs:list = [] + if data.enable_private_segment_sgpr: user_regs = [scratch_addr | (1 << 63), 0xffffffff, 0x20c14000] + if data.enable_dispatch_ptr: user_regs += [args_addr + data.kernargs_segment_size] + user_regs += [args_addr] dispatch_init = ctx.gc.regCOMPUTE_DISPATCH_INITIATOR.encode( **({'cs_w32_en': int(data.wave32)} if ctx.target[0] != 9 else {}), force_start_at_000=1, compute_shader_en=1) ins = [acquire_mem(ctx, gli=0, gl2=0), - wreg(ctx, ctx.gc.regCOMPUTE_PGM_LO, *data64_le(prog_addr >> 8)), + wreg(ctx, ctx.gc.regCOMPUTE_PGM_LO, prog_addr >> 8), wreg(ctx, ctx.gc.regCOMPUTE_PGM_RSRC1, data.rsrc1, data.rsrc2), wreg(ctx, ctx.gc.regCOMPUTE_PGM_RSRC3, data.rsrc3), wreg(ctx, ctx.gc.regCOMPUTE_TMPRING_SIZE, ctx.tmpring_size(data.private_segment_size))] - ins += [wreg(ctx, ctx.gc.regCOMPUTE_DISPATCH_SCRATCH_BASE_LO, *data64_le((scratch_addr + data.private_segment_size // ctx.xccs * xcc_id) >> 8)) + ins += [wreg(ctx, ctx.gc.regCOMPUTE_DISPATCH_SCRATCH_BASE_LO, (scratch_addr + data.private_segment_size // ctx.xccs * xcc_id) >> 8) for xcc_id in range(ctx.xccs)] ins += [wreg(ctx, ctx.gc.regCOMPUTE_RESTART_X, 0, 0, 0), wreg(ctx, ctx.gc.regCOMPUTE_USER_DATA_0, *user_regs), wreg(ctx, ctx.gc.regCOMPUTE_RESOURCE_LIMITS, ctx.gc.regCOMPUTE_RESOURCE_LIMITS.encode(waves_per_sh=getenv("WAVES_PER_SH"))), - wreg(ctx, ctx.gc.regCOMPUTE_START_X, 0, 0, 0, *(info.local_size or (1, 1, 1)), 0, 0), + wreg(ctx, ctx.gc.regCOMPUTE_START_X, 0, 0, 0, *info.local_size, 0, 0), pkt3(ctx, PM4Ops.DISPATCH_DIRECT, *info.global_size, dispatch_init), pkt3(ctx, PM4Ops.EVENT_WRITE, ctx.pm4.EVENT_TYPE(ctx.soc.CS_PARTIAL_FLUSH) | ctx.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH))] - return UOp(Ops.LINEAR, dtypes.void, tuple(ins)) + return UOp(Ops.LINEAR, src=tuple(ins)) -pm_pm4_opsel = PatternMatcher([ +def pm4_ib(ctx, submit:UOp, lin:UOp) -> UOp|None: + # the ring only carries a packet pointing at the ib: the host fence at the start of the batch guarantees the ib is free to reuse + if lin.tag is not None or any(w.op in {Ops.CALL, Ops.INS, Ops.LINEAR, Ops.NOOP} for w in lin.src): return None # wait for the flat word linear + assert (size_dw:=sum(w.dtype.itemsize for w in lin.src) // 4) < (1 << 20), f"indirect buffer of {size_dw} dwords doesn't fit one packet" + pkt = (UOp.const(ctx.pm4.PACKET3(ctx.pm4.PACKET3_INDIRECT_BUFFER, 2), dtypes.uint32), lin.rtag("indirect").getaddr(ctx.devs), + UOp.const(size_dw | ctx.pm4.INDIRECT_BUFFER_VALID, dtypes.uint32)) + return submit.replace(src=(UOp(Ops.LINEAR, src=pkt, arg=lin.arg).rtag(("cmdbuf", ctx.queue)),)) + +pm_pm4_encode = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), pm4_program), + (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="lin"),), name="submit"), pm4_ib), - (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val"))), pm4_wait), - (UPat(Ops.INS, arg="barrier"), pm4_barrier), - (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),)), pm4_timestamp), - (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), pm4_store), + (UPat(Ops.INS, arg=("barrier", dtypes.void)), pm4_barrier), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), pm4_wait), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), pm4_timestamp), + (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), pm4_store), ]) -def pm4_submit(ctx, lin): - # ensure compute queues are allocated - for d in (devs:=ctx.devs): q = Device[d].compute_queue - ring, wptr, doorbell, put_ptr = (UOp.placeholder((b.size,), b.dtype, 0, device=devs).rtag(f"COMPUTE:0_{name}") - for name, b in (("ring", q.ring), ("write_ptr", q.write_ptr), ("doorbell", q.doorbell), ("put_value", q.put_value))) - - # the host fence at the start of the batch guarantees the ib is free to reuse - size_dw = sum(len(ins.src) for ins in lin.src) - assert size_dw < (1 << 20), f"indirect buffer of {size_dw} dwords doesn't fit one packet" - - ib = UOp.placeholder((size_dw,), dtypes.uint32, next(UOp.unique_num), device=devs, volatile=True).rtag("cmdbuf") - cmdbuf = make_cmdbuf(lin, devs, buf=ib) - - # the ring itself only carries a packet pointing at the ib, wrapping the ring - put = put_ptr.index(zero:=UOp.const(0, dtypes.int)) - pkt = (ctx.pm4.PACKET3(ctx.pm4.PACKET3_INDIRECT_BUFFER, 2), *data64_le(cmdbuf.getaddr(devs)), size_dw | ctx.pm4.INDIRECT_BUFFER_VALID) - write_pkt = UOp.barrier(*[ring.index(((put + off) % q.ring.size).cast(dtypes.int)).store(UOp.const(x, dtypes.uint32)) for off,x in enumerate(pkt)]) - - # advance the put/write pointers past the packet - bump_put_ptr = put_ptr.index(zero).store(put + len(pkt)) - bump_wptr = wptr.index(zero).store(put + len(pkt)) - flush = UOp.barrier(write_pkt, bump_put_ptr, bump_wptr) - return doorbell.after(flush).index(zero).store(put + len(pkt)) - -pm_pm4_submit = PatternMatcher([(UPat(Ops.LINEAR, name="lin"), pm4_submit)]) - # ***************** # SDMA -class SDMAOps(FastEnum): COPY = auto(); POLL_REGMEM = auto(); FENCE = auto(); TRAP = auto(); TIMESTAMP = auto() # noqa: E702 - def sdma_copy(ctx, call): sz = call.src[2].max_numel() * call.src[2].dtype.itemsize - src_addr, dst_addr = call.src[2].getaddr(ctx.devs), call.src[1].getaddr(ctx.devs) - return call.ins(SDMAOps.COPY, src=tuple(UOp.const(x, dtypes.uint32) for off in range(0, sz, ctx.max_copy_size) for x in ( - ctx.sdma.SDMA_OP_COPY | ctx.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(ctx.sdma.SDMA_SUBOP_COPY_LINEAR), - ctx.sdma.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(min(sz-off, ctx.max_copy_size)-1), 0, - *data64_le(src_addr+UOp.const(off, dtypes.uint64)), *data64_le(dst_addr+UOp.const(off, dtypes.uint64))))) + hdr = ctx.sdma.SDMA_OP_COPY | ctx.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(ctx.sdma.SDMA_SUBOP_COPY_LINEAR) + return UOp(Ops.LINEAR, src=tuple(x for off in range(0, sz, ctx.max_copy_size) for x in ( + *(UOp.const(v, dtypes.uint32) for v in (hdr, ctx.sdma.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(min(sz-off, ctx.max_copy_size)-1), 0)), + *(a + UOp.const(off, dtypes.uint64) if off else a for a in (call.src[2].getaddr(ctx.devs), call.src[1].getaddr(ctx.devs)))))) -def sdma_wait(ctx, ins, dst, val): +def sdma_wait(ctx, dst, val): op = ctx.sdma.SDMA_OP_POLL_REGMEM | ctx.sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(WAIT_REG_MEM_FUNCTION_GEQ) \ | ctx.sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1) - return ins.ins(SDMAOps.POLL_REGMEM, src=tuple(UOp.const(x, dtypes.uint32) for x in ( - op, *data64_le(dst.getaddr(ctx.devs)), val, 0xffffffff, - ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff)))) - -def sdma_store(ctx, ins, dst, val): - op = ctx.sdma.SDMA_OP_FENCE | (ctx.sdma.SDMA_PKT_FENCE_HEADER_MTYPE(3) if ctx.target[0] != 9 else 0) return UOp(Ops.LINEAR, src=( - ins.ins(SDMAOps.FENCE, src=tuple(UOp.const(x, dtypes.uint32) for x in (op, *data64_le(dst.getaddr(ctx.devs)), val))), - ins.ins(SDMAOps.TRAP, src=tuple(UOp.const(x, dtypes.uint32) for x in (ctx.sdma.SDMA_OP_TRAP, 0))))) + UOp.const(op, dtypes.uint32), dst.getaddr(ctx.devs), val.cast(dtypes.uint32), UOp.const(0xffffffff, dtypes.uint32), + UOp.const(ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff), dtypes.uint32))) -def sdma_timestamp(ctx, ins, dst): +def sdma_store(ctx, dst, val): # a fence packet then a trap + op = ctx.sdma.SDMA_OP_FENCE | (ctx.sdma.SDMA_PKT_FENCE_HEADER_MTYPE(3) if ctx.target[0] != 9 else 0) + return UOp(Ops.LINEAR, src=(UOp.const(op, dtypes.uint32), dst.getaddr(ctx.devs), val.cast(dtypes.uint32), + UOp.const(ctx.sdma.SDMA_OP_TRAP, dtypes.uint32), UOp.const(0, dtypes.uint32))) + +def sdma_timestamp(ctx, dst): op = ctx.sdma.SDMA_OP_TIMESTAMP | ctx.sdma.SDMA_PKT_TIMESTAMP_GET_HEADER_SUB_OP(ctx.sdma.SDMA_SUBOP_TIMESTAMP_GET_GLOBAL) - return ins.ins(SDMAOps.TIMESTAMP, src=tuple(UOp.const(x, dtypes.uint32) for x in (op, *data64_le(dst.getaddr(ctx.devs))))) + return UOp(Ops.LINEAR, src=(UOp.const(op, dtypes.uint32), dst.getaddr(ctx.devs))) -pm_sdma_opsel = PatternMatcher([ +pm_sdma_encode = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.COPY),), name="call", allow_any_len=True), sdma_copy), - (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP, dtypes.void, ())), - (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val")), name="ins"), sdma_wait), - (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),), name="ins"), sdma_timestamp), - (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val")), name="ins"), sdma_store), + (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.LINEAR)), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), sdma_wait), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), sdma_timestamp), + (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), sdma_store), ]) -def sdma_submit(cmdbuf, devs): - # the cmdbuf to submit + the patch writes that fill it - size_dw, zero = cmdbuf.nbytes() // dtypes.uint32.itemsize, UOp.const(0, dtypes.int) +# ***************** +# queue submit - # the sdma queue's ring and its host-side ring/write/put pointers - for d in devs: q = Device[d].sdma_queue(0) - ring, wptr, doorbell, put_ptr = (UOp.placeholder((b.size,), b.dtype, 0, device=devs).rtag(f"COPY:0_{name}") - for name, b in (("ring", q.ring), ("write_ptr", q.write_ptr), ("doorbell", q.doorbell), ("put_value", q.put_value))) +def _queue_bufs(ctx, q:AMDQueueDesc) -> tuple[UOp, UOp, UOp, UOp]: + ring = UOp.placeholder((q.ring.size,), q.ring.dtype, 0, device=ctx.devs, volatile=True).rtag(f"{ctx.queue}_ring") + return (ring, *(make_buf(ctx.devs, tag=f"{ctx.queue}_{n}") for n in ("write_ptr", "doorbell", "put_value"))) - # sdma needs the cmdbuf contiguous: if it won't fit before the ring end, restart at 0 and zero the tail - put_b = put_ptr.index(zero) - tail_off_dw = ((put_b % (q.ring.size * 4)) // 4).cast(dtypes.int) - fits = (size_dw <= q.ring.size - tail_off_dw).cast(dtypes.int) - start_dw = fits * tail_off_dw - zero_amt_dw = (1 - fits) * (q.ring.size - tail_off_dw) +def pm4_submit(ctx, cmdbuf:UOp) -> UOp: + for d in ctx.devs: q = Device[d].compute_queue + ring, wptr, doorbell, put = _queue_bufs(ctx, q) + p, size_dw = put.after(cmdbuf).index(0).load(), hcq_size_var(cmdbuf) // 4 + i = UOp.range(size_dw, 10, dtype=dtypes.int, src=(cmdbuf, ring)) + copy = ring.index(((p + i.cast(p.dtype)) % q.ring.size).cast(dtypes.int)).store(cmdbuf.bitcast(dtypes.uint32).index(i).load()).end(i) + next_put = p + size_dw.cast(p.dtype) + flush = UOp.barrier(copy, put.index(0).store(next_put), wptr.index(0).store(next_put)) + return doorbell.after(flush).index(0).store(next_put) - # zero the wrapped tail, then copy the cmdbuf into the ring - zi = UOp.range(zero_amt_dw, 0, dtype=dtypes.int, src=(cmdbuf,)) - zero_tail = ring.index(tail_off_dw + zi).store(UOp.const(0, dtypes.uint32)).end(zi) - i = UOp.range(UOp.const(size_dw, dtypes.int), 0, dtype=dtypes.int, src=(cmdbuf,)) - copy_to_ring = ring.index(start_dw + i).store(cmdbuf.index(i).load()).end(i) +def sdma_submit(ctx, cmdbuf:UOp) -> UOp: + # sdma needs the cmdbuf contiguous in the ring: if it won't fit before the ring end, restart at 0 and zero the tail + for d in ctx.devs: q = unwrap(Device[d].sdma_queue(int(ctx.queue.split(":")[1]))) + (ring, wptr, doorbell, put), rs = _queue_bufs(ctx, q), q.ring.size + size_dw = hcq_size_var(cmdbuf) // 4 + put_b = put.after(cmdbuf).index(0).load() + tail = ((put_b % (rs * 4)) // 4).cast(dtypes.int) + fits = (size_dw <= rs - tail).cast(dtypes.int) + start_dw, zero_amt = fits * tail, (1 - fits) * (rs - tail) + zi = UOp.range(zero_amt, 10, dtype=dtypes.int, src=(ring,)) + zero_tail = ring.index(tail + zi).store(UOp.const(0, dtypes.uint32)).end(zi) + i = UOp.range(size_dw, 11, dtype=dtypes.int, src=(cmdbuf, ring)) + copy = ring.index(start_dw + i).store(cmdbuf.bitcast(dtypes.uint32).index(i).load()).end(i) + next_put = put_b + ((zero_amt + size_dw) * 4).cast(put_b.dtype) + flush = UOp.barrier(zero_tail, copy, put.index(0).store(next_put), wptr.index(0).store(next_put)) + return doorbell.after(flush).index(0).store(next_put) - # advance the put/write pointers past the zeroed tail and the cmdbuf - next_put_b = put_b + ((zero_amt_dw + size_dw) * 4).cast(put_b.dtype) - bump_put_ptr = put_ptr.index(zero).store(next_put_b) - bump_wptr = wptr.index(zero).store(next_put_b) - - # ring the doorbell once the writes have landed - flush = UOp.barrier(zero_tail, copy_to_ring, bump_put_ptr, bump_wptr) - return doorbell.after(flush).index(zero).store(next_put_b) - -pm_sdma_submit = PatternMatcher([(UPat(Ops.LINEAR, name="lin"), - lambda ctx, lin: sdma_submit(make_cmdbuf(lin, ctx.devs), ctx.devs))]) - -@dataclass(frozen=True) -class AMDEncodeCtx: # encode-time constants for one queue: devs (every cmdbuf address resolves into these) + gfx version + packet/ip modules - devs: tuple[str, ...]; target: tuple[int, ...]; pm4: Any; sdma: Any; soc: Any # noqa: E702 - gc: AMDIP; nbio: AMDIP; xccs: int; max_copy_size: int; tmpring_size: Callable # noqa: E702 - -def encode_queue(q:UOp) -> UOp|None: - d = Device[(devs:=to_tuple(q.arg[0]))[0]] - ctx = AMDEncodeCtx(devs, d.target, d.pm4, d.sdma, d.soc, d.gc, d.nbio, d.xccs, d.max_copy_size, d.tmpring_size) - opsel, submit = (pm_pm4_opsel, pm_pm4_submit) if q.arg[1].startswith("COMPUTE") else (pm_sdma_opsel, pm_sdma_submit) - return submit.rewrite(graph_rewrite(q, opsel + pm_flatten_linear, walk=True, ctx=ctx, name=f"{q.arg[1]} opsel"), ctx) +pm_pm4_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), pm4_submit)]) +pm_sdma_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), sdma_submit)]) @dataclass(frozen=True) class AMDProgramData: @@ -262,10 +240,10 @@ class AMDProgramData: private_segment_size:int; kernargs_segment_size:int; kernargs_alloc_size:int enable_dispatch_ptr:int; enable_private_segment_sgpr:int -_amd_program_cache:dict[tuple[bytes,str], tuple[AMDProgramData,bytes]] = {} -def amd_build_program(prg:UOp) -> UOp: - dev = Device[to_tuple(prg.device)[0]] # TODO: rm this - if (cached:=_amd_program_cache.get(key:=(lib:=prg.src[3].arg, dev.device))) is None: +_amd_program_cache:dict[tuple[bytes, tuple[str, ...]], tuple[AMDProgramData, UOp]] = {} +def amd_build_program(dev, prg:UOp) -> tuple[AMDProgramData, UOp]: + # key on the full device tuple: the same lib can be built for different device sets, each needs its own program buffer + if (cached:=_amd_program_cache.get(key:=(lib:=prg.src[3].arg, to_tuple(prg.device)))) is None: image, sections, relocs = elf_loader(lib) rodata = next(sh.header.sh_addr for sh in sections if sh.name == ".rodata") for off, sym, typ, addent in relocs: @@ -282,20 +260,23 @@ def amd_build_program(prg:UOp) -> UOp: wave32=bool(desc.kernel_code_properties & 0x400), private_segment_size=desc.private_segment_fixed_size, kernargs_segment_size=desc.kernarg_size, kernargs_alloc_size=desc.kernarg_size + (ctypes.sizeof(hsa.hsa_kernel_dispatch_packet_t) if edp else 0), enable_dispatch_ptr=edp, enable_private_segment_sgpr=desc.kernel_code_properties & hsa.AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER) + image = bytes(image).ljust(round_up(len(image), 4), b"\x00") # the program is uploaded as whole dwords buf = UOp.placeholder((len(image),), dtypes.uint8, next(UOp.unique_num), device=prg.device).rtag("program") - cached = _amd_program_cache[key] = prg.replace(src=(buf.after(make_binary_patch(buf, bytes(image))),), arg=(data, prg.arg)) + cached = _amd_program_cache[key] = (data, buf.after(buf.store(UOp(Ops.BINARY, src=(), arg=image).bitcast(buf.dtype)))) return cached class AMDAllocator(HCQAllocator['AMDDevice']): def __init__(self, dev:AMDDevice): - super().__init__(dev, supports_copy_from_disk=dev.has_sdma_queue, supports_transfer=dev.has_sdma_queue and not dev.is_usb()) + super().__init__(dev, supports_copy_from_disk=dev.has_copy_queue, supports_transfer=dev.has_copy_queue and not dev.is_usb) - def _alloc(self, size:int, options:BufferSpec) -> HCQ2Buffer: - return self.dev.iface.alloc(size, host=options.host, uncached=options.uncached, cpu_access=options.cpu_access or not self.dev.has_sdma_queue) + def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: + return self.dev.iface.alloc(size, host=options.host, uncached=options.uncached, cpu_access=options.cpu_access or not self.dev.has_copy_queue) def _do_free(self, opaque, options:BufferSpec): self.dev.iface.free(opaque) - def _do_map(self, buf:HCQ2Buffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) + def _do_map(self, buf:HCQBuffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) + + def _do_unmap(self, buf:HCQBuffer): self.dev.iface.unmap(buf) @dataclass class AMDQueueDesc: @@ -388,15 +369,24 @@ class KFDIface: return hcqbuf def free(self, mem): + self._unmap(mem) + if mem.va_addr: FileIOInterface.munmap(mem.va_addr, mem.size) + kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) + + def unmap(self, mem): + self._unmap(mem) + if getattr(mem, '_owns_kfd_handle', False): kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) + + def _unmap(self, mem): gpus = (ctypes.c_int32 * 1)(self.gpu_id) stm = kfd.AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU(self.kfd, handle=mem.meta.handle, device_ids_array_ptr=ctypes.addressof(gpus), n_devices=1) assert stm.n_success == 1 - if mem.owner == self.dev: - if mem.va_addr: FileIOInterface.munmap(mem.va_addr, mem.size) - kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) def map(self, mem): - if mem.owner is not None and mem.owner._is_cpu(): return self.alloc(mem.size, host=True, cpu_addr=mem.va_addr) + if mem.owner is not None and mem.owner._is_cpu(): + mapped = self.alloc(mem.size, host=True, cpu_addr=mem.va_addr) + mapped._owns_kfd_handle = True + return mapped c_gpus = (ctypes.c_int32 * 1)(self.gpu_id) stm = kfd.AMDKFD_IOC_MAP_MEMORY_TO_GPU(self.kfd, handle=mem.meta.handle, device_ids_array_ptr=ctypes.addressof(c_gpus), n_devices=1) @@ -468,6 +458,7 @@ class PCIIface(PCIIfaceBase): def require_profile_mode(self): return True def is_wgp_active(self, xcc, se, sa, wgp) -> bool: return True # TODO: account for WGP disablement on some asics. + def unmap(self, mem): self.free(mem) def _compute_props(self): self.ip_versions = self.dev_impl.ip_ver @@ -512,8 +503,7 @@ class PCIIface(PCIIfaceBase): cq = d.compute_queue for b in (cq.put_value, cq.read_ptr, cq.write_ptr): b._buf.view.view(fmt='Q')[0] = 0 d.iface.dev_impl.gfx.setup_ring(*cq.params) - d.signal('timeline')._buf.cpu_view().mv.cast('Q')[0] = \ - d.signal('value', 1).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] - 1 + d.signal('timeline')._buf.cpu_view().view(fmt='Q')[0] = d.signal('value', 1, device="CPU")._buf.cpu_view().view(fmt='Q')[0] - 1 def sleep(self, timeout): if hasattr(self.pci_dev, 'irq_poller') and self.pci_dev.irq_poller is not None and (events_cnt:=len(self.pci_dev.irq_poller.poll(timeout))): @@ -527,31 +517,50 @@ class PCIIface(PCIIfaceBase): def device_fini(self): self.dev_impl.fini() +class USBIface(PCIIface): + def __init__(self, dev, dev_id): # pylint: disable=super-init-not-called + if dev_id >= len(visible:=hcq_filter_visible_devices(USB3.list_devices(0xADD1, 0x0001) + USB3.list_devices(0x3801, 0x0001), "AMD")): + raise RuntimeError(f"AMD:{dev_id} does not exist ({pluralize('device', len(visible))} available)") + self.dev, self.pci_dev, self.vram_bar, self.count = dev, USBPCIDevice("AM", *visible[dev_id]), 0, len(visible) + self.dev_impl = AMDev(self.pci_dev) + self._compute_props() + self.sram = self._dma_region(ctrl_addr=0xf000, sys_addr=0x200000, size=0x80000) + self.cq_buf = self._dma_region(ctrl_addr=0xb800, sys_addr=0x822000, size=0x1000) # +12 is the dword that releases an armed read + self.usb_handle = unwrap(ctypes.cast(self.pci_dev.usb.usb.handle, ctypes.c_void_p).value) + + def _dma_region(self, ctrl_addr, sys_addr, size): + region = self.dev_impl.mm.map_range(vaddr:=self.dev_impl.mm.alloc_vaddr(size=size), size, [(sys_addr, size)], aspace=AddrSpace.SYS, uncached=True) + return HCQBuffer(vaddr, size, meta=PCIAllocationMeta(region, has_cpu_mapping=False), view=self.pci_dev.dma_view(ctrl_addr, size), owner=self.dev) + + def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, **kwargs) -> HCQBuffer: + # everything, even host-style signals, lives in vram: gpu writes into the bridge's own memory collide with an armed 0xF2 read stream + return super().alloc(size, host=False, uncached=uncached, cpu_access=cpu_access or host, contiguous=contiguous, force_devmem=True, **kwargs) + + def sleep(self, timeout): pass + + # we don't own the sram region, so the buffer never frees it + @functools.cached_property + def usb_sram(self) -> Buffer: + return Buffer(self.dev.device, (b:=self.sram).size, dtypes.uint8, options=BufferSpec(external_ptr=b.va_addr, nolru=True)).allocate(opaque=b) + def _mock(iface, name=None): return type(name or f"MOCK{iface.__name__}", (iface,), {}) class AMDDevice(HCQ2Compiled): - pm_lower = PatternMatcher([ - # prep program - (UPat(Ops.PROGRAM, src=(UPat(), UPat(), UPat(), UPat(Ops.BINARY)), name="prg"), amd_build_program), - - # encoding of cmdbuf - (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_queue), - ]) - timestamp_divider = 100.0 # AMD GPU clock: ticks/us max_scratch_psize = 0 + pm_encode = {"COMPUTE": pm_pm4_encode, "COPY": pm_sdma_encode} + pm_lower = {"COMPUTE": pm_pm4_submit, "COPY": pm_sdma_submit} - ifaces = [KFDIface, PCIIface, _mock(KFDIface, "MOCKIface"), _mock(KFDIface), _mock(PCIIface)] + ifaces = [KFDIface, PCIIface, USBIface, _mock(KFDIface, "MOCKIface"), _mock(KFDIface), _mock(PCIIface), _mock(USBIface)] def device_props(self): return self.iface.props def is_am(self) -> bool: return isinstance(self.iface, (PCIIface,)) - def is_usb(self) -> bool: return False def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - - self.iface = self._select_iface() + self.iface = self._select_iface(device) + self.is_usb = isinstance(self.iface, USBIface) + if self.is_usb: self.rt_nbytes = 4 << 20 self.target:tuple[int, ...] = ((trgt:=self.iface.props['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100) self.arch = "gfx%d%x%x" % self.target @@ -576,12 +585,12 @@ class AMDDevice(HCQ2Compiled): self.is_aql = getenv("AMD_AQL", int(self.xccs > 1)) if self.is_aql: - self.pm4_ibs = self.iface.alloc(0x2000 if self.is_usb() else (16 << 20), uncached=True, cpu_access=True) + self.pm4_ibs = self.iface.alloc(0x2000 if self.is_usb else (16 << 20), uncached=True, cpu_access=True) self.pm4_ib_alloc = BumpAllocator(self.pm4_ibs.size, wrap=True) self.max_copy_size = 0x40000000 if self.iface.ip_versions[am.SDMA0_HWIP][0] >= 5 else 0x400000 self.sdma_queues:dict = {} - self.has_sdma_queue = True # self.sdma_queue(0) is not None, TODO: think of this + self.has_copy_queue = not getenv("AMD_DISABLE_SDMA") super().__init__(device, AMDAllocator(self), [HIPRenderer, AMDLLVMRenderer, HIPCCRenderer], None, can_recover=self.is_am(), arch=self.arch) @@ -589,6 +598,10 @@ class AMDDevice(HCQ2Compiled): self.max_private_segment_size = 0 self.pm_bufferize = PatternMatcher([(UPat(Ops.PARAM, tag="scratch", name="b"), lambda ctx, b: ctx[0].scratch_buffer(b.max_numel()))]) + self.pm_bufferize + if self.is_usb: + self.pm_bufferize = pm_usb_bufferize + self.pm_bufferize + raise NotImplementedError("usb amd is not migrated to sealed submits yet") # a usb pm_lower can override the whole submit graph + self.pmc_enabled:bool = PROFILE > 0 and PMC > 0 if self.pmc_enabled: self.iface.require_profile_mode() @@ -649,7 +662,7 @@ class AMDDevice(HCQ2Compiled): wg_data_size = round_up((vgpr_size_per_cu + sgrp_size_per_cu + lds_size_per_cu + hwreg_size_per_cu) * self.cu_cnt, mmap.PAGESIZE) ctl_stack_size = round_up((12 if self.target[0] != 9 else 8) * self.wave_cnt + 8 + 40, mmap.PAGESIZE) return self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_COMPUTE_AQL if self.is_aql else kfd.KFD_IOC_QUEUE_TYPE_COMPUTE, - 0x2000 if self.is_usb() else (16 << 20), eop_buffer_size=0x1000, + 0x2000 if self.is_usb else (16 << 20), eop_buffer_size=0x1000, ctx_save_restore_size=0 if self.is_am() else wg_data_size + ctl_stack_size, ctl_stack_size=ctl_stack_size, debug_memory_size=round_up(self.wave_cnt * 32, 64)) @@ -657,7 +670,7 @@ class AMDDevice(HCQ2Compiled): if getenv("AMD_DISABLE_SDMA"): return None if idx in self.sdma_queues: return self.sdma_queues[idx] with contextlib.suppress(OSError): - self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x200 if self.is_usb() else (16 << 20), idx=idx) + self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x2000 if self.is_usb else (16 << 20), idx=idx) return self.sdma_queues.get(idx, None) def tmpring_size(self, private_segment_size): diff --git a/tinygrad_repo/extra/llama_kernels/__init__.py b/tinygrad_repo/extra/llama_kernels/__init__.py index d16187e17..9308bc169 100644 --- a/tinygrad_repo/extra/llama_kernels/__init__.py +++ b/tinygrad_repo/extra/llama_kernels/__init__.py @@ -1,8 +1,6 @@ from __future__ import annotations import functools, pathlib -from dataclasses import replace -from tinygrad import Tensor, dtypes -from tinygrad.uop.ops import shape_to_shape_arg +from tinygrad import Tensor from tinygrad.runtime.support.compiler_amd import HIPCCCompiler FP8_MAX = 448.0 @@ -12,7 +10,7 @@ NUM_WG, THREADS_PER_WG = 1024, 256 @functools.cache def _local_abs_max_fxn(x_p, device): x = Tensor(x_p, device=device) - inner = Tensor(x.uop.replace(src=(shape_to_shape_arg(x.uop.shard_shape),), arg=replace(x.uop.arg, axis=None))) if x.uop.axis is not None else x + inner = Tensor(x.uop.src[0]) if x.uop.axis is not None else x # the per-shard view of the flat param return (inner.abs().max(),) def local_abs_max(x:Tensor) -> Tensor: diff --git a/tinygrad_repo/extra/llama_kernels/quantize_fp8_delayed/__init__.py b/tinygrad_repo/extra/llama_kernels/quantize_fp8_delayed/__init__.py index 6e1c25a1f..f7d42b44e 100644 --- a/tinygrad_repo/extra/llama_kernels/quantize_fp8_delayed/__init__.py +++ b/tinygrad_repo/extra/llama_kernels/quantize_fp8_delayed/__init__.py @@ -50,7 +50,7 @@ def _custom_quantize_fp8_with_amax(fp8_out:UOp, amax_out:UOp, x:UOp, amax_state: else: raise NotImplementedError(f"no atomic max for device {device}") amax_idx = amax_out.reshape((1,)).index(UOp.const(0)) max_val = lds[0].load() - atomic = UOp(Ops.CUSTOM, dtypes.void, (amax_idx, max_val.bitcast(dtypes.int32), max_val, amax_idx.load()), arg=atomic_arg) + atomic = UOp(Ops.CUSTOM, src=(amax_idx, max_val.bitcast(dtypes.int32), max_val, amax_idx.load()), arg=(atomic_arg, dtypes.void)) return atomic.end(tid, wg).sink(arg=KernelInfo(f"quantize_fp8_with_amax_{n_elems}", opts_to_apply=())) @functools.cache diff --git a/tinygrad_repo/extra/llama_kernels/quantize_mxfp4/__init__.py b/tinygrad_repo/extra/llama_kernels/quantize_mxfp4/__init__.py index 6cdd983c6..48682b8a1 100644 --- a/tinygrad_repo/extra/llama_kernels/quantize_mxfp4/__init__.py +++ b/tinygrad_repo/extra/llama_kernels/quantize_mxfp4/__init__.py @@ -12,7 +12,7 @@ def _custom_quantize_mxfp4(row_fp4:UOp, row_scale:UOp, col_fp4:UOp, col_scale:UO mem = M*N*2 + M*N + M*N//16 # read bf16, write row+col fp4 + e8m0 outputs = (row_fp4, row_scale, col_fp4, col_scale) sink = UOp.sink(*(o.base for o in outputs), x.base, - *(UOp(Ops.CUSTOM, dtypes.void, (o.base.index(0),), arg="") for o in outputs), + *(UOp(Ops.CUSTOM, src=(o.base.index(0),), arg=("", dtypes.void)) for o in outputs), UOp.special(256, "lidx0"), UOp.special(M//128, "gidx0"), UOp.special(N//64, "gidx1"), arg=KernelInfo(name, estimates=Estimates(ops=12*M*N, mem=mem))) src = (pathlib.Path(__file__).parent/"quantize_mxfp4.cpp").read_text() diff --git a/tinygrad_repo/extra/mmapeak/mmapeak.py b/tinygrad_repo/extra/mmapeak/mmapeak.py index 485e09efb..4488346ea 100644 --- a/tinygrad_repo/extra/mmapeak/mmapeak.py +++ b/tinygrad_repo/extra/mmapeak/mmapeak.py @@ -3,7 +3,7 @@ import os # TODO: there is a timing bug without this os.environ["AMD_AQL"] = "1" -from tinygrad import Tensor, Device, GlobalCounters, Context +from tinygrad import Tensor, Device, GlobalCounters, Context, dtypes from tinygrad.helpers import getenv, DEV from tinygrad.uop.ops import UOp, Ops, KernelInfo from tinygrad.renderer import Estimates @@ -37,7 +37,7 @@ def launchBenchmark(instruction, vgprIndices, dense=True, accum=False, **kwargs) gidx = UOp.special(NUM_WORKGROUPS, "gidx0") FLOPs = FLOPS_PER_MATMUL * NUM_WAVES * NUM_WORKGROUPS * INTERNAL_LOOP * INSTRUCTIONS_PER_LOOP sink = UOp.sink(A.base, threads, gidx, arg=KernelInfo(inst.op.name.lower(), estimates=Estimates(ops=FLOPs, mem=0))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) dummy = Tensor.zeros(1).contiguous().realize() out = Tensor.custom_kernel(dummy, fxn=fxn)[0] linear = out.schedule_linear() diff --git a/tinygrad_repo/extra/models/llama.py b/tinygrad_repo/extra/models/llama.py index 4bd2d0d06..314e663a7 100644 --- a/tinygrad_repo/extra/models/llama.py +++ b/tinygrad_repo/extra/models/llama.py @@ -5,9 +5,9 @@ from tinygrad.helpers import getenv, DEBUG # https://github.com/facebookresearch/llama/blob/1076b9c51c77ad06e9d7ba8a4c6df775741732bd/llama/model.py#L47 def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0) -> Tensor: - freqs = 1.0 / (theta ** (Tensor.arange(0, dim, 2)[:(dim // 2)] / dim)) - freqs = Tensor.arange(end).unsqueeze(dim=1) * freqs.unsqueeze(dim=0) - return Tensor.stack(freqs.cos(), freqs.sin(), dim=-1).reshape(1, end, 1, dim//2, 2) + freqs = 1.0 / (theta ** (Tensor.arange(0, dim, 2, dtype=dtypes.float32)[:(dim // 2)] / dim)) + freqs = Tensor.arange(end, dtype=dtypes.float32).unsqueeze(dim=1) * freqs.unsqueeze(dim=0) + return Tensor.stack(freqs.cos(), freqs.sin(), dim=-1).cast(dtypes.default_float).reshape(1, end, 1, dim//2, 2) # matches meta, non hugging face weights # (a+i*b) * (c+i*d) = (ac-bd) + i*(ad+bc) diff --git a/tinygrad_repo/extra/onnx_helpers.py b/tinygrad_repo/extra/onnx_helpers.py index 7d3af8fa7..a0a0363d4 100644 --- a/tinygrad_repo/extra/onnx_helpers.py +++ b/tinygrad_repo/extra/onnx_helpers.py @@ -1,10 +1,12 @@ from tinygrad import Tensor +import os from tinygrad.tensor import _to_np_dtype from tinygrad.nn.onnx import OnnxRunner, OnnxValue import numpy as np import onnxruntime as ort ort_options = ort.SessionOptions() ort_options.log_severity_level = 3 +ort_options.intra_op_num_threads = os.cpu_count() or 1 def get_example_inputs(graph_inputs:dict[str, OnnxValue], config={}): """ diff --git a/tinygrad_repo/extra/optimization/test_beam_search.py b/tinygrad_repo/extra/optimization/test_beam_search.py index 133c77996..382fd2900 100644 --- a/tinygrad_repo/extra/optimization/test_beam_search.py +++ b/tinygrad_repo/extra/optimization/test_beam_search.py @@ -89,7 +89,8 @@ class TestBeamSearch(unittest.TestCase): s.apply_opt(Opt(OptOps.TC, 0, (-1, 0, 1))) up = prod([x for x, t in zip(s.full_shape, s.axis_types) if t in (AxisType.UPCAST, AxisType.UNROLL)]) actions = get_kernel_actions(s, include_0=False, max_up=int(up)) - upcasted = [s for s in actions.values() if any(opt.op in (OptOps.UPCAST, OptOps.UNROLL) for opt in s.applied_opts)] + upcasted = [s for s in actions.values() if any(o.op is OptOps.SPLIT and o.arg[1] in (AxisType.UPCAST, AxisType.UNROLL) + for o in s.applied_opts)] assert len(upcasted) > 0, f"expected upcast/unroll actions after TC with max_up={up}, but got none" def test_max_up(self): @@ -98,8 +99,8 @@ class TestBeamSearch(unittest.TestCase): s = Scheduler(ast, Device[Device.DEFAULT].renderer) for max_up in (2, 4): actions = get_kernel_actions(s, include_0=False, max_up=max_up) - for up_opts in [s.applied_opts for s in actions.values() if any(opt.op in (OptOps.UPCAST, OptOps.UNROLL) for opt in s.applied_opts)]: - assert len([opt for opt in up_opts if opt.arg > max_up]) == 0 and len([op for op in up_opts if op.arg <= max_up]) > 0 + up_opts = [o for s in actions.values() for o in s.applied_opts if o.op is OptOps.SPLIT and o.arg[1] in (AxisType.UPCAST, AxisType.UNROLL)] + assert len([opt for opt in up_opts if opt.arg[0] > max_up]) == 0 and len([op for op in up_opts if op.arg[0] <= max_up]) > 0 if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/extra/runbook_digitalocean_mi350x.md b/tinygrad_repo/extra/runbook_digitalocean_mi350x.md new file mode 100644 index 000000000..32df97ba4 --- /dev/null +++ b/tinygrad_repo/extra/runbook_digitalocean_mi350x.md @@ -0,0 +1,197 @@ +# Runbook: Llama 3 8B Training on DigitalOcean MI350X + +## Machine Specs +- 8x MI350X GPUs (gfx950, device ID 75b0), 288GB VRAM each +- 2TB RAM, 192 CPUs, 2TB disk +- ROCm 7.14 at `/opt/rocm` (NOT `/opt/rocm-7.1.1` like the submission scripts assume) +- Python 3.12 + +## Phase 1: System Setup + +### 1.1 Install packages +```bash +apt-get update +apt-get install -y python3-pip python3-venv git tmux rclone clang +``` + +### 1.2 Install Python deps +```bash +python3 -m pip install --break-system-packages --ignore-installed typing-extensions numpy tqdm wandb tiktoken sentencepiece +``` +Note: `--ignore-installed typing-extensions` is needed because the base image ships typing-extensions 4.10.0 without a RECORD file, so pip cannot uninstall it. + +### 1.3 Install ROCm dev headers +The base image has ROCm runtime but NOT the HIP dev headers. Need: +```bash +apt-get install -y amdrocm-core-dev +``` +This installs `hip/hip_runtime.h` at `/opt/rocm/core-7.14/include/hip/hip_runtime.h`. +The symlink `/opt/rocm/include` → `/opt/rocm/core-7.14/include` makes it available at `/opt/rocm/include/hip/hip_runtime.h`. + +### 1.4 Configure ROCm comgr +ROCm 7.14 ships comgr 3.3 at `/opt/rocm/lib/libamd_comgr.so`. tinygrad's DLL loader needs explicit env vars to find it (it searches for `libcomgr.so*` by default, not `libamd_comgr.so*`). Set these in the run command: +```bash +export COMGR_PATH=/opt/rocm/lib/libamd_comgr.so +export COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so +``` +Also add ROCm libs to ldconfig so comgr's shared library dependencies resolve: +```bash +cat > /etc/ld.so.conf.d/rocm.conf << 'EOF' +/opt/rocm/lib +/opt/rocm/lib/llvm/lib +/opt/rocm/lib/rocm_sysdeps/lib +EOF +ldconfig +``` + +### 1.5 Install geohot tmux config +```bash +curl -sL https://raw.githubusercontent.com/geohot/configuration/master/.tmux.conf -o ~/.tmux.conf +``` + +### 1.6 Verify GPU PCI access +The AM userspace driver accesses the GPUs directly over PCI. Do not load `amdgpu`. `/dev/kfd` is not required. +```bash +rmmod amdgpu +lspci -nnk -d 1002: +``` +The MI350X devices should not show a `Kernel driver in use: amdgpu`. + +## Phase 2: Clone tinygrad +```bash +cd /root +git clone https://github.com/tinygrad/tinygrad.git +cd tinygrad +python3 -m pip install --break-system-packages -e . +``` + +## Phase 3: Download C4 Dataset + +The C4 data is on the MLCommons Cloudflare R2 bucket in Megatron-LM indexed format. + +```bash +rclone config create mlc-training s3 provider=Cloudflare \ + access_key_id=76ea42eadb867e854061a1806220ee1e \ + secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 \ + endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com + +mkdir -p /raid/datasets/c4-8b +(rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /raid/datasets/c4-8b/ -P && \ + PYTHONPATH=. python3 examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py) \ + > /root/dataset_cache.log 2>&1 & +``` +Leave this running and proceed to the beam step while the dataset downloads and its cache builds. + +### 3.1 Smoke test (beam search, 2 layers, fake data) +Always run beam first to validate the pipeline: +```bash +tmux new-session -d -s beam 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=PCI+AMD:HIP ROCM_PATH=/opt/rocm bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh 2>&1 | tee /root/beam.log' +``` + +The beam test runs 10 training steps with 2 layers. Expected results: +- ~0.29s per step after warmup +- ~700K GFLOPS, ~7% MFU (low because only 2 layers) +- ~380 GB VRAM used +- Loss stable at ~12.55 with random init + +Files downloaded (~85GB total, ~6 minutes): +- `c4-train.en_6_text_document.bin` (79 GB) +- `c4-train.en_6_text_document.idx` (870 MB) +- `c4-validation-91205-samples.en_text_document.bin` (159 MB) +- `c4-validation-91205-samples.en_text_document.idx` (1.8 MB) +- `LICENSE.txt`, `NOTICE.txt` + +**Wait for rclone to fully complete before starting training.** Starting training while the dataset is still downloading will read a truncated .bin file, causing `ValueError: all input arrays must have the same shape` in the dataloader. The stale `.index_cache` and `.blend_cache` files must also be deleted if this happens: +```bash +rm -f /raid/datasets/c4-8b/*.index_cache /raid/datasets/c4-8b/*.blend_cache +``` + +## Phase 4: wandb Login +```bash +wandb login +``` +Enter API key from https://wandb.ai/authorize + +Alternatively, pass the key directly: +```bash +wandb login +``` + +## Phase 5: Run Training + +Run training in tmux so it survives SSH disconnects: +```bash +tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=PCI+AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' +``` +Attach with `tmux attach -t train`. + +### 5.1 Full training run +```bash +tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=PCI+AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' +``` + +## Environment Variable Reference + +| Variable | Value | Why | +|---|---|---| +| `COMGR_PATH` | `/opt/rocm/lib/libamd_comgr.so` | tinygrad's DLL loader needs explicit path to find comgr 3.3 | +| `COMGR_3_PATH` | `/opt/rocm/lib/libamd_comgr.so` | comgr 3.x uses a separate `comgr_3` module with its own path var | +| `CC` | `/opt/rocm/core-7.14/lib/llvm/bin/clang` | System clang doesn't know gfx950; must use ROCm's bundled clang | +| `DEV` | `PCI+AMD:HIP` | Force HIPRenderer (comgr-based) over HIPCCRenderer (hipcc subprocess) | +| `ROCM_PATH` | `/opt/rocm` | Script defaults to `/opt/rocm-7.1.1` which doesn't exist | +| `WANDB` | `1` | Enable wandb logging (off by default) | + +## Architecture + +| Component | Source file | +|---|---| +| Model | `examples/mlperf/models/flat_llama.py` — FlatTransformer, FP8 MXFP4 weights, fused QKV, flash attention | +| Trainer | `examples/mlperf/model_train.py` → `train_llama3()` | +| Optimizer | `examples/mlperf/optim.py` — GradAccClipAdamW, master weights, FP8 re-quant | +| LR schedule | `examples/mlperf/lr_schedulers.py` — CosineAnnealingLRWithWarmup | +| Dataloader | `examples/mlperf/dataloader.py` — Megatron-LM indexed bin format | +| ASM GEMM | `extra/gemm/cdna_asm_gemm.py` — gfx950 MFMA assembly, MXFP4 | +| Flash attention | `extra/thunder/amd/fa.py` | +| Fused kernels | `extra/llama_kernels/` — rmsnorm, silu, quantize, fused_ce | +| GPU driver | `tinygrad/runtime/ops_amd.py` — HCQ, using the AM userspace PCI interface | +| Renderer | `tinygrad/renderer/cstyle.py` — HIPRenderer for gfx950 | +| comgr compiler | `tinygrad/runtime/support/compiler_amd.py` — HIPCompiler using comgr 3.3 | + +## Troubleshooting + +### `'hip/hip_runtime.h' file not found` +Install `amdrocm-core-dev`: +```bash +apt-get install -y amdrocm-core-dev +``` + +### `'gfx950' is not a recognized processor` + LLVM crash +System clang doesn't know gfx950. Set `CC=/opt/rocm/core-7.14/lib/llvm/bin/clang`. + +### `comgr not available: try setting COMGR_PATH?` +Add ROCm libs to ldconfig and set `COMGR_PATH` and `COMGR_3_PATH`: +```bash +# /etc/ld.so.conf.d/rocm.conf should contain /opt/rocm/lib paths +ldconfig +``` + +### `comgr not available: try setting COMGR_3_PATH?` +comgr 3.x uses a separate module. Set `COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so` too. + +### `No such file or directory: 'clang'` +Install clang: `apt-get install -y clang` (for CPU compilation). +For gfx950 HIP compilation, comgr (not clang) is used — ensure the ROCm 7.14 comgr 3.3 is properly loaded via `COMGR_PATH` and `COMGR_3_PATH`. + +## Appendix: KVM Virtualization Observations + +### Virtualization detection +``` +$ systemd-detect-virt +kvm +$ lspci -nn | grep AMD +83:00.0 ... Device [1002:75b0] +``` +CPU flags include `hypervisor`. `dmesg` shows `Hypervisor detected: KVM`. + +### No fan control +No `fan*` or `pwm*` hwmon entries exist. Only `temp*`, `power*`, `freq*` are exposed. GPU temps read 56-63°C, power ~265W per GPU. diff --git a/tinygrad_repo/extra/sqtt/examples/generate_examples.py b/tinygrad_repo/extra/sqtt/examples/generate_examples.py index a7014fbcc..465861527 100644 --- a/tinygrad_repo/extra/sqtt/examples/generate_examples.py +++ b/tinygrad_repo/extra/sqtt/examples/generate_examples.py @@ -1,4 +1,4 @@ -import os, subprocess, sys, shlex +import os, subprocess, sys, shlex, pickle from pathlib import Path from tinygrad.helpers import temp, getenv @@ -23,5 +23,8 @@ if __name__ == "__main__": # AM_RESET=1 gets a clear trace, does not work on mi300 machines subprocess.run([sys.executable, *shlex.split(test)], cwd=EXAMPLES_DIR.parent.parent.parent, env={**os.environ, "DEV":"AMD", "AM_RESET":"1" if not arch.startswith("gfx9") else "0", "VIZ":"-2", "PYTHONPATH":"."}) + with open(PROFILE_PATH, "rb") as f: events = pickle.load(f) + with open(PROFILE_PATH, "wb") as f: + pickle.dump([e for e in events if type(e).__name__ in {"ProfilePMCEvent", "ProfileSQTTEvent", "ProfileProgramEvent"}], f) PROFILE_PATH.rename(dest:=EXAMPLES_DIR/arch/f"profile_{name}_run_{i}.pkl") print(f"saved SQTT trace to {dest}") diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_empty_run_0.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_empty_run_0.pkl index f1048a1c6472838253d704120d170360b6538001..8fc25f57068e1bbf1c63771169214defaf3f4031 100644 GIT binary patch delta 787 zcmZ45$O^(ZgSonOB)!l$fHIl3JFToH}LlPhYu-wp;?tj=nBa zdRX!jb5p1EFuG3Jtm@|~FnM#FxkLtg4@-7xQQj19h91`9N|Q3g zH5;eY0!`uQ;SCOq_w)-6362kT^a-7U-y{ZRB$FmLB(Q)S(3v1h$N?-!+SrW24w%f6 zs6ap)0~?Yy4--^(^ScK*`Nq44dd53Chj@m$f?Uo55TR-?#~J!>voz0|0?`=WGA~ literal 66725 zcmeHw3vgW5dEUJsNP-j}5M|4fpV#z53z}Lk7LQ#pq6)r5(I6-+AFegBx>zh0*ad;z z)$W21$#P+4>?)Z}!ql4?O($(&P7pxP=IHrMzEb-_t>?3~7movBqj&z?Sd z3b%Y}E>p{z)vaersdO$sJ(9Up&%JFjS1!(`E+sivbz^3(Sh<8k=Q5YdwVvvRQl?Tn zS?#Dy)Iy=^-hJ7^TxQ>MspnI(`_hHU%sy$FBaq3L3#D-JQf>5Cc2v8P7fPuj8eOYK zFXSp&D_xvV7FFXz_4b6O^5G;mA(`yiYwf0%@9x=OtDrxRH6fZQSN2V$(&sYy$$b?( zB-8WdN?|T}E>p^9W~mV)Gu5q}qC56L$~s`3QElX?P;U>D`S8B^a%taeZem~YQYBl+ z_l0|-`{whx3YFfMPL(sVZ<7~-ft@?hvg!xil6-GUwJ*m=k zd9Sr+&$$beW!3zTG;Mx)K3C!Pk364!e{~xX43#qF`Pqu1_~DT=*$=itC;Q>Mt)cqS zrY%MH@);JNk=uSvtu}v8bE#aO-_yfpwF`C8Tf+D92=iU_81J9@e@YP|c5))cDP}}Q za>V@{-aT=Drz)%K3aLB~j(z|H9-Avn&d+8ZM<$V!Ftva)RciAS)sAwdR^2?AN$2KL zvj`p0p)b^`-2;7ty|KvP&_MstV0f@ER{LBHpKC`ZU&}u3NR7v$;pkwTQ|BkL8`Ni> z-IU#&?aFS|Pur{88$`dB-Px1vKAF8g6w20hv!I5*>cU7Nsd^Zi0LaDvh z)W!UP+0@*`WXekJx6VFQ43~5Hb0J9Sff4=^kLRZfp`M?a02Th3^I; z(E2y?c8T!ae+1b_{iQn_8GYEy_O4o9v;DzhnvDfXXZA-L1Z^l13lH`Vv7luSH6{&3 z!?8h?)Irta)+pSA{TM~#gS|e&J=i}K3CE%;sX@4@=AHTZxrt24s$?_P ze4$u&*E`S~3CDHe#*Ze}hmWQ=KHwd=sowWyQl;5TR;856m!}G)Im}6+p3hjMvACKZi$^2jp{Od_SZh4e3vLbh%sJHb zZKVt}#&pJ-n4faa`BdPfg)f=6f+aX8+)GH9bLAA8n#xYSnum-u*tje%38s>8x%^ z))m{COqi5s(kpERJ9IAWRIDv1h&jxO0r1UGa zM{DU4QSWGI4|<_X!CIIsBa*pEe)T~r~L-;U8C2DArB)PxEs{H2S;Y1Io+2E>$Ym$v90{vd&4^z7;?-H zjuD9Fhk#*xS9yM-T*+1DD?A)xc=3#ggAp3Q95vgDe>^^j$zve8(h#;81Ffprq12DO zulmj;OzYW8^o`SZt!5Am10QQhJfa0wI|w5o8uzU>9_tMc25IrzNV5oR z>i__jfq+T}ZL7i4*7uh$%}o?$bLo$jpoyh3$#h|MHdQQVJ~o-b0O?{$Uli)OD=XZm{9j++P~VVU)LkpM zH4s(e=wZn&)k?BBDg`LFg%Xfw2g*PVj`spe@Du0$0raJ>H^A!H4wxEjP{^+PBk3QA z#>0Jm4zoi~vav`%xD$@Y18P-O~K?=1tGHTWSW;(@gpN8!2QTYALuVE!)WWM^2*}{cPiQ7oZ$}Fg+plSE30JLZq z$&j0Wp(!Zim0TgOj1r))FrKiS@px@yLv;tNogx-QCZP;f(%D*d%VTr7GE5;P6X5u* z(QkK>rNKii^!rNb(tff|K$i=IU;^#6(v^!!qxsfo_r~hh;vB3DUE<7SX3DBZC?m@kM-!|1bTj&A%C%X=JN!GDQvko*8>}XYYJV2(MQv*gVwVf8KIlXq4_q!Lol{D zV{wj_6sT@8w?g^Mg=D#uPUiO?(8*GUlfWIxu2vtuH9Ehsx}7CZGS;eRw0)zgCDEAh zh1Bdh$q^zW5*1BZ6srtWXI`M)&ep2zn$b5ptM8z$%QBC8Bbe7%%_4K@qQ5z)B;_Zn zyXjrdW*~kpk*HgMwW<~%HfB+0L;pf`GZ=47WLR`pd8QFc2_Nihte` zh!|Y$%-*cNN1J&rW6){G?j-&Bpj5rrf;|qKe6~z-y<%_d_O{jO3w5Tpt!IA#`JPj! zF-Z;%4TU2<;4PA3y@TQY03Fpam+exYB54lt6*;CdH8kv-`6}j;YG2c-$XkzVCYpJT zDp~WqGxM7?pKX}=E!<}Q4gSdVGV=|px0RV?=9`rBt({S3eurqR=Oi-oI=A{QojGQH zk9nxa+vfAc%pX$T_B^mA)U^$(O!~TAw3mD9)Hx3TPhayAaSg)V?+#g5MfHvUW=2G?kVycvyD^tb(CNx&?VE@Ow zSWql75FP3d_s0XUpaA3tMvpbzunz{=R~j=9WSp>)Z@ ziW@IY)K}KkoVcil!P8CW#Kt;kKC~xUGJu#uQd0=Y^xIZ}%WGI@Y9q=kxvH&$2gE(7 z0uS`X0bWJ?#;1Y)NFPQ;|53!D0>9(5Mq9A3su3w(s-F4D4#Tils{v)aLPqJ^zv|q! zaT(!gFBz@ii~9fL2hIuz++~IWKJZu=?livZxYg~9v>$>4v32nD8K+v)7a4@x=b#@P z&PnheiuVN}?aF^>u5dmx2k^>rxh7G0Am8DLb}My&_?$F|MV^5GU!3i;h14WqE47&6 zZZHgbG}z-c#I?p^aGDONb(_NPc@d@ozUPudSiFh&RvL?g?SUXWIqBw7vz9EyxLZ3k z5DyQ;915vY%|J8O3-)SN)uS}-Eaw10VqHtEMCta{%SPdK8W=6 z)NBg=ekCV^PLmz`C&r0017V0D3*jRdKwxAL1GDtTdpx z@!kO}E=E<7r@393i^Wua5)LtrH5@860M@~+%e^E?NrOKkhm!QO+C#Nb}iF3Di5PlYYTHG5s8;{4ZIux|#bRV^Mx;Q_X zn{$gSR+M1R^n2EY&?N=4_iMnX{@>dMswFWi0>#8@mM)jCt3{u^8lHBkk@T2Amdo?oy=UcCM%bU zjjPmDM+~cMkv`ApNJ&HScz9?aKzJV@;ng+!%Wo_afqfT^`Ft2i>c#6D45)ht{O$q1 z7s?nTux{PR5m2QpNcR(aa;I{F639n78=`^@m;KlS~*%Y3z zK2nS%3>}i>ZmmPhemAWvVh0k{&Q6a<-}B5+WbQrBL_ln3)jW!w8M3R193#UtvsO1? zeGi+aLfAVao3W6F_@|Kih1$pt-n+5GPUMfIR^52y3_cbr`0+P!jTqI=^M%}G?c|7g zvVz?nJv0Ao13=8J^<5J7&McrJowO&VR^6Jqm@6mMSJzYBIf<88=O<1w>CcP%>krDHXi1N*_P4%P_0+Lb2S?#yr#$6oA$2#N^!6Am zwC<&-{Y+jpFN8>Jj0r2kSRSUe*Pgv(Yy>Pee5$j$Z8r7%C3u|hs&|b&QAsVwh*Y}t zOz*Rv_3UqVwzNK>`uTb5>?2 zhXA>i)q1_E`J3<_cR2ah;0K>Wt%5m)i-lq+(i7%dH2}G5^u1;qu5QECVHyn;D2W|< zcfI2Yn#o{aKWqy>Z4?x@Z2TN0Z6HK(ic*Dmdb8*%tY1Ah51Tx9>e*wwd$r^ zUI*u~6Bl}nY!AeYI1q`&VaE9A6b;)0vv1Uad*S@VB}{103^V!E1RNI+Ky!TJ_=%&e_qF7@DZK+blc~8$ z?%CgMz{L1tN(q}K0|8(F&KuyaiZ5b1sTM-_>XpCq*gF}b)}%_M)FpQ=Y~_AFgRR@v zhpgu_r2>2)(*oW=0mL3NSeIZkHojQd6c$4Zd2DN(MZQ$25+)tCrBfpme^_<5st+ZT zq@1Ty`9eMium}y9Ou{vOu2zMvNj{s;q~TgeTjkQEWy5h3kMKsv20Q5X721i0MNMqm zEQf&O;my+yFBM9vE&HLVM_!l!B>N&Qcaw~enU{Rw;){=^Asg=)PN}$rriWBl_gZAog^o1Z99o|*K`fv{)1E){ zO>I)!YdryfLgkpKt&y28QN=${J7ngUsR^_XZXK!0)wyUrjqlzX(cynoDEeCS*bnrA0m00>Dut zd#D(eodbNr5N0j%zb}SSHO|l}hEwzK$=8`_AN7Hv6s0Hjf+q^%gwMOR5y6uL!$OzT z2>%RhJRApvc?;|K&(LuHESl92IiVxfEOuC-H4FtzZu`Vi_SrGgZO1M22PP!`&w z{;B|{_NecBAXR(Rf2&9RZDeBg#qejSPuV`aZlj0%zQHK1UD->oGY4q2ro$@8rqW2Y z+o+nk1q~+KHS>8=WHw;@svWYr4db64J@G_`X%3?XzBU-k`uI5YJgjbpwM_*M;x=r= zjV<`wZR{{M;hD+~;~!na{I`DpJ{+y-l6#kWWEw(KM>X7rH>yXugBV{_9VZcCx*k+F z``)3S1j-3Zq)n~di5C5fYks-!|4tK-yS%f_DDxGhS-8W#J~npNFn;uxVM=Dp zDcBx#gkJfl|J>O13t`%obLH#d=~7|7m@H?e$>k(DmAeSLjLbvh*Eu_K*@>{)rt_U& z^~7`be;tN5>-0P(^jDuMX7YzmSciYbj!K?~-G=qrf8JrdhSSt8Cdp|cd7)4`r^$n$ z`3rpH$`_CRX1G|&o#(BJ-d24%T%ImsR~Ol*#;>S{GIj~h>Zn)3YQ_2IkA}}pz80oU zZ8(XCNny(-ca>=UY8b0+<>cfg*qXUC9Ws(W_suVd&pYep2WQ=1!G5~)nW+*j)={6W zuZQbh00cru(iBQ$jb@Br(k!olftkv0;AuLYe-tqXm%liA0_m0uSxPJpAO}ZgBHKc)Fy+~T7Kaok8Iv$-OUY|Kq2hf>DNvKyDbqEjKAOrjX~M9wraY%GozNdw%Xw41+U(>nx0hkHT{>+-pS#`uqE>XlyVZO#y!% zjAZ)yGJUah|5RT(J(!xD8cIQLiuLtP#iug8J;n(vm1j!(2_3dhoH(MJVTJb17H}>_ zDOpTavV;&O%Qzk6s)Q+%S!kKc@;wOa)2Uq{%{A zaN+Vk;I<7d=Z94^*Jhx9K#iII@TNQU9US3}|3Ek zQh`=l4J;U6tZ;6cw-KG0Efw&>Cnxc`jqpUdY=o8C6Q-#gszz>Zeh!fkSyFHeQV%Mr z3EZ=&SB2Ca^Or;KP&&}jFpdgLrcf19DM&hj3Id5HO-VhoGI2hgMiK{Ey(y>PaxK5NIWkbi7GU<8NjEt4%&*YJ(PaYgSeptVOyStHGlCCJ)Mv#6fRNTV`m8CIB z%G>j<&N!|M(l|x5NDrY)#;=_S|BpA{EJ%47r{pf;Ch1b1E5V@Q8oo!|X@ z3gz8#L{8{a*m=40P9u@x+-jwiA959tN4XP!+$<(~*S64K*j+j!$tlgPx?8sVUPPSm z4nh0+o}P$*+>TKJp)1js5%4Qt(i8ocB%{8&%afN0*1BpxzR4H|BdqF%N@}_`VmOyz zozMb1v}*W*b`cCX{4F(n!O_67quT4V`jEXotdUe+P=oz~v#|x#7(MI@XwYHvr99eG z-N=4maCr&M{3$-Z>Xxz5!>3bp7>Ls?a>waJ^3d7SPbUvQefY%D(^zDl0Cao~M`xc- z!1d;0I$awvtDSV(CIvds^N~%srD`KMhyB8Iu{P3K-B7@?H2GtaaaJ2$FsoZind!13 zR^5d#sEi#w^~B+)&YnDT^i%|=dC&Y68l?~1?y7D_hT|tspE;fU#K9A1YcqdMEv6>S z{4GKw=FH!z+kd6orkVdnw~m>=r(5UDZQMX~<{$L^4(>iwdz(-_y+*3WhLxyBJyds{ zP8?32IC469{KzMh2M?b){)wY&>Tu^ur29Oi-zlUI9XvUbJp9zjCr%tcd}b}wjZG_2 z@ApvOdF<4o(d4nS#~XXyNzuM9+Eg#vR4>}pFM2am9_gCl_xuh>7uVZVzPgT`h#pQp zd0hJ1NzaWmtr-?BUz#r|nxls4AWAhhCAx))jvhaKT61`{RJ#<9n^7g9&R|HIEdak1 ztj#rqt?DuAH#o1VhIZyQ2ihEHbD+(EHV1zAIPg2a?`K{7@bO`#9&2+iFq_K^_T+Qh zW$p}Of4k%5*{$ZdpCw=W-sV7?18okpInd_7{|g+bRdLcpCY@wd(sZJdP268;$q!pB z)Gt6eHijto*8%zgMcVd&ZFIHjmG$4^;n~_DjnP{?uqmX{e2<6nf=zYc$ibN)FIcob zgbf$8VPpRRbxxb#ZVh;ehIKZa5wCY_Xg}A+G{fQVfE_kc>N z3$#b^*}c{@5S!F2uS^}N%mbATIbvv!Go9?=Os+?;7j6AfohVIbTB~i&>_+^0ml0VKX;Y`Oe?-^Db}XWsr0 z?g^vAyz_?x-w7{m=t!6yiBA6C71-!!eV8{l?KVN;v%KDDMh_07lAVZHparajr9VMX zSxHY{R;y&$Eti#9mDq~3`?zU{Q=Y09w_M)N?Ly2B3D7DVykNInR?CwJD0eSp1_xBq z103Kg->HJ#sQp2M4&iup%jI`*Aa_|UFeBn&NqIL161$OGu_^Hog1fnnOP7f`hVjyt z%O94&M^$6spAs!aS4n_;DiKd#g=(|QUXtLAh8{C6}ds-p^jd@ zXSZ}jbwnTE6aU0`4z9%zP{X`JjFKmch%7@6NWP+Z5{*^M*WxYIvgPud2KmI8cb-Qj zsgER5dlGo07>o9J)ziP|P9GmbF3=~Lb_ui{v#}%b>nmk=27Ea5+f`Kd>Xwefzy0E0 zL@MT;FHx81%RBVjul@C3{;duo&%#TT2?6zC9^xR!5hS4VQP7S{Jc3?qGtT3CIF4_? zLRun(D3b7R+&F&u0fQVE6CXm9lCz}`AP9Nx=r9r%g7k58q3r!!c4-7r!-z6AA16Vw zK$T=%e;kh@xEZ7j6zJmnC+V}Ki0{)!Ntd}ygq|~=?(E`c-#<&#pwDoUx%eFJ^eV{x zz`Xu7eieUg8YD>+P2)i7qwh0soY4{VN>d-DmuU_k@Wdek)f7pfNPe#W4l%o_{Fo7P zcoJbyHB%)`F7qaR3oM`CqWU{n4)7O*ARmnZpL)#OhyMINkksSfogKf7N<8Bq@s-9u zy5oCRjMn2HRf>;Py?P|7caMPo_}4|5)jLKWB^?N8@t`4fh4Bya@~UV2W7)Wu#=p&I z<3Yo)m|t)I_y-<)#=oAX-`_Q=A?i9oAiF_{1=`E_J66)e@Fj?S1s^=_}85D@BBT{SKV{H|Aa69 zmwfxbiuZr>>zY2gxM*H~-&ej9Hm{LNL(B#%7tJx62IvWbyl4(f8j3BNx8(`ayJ#+x zPDDAOoi3USlD3JS#wBetJzal<=yXdCbam0ZA~~QQE}DjSvcIoIjrme+#?=ajh zn$aI2IyN`puIK2%yXeX0DW>d2G)G@%wGBRt;7bDaNE+QuOVY6qW6`{mcBgks z`gJAF*bqcdp# z64L8j_K`1JCRUWMqP(k3RI^LIzDsm$S!MaUCU2AF%a&D^FGJE;zU-VVUkj4P@^wR= zlzjaa(Ye+jUv_?$uZXmg<;(siSiWpJE6dm46P?>q7t5C=Pb^={^5l{)TYgx+?rgl~ zlCK4MKP+F-hurCllCI=yk2~Fd|4P0d3ruJEiazB|@0R*mzDg-~`mm(4d|fZQ)2~T7 zD7t$6uXyPdhy7dn&Nom=UH_v&+ofnjSyTGolCpo% zRMH7u`X3aKApH-SUHTs~-=+TdmfF8>i2ch(ud;s=#3N<@a;)3_-TazI|3kkI6SGJG zKEeUS9F@Qm5;!J-CncclUzBRy{zc|M{qLgKzdap&H;4w?zgqv(_HSMPTRwn+jO>d$ z|C>aksr|B);h6wcF0<{2PlBq?u`I_BOnvmvQhWX^u}88Ef1l2Z!|Aj*0r{nwhc^Nu zx(hUYSN$-FfsPpqwcX1&dWCTGCz|2t<>xAuSf4^F{2X z!TzaAs!~>mFSapqKIWs1k*_~Sj>G7}y$TBFvl8kU`4G-Zurcxk#v<*8zlrxPG{E6z z6F9tM*t~HOBd4IdG@v%QOpEMy9Gc~K$$SX zLFrThr4C?Xf)S%)2=U`QP%Tw};2@qK?-CC8hl6sffWu!FnDV79mwUKX2<+j&>!J=Y zW16W0i8>tq+B(7E;HXju5K!s>I7%uYGxCH4MkR1k0!kfN0@k+@9KMva0dRAX#K{SV zTYsU|fi=S6%PYX)H{TvOeCd53<9f$!@uVbhXeQCW-Q!L%V&T9%<)g5avKU?@Q1=Jk|HM*e$xjEUkHH0 zsr71orByim=8z^=2imEUCYL!b=1_EPaQO0q2M+g+fApqh-2tDU2M))#sa`!2?*xiXQsxi>Z?!oD651Tv zW{-cta5#;BL2&p&0~~(c1BV+G2nWGoorYev2b$^(3Lt&TSM_Xs=-+_=lfgBlZvK^c6DRMMUfL1-fp5 z{>?fZj{e{IS)#AH$Ng8xG}BjTJN19b(SHYh-Bw~n>-}GD=zsKgiN3BAOn3BOqunn4 z%hCn4O5k_|Tc^Ls^OwB;>;JQ{|N8Y({~h%G`>$6$*iQZaA%=iEKR~cs zBLF@wKpQeHn%x49chb|cfP>Ii0YyvBP5dNjo9W4h1sR&W6Cye`FwFp^or3|&ZYj+G z<)S<>Kxvm_fYMH5fbzyiAn-QyeLc$kkhJptS4q0VuCY?-&5TEl&(h zUVn_}+!Q(tP~MarjR0li_h{O1UjQ7wEO1`{96m1T0dTk_=>c%KT|NK~x6=dQa2xIp zfWz(b=(oV(HgwA}7ZHi#5a|`lzSlX85 ziKWewCzduljiqf&(pcKA$P-Ij$p;Q!lpHK=cAI_R@Eekor7a?@<>i*5Jh8Mz84kC( z!7_JSO0%@xko=9(X4>y301m$*@(}=s+v#hB!|n0`aQK4I4}imqk{$qu8!zhl6Y?c+ z_;Q@a0PRB4+$+Mg$OIz&a*c5K?GJh2aDV-;nL7m3jdjug*uz5Ue~E6d{>ScMt8j-{ zjs5#lu>EW6f6SO>?qGHM_XiAzH`D*TaJWPN8&mdg9S&c33?{tD z8p;9EKsi84I|q6s5DbK`=okK)2M7n_4s*o_9Ff3L2|U4pi(4)qlb9zZa9jck2uE=j z5ROuTKscNiRs_PwZRp$U7vBBnS`pa29)97wUtnirhhKO%11aJ+e#_$*zDR%; z7ydmP2){vuR`3f?d>GCC4Dtc9E;fMSSH&;f1BNg9fZ>qL5I_-;jqd5@1YQG?R{KL5A z8vl^A;`oPBuJI40g2ul@eIZbQaIYamcmoiA%L9aW9|iuSfangxyn38~ zP=zyh0mjrG*xL})IJbeL!4to)N+75)(JqSBpfSa2(A>b-+x#+z>p(Q<@4f)~QYX!!+>c4IZ^?&L2>iu`n*NXI6w1W76Qc4uu{QlLNM3eXL zpueW~zkEQ;AA^#+eSq*Wf!Ode-Jak!7zb;7vFQHsHuWcr$3y<^w~M_JRO|l2;iBx4F>_ znr!?YT|oF%k6*YUKp+E^yC(s-J8Lj+ieEPal@lJn@NOy3K=@UUU-%6v&p_oZk6-wh zlxLuF!s8cimuH~zs>d&US?Wit@!r`l+y-q~=7velpb^jq2)`{d(MhFkP?@F8esh~B z&3?x$ZC8DO@I{dVmbPViVrjFbhJo;A(q^YI5I(^`c;n*Bg4E2?)-63?AbeP!SlX^g zP6onl`C=e^L2|ORO)wDND04P9SlX^heislvEbotj@LL|g@Q9=<`SSRM+w!O6%i|Z` zC||cce&M$Kuzc5lNg)c1q zqQm~3aQlU;ebJEfruzb+fGE`;0p$n*B?SV83o$z-pj3RswBQ%MI!6e9`*)r6zt=_o z^9REH^}koe5dz&k#0&sS4htuO>|bO)CV8HefXn_}mHtN>^@{ed0|@7Btk4GSeSzaY zf@zcSmDTx$!}mJ~2wx&UDh-6+IEP+3JdL=|jz0=0|FOIE8MoR#`@(dNsb+CVM1?mcZPU9%0oW@(hictB>tHjD* zx;w1=4bc@GPLu0`m4ivGV@1dugq4GTL0HlESJV^|6TwJ10jT0MzDBJ4#x1-sfj><4v$~zcW-AKPR=>?+leCQjXS$l_xBeRf=?LXQ(Va ziUOPAiCFh#{-pRaf8WN+Z?6(7zv7RT54#=5Z~I{7<3U)tLEt0lnr{(SK1R6dU0~(L z>rSWf-8%_u+X?PAY?^U08sA~C(>RZR7FvXWvXzj1MOz7()gIav#=l^vA&0HB^blIV z8%;0_Su1?$?#92@W&HEEl{i*8jko4Bgzg5}O5oOtwi1dftQ?uiV~m--;`kSgl+*a9 zk#e`Cgz?W<0V}`qPS{ckE8hUDG(lMD?{Y*JkZui|Z$nVS=7x#bf3Hd)sA1zS3M$xi z8wC|?+`?~0nm6POjZV-ouYQK)S3%K4U&H1^U&F>kUpKImf*RMRpvH9()V}Vm&|iMd z+5cN#rv9t$asL%;&GeNkx2ykp*?{}6^HNh<@Bg)i{!jb~^wAVgI4-~v!VZ2HvtRv@4w^y>kQ0)-Bu#%c>fAEZ-xGq=P)>t{Nc7}Dy+QG zjdxs7IJ_v}m4a;gpqDnjx54%j!qL+4?eX#OI^wNG}^s>=`*AFYdDLrmPFL}hV|CkGcRWg?8#ma50 zFaRsJvB&_dyeRbtVC6Pe8Gw~rQl7EQ-5#vm#!>^Ya{G)HbXl-+o98TZVSCF*bFj48X)JBsB43P^+ZZ-W+nA*JNSngS8)eR>1C4Caw4@C_ zPUE)ZGFCn=IT$A8W+Q$EAIcFTu*Yhwvaa!wOF0!=nGK6)WHCbQ*VK<@yX!(*M@YX&m~C!pfn) zV9>$|i-@_)GgSP2C4%+8m$zJ|uVU@KVyyIe&QMAGW4yLus<)ZbxL5xJC!U~Eq&6Ru zz>^YCSUK{vZvP@PSuKX~y_N0XR9*iYCS%2cltcf!DNf_JTENQdPUA}-11IoF;PRfK zBIo5A!=^Qh98Iut^J>=(XcWH{lxLl=^x-y^-p0}k$u^eW#?q|=7UAZn=JR=+?oY?s zzeQMj-%ww;-*HBH9ZT;Y7!s5nzk}M`@4QoC>EjM8ozHZjwaAU|JO3b$udQL}Z2qr^ zHKIcQ%QQac#L}ZKzw;K)BgfPQwG|iULTIci;J@z9Fgx^8m4|c*E~T*a1XhVxI*%9m zU;V*j=_~0H#%rtbI|u(()Fr@LrAr{Q!jcekLijf-fs+zYy2R3U7^aaM6xq^w&R&Y#Ddt{=?KbHPR>sWf-mU?wfen;R$kSzs{2H8?5bvNVR(zA7I>Wa&n zG9y($MA}UV8^p=)JaJMYFg|ww7aG5S{dF2Rg5~-r2}@Q;Xcr>nP!9^qdEA|b_Qvis z5i93mBUYXIv5jKoJZ?B>Hox@mISBnbpQH9EB%0_;ofOn{QuPWoCi)sR?xNT_{luT@ z9HwByY1bnapep^;xSlVt%T`X<3 z^suzukTjMybpVT9n%8f}C5I1|enr~M(q_|HQQFi2EVrdJ%bX!kEN#p3)VOkEUJ?GY ze0k3!Z%R5c6aGZ0hL7Qe^6T>Rk#pP?lK6hJC^S7rAX9$ zDOTfmjt0|r%#Lb39C)eO0i^!EWUcv9fYcGD08Sj0z!MTUCV?jj`@;Q+_sr zkUZG(V>I~R1(sfXcT+4~{k92FTHP5gfw}}M6$(p7p7p}gSGz6`J%Kc4$GZH>qDvsD z1zjQlOaBCBwqdHb8J4~_T>|RC8nE=lHcam;VChTEe%o{5F0u40x4;YmUVX6i zJ8Q+#S3JX|?w8K)-2V($YL2&vr7!#Vr3e1DCxeq@>$LLQp0`_fNa7}A{JS!O5U=&D?F<*=z+bEp zOXr0@*zUOSMF(N&_-tLa6f!GY3Q08nt%qOI-P=+xt#15#+iWRtOW9K3RuGnsBo~&B zQp%QUdHf6dZKs>nAdcFU8S;hVSl?Oi4*YApm*84BA0v4+Q&IOj+Krq9XHsrV>SZ;&Mn+(90 zHfXq+p6pr}xKv>IqQHtb@| z?Ux&XmD}k7SoyHjAAps2OL_oSZu`BX+k%x}5k~r8<=v9T!e%oWgCGvdmozMaEjB$7 zNMYrc1YqU%`)B#`V&!8}J^(9^NO}NPZl?!e<(8BWz{>6N0a*Ehl!tr? zR(@?2SoxypcWcDTi#;B!d}$|iUX7Jwpw}BbJ-!la)$gEaMye(RTma;+1V-u7F^&#vA(XTbc-80K5L z^nvHE0OK9ZRNzv~KE%b+bO=9rTHb4&PnDAAQ?m$NNaZRA=#Nz>Sx1f@I(y7|{D2j8 z{48qw)b#eL>Dfo`!Bf*?I5j;Q=i|}&)bzRodn_Jvfa?DUA4TIYh#p2v_VMcWzJXY8 zbSM%H$D;88u4v{Ml3uKZs`sbolf_IaSQZf_tGbmR(}nqbr8aV-dLKVh zw)A{$q@#KtXH4q!ZPj6`pD?P{gK*?ixIuy=un@75i_rDGK~3&O!)r-grv2C diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_empty_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_empty_run_1.pkl index 07a57e03f9bbc89ae225d4f019ca8775387539a4..5a166f28fd3eeefbb9c83bf1c927c83c972a8732 100644 GIT binary patch delta 873 zcmdne$O^(ZgSonOB)!l$fHIl3JFToI0gPIG`v$Ei)$-NCCxj zC+bTGFgyCXOzC0COUwnzGrCUMtmo${FnNERxkLtg4@-7xQQj19h91`9V2@CFCQd-?^31jmOt`h-r&knUj$4xEx9>&=)U2gLH;Y#9pP zOc{#aEE!6^4U8GeAVQ@_s3l2{Dv?v<2J+y zYzW+OLcxK~@jfoW@t!Ub@s7?Ro?)(pT*#84EQia9#t*@7b7UR*fW&9IWklxzfP32Lke>acON6?cn{AIkY`vx9GD3T zzyL=!p#x}wF9R^ZCnqFH+9Mgk2{yvlGdP&A85{^RL^r=qQW9X~o6K7w%a_5PA?(cn zw2>jh(}`hn!ZMM~5f#4#3ylUgxH?9-x{Ur0g7O)G9XORTWq59mTc#$Y um=V~CQxP-JJO{A73-%h|SIPoZngCY%=1Ks5rL1sMBi@I literal 66740 zcmeHw4RBl8b>4dvC6W52xa;+H~Z{1UV={9km(Enp_6z5R z$8pK0<}%f+Ztgl?N~Lr8>5dhABGQBUSUP;aNrVEpqUez>PAd@c_O2Ohnb@cQ5%#P&MQmTkXSIxew zxk}bZ7w3~j(fB~EJ%OowFv(3wCVLJUPf^RC>N!%apg+$vAet#xdM~BYmoxdv-U@D# z>G^V{FqgcXDdjV>)QFK8a~G$`jy;kxjvD7h8#yWv?qM?T>zyx`dS`Q&dW#E{Y#|>B zhWmQw^SKI@?oFr48Sc|)o4G%kPR-7qNAze0IVY>;CZ-BUd)^g^F>ukk4=j zvhNywr40i)pRbxmIWs#Ibd?Vf1yL0@1qHs)hLPiJ$z;`h5)Wa%loy3jPvBsxG+jPq z96WgW>SS3o|NRY{U!KoZxcwuqWS=zm5WzqxQ=Xr#2#W6;IiLN$X6R(!cds>=A86Q8 zbT6M__8GbB)l{?bbDB%#^8B2h(9I6iMNbLu%LjSfMUV0PssH1G5Dh0MQk)o!&`6F( zKZj>eqrYF2m30L~p2tSN7X+S}D@@MMW}ZbR#VJ8*0cWaI=P#LUFq#*3JJWUKFA_OX%6*>{>Bw!JqFN`cfNW9nM| z=xl24(qzg=9x*OFUksLW`O5)t>Cq8>6Hn)-3W1)Uo|i%w>16|lkG5ss-NbtX7HIyR zdAnHn-d}?3VQ=ovhDM*zv*AtiYc}f7rr7~M?#%upC!=t(=t(hN!9Wn0He<`3at=Z{}b))EA8QiDIrA!(sGbz`I7<()b9Z z<#eW8Hp0g7`Kc*9&3qUOWcGJ{v?|v~^Kb(pQ%c{xc>KNK02VJk({6SqYqIT3CP?x# ziI$d(9k?8{%hm>D#2l8zawU~dX9AQx3Ayf&QLdDFjLd8~V@y5uv8nv42H3&)n6vP! zPaQHY2T3Lu;z#%BZr$uIr?VN7KawO_C$FSPGZ?>=n@-Z|d&szyNkdvMC+AW{l6e%T zB`XD%ECbd$eek7gL3EA@Uo&!fgC(LvJx|Y;D;Ikd#n(xQ7(p-2j~_cRJUV>-nDGoo z6pc;N&Bz+xO-{&QGrVIH>=2``EKWhsKb9;^i(5m?^)$sXz z+G`%~9{mCh$P>th(e>-zVi)U@#=52bW+GjSmF-`@LUEvttgpNiJEO za|%?kP<#-w&p@1CWk-dva3tCn931qK1rLxcm;?TR#qw$iNR4a$h=rq}a4;r|J8SL> zN8`b0q+U3N1l(cn#)6DW#p3Fah+}=>0kl>Ybk`Y=2IFy`Iv>Z9S)QMR6h(psqy-2T z)0y1Vf@?VXXndnF4~ah3hgJvUVIKnfct-QBwhu!Y8t|+(I*6pGw>${36{3Z2TVv5+ z#JjCgJk`jcPo+!JDVz&Gfz)uFhRP9Bp!-beB~VTNPi!V zxetfzoz2bVDkN~%uP(7je-SI(kkpS_Lw>cg%suY?_#*unl8~3ufm#P*L9nFf znq_v2R+83HDL}d{lz=|lk%m|#7J}R!^J;K392r282iisKycbY4hC!ga?w2GQ?gvmd z;DKMnqTv`u$gftRJPM%7OL;``IMCp@XU%;ffF^!5cj5JuSai}uvBQipfGMcYYsQH7 zh2p_}*&DaWVUr6UsT7I@D2^3_g=7}}ZQ!5T--Ke5$xr9JbSFoEgGH*UQTezAjaT`fknFZApMD3^uK#X>g47kTHFa>G6 zk}KqeRssYT+7q@ko~@2-GxtH=DPmJ(62ed=ovoUk&&=h@(1nmppySs^ztK*r22ZgN z@GGT-Bcz^yE*A(u2RdY=E7ycX^YziL?dGoH9MlY1;{0T0%BV#MEy=ic1d3FTarjvR zK0nT|zfyYU^8|m0pX0A|LFtF{$n-jI|b+ITei+S)| z!$r|q2Q7%!DoYv=Q%Fh*0k!_NWZ?1^wwhXy@<#S*_rVO}VU&U6aqRQNye6prP#6=` z#``>vo|AA3wpJx1#hcYLKiQ^f)_ygxjL(o(`u6WTx0GClI9hu~8yKYiy?nt*0tsU? z!21R7BRdtJJ)^x0wI7ESv2pC&dAnQ_!{+cHQ2M@Toh1$@4TPXo``Gt%3nQYr!j;S% z&@01bpG4*35!zm^gRVtV9ClBBQ2T5lH3{5G>}I$d9EX)pT%z7KL5yqdj|2yNaBFjq z0PuMi#)02+Npuw>;od$&QnZgZ$$g}|F0$0krDhGa7vpMev_BjQ_DRDdXKSMak-=bp z+~=X~FXsS4VrNV2MY$Us9q8|a-EJK-P=n)r*x`uy)Vp_THU+bCxpU@fZGS)bC|!_X(pjGe>KAER|EERyPE+>Xq(Vk$ogqZr#Z z4w@4SV@ZqD8;o)i+0@BrSy$nK)a0ZwTL9APC604(Fp=5?ayB&cOe$RhHuE4pa2F+lxI~Jg_>;H?Eu19luS<-8cb~Nn_&;NSdq80c`u?U8vh}LY)JB zeY77glIot>W4v`^0364b?_}nhFc=Y)^?7cO5bw&0 zo$PBXPp_(NFl;HXpO?lpb?x2<+k1fTg)*iH>|48r1IC5XAXHY5IR`dBFbDFRdH1n7 zT^i?3Fz>>$-}zRC>I^_8x58`|R7CgV1*8*C5OEeVj8DcORgTh~r1;hp;;r_l*FcS6Q zwmrOD!fX?JIu0`nE{QGTXw1fA;v@wp(Yz7yuGFSazl-)2aSDlQXRF8IcfS-5&At1j z5QyzJ^))0>(Ucf%Mw{QmmbNaug?-K3Mb8;n78`UvQ!Gs%F(gl&D`wKPWmw~bK&lL5 z+>?BE?j>=A?jkm9iK{auhCycj6`s?`A~v1HSr;7pNd_*Ar7D2{I3RfL^x2U^28G7P zpFcHzY!nuD!cvYlQlG+|JR*aQ)4WG@1A8a@G z%%)yhfW-;#dRN&Lm85daNTr3B!WZlIv%k^a)c%C%=cml=Jmpu-VcsYg8_lI@+RemJ z1;%+;sij~-?CRstmfGIttUR0?0_0W}`}LycTf!7aK}=QH!sk${x(^l$#XzVh$hArU zvU~JBdJC>@!PP;U4Fx8NQ+oG&>Iq^=WDwhyv6#nFjU{Rz0;@Y~HPNkOw_}A|zT!M~ zo84hP0HM z%eE4kI}7ub;ykAFi&b++E-!=gIE@QAMja8vvIom1s0?13oa2aK_RSh@eD8+>TXyYU zrve^6P_n;2HV9o*if}e%Y9I{Vf<-A0q1?$h#rwxNt&6(PKwKm<(+K`))d^KqhuP#nK6gQLT#(H=FS$WJ8Ep5Pf0bIA1KnQnELj zewhz(9cVLmLVPMN0Gzf>8eCiTx7mq@CeD3s_ zVHW+Gvffm^!*C{3bCcY&zubnk@yCT6HcL7JzyP=%;I1lj#B@^Zg&vZ7e^+pXGC-|K zl}f1vcP^ahekFqw+{Sy2S2Cpn%plVWynzgeb7oL5G0ZsoVq{a;4=vD@3NPbNt|Pp9&Qd=6j{8ZeoJb^Kh_gt$p&n=hqdT}Q{|(j;iZb`!TS zNXH>Ni1!sbjfQPaoZc)40OaAx(=HoPS82zac!V^)&h-R5p?3{fYa0C1jA+u`zSW*{dlNrlu~n)wW=d8gp4VgXGLh^`(oNT&;& zbgCUvq%VOjAV$-NYb@EgWpaQq^m%kN}vymMhHMNg(it3_w`Z_Sr9l z(jVv2ShoqBkW*<5$#LONj2G!vKb~J=li%jr~K~xP_I>lgW9%lJ6Go7VAT2w{p zjt$|7g0jQs6WfU3MS_u`1u?_F1T_y{fv|32C;ugy?k}QQj>txcV718|~a-+kOiTVvQi><9e#D8no@50)ee(KsGZkdLnDWe>h zq3z;UT|tb0R2?S~VfsBLF4nb8-m9yv92Zdy2*`Z$T9mh%N_Ukj%J|}-$f&NWnCzye z+DYV_p{M97vg3bRndEU%5dld-UR8e8PDM9?a%xCao0_>2E&4Yo$IE^FHA16OF+r^YU5+K>Dal*w#41?_`;>6L%_kF@Te3)0b?>t7B|mkRU6WH~cUW+%z1 z+%;%sq#$Zv;_S#}B|>kT&bNQDE}pahd=M6`)ALx;Z$4kl9{%-Uv6Q>Q=N0Q)_1R!~x`?A)q@ik`7dK@b z6`Yk(p9zXh=YKgIygd0vkWRM2B@dIrxlQgW(fZ{ew%p3e$pvVexiq;NNngJ8+29p> z-F#!O`*S!{cO^4bq8&Txv+?C%tqTA_$T3Z!MEYn(`$akA6%1ge@@u%8PUjEfLjUkA z4j-PLx)z26sh*7*Azi&7cz_;1dhFfX@Ra5IrJg%x5ADDK#u_W(s+p>Y?<&VNUPt?3KC0m#y|V>4rzj=RM+Y-IW<9WI>vzh!)8q^i%WKeuV$mTW=Lqc{NX|t712muF7*I z`11t*QQazC&1>rFF0b0JuJoRfOry%Hc~o81Jc=y*qdwSwAH)sCsq#whQr97SW`y1M zy@;do);zEDB^6;+j7;=%HJ~3+l$H9xKhx3G=E}8vVTq#bKFUs;tpSn&D z*!FXg*G7@%D6jM{NyweweLs%!?l>Z+=u_DKbLX8#BE`AIN>x7YDxhxFmBzFC=viSEJ;?&q7X#Kt|7vi6=VnjgERnbcbc$F{7h5i>*MtOCYCsPyb zc2$33hcTm9TX*1UI7tL3x6S?FeisFhGg+N!Z$a+}b3T za4PSrL4|>zY=vu#p76vp$nktBk0zPh*&Yn0FM*kF;r%l^$3{<_OOaC$yLsfkbBW~f z3+G-;o_O)Znc;KTX}$#f_%hsRpG&|5=UO^l9nsBpa=S@^HgtYu2QI1V2z;_%oi0{K z+RbeR>`jwhCh2F@(M8?tEM=z4f|%KjFsO_Tk3Vq>l7E6zSu~ z&W3=y_2Rf!_4JL^EE+pRMTxlsm6vxJDKR{>2v4g z2yd2ZhZy5VREeoGn3ZM=fG`!@=9xHHn<*~@iT=*>jOA-L8ms392M@`ypC+ZR5U2F!B@OIwjqsOooTj1&Ch`F zlH*JRtWN{1F=cX^5)dh$~;infGvj3I+J@3dvYy;O=)Yl z!m~8_wid^o*_QS*Fiec`84qEli`Ed~np5RN@JgMl9Hl?x68q%v@e8MnXO9|vwqE_7 zF!~H{Sc{3@M^LCqFD{9lvTWy)(5u#VA#FD|4ROL+6@k6nE(G>*;9BPr?X+R2M#K{;2RZO&=hD-H%05(mh=U0lqrl-aE1dYMZ2~<#%ERF zc@8A@g0DiW6ppur`I_x)>yr@?z9#;$_A<u@zM#%_>7xOp=IgT&^S%iYG zQ$7k%S-b>S@IIWzJ77?@0HTP)zp@$!gOd?s;=PCxe3mc}1V6X6X=@aum!m_AzD+MB{q!QS@5|bY?c&AdNcW#a z3N`%@$LeEOaivGmkE`B2!jIyc`t@@%n&yGj2fjx)zF$VrBTapP9;Uu{v@Q-2sOCB` z6!FjaN9h4HoS$@qjGh!Th?=RA20wkBUID7VJ4N-kF(2SB2!TJG0grn0;>-eG5jFpf ze>r&6KL4%#C!~piruok?&^%d*p~m2gM~^pAJbT25`40id{MSKGN<3p^7Tkdt$sSG2 zf8dv!P0oM0P0W9@c_ zEXhl=wyux=KJmw>ia>htCUQ|aPoH=G1nG%ya0I<|YyW|RR2i`nRRbPcyDdr}nD{-B zmty~;$V)*iyFwa3!G=G5<#V!qOuu_~FWPrU)QN%#StkW$okVnvGqk(tziH7=e5+~t z%fDi$Ki)_5MfKGG#Mb(++R86nP{c}xGeu} zO26|r?(`K^{ym)D9&*)hrGF2l-+0uWuBrN;r1bIkxznxo!EO@mi@MX-6@7FD=_iq1 z;|ve}vPPNt>#iC#5B{1^X*;N9>sjrjyBlYSj-hgR@Yh}SY?;5-6^Af?S#>dgS^UEM zW$_F1SN8>?!!n8we`%@>8}e63`q@+>C9i_EADiQKJ!=isyn?)@gMWo zov*vot@7yAx_;x2k*<#o;-)zh)OdM3%hQ3Qq>Gs$UVU6-8GjI z{cSS_YI~T0^a=s#&ou(l%P&_d$+;^&K>F2Ur3CMr#gD8*H=&5zOwm#CcqJ zI5IbdhqF9@%3p6J53JSj@V7@E;Nhyn!{1PNI6C)$QFJ^m;se@dC<&eq%g5u@N4=2Bc0R z;rbnd#u>;17*9=&kkVe z0xM0_OZ^4K!XhvGL%AhvStL`Vew?&wn;N9GP8sIICFl5tCBpqYvFw^WW>8OMmO&~L2|T=+`H%Jkr9M<7n8ksIKU%{Z z*99J)u;bxf3NL5}uj@DdocQN=S>K`U6A6v)K~O^DnvU4t;SB8*6wqh~f@=as-$Sti zn)N=|>3AFT#Tp(=^bY_}TdUER&^XbT&@|H*8JNDrZi)Ukxm8SGV)q&#)W$!3^$T)* zxc_$sLEqkgQN4@)+Q0YgzksHRzT6sUrvJA60QA3mg8DCp*FFAXYk>MM2gud`M6Lho zSvArBMMwX~f6PT+An&f{FB{18<=z3cs@8wGjnPd1hxW+vXKeDu&x5|+^A- z%Qak=5y};nww=<36ehQW?)X(8lyE2AS+y`W+5H8gGp0&20=}y5JP>dz=N_u-F0Bbv zGh{`yNR?xR^0rE2?EA*A5gkM2z#3^oP8p#rsx~tMeqH5c1l+ zuM!Y@%}#{U`9A1luVCOC;Ne{=-3JdZs&pSbd`PAH;Ni;)lD-cfZqfI_!wprx z4<0_D($Q~kJlx4`))+FkSqLz5+imr9nA^tG^VmT(TO7jNCh%}4&rPV(%xzX0bDPCA zjE7rXz}%*(avt2aF&@66N;9`Dsy2IYTS%obw{@vI<~D(cuc|l?JiMsVoZP0bs^{y2 zhlkYj@xjBbbRRt2O83FT?<)GtU)^RCx>3Js{ z>WhZ)F+}NkX9Fy}O<>^$R>C^l@NGl->Pwg=-!3ft_NPB$lLg#ZI2oxy7NB39ZMY2! zXIX$UVR^b2EW8;k2vlw=EC_){SdhOg@OtMGEqYjmI+sWYZNnwT4T7FnxP^Or zV&NqGII(b>ZTP0KaGPy-oEXW#AP|l&fzrbWIe*Nqs-OR|$~HW%Y{O~(^TWa?9+Oxj*i2&K4%_h1DDYxZZQ22x z>fL`uw&Mco?LdUI0^dVH0d;C{5x(}@q67uM&cO}@6>d(k!lc;S$soV1>W_YxKF(y( zAAcN}o`l-n^aa$*^c5Bjq6&*eP+_r~qTf}sjYj`(e8@>(LjBZ#Sv}ED*yz{#FHOq# zQ2*svY3N(ypT6qo|BWfox3F+jq_AiNz5B2BBG4+e7lB~&czeeBQ z|Jw6U)cP+?%BlYnmT%_y4?QdS50L1xF0gRJfrZ~yXdW8c*c#kI`gTyxqACaZ57;q@ zNyKM1EZoY$=%kf{(Mb#MW^~fRG51hiAyqS@lgsLkvG7$7EPP1iU@Uw^-7z|OUA2qR zNvkf#!Yw?K(Mbz!WOVYbD#y5_z{2mUni&gUQ8}FGq>~@@6@~Tr*oH5wbRR6-+Dr4n z!mav!u<+}uejhB{O83FShg5kVEPP1q^`YMi3m;M=!#o#qaGRCJ+-8k3ggl!aKV#t` zRm%=avp8fY-3ctbOXXmmTU1Ynx$U;PV{ThlT;qX-8!9Jrn^lgv?aq0!MZT^|^Ztv~ z66Q85zXuk+qH1PtvwF_lW~jE?u<+X|&I1cyQE5(Y)7RDW@xj71^?ZD=@Ksgb2Mf3O z!v_n$t;#ch)!T+!={{Jv)qWo={I04W{H3t)XjQ^|i2>6*vFHzofe^M?9wqGk)>dp3~u zUn&3b%WucR*T02{tVt}~W*biS-x3R7%Q7zyfTyKxxD5*r%}Q3^$;o)hoIin;|MiZy(Z)4`IG+7AwDf7@0T35}gyU?M|Qg)vaRX3~WDq zepDWy-M9@a_m&kNKCJw^Zlt^pDOyX_kn&|x8f-{8?0y&T5i8fegDREW>xGqX;76r7 zyYUUN^0kjb(HAJWup7VW;Ya13u=1zTOM#WgrR{LjSoxZPve3*L`%!6(m48%W<=?gJ z#+$;*lOQdz@-YWie&-|7k4ha@u6bbP*N@sBvrWCxyNIPo)@;UZeCUXGEr|8`)y~mw z5i4K(R2^2nMkd?h+>D3aI5ZrwDIw=SY8c}D?1muF`}t8psmKrZf5Z5_co88cT zoBv+#T-xkj;oFOqV`w+jm8|*CUsIy_&&O&A^WW`ytlVKYzWN>uE8V8)x=lfet0E|2^F0(4ux%6Hj?}eZwbQ@xBw^J8itPYVX(3JY zC2Y*}MFZQ3l8wHAjfuX(bu~a-jeq*sm+bWK3=>vd>%WApx&Ok1+|_@%ZP4NBzs%bV zeMkSh=Y0Dwp=kg9OW1mo{a<(Vf4oZcYyFqo7OwGMvps+LtZ2{$Y|hWW`Th^R48W2w zY+tP0LhbONmTR6y7`4TM9SoIdMatn*}!OE{ItkMT7x3GNlTVdskYFs_AawoUlRyjSm z&3YcpZ9|F+b`l*gto*vl;juGg?XWObzN~UGw^_XBu`{zVR=%$4+7K(ZxQDrINcG%? zmEToyjFoHZj=60`-8s2Ux3Fj*tlWA&K3KWo;IDc=Dj}8bgOyuYJ7eYbcH=9Gf3}E~ zTkQjXDXjeVCb06=d&J7Gf3_i3-oTHFv=yWIz>@zUk~Hr}Wz+KCX6?rReq!YqV|wO1 z>dni4zFtIG|Mi#uCN_bU+wI0l9mFevUp-c?eCEP#d?Tz}?@p5Ymft{aZ)rPG8ys13 zt@6)(!qSIYSb7UfFC<%7dJ9W8jv9oUkI(1xaPg0itOgzsmX5Em2P1=QUGBotV_|q_ zX%b6s7g+j)7nZ(#NMHRvOq9)H>B)g+OOEL^jj{U4=I+O zXvPN%a3YeCw{CatjiuMHB<&4lcMgfkZ{HnK!7#T1d7e{&Q!4Ob6*#Q|AK}2JWQjEx z)w=A?>(TU!7Jja~(R6)GVd=NFW_Rwy(vR2qB=2(Aogc6BNq+kRI=$gu(S|-)^jqY3 zCDGU?d9zr0LilQdCW&(k^*+fJmd@w!1X}9hlbiu`C{8$2nW$mu$_I^GV+I7Z#QbFC@4AFJbA@mK#X0@@s-1nXu8x zTOC1xnzkb-ouqV7tn?pFvC1boAkM}={rcza^xNJ~SayxRs*{?ce3Bz5GBAB%;Z5|V zE;0|a1$-S%WzuYxo`qF>mpg2Hm^=$_LKI^w)nD3}RvF zOYDQiX?Jd6e2k^9D$LHq61b?+b`YLfR45`;c()IhHuk|Xp>i;~xvK6MOV`vLqnlP; zo5a$W*au5U)rI|LoAvUBcIP409}l~8Czjsk;e%yam3E?=&iB!Ubsn$tNp4~7jCHov z`6RcnM8?wFz61WTQ>fV;GuC;l&L_Erg)`Q1{8&eR|(k>C9iZ>U@%0dpXQsOLacUE&gEsTB`F&Zq?8Hb-d08 zi9@!yxwqZ< z=6vu$NWs=1eh3I#3Wy|ND45G2M)+Aoz-3Fp@H{qO@Jn<-Fc&`fUiX#%kg~ON*RO3x z|9w-*f2{O2Ja^s52OkFD1a}SnKB)r3D)5{NoZ zCt6s03u|v-?QQr9Ys+r^ZNl0I`oikdpDwI@U~r&mf9Km9*sa4=7LEX6%af2ivYWd{ zyL2sT$I>_ReNVYBPwl|+2C)@CEM|~c@k5MDmOu>25)a&;`2)m~yk!X%z3Tj#-&9yS zhQtp`$B;Z|St79q%X?^*#@&Bs-u8XZMN&#^mg)M~hoA8;$NU3$VmSs`dEEIn}sQ9~HI z+V?%Z?b74!&rW{Z!qSTeQGA2%d)j@tXbvphvP-Y?X}Q#?yBOfkBVkqLDX6(}M z#_MWEM5CX+qI9V(+NBqBFWY>$@b^6%>r!}WH)EHM?9J;^ygey&DK?IG`7`&oOY;7{ zvA++i*P76!UTYGCQg`10 zU5Y_q()IcKo({Y8uHU5j3n)&zwq4V2v=f%9{E2V3{DL4&`32FpYtmY~or21T2!hIo z2=Yqb+l@G1r0JoW4;9eA^EA;HP<;mxm3~LsRgTCA<;qjNxl;}&pFw|*J`s!IxhFbr1gMb4IEcbF)ep`Ws3}3D)hz@*d1D?0j z-GoZp!FQ_MPP(&zP6jT8!*UB&WU&0M%D+jl+^X5v@I0hy#v-^}Gl6IL(rPmso=>Qp z9`L1=6Ei5B&IIJRF>vYSuzXpKMgy>1FDk6k$L?HH>5OI8f4|eR>-DiaUsUB8%e+&E zm3OIh#xm=_-?^gF8OyByerHIf``Dc?t8|Q(vOBj1(gQ0u)Sxl5t*bj`wsrMznAt3B zc?Z>Os5E9aVRvpZ6ti4Whi$KudAFMSoxw#V`dZn$|pz$p;rhivCVTjGn>^0 z53Jmx$0(GHlf%a^oHCw0YV_GYjoNaY-g2Ct zJ^au(PT$ubiG`#1FEkh*2o8AJug9YU3T?+XLPJ?7QP<{O q17PktHhS{Nxs#*Ej)bZ+av=7|frthB2jjJIn7O$_)8;Q_g8v^93Auj& diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_gemm_run_0.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_gemm_run_0.pkl index 7379d89fc6699fcb50a65b3f63bee010e85c7cbc..acba35dd13055c7e102e247d9609d316bd0d6472 100644 GIT binary patch delta 805 zcmX@Rily@#lR^W_)PK7ez#w*tMh|~UW?p4_QDTZ-N@`hVa_W@Hg?4fiZMmeG9erJ< z^swY5=B7^R;VFtYHUc3-vv{K^o9EhX6`DLR&|D&ey@w?`wJ2|jH$x9=adJj#$`qiX z8GMsJ2TpLz5Z7#+QVZmA^za4;#(Vk&hXlumIr@Z7$&l`03J#o-A?wYUAqT|r-fS5P z-b@*a-YgkPz732S${<2za$txso8gqwqzv)N|AHjc@oQsXM$+bG1l9(3fKYItbG(mB zaJ;8WM7*POh-a89Ay=><8OLS}amD*!1wyW1LueCc0y;c5M5-R>a2EF+KLVO^2 z$WmUV<(FiQDLIB*JBbxmO&T|0WJSI9rrljz+wT1|ZMm^{8@Jx3b+z8zZti|`Z?$gI zrcdAB|C~94!2lpB*>c=OkcM;KuQ_vG-}65+Gtl=+4N*NObeQ=tMgEY_|E4>=)p?Vko+AbTFFL9sRpD@5F5)G9Jyw z6uoM9GBP+iG13)1kaJx#G@43`MGl0yE4@5Ao=6{nr@hewscf@emW-yeJ-V76K*-)T zvB~l1n$JW&8yQ{NnheXcKdgi>(21Q$w&fe&g#L3N7FI)U?LMv zh_rih$?*(N_`*B|;c&Cp-9!m*YF?X7BU0zbQ|Tm9{fTUs0?qj3P-ZOpiLActXk#$!O`)^7*v}K)w5Z>UTbL!gtYdK*0v7K7tmU=k7rRdYIGtS zyCr9<`8!&DnqT9#nSofDDAHI(tTN_|RT;Y4*z#CibF986w!#y0A)B$4mz%rTwOvZ= zx>%#gLF~r7!~_Cuz79M+uwQFsQq*Jd7hsPu7@yu8_oc&Oe9PIG>w2(-b zEy7Vx#T7J1Vs+?%426%Du9F&Oob z^Dz1yuNy6YvpYJLin@oJJ~uq^1ve^#`*Zo8ztH4$@AXl4)lB^b8sut4cclhnQR*nd z)a!)zMX1x)4wC(BbT1l5A63Lb3Eex z?$HUiClwtV_BP)=mP$X^EMt70x_~a^V)xGV8#iy?ynDU-<0w%iHcXmL>vymBjYQL) zW@OVnF`0JvOio1Cx{=3Ba>6}4;aMF?j-dJ4B3C~OnT9ZCxCj1N)$n|iB(k+!ev+GNv^X3hb z`TDiB_|Z!jYd1PtwN_uKqeQ#mN@WI8>Ctp1&EG&&XZuTt{xskV6@O(yEg@fv(Rtb5 zY|ATCiFaFsEC&IobX_;=4PlIQ#tu-OqIx8nEL6L@7aI6bU8fNbwbjOm2Q|1xiT$b` zEE=B2Qgyv<^1iOgNKlikHdn4X#(w=uR>RnDboIr)b;YVzu5r}-s6J>+fqR5pXqH8-yuZ4(q!`{xHT377o!ueoaGPkDL_mg&NTu zq%z~^A*j(rONh>SBsw~LAg|M>P+D4j?X4mdYvGbro8RXTl!yQ=T`CqyV5(vn28aA@ z^h8=(iWF|3EdZ-viNQ&nVFr~KRG_WB4MT-?(aq*1Ezll-mDW-Gxfcnly)78pZ-ryuBOY4!VBTZ&ZIARa#kP3^@WnO-|KIzF1Faachw5Q0~v zu*G=Vd4&=TbhP0mXfHBP;e$p#6yvk*f|?Jn7)G}is}sSXzoo+$43r4fB~nRa(exw+ zJ(Ed$X$Sowyg5dy@)L~0L62t44eZ^PUb8ovoQRIOF~=WG=a+4;m4>pxP+>P43bwVi zw)wPDf!6ZS+#+aOlND@jZ3$wcXL(;`?GCnuf~bnctDBglbcxhjCMg!r9RXkA;2CM{ z2p~dZmQhdu^=JvhBBf!=^uc^Rg6*MJpYaal7i?QcFo;2U@fOxH9$1;4OiW_UBkks6 zF+Re~&9*(dwuJF>bYf(5BDzKn=J;lz8r(;1XiPl zXtiMwvrDXJcw#SZ|FlD|8H;@O0NN04pXEbB8uDVnDtX|6z=Ix)t-bES^nNjPd2#z6 zIcUVo`$CK};fRltkuDx`f%E!uy(%$|(XQcR_fT}$oih-Fb@%?Y7|%Dm@A-rq_s{2z zy)zTE-oV}Y664K2BP7p$Og=Es_gRwmV7XqGnHU`&MLx|4H(=1u9VW(UY)@g+u-`Kg zeK?#-4u&Vzt~0EqA8v)xlg{}n7YZv=uGjJkDjjC^-NrcEh%Jo7_#Tdo?UgP(G(|x` z!)`Ja6D6x#TWW=KK1q6U`&oy64P{-f#)(IV0SF&{(Xf4g@o9;uNl)ud^iap5sGhbe zlbeZTb)$muNfpA(%bzDV9m5lobzF{VrJhmkXW2lEklw$o$ zyT1j);>Fummw5ElGs>rCX<9{9Toa!{+orcH*fbV~V`~}Xk5VA-s~wC#K_gDH?Ta6x z5NQ&;NQ)OgLJpVG*u|&G<&|yZ;?GbVA7r~0&+=4X(Z($PE4&PiCSc*FEBlryxv-e)m-0r{?HwJVP|z3f7nyKjrrM67dTWWb z;}wZWGBTc`fQ!#s+FOz6U|W%yb+Ek+AqV{>#?~t_vz{E=7d1Lk%Yt_<*MSh`Y=L6q z>-M$|zz1X2Y#(1+#=*6!Gg+ z#!OOyLG(9iAo$@EhpXdCKCAe!mTyDtK_6N?zL1bVXKxP$uuc*xUZM0_zIwAE0-m&f z*KTXOjg`0d_7=Y{RJeM>t+2f750+ZJ5wLQ6a$j^D6BM_NgTg-=Px5a`uvmVeWxba6 zHmtc8T@#7ljd6-)1ryE#e!?Hd_61AU$*MH~WLe@}!gAPTWC)WS>sp>A5gkD^{`T_Z zfr8*_#vTNF+ppJ%bs)a*n70hXeYR;>1}<33B&$$6A!|__1%cL&PK>5UBUrLB)(P{U zQwMrE^n#&+l~)AZ8Vt4p4wZOLb!bBd_lB{8JrWHE!a-ZOnjd>Aw89<`;dWrWfTAtF z5~SCqMllh_a-dk$wHHKd5KFotW8*{qBg1xxVCnI|4XN1ZaN6zhdz;;XHq4zTGYQQ6 zX;C&W=|MmEJnC0VVUfTp)X`RY;jwmjERqJO5-GQZ8fXi(0jw7c39!J6b?SCsOR&TW zZ#|a7uoQ*$uwqXTzB}5mu_aV6jtK>W8m*IRq2lj@?u_nFL?(uSAT3CPe6`S;t!2eg z3~ON1*4BYt99oI0wjwe#uUi}HSFlLa zgB9;q?CWSXmb?p^o|RN_jgX7iT?9a{Y_XT3HN=}W&yAxg~2FFV%k@1&p^JEBOY|$*XIcMq7E1 zpArF^%R6h5R1`)7VI~XdLcm7HkQZ>fpI#5GRG(49M-12_gOy7I`pQ7z6Ziy;Bu|np z&=6QfO9^~p8P|@}>N>VVQs{h(N%F=AI{f3|8z1nG-}pdF>v&60C>Js82#g2X3S0)p zbH2$ED`$F5GTaiR|A992fR?SLScv!0$VD&18acLydh{B(xsx`)znfkN3TYUl-FI`zA++vOQf&Pa1nI;|*nudE$-u+to%v#NAL@O&f}u zuH>dS!maM?O()G*x@x=JdO0dl1e-pUVQdcViQi07kVU+i!d{I(GIk!ovh_nl$z2KT z&!UVd6mplcojL63bMiQY@ZQ|TdJ2;7=GHtip4aW}GTIu0_NZuXZ{yBCMk%tCL2FUd z4%4)`%PnGl9BGWNgm2xse*b97DCI5(xqKpPq;4bZNn=gxFHwv<+D%Y(squRaN0eRD zv8k{oZWm96mjVz;UPnrc{!W{|mGq*(l)OZQ$bu)Lhg2f8T*@fj$RLYvfugIznkO@1 zro!;OK-_AHxGkCBX^Bu;^2@9bMrKTfia2?BA{(dth)7H~@h%FD{68t5z~fo*#Hz$( zb=$=7Bx;85TWQvtBneIgt)$QqeFUk)*PR^%~kMK)4BmDcX!( zOLxL{d=CM(WiLZG*z8REMtU*>GF`^Ty`R-|cuBKBy>;nS)Y!+uq3C|SirP%R&oh>v=RXp*lQ#h<}*>zcyG^0CPH zz)%Dm%fp`j-bmWxL3`)hwXLVi>n2t2&K+BKuHTNmT4K)?p|>XJgf~K%UNTzCqZi5Z z!0-Y8gFX>4fFjYlW-p&A1Djq^Tp;rGtX|GXOj&(1N2vluCP#>Op=dlic`;JTQ0c&h zqe&Bha;LobxC$$hDM5Ti$iIW0#IhYRi!HmO2vx4g`4W=}kH6W+L-j~tc9ZgRuzF1E z0enTYc+cN~siHqOEl^BRjo!)8iFCfGMyx_FTfco1?a_8b_eTfO$zmUCwrjm!25QNL zIh^CsbOgIMBY6Ft5?8`TchD2bLOi4keO#9sMHmlW+0lbuIhjc(GH8h&M2j8KD@G>_ zork9&11S-VYo#~^7XE@Q*uBCfn}XL}+623u z)k7A>qjxN!QAe|ZJ_~zYGKmB>&8&$Heui%wU!m%i*eH@XFglUdk8(%=E4e*E?9f0< z>UDo66U{_!iGc&?OVO%FCn5vb&$14!{+4asn|YhPq>Vf>cPP$qWPFI{>_Qpjpu)7Q1u-1` zuxMGwsF%m=*k|NfZb2Cg3;5$TV8=elZg?nJkz_J*!0rm0)IS?-#%7Y+-JgvnC$aTq zP%dSlRmT>0Ot(ca26vHU+D9GHl4QC>LGk|Atv80l)WZ%&CMG9F zv2=n2427|Ebv&!11ECGy4-8^Q1?|)yq^=U%d!WUBQS9$W2c4#k@ZdNQ8%cSvaE2;C z8_b0&ETp?U!*_jd0IRfL@EOHPGn4S=_J83n$ zR5CrIVzZIx$CB|cAtZh4AR1mI-6OKk|pk75id;>RiApQph@{6)&q2R@RB&rqth5(h+s@&LwdvB@Dclb{X{Q0wv_l9gvTU>J%< zbEDWIIkb}x$YyZ_V(Ox~S_JsHb5 z4WqnUCk2e3ydyIHBY)+js^nV6uCv{dJ^qt-i$5M@m0`a+|DG_kJVT_m`AZo75^IG+ zd6(XdY?}U5SS%$z8Lupto&O^}#LH!g>6G#I!JkoNGD%CiQ(Sa~D8WVUUp36-T|P4^ z{zI^HJLLm7z#gVlI7N%MzdkZH zIS?89rf+h1I2BE^Z}3sD^V>db<;BckFP37zBb*px=|uu?zTW`q7Y*RId;;I5q9e3N zAUr&}AERa(ym!9JbBv6Jr6tDHBNNV73e3rQ<8Sz|y=o+bF7TxtiRi?}ZuiFTnnvM$ z7*zu=eMx2C^NmOLhl%foAD&F^HKIodNGuJ@XRRRM-}w^B(S2Ohpdf+l+rHFD0=uqh zBH{Y3(4?^4e@qy$Z~6o)d)XSIZ|~6ee6;@@(%U=%5WXOOO?gm$(}!iwRCwqB5;8hS za;bxRe+Q4xjHc?lLSv2QWo&)l7adO0Dk+8H{w-fFW0=2^6!T<~W)M;K9iuL#QLNGQ zZ$mdSIB^edbfJ>nbI-`|{y@O*mz7SIn_|>hRLmkjdS6^+sJ%wJF=o6#)V!@I!y9D0 zM4g7;TgaSMG5NmB_l@rIx`DgxWSy&>-0Zohj-ER6Y2VlX=R*Up$oGp9MNN|Q&%Idx zSWBK+HFyjx9FC19!h@O7q3|%#;Al%^D4-1m@y8m*WvDw1KaJAw3aKc5e}uh!_N&N* zQ#J&}o&D=ERh)l=#QwDIarH5@gnxr(A6?b`f6U63;dyQOV^-V2T*{NLe@%6?eyzgU zN@|6D{7(+5D(rEuP+8?a9e({wWFBXKQUUWyhof_)`sC|n&@<`pR2THv4 zt#Xt(#WlsUU$~on17ZBrk6@wJi+hE-oIUb_316G{o2**BUT1-%#Uw^6#HOQLMlI?=UacUsGdVtiNVHoY@g&J`7ke)M37niHt86NyV7) zm(0^i=1WczsbM}^5#+9Ecmw0;SaK2%d}s)doB0M(DdrQyULTEkFismC&y2$mRhCAF zj0RNc$N=s!gez3yp5M7!25o#uly8f(dBPDyP-sd6tpZal$kLFsz={vm_6-i^d*G&g zbeS4VlF(W(4wBKq3?HuMS>ZJ_io+eLZbV>qPdjpCB?HrZa*!R&9}dv8+um+hjwdkp{5 zUivqT+4WFR*wS9Mqw;R|FVipY^uE(LQ~Hmm)_(!+nleIiD-zr*H1YUQ2E zM?QON5T}ifJ!CkfdP-T6&Zr=ecIdx=L|NQ z*wu=sFtg0+b#fhT`^KVcXmqY(aso-x%lTFb0{HR!@&4({dbe-f6(Nag>rSENyZXW# z`geUQyzx^TyEpFwyB)yd_+Fe`xT_Dl1@;dPVt1JCprf-$p(68L6}UyRU8-L8@JJ%t z<ms;rrKj_h;jSlwwLkJPH-|rwxnS2;C~;F}kVoQMx(eaoiATe6MjI!@WqXfdyjS zb(yiUvc+Nz7R2h>)weO+y=hl?+on&3*Kgdt?f%V|m0`zXQMVLC-5{ghu)e1&ym3d* zmhNpEcVABItYWd)TMJ^Z+q!eZ_VCvJZTY!&ka5umt%wJ$hzG4054w`a+~th(=X?!n z7Y}#A`sLi(9o!hccbm+sgY@}vy5qcZ6^-+dh;utrERj-(3r4z(N4kC6u3bh6UuvvQ zQN|0!N|ic_RtXE1*$uQTyvpcn{*l^H{BAY$HyHQj#{KUM`vyZV?@mMSHttQv{bt+! zCPRO{ahKsMhW?+yVQT*(5H?2X5(%TN4mHBm->t?Txq_TKWT4xzwDcw=9Rh|-Mu*#7p0b) z@)_7!-mSIk#EDPEgdngR7AJN}aS1~Z!r0H7Z)Xwo z`*{3Cj44Vx2o^$=nacCf7i8l=dupq_o)5J^=v9tAFL(1AZ2FxGK z?R@3y?)Aos{jR>Woqv_bPgUh(Vd=>1fdD37e;0zREOb&(EuDgHP zK`{$}iC8^@LhSH@0^5J_gJNYa$Of+Lt4HY9GCBp|`DF*Mw(Q_Wsn{eHV&(^{EzT>lcDxD3tfuH5A&ry$B?7M_*)hAGBmL^OvY+Is1Kfd-tc|3h`5& zwpD1;q+IQLbrA>3nO~;l{~MIXhTqi@2_E16Oi0cg~EI+>BPg$S+=khmqZFcn8R?&jTYDdxX7yb{i`AS091V?JyGctj`T4 z>N9rh&EEAKRta2MwBsXH9Ixb$0`Fo(O{AYUap&(~HEil_Pff+t+g>NGr#l)|+tMhpU?cg24qC%)($62HuSn&}RD61>JU- z!ekZ(v(V2$-&ivMiy>TbTw}0+-lM}{76!90@D7rzDE-+B-8f8PG7E!Q=x3pK*6e}B zFs>M`2?PwiM~A^I3}#{Ajo=!gAV#5>gdq%OVK57QqYi`aDOfy#>kzJKGK9ee7|g<8 z76#r&P%ckW5Qm{T3PTvo!eAD9j}C+GgRpoE*HgH3Dh(K1fWa&bW?|rc6hS;rK^%eR zX&Ay_76!A>&qD91ISGrem1+FHUBMsM?b!AOkFJx-^h5a5bLxa^6(duJzwwCj_Cs$UQI06VRq+37 z@L$hxd#GBS#{b(@{DJeYBL!bqPMv6+KBdf16J3i9sg%NM)`)=FIp_3=BdmwgafI#c zbihnGS3i9-n?CC5T*Y23Q+iyTt@YLO?rP-?*Pf5L={o9)e9T>~9(ApU`!`&wyZ+T_ zSL8>|H(cv~*!Tl7eS=aseMFf)dPMob5#{L1O&c6G{pP#S8hsJDLz{D+`kz{+oOue(E#>px=@a`Y*Y{3;Qr+_=%KZqt_0O5hU3G+= zII+V6(@6O*4V5$3p5@i+(a=>Z52Da+TU&qPM0mp#JJIAWZ|kggs_t?;;ahRl;jQa& z4ZP`eF?ao6_TVp@aj~kNx``d(hi;~VuU5y^`_IBp)9QL>nL2%<>9+D6uG`8{4Q|C% zheBpOO}J3lNMX-J)6kpsJ=LfttKEO;QmX1sgsa((*H*IW@8AKW82^Lfy1mME5M?^q z*0}=VzUiEHHEH;xB1Dx!L=D{eC_+^cs(Ox1*WIk*vATwqQ@De7xMzOgQjz6bt63O{ zo!)Wc1fn=~3YM#?T#vE)J6+D{9j;a7l+0C|u|M18V0Ei15D=baeO;4-Y6;AGNc&Ke zTXX)P>5jGaJ*X;muChOEbSY~ofU?#8zeK>Oy*t7v;A+-{$SAI99$C|)?EbTqfpXMt z9@p)gA7P!HC=^$j-(7bCH&$0Y@5GJbgIN!h)yfZ^Wj!d_hg|B~dQ?)!T4ybaYpokm zI@ZFTM>o9#MRtnEMmcL*Ro+AqmH(-`&Q)G@YZw9SxLdu4zgts|YV}10!8IO*@0IF~ z6Gs$R+1mOS8D5C#9Vql_QME2eao2|#m&!fMo_Gwj%k(bO_lXCrn#8rV4A|T8m2i}!%C)AGwnL2EoTko zOna0aKf<&r*8dpODwuYVX*$zxVhztSt&wSmSi@1)U(XtDW!f#Q;Yp@lgK(6FGDSP2 zG~CFvW2~WC)oRp+uQ9C_;VTWxnDzu~c$#TnW!m$s;V?o|8jdR3^Gd@Cre#?}1Jj;i zS`}+J&a`RP@GDGv3h^opD;13?4cDof11_Of8*XOW^-NQhh8K}GrJ+H&9-}x7uXn?> zOsiKKzQVLGAs9uwNo_cyXe(6hSw(A98fw*u`%Uc{rQrpp-N3XHtl?%w`!Z{ILD6ne z8g5mz=UBs66zyw@mPL9M?Mq6-VMV)E)m%!$&8nuuF%kk3Budq4743x5P^W0i;aJt4 zQnark>xx#dHe9c0(@I0NqE#Vvs`j*^In{q$JgR8ds9KfUP^W5-phy($MWw${ zZFok}ZczGFwc&9*6GeMW(QZ}ypMqA=D%AcdrQuf;ty0mRP_;6(-=Q`8Z@58yC97)3)&A$zpS+;Xds#qb zFG2c(p3dLM+JAxTEsxLPs|@FJj{{A&>5Cu>siP1F_#_*bg6hmao z{P86@e;hT&+c2=rANz9i$NAj+anwG4e16%%4`}{4Pm@M5f8?`D`~0!;dwl-r<^^Dz zKcWCi%^y(`w)rC*+vbmOW1Bz14Gm6sT-1MaMZx^B!pMvd^P_Y?lTYF!+zdA=&fkXR;Rdeg8K%pjk`bA9E#z zH~*IKg*wzXFMN>lhr%r>|3tWja{f<+TVCNrLQ4EqW?n9{#Q$zPr7su%d@lYzBTNcD z7yta5x%jbGf_&6aQvPho@%NHYbT0n`6u$_b&)P)ryl6%6l%@G|!UrjH`Jev)@_$Cp z&%d?&bNM$O5Q&6Y;_nj=i1I)Gqb27*@qb$LAN;rU{B!y5E1v&1a{2!wu5?nQmlZ$z zEhBzZkQwF7e?|PtFbE=IZKTe zr3)=7av)uxRi05UN*&hZW|Y%1MHpy9C+)C0XGszItQlpGbcyD6MhX5i#p9MGg?$Vt zH|YX$t~1J6sl$@T43cB~l+&L>Aye8Qb}*wPU0;}Lm3DN`+26qJZ@1f@mHx4q3jeKk zd;jN+@NXjfGn?)9rvIC{{rz_PPU#x1W;s5PLv*@3-5V{*f8jABMfHewn3K zq<%43qKnqAWAa&5Qar(DDIT+~^7@sKE{fEzdTGb&ml+nXU%k?f*RP9G$Lm+{s}xUu z{W5DhuV3}@x%2vUOzL?3IxEBC^~6_OtGj|KtFSmR?qJI50!pW;& zd!#+DUuXW%Zhtw>RVG^($EaMO*rMrGH+( zg16c2&H9U6omYZE*ej<)&=eW$Su<{^mCkOc`1(&NA9<+?81 zH1ooRf)-#I`~H&o!TkLwu1@mvp^pDz1>-*ycER{>VSXUSfAjxfAOD@6Lq|#V>&*W{ zRbxRuH9yRI0%ZR@UpvPuf!}+)%Mr{%;eO^tsUp?#D&cv=+{ddortJoTA$Iy}Cw1Ob0Ikxt?$Svu>_0)Rh~bUS{M z9>-Ok#N#O07d&em_)n9Ri&6;Ch}p6p3ve0%fyF=6kbrvqhS0m5O_%!nO{90B!3K2ynY} zEY>05M)X5iXq6nhU>%~_Scm8fqw_I1wwuaF>Y&6}7*&~%b1si#cfN;EEKjy?a%A`S${?)wx<;|g(Xg4SX%)=`>c^Lk2i6XeMa4<$J^XG5Lj zuCjwFk{kVbQI6eV=h#0rId;OzvHR}i9J`S?cDEBKa5=Cb1Mr02sl#9v1`=d@N`mZe z6|Nk(s&LhiR~XE}K$7f6BH7+@;L2*;YheLBk?af5FHEv$SHqP7P8|lNQFh?@Lzt|T1lry;2<0|hcj0QLfMDS5#I*(2y|{WPg)o?dfdtyln)S%b zRwQH_Tzic@kN)lkB;}8_%!G;U^dV-cy<{fZ6eu~>?feqAkYBGAka^ zU?1i18Pxm`XfxO{)2?L=0M%g80M*BtNvXfe2xJ2`10DmB15tx?gCc|b-oObvnDnnO zV%6Z*fY6}ifYy&9cyM3V4{i;B4TcUBeS=DT9Bde<8@Lxp9e^3A8?f2MK{cp0V0Nua zgu7g#;^#QeehPmHR1=+V;H(;?TvrI3gGPhP1Azl+17m}8gJItQ)Q#v=g4v+gRiMip zW&^|nc)L`vcw+Rx=V0_;>R{Nw;ZsDowMMo73C_Yn%2gF?8&q4v4grVS{}{1q4SXML z`w^udTL_*~`+>U4kY-hTQt1bhzZPs;{c*#W@pRP8A@!BRs+Lt>c~0#=u4>P#_cpws zVm4zS?b934GSYar5_dw{tO(LRyJV!TTwH~479AfeQ>8fed74Ym`2I3Ewtw}@OXJvb z!zHFB1sod_m{RMQa1Q!{AKbj(bxb~g#JI{C@UT9CDQVk-fs|{YG>|I89rUK~}KTnBgS^iPn z-=+DdjQ`zFApZFua07~8hDk}0934vGz>y!nfpjT;+WwCqml8kAacty2Il7em&o7pL z11T>h{`$Q9pLv$zmmC=x5ty8pfARbqZmB2?x0jax^S@~=|KMv&$-nja8+9c={!4iN zGqpzisD|K-?;w7~q=%d=GD*+i5ZqVluq-l@qs`@{Gf7+p>GnuUhnC*r(n-4D1&Svj zU2w8!5<*U{FVaE`RibKge@wcpAzi1`VXe(huFp$AwWWu9x;DIQGTQ8tqdFj(tq}FU7H?J@Q+eWBX}6h@LdBZDtwr+ICKsIj?Q!y{4W1G*f1jqJE|Ga(`a%__$m*CiD z_$4@YuMD5puXpS0oH`-xOK|KdX^%>DIUKu}h5&qh_wqP)Z+QX7<_qG69NY)Tu@@Ns zVIWhmkC8)+C3Eb)+&)Ij`0r9U_My@o`z^__`Q}9-2UlVr?Yj$=cD zMU1Xkr;?5THk!t(4BPhECX(X5%CN5rXs7XMEF5!iBfd77L)AfiIlZGO#BK?gqT4ux zbYY0C3W)7S?^_^@UDm0bCT_P>h<%Po6*UqkFHacT&#c1O12$o7nz@w{#_lU3jE%+* zLmm9kKw^|56nTpf+{y;A;cFp?{k+^vB_Z}p1&a`tPUtcr_Mw7K2+>~ZW~#oNFgDV- zo)-Wzw?US~MyV*(3BmDJ=|*%yD9BQs5FD54gy73qgvdc`xtXe9GkqT|LKqO+PvcRk zgA&sxbP`m*-nl%8y(bT1H}%;>OqBXOi2Y6h#O|e+0r4*ku@@J{K6!A z#OPPRRcrlJvs~| z#rBqwt8&;@PrV&jHiZe4m1WPu^g=B;X@ZkgxK`ux!awx0(0g!|$EAJh*(Yfqo8prw#+FSax?aywt3L;cd9?!Ucm_=o@P~;dKkHdvW!^ z5c)2YsUs11XX1jZzqUfp5oQIEV`dQXrc!cK0xR z$8b%+9C{~VY^M$rt3Y=5C}n99h7nvNxL_~`ePhiO{62x}5Uy!hK;NjtU=9XSAlv&S zxjc+i9EIT{xL_~~{RQZqH3#AKFCXlFl62Qzj08qH-C4?&>Qwfd)$^&UiK#-eQ|1rW!{kO1w@F)N! zpem5AuM+wpwsMdJDm6eSpe;ycR;rnL;3N_?B*DT8E=dS!N5R?8(LDE+Os+vCXfse!wX}E#N0WEg&imwTLSL zW`SM-A%XhcL6|wae=N?NQqA{D=?@RSU>RB%?gn&Kqp`*up_V_FetFB zDGsS_Rr(=x2_6Lo1vm!*N$@D(EYKw2FJP!cTo|)T(Lk%7R{DXzfWm-}0Kq_?z^}lj z>Qx9`HUL9`UjaRVgMnN@Y7&UarS=0!fkZ*f6ZGjAhg6Wn1RR5iCD>HC3V4-yLWKZk z|1IiqaIHhC_7x>VvYBdzgDCAe_08j)LiOLEz9cZzp@!#GrFRRjSUD6`zbJ}2SyZ}G znOTL97VY1ZbEP2EMVgRO|J-zW!ju;kD})*#VM^;FgxJJL9nD8tn3DIaAEm>1ia;og z$x4m?Fp{GR#mB~rj{o3z(M5=b#(xEe@yzESRN*4TR33zShlDBfAXNQYti95jf`dWo z_~ebSHyDc(1_ooQfzse13|}go|3l#eN(rn;DP$mPNGW9Sihopt7nJ;?^yY;55P#Da zN}q&Kh)FOO9={PL#XpyeU&u8&DN+f+pp*~{3or}+6pJt=q=l@t<`49YG`Om*k6fc#( zCI6P^KQAFve)%sw|MlyP`Ulo9qcr_D#Gexigp~y!;b8*#$c1oBYwY5ad!$_j*_liE zm882U7u+$%m{A5egff@wm*u(O5Go%FB&6>ea_N>jjs?z19mfJDyuz#*@o)%bx<@-~ zfmTJZz*!j<#{wom;8>tn=8i+Ci&Dpdz)23FO!piM%t#jtLMU^QpF@Iy0tjV71P%!X z3WO=oNl3&Y!9am9Wv5)E=aArJfiR_8+H**7vOt*9Odp2??-U4A&Pe|p61-C=OeyWr z4$LU;aA8WbWO$u3aUQR2epza~ww;%{BDKwY9u*X_SwkvG*TjV>ne@%;+#dOCd2KUm z4X)EX4Vj1+XgsrCpKQHo%1` z&q^0XAk;BwmtWfy^ZE1ob+SO1@|b)+yndZ55T-PrKd)c!6be&H|Ga*^Qy@%f)(>94 z-YF2KG{fihtEoVk(uAPAel-;cQ=0x!zs@U7T$u7)2pYMu-*R#WOzl7AoH_pc*kuV* z?)lqO79o}a5~j3{|8fwjVEi|qJ4^$^jZ&MCV1+B|iW4d>W;x~*fQpU_mVm0!3aGl55gn+5t%9o(7Yyd04`6nLQ;myZsz%)b zcU8D*KorV|K$JsKji_p22m=zfBtYfVVc;xAs4lD`0I?d*yzqKEt~+quO<~;wLm0dd zLe*#|RC8-68J&n>3$A-{fl$psUqGllM5w&maP2@6EQE^7Sr#Q!9(@y>-iNCj-ZtXe zOxftVR6^yn5~{H%g|Qc5j3eMdTriM?%2~4qv4BWf5fzuTybyycFe`avu#+S$$0lKD zAygNjFG{F9M5wHimJ1RpftTD%L)A3_)lUln6(&VQr*_w~zLfx1E1bJQtX?HXbuVU38I=zBJPt@z$yrkkI}Sl9 zu&HG%!>JS~)0+-10R`+cMa+uM4F$FWfa1`p|Df`UL&-qy2d8*C;THlr{Nzq>CrC^| zfC?-NOsNJig&*n(?gI>USm_6S0(yeb6sQ+q7i6ev)&4JW&;)#w1)&1;;bK%kOptj3 zVgld6fuKN4KtZL6ctD@(Bo?d4-!jcUZ zuZt6^^GhRCr%Mqke>VbuAB5^$^o&JJ%1)?^9ZEECD_EiEE3!g?fnY7|rYOBb$w^kW z6$8nB2dF#W#pK}rE@oTL=6 zqK1?LRxyUL=fXe5EsuM2AP%KzJH>B=Ns$UzL-88`isGOD{apM4*6{cZnnLj#G-nC% zv+r5+KkzUS8WB9@-+*;IegVtu`On2KX&xnT>G?mM7ymoInxB7Jrj!&TOv>J(`8R0V z67qlS-{$3iuH^GKgtvJ4|6Z>A-|zE3`x#&Ys(%(jRqsKiAv4YlOzwh&f=S7haLx=& zJ1)3luD@51OOpsxl8#A#oE8)#RHh5g4BT>MzX&rh?J!rKQOtFI&J6O!qy(WdQ-XGQ zVM29U#>$z2U#7VTGce1jB%wMhedjZS{P$5YSMW;^swqhTN)W0&(!K4MibGtHb(nK|c#$_#6Xgvu0O zEJCPeWVz+nHpP7YB?wigJWjI&p}HuaFRx!G3ka3zzXYK&kDDz)sHSB2B?uLh=_^5~ zOncNXNvK!{wO__3mKGsYZgc#1d4%fpXJ57xs`)xZF2tnfm$pL*Lprha@v|ELU5Op) zhvW|Rw~B~KS=Rsfs-PJE<<|c$ZHE%dZwqf%QGQ#w0w#-&|KLW9|KO|C`0vMZ{jcQs zkFWog5R%9E@9ZC{Ht6fgbqTL7{kpnks=S)pYqOM z6Epn}rQCc#1grBZ!FmSIx$Ejtq3P7Rp~-M2F%(Hh!>M#6ZGxkJ<=?@~9Tlpz2Yuo| zP#eH%Ygytsp$-AC_E-Rx`BIld#OgM@!b=sgO56>{?*m{>ecKAKF6Dcd#sUTgakM5u z6X2zt6AH%|3<<{=2jK@341w_DK`yez0?uCg838QbYhCL1E|IwnJabTNY`zvVIsaLTrF0keU=Cp=Xe?_%Sjxp_jJ>G?yUg!h<^!z4?_C~S zc9#t{E7su##DkF4mbd) zk^)w<&^PKZkmRbnoPw$@Bv-QlR~`aa-K*ezH7;-}DO%+uaOKo(+!k;hKgi4CbKs=rFL# zS9uRp7)N3I2rQn&1%p}WxqQ{wWAOeIE**vkkpbu%br{HFLP@@=`v?W`Gz=fb1p_Hr zfH$)fh+zE z#+{5Bl~p{5wH zl)72p(z63En3{QGraWx!(75VxXl?-8y7MFNw8=vNj0W*xW{iDdCl(g}10o&{+% zpfk`b&@rGch*$y2-N5CmfW`o`K(8Q5g%kZisere@vjDsxIE7Ex0aQV<3ZP0H4hSJF zAg5}MqQI@5AQVMk$phrV=kFAi1g#*0^_bcZhDFj^{JVBfDj6^=NL)e03K$Cv>oA@2 zhqL}N$JOJUSm8^C#yNje>v`SE30JxAHw*|Tpqew=+kvdFLk(!j^C7-QUJ?5 z=jC$G`8)H91z_>-)$w7Dan8%q7BDbcS#*mnIw;Y_K_U7$___k$tFvseeShctJueq9 zzAqOrE@AvvWB~)&72`h?zy~t^LxA0=JPQJ>rHudh_s&ZoEXDu(pa28|5YPD{->bX$ z7AC%A?*Q#k&NLIi5vaxq1LJdWHKa7o&QqWisLDY~fl8cE8i(7}kWzjxt(@m?fj?Fe zE5sk%P4NpKQF&mw|M+Br>*%q=zr8&IKVP%0k3U->6_!y zV^YU!Td&k{fMwQF4zR>If2Z#-a)7PI7GW=5+n7uPuWe?|i_|tV=e)L=E_rPe->W+> z!{T+$FLk`OotC=%+NPYB<<9|DvOvUYM%wfGRrtL+^ZD}nRpgvM>7UoH!td2xl=i%S z6@IVItbe?I6@IVItUtVd6@IVIw8tZm=lo5D=y@y+VC|tH0AHWEymS7}{m2TilD6;F zeKY~qO9ktH7Jy|L|H*UyFp^t^Qo?z1rN0vdD|(N)n`Wt?1(f37tK-yafeq|M#(x<0 zZRFuWA;y0gz-^W$Tcl#ERNN~S+oa+?u7EGw26hBk3Scc|{5MbQevgbdeA*1@W zlym+@%#k9AQI!<2!d`DNpzKsS$B?Im0M;1}uwF0artg_jHoUCZ2L!nGTm@WL0ha-^ zeog^bPDHD{rN!6U%4M-Q;mV<}kl$~huSJg$EH4z&G8q>;Z!`}WKjEjmUxsyTT>-tmy(QN$c*j-1u=vV`eG_gGAuQz5)`!7F^kFEZMf)(g5q%hZE!u~{ z@eW?JI0s8IEGdRna7>K#TX%d(&C0Oq-zUR5`ySFE$74mlb=R9Gg!OL0F@g1;hW|y^ z_!bw!5)A88D2*$_u=q!2#X_F^)*YS6Ypl|3Gj^GOs0?e0hG3=;)*aYqO?aRJ6u=2Y zup9^i2E?%DpuYgUSHV?DC}J6SNgdb$45SR!1?UT;ujW)3>JGT7!c_xbWw^kw#(=QA zwJ?KT3SW6h_{!UaYZb25xV*50zEOw41sGUmug31C`0jz>?YQnhDq&FiLwEX}ph_?B zkKK9T-iOd{!*v%0(v0-1fgube$8zd080&M$6af{Q0`9l|wD zB>@Ac4il>cR`-(>#$h-)3X4Z@!GIW6=}+A`^~d1kDO@_dJ&Ncbr>q=-=4lwhzzVS% zN&3qBbzI-VH497V`KRvYU|>BS5Y4gp%w6Ap3&5(+16XJK@D7*&tBB;)51p*9A@69w ze^2xZAE`PHXjM+!Dg$Wc;Nn-HRF&+_3PY$$6OvYt!vc7MC{`7HtqLEh0L5{A#Z-AnJu>i0bBG6up0ECsqmqF8`gKx!Ojfk5H> zL0~EHDF}N3OF?o9&I!5E-?)Ng z)^kcfgsngYx!@H6tXtH6z%Nd$@I@{_1wCusFlAjQ{$u@Qh=k2|mg5XfgZ{ zn?Vxdzkq6Ppo0g+r_YyW8YtdVSO9Rvw~rjgJJoJp~z$E$|a(zVNe=JzbQ}( z$s-3T4cdq;v7MW-@rLeGBtP)nHt}fT0;E2dHH|$Wr|-QLMKIP zlqr>eF8|i}4c?a@|0U!eW@{6_Io{02_5KmT0*&2N>#ZSMISIFYii^wMPK zvLV$%*uaDWl?+Sw=HfmF2E`ziTQ0L#arY7p6u}0a(hiehi@XfS2IumCf%q5|lkPb- zxF~fT8_dX9IX0M)b{tZf7@!EGGC^KRNOe|*oR1CiDxK0R853s+h2Nqw?MpDKg!Es6 zQMn}}D8Z;sOaCPpmC3+LFsh5ve+fo4BkhrG$*7LWg5iwHPixy$0Xd`Uly-RS?O#4K zA5J;h^-7l&q%*lvCFw5pTU37e*m-3$7E-j2|DzD1*N*BDcnbn?Gwkerr zUfIq{m%Or>IWJP#7H3pundVnE<-9Ea5{$~MA0-%-`TR>Ts+qj{RmiCFg{%r0m05pz z{VMzxm6`q$jLNLPs6>)c9lHcZbxe-$-n~3Vwdc7dGOEiQ-(jRtYJ7*$Na;OFe2{5d z-zz=7TbNND`mh*PU+!B}FY!Z`+ z{AZ6pRUv}nAH{m#=M7$!){0++SwG(}%Yv*rT75;oiPh1n`TT-%0cMdDO7kPxFJwKj zia33`H~zoy2cOrfY0=vK?Hz6HzJMlN$NxXrz@ffkFcVHhli}3p_)s{KOhyi5yPSFz z*A7l*CeqoivwAhxlC!}~woBEkxx=twU!z|$9@!sGjZLOg;nZLGalYQIGF8Hbl1r6 z{%|^y9EqmG`=ZIz=;TE9zSG}PSjLury=wjTO>1{;+P;3RKP$_j)+h(fr=jFH9OMNN ze-r~n1aQV)fH9|D&c9xljnf$yT`W#VZBV(D52X(zqS?3?dR0& z&Vvkvg6+XTFyw1D%Di|4@#he?2wjn($Db!N3OfD*RKTtCtjXOS|6} zY;Th%Sr#i6$8a)=?DJBQ>9%DbWgKb`_)rE51Qh?anSGhYBEfV7e689NgOSguAQ&wa N3iw)DWVWKd{~uRSWxfCa diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_gemm_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_gemm_run_1.pkl index d7e1cbd2a4e1fa882d6433f52138697e71a56f84..c1c3985e950ce279e97c50105658a2b9a4b76d75 100644 GIT binary patch delta 810 zcmaE`ily@#6JG<%)PK7ez#w*t#$-bi;mLJ&vJ-8&q?sLkU8eM~xkgigtj?qLcJoRT5y&6ptv#PZ&384BJ^8H(O48A`qlj2X%x zLS^#1An_=yrUVl)cYC6#V0M{TlXa7*Y5Z551DWypn;*%SKWz`8dg$3r6 z`@!N&KrO$66$ofyV1sFi4H2(Lay`F$kdtq`d#Go;qjQL7m@6n0SU_UP;lKzC2S#8x zFoME?Jww@>BSWP}1n2>r(XqLs|01 literal 71008 zcmeIb33wdWec)MD0LUf*5@1u1A<2?j5G6M$sM!smanP0nUZN;~2S`Cv`bZ$?qQn+SQB8%uXvDE&$A)Fui9Jqc$7bZrH=bFJMQ9@16CdI3X0rZv`EBT# zB;(0uR{Q(Es#jfo0U&kwVv~iccOUiY)&ITU`@fnhKlSG;O+$VUW`gOO)ZobB1LN_& z&_H5nEHOTjNw3;HKH5Jxl-Ls=Igr@;aAG8tc|NoD(acBizN9bl@L+EulP>SsyLAtz zk@#>TGhn8d-!~rb9UM8(nK+#FUDY=@F*X!G98bp)j1DK(e<1$B_|W>^(Z0lboiooxVq{`;JT!JVv*Rl()4tdv2eH`NhCsb!yngM5Oo~K(@0m!Alk4|pI!z42qkWS@ ziT7sG%ic3QI5D9dQkf?+ll#-wL@JYB+L!1Z9F7lhwZ*x5K9jDAHnl|B+FP2NTN_(K zjZM+a6B(+;N{nO%ZpykwBhAgB_U2aQI=O$K%&yXbB?C(bd;`lJ(~5zW1J!E>YIY8+ z3J&-w=D_MJE#1Jio#wzh2I}k*4BU{LnU>a2q{+_AKz)u=Q)62w($=;>KGVx%vEg`X zcycHf%e*aFZfcH(8e8pra$8zURNb~_ zb#HR3p(WZL2}Rm%r<`ioPt4ULlf(NH+FC5$*5;;YsJXdF z3zm4_nTU@M9S)?%<0BLOqvOMYl%07x!Gg@UHb>h+%`L^+Tw6;x9BK(S6}CA^WJUkr zNPKAUgNZuoKK^aM?I(A+9%Yp1rFTI?>Qph~L76LhD&iNOBJ{`~r&18NC{ zqjt#LNa=K

S)Fo=38}Ox?lFh1qOxX$*(j!o>1?=jvy4FUT8wSmOYL?X~%|Dpboj|6BAfe+;d{z!d8;9y8z zSZn2tL6=-cO=-#nbW<=%f_)?b*0_>yEAW zZ4A7J8YQzaF>Kj*-^S2^L@KzJVg^P=Q-PhMBZ&^N#8#{ITF+53a+!C=#aL^7BskyQaGV_o%W5!Cdc~XsYHzGO-NJMk)}|jk$3z^Dsd=}sdIV|D z`A=bcYjcYmUOk1?j&u+Whnpjz#^O)5=aE^;vu#JL#Q+*z=Z)#w7_!dLVQEv+j_gj0 zW_QPYqK~w7k$I%Ic4Qt}aFsdm`{~A_@dZ6iw;>?G@2Yn)RIi)OTh|;1{_w4Ah68`p z**)-ux4wC`x3sr~+M=F*)7u8xLdD+u(h>WRrBZ{jvEkg;MYNGdMQ2;2xkj5J?d>>~ zc!REzH{D=jB1Kp5$oOC?k=I(IO^q#$p~l7{@6c#866Qf@Eb)4)q`M(fyX(k#xCj_+ z4Yfsz$I-)IYNDWRbIWM|sw3SRp zT1zBAUpFxjA7i-U6@>{C35TL>jfDbAOG_ge=4D+p!%2ftR8CYaEp6>Y)m}uZZfR|7 z6LF<|u~D_QHql!|ib!QGZDG2vwkA8n`7ht}O4Uu2s647JN?R&+cMzW&n{X;2B~;1l zq0}9bkxg^4{**|Y5EPtoMnPVy$9aqPCStv#LqqYgiM%`08g6T94>gC2h%~K{a7!eF z$WydB(>F~V9^OAXG}wDbUIKUYCFsp^8$b&pS8J%f-M%$B4XQQLijdjnI2B}n#n9mJ zU`m9s!Z!gy8(TwB=Z5FT+8A!7Qi=^%l<27BF*W!uX#A~>;Wk9aV$GtpG1Amb3o8`X z2P9U%olB8OQ=>>eqvQFt-N=K(_^L>YYh`4MY}sC-#Z?_ljE^LS0u1#BQ@M59+T4cd zi9D|EQ*JG^@HB)X;S!OqP_YHqWNtNN83X!j?`Wda}7OW%+77B$Vh@tyd_#(<{-_Mc4UU)U9CqlSe+Uj8)g0@ z6;M){k~OmDgl9$Psin+jhO+|Y7*f4Uh6XL$UM1LS_VT>s3@Zi1T-|CPtFVUhABZW#as zM-q?3CdPYXBOC5=oOKxGMD1zsYQ~Gonyg5#P%Tt@%%tygByK0Q7?}w@5+6FKeFSB& zLPY%mv9*UxUcY&{wY{t1(kFMEEl*z~Mb|TP_N_w{QG%EV@P~>IQ0$iUY`R`<>QI8_ znb%~pW0FkTX<%x2MVvY13uXsq+%et8wREe^(k^*F0UT-4xd+r|8?#P4jk*h4JL|jLF9&k~h>3CZ80^)9w3` zMR+ah;)KODI}m zswur{EIu9|o{)Hpk6cKSj({79=n`oO(`^jsz?JY9kUa)Chv)35_F}v;z5*4pVzQZyM z7g-=fe@^7Uw;w%R6)V-C;;o&>^>nbzdbAboU}ch_EnK`krB|p4oK11K(#E~_c_u{K z8^h72Ryx;0OlLc_%B)6lLXh^`*m8LE;lwb*lRz#JmGIG)CZ-sp`R^KW;;~@fxkTTZ zyaO>sO=PYVR-yd2M=4<5OG_MDnKEW>yTl~ZilNbXA0wQC**rTB?UcODuC(m)X~zm? zH*aAYtw7-!bXnJUb?Pp$2MoI<=z znl0f7@AM|8o(m>|s_8;{55}0sK9Gn-VvV`+wuIZG1TKK85^prZ6j5u*xzdS2hQZ7T z+EcrE6+xz$Zfq);%2)9=G110kYVq2=eqvy-KNSdu8`cIQEsUH;C&w7~%e-t})>~Sb z%WWsIkOGz9n`iJ5vfR&x|yjWX>qqjE}X>3doHZsW@Ek0+Su3<8ai6~~n za_*pdd?inoo9@D5GgTv>`NRPi;wg{BD0hG#K%lF&k*FWNQ8|oU>FS) z3tGggcwb*&Xq4H6A_AAh3}paY6aXG^YHDhuc1q22uNX&Ad01$6;q7W~YGyu|I?U!* zR}J%WZ&R$ULzwM4~1S2lJm5+M_Z#rS!#{J>UHY^b}hvFCIUfO z-nBNbdYLMxq|WlF+_+oXT9K4XO+Qz%BrVa)gf@fsyf~S-&XYEsDgbUZQRk)b*0pN# zGQji?9I7vIDA1QWJeEVxsDyAU{EYrO{{?|l1SvP_$|3~~VzsnfC+6J7tCo`45wtZy z<_e#RNTj)yE>HJN1#z!ri5fFZqZ9OQ@U)x?i7-zR2LBYPVomnp+G;kGB>v%lqS0N0^mor^N*D|;Zi)c#+}?*KQ1+q+e1ibvZX_Nn{b2` zJuG!f&Nx@tVu)F}TWExZ9ei%gk@ z4~gWGE@Or~^E^v1tMoEYSqOvuE?l3_bgoouC|A0U>IuxGE4JLnlPCf2eVt#YkuHCD zbg(b8v(wy}%IwJ`Ys(BHn5^S(XNyFU3~=eStu1Q1TG`%!-?VRcYTQkwb46z$T|py? zv;5QSXPM~ErBCU(k|9a}bMl5YxVB=6>;bBVW1ZpPdTC-gjNTP4!=6hh@CE_unV zvu>>1lnO7B66DEDQp#f|;`&K;N^BDTDSZc?$k=ymxjk0hKGHu*i*ptd`_kns_@LD; zk8?j`_6uxny1H+4Bo&*SNW_#xGLd;YT_r091U4Z|$+GMfau#T(RuZs4dl~Vt>{-^2 z?3~=M^W`ky`$dh07c~e>H!q%vI?GuU5g%cE?k}ApKov z@AjRY4FS2@y=T|9JsWqhX3JiCCE%?dlO0R|W`kI5P|4fb3u5^|_`#5!FgQ_I-P#5< zQU;p7rLe#**YoKLB`{^uTNOyPLFD)WpaF41+$r+6fDx4U1@u%{ts!v@pAvt05 zUOVQOd6RgSC1zN>J4md0LM}8mIu;DC4XIdz8kfD({MDa3M)qecj_lu5u>3^{1#PU= zVLiMS)=XDvWB2IbNGexpV_-?TY~zkCvRb=5aVXKtOP2Mlna+*rGMJ@O>+lUHQgK#s z#(DgG8dl6KN*39!H-GpBK-tFuCwqx|EiD-;bOyUQzVZJT%ZAgZPM2K!lSbyM05 ztEkcy$e>7fyhF0eFX`np%?*3E?AW*=Obhi$t(CH9Rn}6dwkwNacNjHj#RSvsi|Ewp zHh5>T(q(dNj3qPc2YNrCmXEKp(o0z;GIn@yB$Ga_7y(Xtae}=*gPyb@@PWz1WJ0|* z?S&Osh6Wx{L1T^^tDh4Sy(|#Wnoq%gwbE8H{UOz#Ngxh1reqyC(C{wSYJ^7iALd<3 zubvo*?`Jj3UG(~QZ|~Zw`s_t5!u!u)Xn-(UC!>PRKxCj3unY znKUmDS@``xFY75}#eT26Dp}lvkyWFt^XCPfk_GW#927_k$h#=%Z%ek5|5IMuH>4|7W+Zi~+MolNRSlQQW^mA6YGl91 z#*-(BDZQgWL@Enz$rT)&HB95Q4_=I-dO+C_+E^>!5V)H~B=%|)N2Exe zl8Aqugo)%Qr9^MKB$1qvXnzCb2Q`1fj6ZV@5f=2?uSM0vdt8`MYcHG-zO2r;{RZib zlm9@q?pS3d|52*DmpUuJf%Sl>uD5o7kVA;5^|fcH@g+Yx-& ze(V_$@E|*n?wHfbBlX*_o#0qVFJDxer}UDt$Zsq}?C9Z)h2oPe{Bt~u`p{itI;fb` zQaedOuk=$&N?52)NsT2A+dawy2yO$Tee{!HhXS@zhVBJ>gW8%N_T1x#uy*s0t+4dEv&KaAO;6kNT{GImL9tSvV{v zJ)N$ump%U_)u-xZk@?i=b`#F2GM%MGoh2SfBeM|=zv8`6R`=IzWu`kL!#D>QG^QPqst=6#zE)r_-s}D-#QLfOle_~zziTWpvbig;`75qf>|x|;_|i$s_(o_r zekdk{H}=Tr_(3OmVj#03n9q7qz`qWSjSoJoR1FF;XnZj=abS#9*D{pwf5kRTu-t#h zwlY2!vSHbmy)lLk_I)EH>(9~TRvAG1LI}2$!TjS86P**WzQbf>uvauxdk?&wheJ_;g25+}p2? zt?B!FGW+xDC#=Wm3ICpMpI6oW-!saValh{Vp0T3c@S9J4>r+;F^QV^hn&sLw-t+xw z%fg-W63bZn&rg5rq}V5o?=QiAX?c0aYU`n;H+{a2-!Z1XX&Cqa$TU7|n#L2~ zGK@VxDl81 zlGc=w`N$o{XNlvVe~5!sW1p8;D~(4#TK?3xj8BhVif2U=$Mf%809s(WP^x<62|7IsIR3oVv7Vvz9<|KIDQ5~R&>ccL-& z>3~jr?g?Rm`6S+d0jI+buoVXX4xsKC97v_cCN`{Jk6~bPe~9^-^?TxdiO~`Do@9K* z_5kaLh8`Zilea~Zxu^gyjXPQ46N&^Hqs`6DfkqZYG{#x+(-KZJH6@x_dz<^4dV5>q zef{lmcHLl=bbniaBC^)#0_G&fH^>}lpsQ<(lZ`-d{m>}N(c`hPcxpi2vat!a1Q_5Q zJU%u&A+u&NXW|ndy$9nc{T;wP5flB_156V3kH&ao#ux*~Mkn;_=frwuCv$SH4408z zw*re_*w5I{Ue9;ESy>d?+wYB^y{_-44%g}Ucf?<9ZS*_#b~>)L@nOe4Yhz!1yoCM# zQo_DQBDRo%*V*Cq-|ur#?0x+=*cW?WOJ`o}eQku;y(3|S5Lgg(jL?CR$lt1&_{C#Zajc!PCfXnd3#-q*+NHbVO+CXA3R_J%~_K~5VSo*c%KCM(jR zAp%t@zMt~|@!D7RIrru65!#eUl+c)bw{>IMIjAU%0 zg~0rAjULD7RCIr(%@S7nKpkH9i~79J@hIJ!q@~Agy1%T(?jK)&eN>0n{h~exo$%US zhj*-v>$s4(wY%;|^*KMh&c8m(^Dc6x4zK%deb)UpUequDd43nR>QQvuQlaXJq{9H2h(!4J6_l0avecl2Bcq-u&y`VA2=PA9xv^Q2U%i*GVyDB8IpOvA z{e%vPANl1aZ|uM>C%hi-JL%{9&o}QW5~rl2)0b=SFF*VN9K|l|^~ALf-<}sh8}(W0 zN8fvI6fZe1_BO|(v{NaPu0P#=4`HY4&Kv%xRkrCFuO-(yU2Q$hMX~VXD{LRv^WHH zf>$q~Vq}>~SL=DS9h-~Jp~)VL<0E7#U7;3B2*OX^&-0gFwtL6sy>Zc~_O2CLxwkvE zsb}x|W1HWTsF~?3yZ>cF5wi^hz=RcX`-dLxg>okKc8dPXU5~|_kTO?reoIU8y)j?&iR@J z&ee|lD(C$7PQ0~_d8KpK{{E}uuFaK>yFUNiabN1#Uz>N<}7kk#>^3T7i;TKzO?ayiMS5Lp78c6pb zee%23Nx#(jKF2-joVEEojydI=w>szic(nig@Y+0?7q7NoD4ce8f?x4T5A#Z$o!6#2 z6t`!8}5#0s*+YTO$3wx1o)Iz~PR3}i-$)r!b`MMJYIuklG zj_w577ODA_C+xMpamlFKy-}9yCrhpPr5Hgp8D#C&hP&)7pNgqLupE{hJGH`uqYB|H z=*@NWsQSGp`AH;<^mLA@0E=wZj{lNkcFX_q_#>iZ^vR=b7~#e-G*iQfA{>Q=vVMuvg_f?} zz+9MPo4ZYtcvQ*g=E)!ZkVvYrm=d9Us{60F6d-9B+ox@5`p~lJWy+#^Ic}9IYXi%s zZ8(FYEt6woFW2S59tCh=&#K|aUg3oe%YXTCu{s-Nt$kTT?AIEy55VV_O}|539G#W6U2t2!T9HoZn)+@>#X*B8PY4TCs?+Gef3kh(RD z?<||XL#5I!71NNtywgs5CAJ~^@_sHmB#iNmWz!qg1%=zBE-3eAZL>vR*dssu+^uc4 z>5F^R1);WUn|t+zy+ndgUD{@c_PkTu>{1uqD`;2YvvR%Gim3CUWmcnO(WK7Z4;dd| zF&0gd)+hC&O=2-~EB2~=E&I`xw6F6&BbH^;uN&7$Y_HZ5N&L3%__}V4V$<<|(9T@< zlkYBY@4*-8OPkJX(5_8|)&2E_JUl=>Z2ERulJsvOk<-hp&ENia|M_54J$akHJ@$dS z|C@7(pQ$6_%!U16Zc>-ZE|9g8OJygsU)}Gb6jjC}JP++W1L8Ip#L90xYJBVLb{hC6 z9>q%9sol=H16BxVc9ja--GyrhR>sop(vSj9So`6ayA#)?YJs>smL}fb zCy%9E{9GPN`I%v9z8^5&UoH>s+-*jcG4+#R)sm^7H264HPW@yS3tt+GSuAF;XfUUK z(zR54EyGZaB^I+-%wn!fW6@>dP|mTOqe_BfaS@AIEM~E2D4Y7pPzAoLDlx3U5{p?Z zW-*_`Tvt_(!y1lt91S>N4yLh~#bOqVhFdvq6Q6gG;GI}vF^h#SjRhYS!Zv`H+c|<9 zAzU&0(pb!5F^fgREgbKXIM#Bp9!o4{v6#hNm&T&21Bbgg?%~)emRMZGVit>8EQU4` zXVo@*ZpX(iEU}oyJc~J)#-eKz4qG_x6`x(WU~v(PSuAETH*6-Ft&)pQ3_GyIVit>8 z%(Iw-Rr_)1<4AH0NiMLsh{Y@xvsg6rawH`FgTytAB^I+-%woQX*%$f7b?-Kes(twD z=NRA^p{g+l(^$-6F^fe*oa2DxWsr+eEU}msi!>I&G&T)W98Yo_;W&mX=8KqTG0$Qi zdX(cS%KbFQaqKY9Vm^o2m&T%Nnt+dUJj0QesIfSQ#Vi)HSTsB)C3!+}aTL?D*kUt_ z#VqDo%)zQp6VGRG`~v4$GJyFa=2^_Mm>d3pOni3gC*9kO$mt_2R^gel0tRoA-+5?H zfR!_q#?D*$^G*MZd8~fM^xwkjS>xE5k%wNauQ!f~(F*&H8V^16&{5;4(I|GmZyswd zH#j|1X&&SMttNl`x3cVufc{UL#~$L(&KJ-4*BD}39Em`f>g<2rBO%3#(CedGe?b`l8>Xto{n-bxc!^TiF{M}cl}$+{rfAA z*Z2OA7`fiECh+xFu<`d+zT)4!Cj9lk{t{_!F6X-A>qOC4dEDPu=}%O?@^|(8;o?(Y z{_@|+Z|+`tqu-%)^f~A4Qkwo<^s!RTdybv?xs>{zWAC%}y+*wsHE#Z~;SVf7YMeQ< zD~N5p;QkatzoOaeqD{P%sO`F{!bX!jFyg7zG}|INDJKa7_nN!YMnQZRo`fFTm5}2CEniK0yE$BTNL@`N+U*Q zkL@~hh9q8m5yv&l{f`?j?((mxAd4*>exJ2wE9P2YJb^Z)%nd0zGJ)mW$VI!OT@I^T`?-S(x@jfN$Z$Oto|{hzP^H% zt5zSEdoZA9N;MC**`V@7m|5uG-oPa4rFF3jjnM)Yx` z_PEiLHljz2=yId>DWWr@$Bf$NjcAQg`3 zmRY;hiY_s0%gpGbMzqq3UTf4=o6(a-?Gr}y6XeW{zCcFJ+Ew_sYHN+?b4K)(5q-v} z{XH^j*4|)5KV?L(H=<`q)Qp}sYTsc+KW5b4Xhg3wqVF_oKWRiyo3$S`q8X$1+v5Y!u=O!{>MQg0u zQ)cuzGg@cXzJsu4?P}68Ygbs&XUymsBDA8%Fq+ZjX7oA|vTAQIqiOPBMps(3%gpHY zX7m%JZbf}o?Qy(O4`xrDRa$*O(Sj2^L~WmeDcnLSTiJtxds)9QJ`?5VJNo-%u0Fr$xK(I>2)>#S2Z zSnsRNSd*u$Q>U%y_15G^t?u_3l#Rv3=Dt=MhBOcvP{nV*(G1`3Scq&Kzg?ta^ZZ>N zJJbKZJb4($oS%7CMm7t|$L5I*c++CGos^2l+$}>0k?&9TlhmB08oV?fo7J$|H=n^om=kvD#%CQ6MT@tik)#B-_fBc4l-AKRVr zLjM%vGuvXD_r_xIGLikhhinIU0}(|z6!!R6e4vi-{C_iex8;fJAz~nNr*X5`ZQ&ozMVl)iJE)O_A810KwVbJ<&SK?i%9>%AA8f^^WSpI z??j(h{<&=VyZ_QnU*}bFwfOQ+<(B`(&c&2Jn||^7zl`*E|4B~y?;-u}1?zvI^e^N7 z2maJapSwL{zB*3&<_Vb+lABFc&X^rR$3#P(F?VY>g3Xwhw29U`V_uLsB?$*_o-y}n zw1a3JRx{?9HqqVAm?w{^ zbhI7o7-q7iL(kO{GgBTjWXJiLyFW}NOWqjC5zikGk9L!D*8O^MfAvrD-8*!6CR1_$ zxB2cDKIFvzPUZf+eD|;pf1|kje_GNVHC*cRf8@Idba>Pra9`e)mwz{Y3PbsBz&%^b zs(tOAr+r<}HLBXzd3`@r`*QEq5{cPuJxj&(>KaMM(4kfPa$AjRU#?q`_T|Q<+E<58 zRkbfS52}5=dZ(mwA*X%q*8Wxda@%o{_VtqXsoK|Wo!cVq%ik*LT+%*N`*LGe?d!aD z%WGdJ^!-=u%il#jIqeH&SiAkH_O<81eD_N_ylP*2M)KWfw7Y6wCx5r3yJ}x2PvpCw z*Wp$BdiD47-QE1D_Vwyl^WEL>lbG$>r`j+S5%#0iI-;RmnodVokg{NOU>KLik4{zFJvQ2xWSE&t)UK>kyiD8>(dndQIkf5C0*kXtjK z{6}%~WboTW^E=)s+LQ)mH)85UjuWCpHHPKdPv;)EdK$ZG3#$-UKv$ zw+BsrooaU!xRxlTw;Ygd&I7Wo4s+}f&#dOM@+i;qd&406!d;gMvgrqrg!s)B0ogbr z?(m~;EdC-=y(K~oKRnwY8_$R`{EAORd|ppP3_2hii_#$bmt!I#3$iKP+dmOOqBh88 zrd5ON(i0K;c>9zB*&S~XWY7GJ`>KViA|QL}Z35ZvqaM5<`}0LWcDJl1QPBGGLAIjS z3xe$KTd4~fxX{)ETq1QNSgA|Ien9qiXOeC%3$oj%HyTEO01}pABfrIg><%x;CbSE( z>#BSJ!IgkPgJX$MNGvqTZdeK_TqeM=8cUM{3nAI(FxORC1TE)S&QXN}=3p9&b69AS z-B2OHE9K7$Y|A*X5tKcPxvO5{Swr}B91U1vaSn4`8VfJVZn#5y-ihU{9Jg^`p-6UJ zRe-R!a|Ag;xL}^eT$jeei?SQm3LRaKi)!i%ySwu;YAYIO$=n>n!1D0^tX zIQ9`d$uWcj7Qr+Y=dkdi?1qC< zT*Q11vqsqspA}DEglGRtBXYvW3#EJ2uLRHDCqsr`OP<}`N&n>V?5QqQ${e1JyzmBy zwqWdIS*nfTaG6BA-lo|t0=C}}(H4Ad6;kb+-&CUQcWJh;?ee#jX!~<%b_;-8==fEl zeZ@q(JIbTr672zA+LPeyGUF7ew!-LvXalm#jh>Z?R6j1LxyKJ!7AOtV{s&(cW%pDX zlhY;yy9a_@1<_Tw8)OZdhF628;nI+8aI|UmK&Qdr&}kqx1RJ;wmwv$zE)CfRO#_*s z(?DkMG8`D_{j|c%5O0w9O^Qor%v#X&^@gBnxH_;JJPzvyWdo=o*uZMwGcXyl4c3M^ zU$5Zv4VI8+7&UAerVXzKR)fI>M8nB}(iU(#3N8j#3xqeLplld8uo_?v^ahw82c8S= z7A}31_!YhexkHa3;oxhiICL7a4YEF}P&SAg=naI2V8f+vQrH}PU1tfw25}ogts&I{ zu_4$~3Vp-0E!aMATR3*~DWm5x6BZ414p;`r8WJ9@FneycdX`!}5OUaft=03WDFoZJ zqBlXd&7P$SVn3zmH=w)1I`xDC;76?2GQe{y`hxYfQ`Y2Z>mNYceS);7n4Q1WO!p`) zWGQDs+8srZ_JzeD?ZtxZeWgJ5dFiR&0LTu%zW`(_GV3gaUfe_`!xdX}R^!J2IpU)D z_|YdrOB-b4XHg)V;iA0|T8$sy{)x=*%nz~^(=GtnWZD+{DP`XHkqX#C6{$+vsxL}S zWD>m8L?-db%~$b=RH80+>WjsXA3YP9-T$0FkvaD73PARp7i7P5zyS&625UV|epW#D zD{7)r7U(*hv`Q{*rfjgttNTAH59M4sOxGv2Hq)(=OPeVd-%v>vQ2#MceANZ1r0+j3 zAUT`9j+6AeZBDMzw>_xz9q1~F+Du#0pIdzT=d&PJ%KuN2{&%zGx1-N1|J*sZ{M`=p zmqfGaUq<=Ia?1be8bF<{Qz^7lr@7^KGN{U5H2paT`WxK)`G00ub~hK0sqX)tU6Q`1 z{C4!J{C4pcb^kA;{`akP(x({$B2%Pq24sXTc_RDsqiYxIDYUpx!?Yz5ZboA|I{BF_ znD&Y3(Z4@vHB zl@w0f^u;~;B2WIqa~{YhRG$2YpUaT{Vj}dZg^Y6r3mGp9WcRAKF8RLD4XiE+vQZG@ z<&3uuGn4!?y2h&zJMY6yqQ-p{VqewJPVsRp_BptL4^CEdoy7t}=FI|x6e-a&ZIdk5jU)H?{z`EwA|sH2Qsl3~wz7Z*kAe6k3ytH6}vdb_7%g} z=bj|VBSho?Y+n_`x(i~h3HCV;!LG|E*ypCDIFA$j8ICj=e2n7>DaujF#%>4;Jz3=1_%=n}Z9N~t!2sGwJHE2VQO`jo<_6vIM?5+((2dO_(m zLZEyK7=e(WPB1KZ6U+!rO7Iq{oUkes9HDGrDJUD%3H$_Uf@Gof2t0yJ0h@qaCj>$X z_ChTZ4h3KWtD!cEx+KU6N`;c>lP1chLW@w8d_r_L9UV;=li%tArNOgMj=WA3Is@P* zpcT9c>U6CYMI94GNhlQ>mSswn6W(-<1&tE*N`MsVl@&xN%9MhoUa&r2t8^;Caz|>P zu})>I=+o9~r>vgS)@xv>2Q-EXx2PsIKZaUZv(gN&A)W=_Z|2!j49cB@xI$&hj^|#^ zSEekK|K#1Icl|)dD6LG1Y?7x;iT7$kE3wah#o?zEgIbLI_x3ZWAIv`ou>gaj0`lZP zD&Q@W|A;wH{_D=JVJx{BkCOjN%|T2RGN>G7%9rF@Gd6?Zn{Ov)g~8Z}$!Ay^F;#Kt z;1Gi^70&&yb_lt2cmgzA!LP`kDb{RYv)R>+lUn1e$8TO@OGPE0`V9lUh9}I|RvXn|u0Yk^EsKA&uDJtO3 z@-tSR$*MduV1`rV@Bo`Zb?DG^pq`vCik>lNbiNc7n9)8J6>#&c)Z2DzpNc`%1bMLJ zx%Dup8tp?ds1w?hO9gVDM{`Wi(U)LQuKN`-s2UxA2?jNz-AgbicaFaVgL1={U{G%U zDL0H$F(|jrRh!FIrVQ&^qw7L7R+|_^d+v0R*5=;1rQ*|OP#5&wRBdjzHmTO;-dok$ z0@_Wrwkd5=tu3rgsAK zvzkwZ@cgkE)zfcO(K7IkfmOkv-gfw^qUA5uHo{jGE!V9(_(%Q8pN(gUg72e-V#4E&aQaOc@4IQtZvyQL#taFQb%An(fD9qRT#s+VQQ z;KE4?2~+^7oR<_bg8h;bPxQ8kLiPnxkbPlKQaCRu=-;JZ)ICjE3#v7yWY|wt7xgn6cTv}Ma#>DR5UYt?(nDXj=?#KmUC;eik4kI=Ko<* z?vBCdu<$}EUlo*UC=XKk(pU^x1TE)S&QS#k!CV?rb*&)YGL8z4N)F7vs(PYXL-2JR z4On8nh}oB~W;t|+_`VY#w{qMDufRg7Sq=pV3X}3`Szg4vXhv0_WO=SbaLV1Ja}UQ( zqTVPdWg8dUAu_wL#6l}s)}^r++Jxg4j(fT8B5}+YG3O(ypdhMZ+XdHE?I-F!6dGT; zt_ggU#M_Y61J+8_vLvaR?IoE6$3c!^$>?H9)ewwoXp~TKjsuby7Im^+a7jj`_X`#! z%b+l-hNn4>b33pQMs*JJIn2Idj4GIZoRps7ND~Soi>QV^Ert0EzP`XQ ziz61nG#2@Y%GR=MF(QFg^$5cEdJvV7gQ#lu&;bde+9ZhTr-guu;gHa&RE;s`2VAZ4 zrF=&BpQt`7O3>F(g$cX{pqg~Gq{?K+&r{{1B_(JIbtyoUU(}^ep(_P~f-?c76uXK( zYn%dHrDdznQ@|%QnrxjYG@u|)U?(V3ho7_Lg5O01&9J)q45QX`j}!=fGCg^bP7@hc7jNu z-2{|^X+2}BR4Gygh;kLKz*^{5LCauffHCOQ2}QZkfPzbj29$zW;Hzq@2c0hH6dGKx zs87gVoggkz)`Cs}w4hc%EHt}7Q;;pt6kMv@>bYJiRjYxO*-6szD; zkSYKbXzGZ??x5@@dg>;t=LHKyHF<;e57%2g&zQ{cqbt>u$wI2R4uMpsQHcn*JMXon zoJCiv=6=UQR$DmRf1bU|V`o}Lw*NGDFAh?b-0?FYB7h7cr>|_s w-QAJyKcKqBV zJAQiQzw9K%6+$ZBOF8nNXU9*R-Yi9_E1sl4f|S{@c^ArlM?)%AF$D`{J@Vh%vQU<4 zy8`*I^b)0crYK~QQVyhgS$6!)QIwkTiTuW#Lpemnoc?2R{}0N-hefx|psKiZ_5n3a zY`o_GVPMi|`zt5)c02Q%y zfYRlqANWR2`F#SZ?C7QZ4lk1OJNsa&@_W*^d5tQ+Q&*(#>{+^)@}JL1-)a&_lr6u* zfK>h43@kr=Px&3CCMmxI)D}_yyZ=>A`5%<@v*mYqUEckl%ie#7mtFb&zw`lTl3D?S zO-fP91uzOyxs&H)G=GoJvwA|l2sSvO-RR+GvIt56gK4EG~D)&P{s+V-ls5Y0o zqpVwpMOvGCcT{V0@7xlJ`Gmf6OU3lE+VRu9)2huazO~KhxKwKkXp?GfXSGSSwwKk8 zpSh5#L;EO#MBO|WX>DV=R*JMXOYQi1LHkf`F04(ewYhG&t<9X#_gA$qYk?g6leW>;|U9jWln2ukyua^rTmFuqB*UJSve(uxZX(QEKiE?qn%QrN54cqSV+B*_;PbnJ+&|i|qb4rTn}}5Ub-V#Cnbw zR_E2-LQ@lW^^L|R$NJ)_L~J4zPr2;qSNv0$+1)~0+oL%UE31Ol))EOdDY~_wqE)%g zu=aTvmeTPM1;~mH^Xx6^xn3N@I{OA0*31{Y3@cyJ%JbFB0*0ktIKL!{R-Uhx%+=A~ zF|En3w6gD!MZCj!&q>`DUudBwE(*R{lAXhN3mMjBy~E~KFu)0)m12~nSo41MGH{#2 zu>4oXuxkD*NxKZI@T-^m{zk^Yr4_BhJ%nG7(atT%u=2lpS!}b=H=<}IVM{Zt?GD3I zb8SJn8%B4405-!aqG+|~uU^ihXcds=s}-%f?XOZ_N&bU9v@b5ubd$_NUxD*D+=fmlI-SPg49)^Rit3Ugf=i;Gxz_X;h5 zSQmjQ=rq7DJ#b?|z{di=nKbMp+HJ z2)>bH8^?CZ3Ksb&>)a-(sxA1qm!pgLH&Y>7B`*uHtcE1Mha}$p5=S55vGM9#bpfrE zzEwWO3JS$)=;s*V7$FqqI?=Z}hlN+)$|tH;U87jWISz;~EEL8Xnj-9z97i~g;fnbp z=8_a^_R%cGx_F$rm?q%k9M5p135>Zw(W>hSEJe|(s|dvMRef4u*k^G40>>iVp-qc5^o=t?8<4N)NL9`{hJa1O=Vrzw_FwEBjkSa2&*vl4u@$~SNg)ao5MdRAXl zeUeO<#BYCXj@|v>R%l(>d;D3(YuG5H#YgNWSBk}Twh@|7?z(A7-}UD?W128v9gA_YPceJY?5IH_Fij>pC<*R%! zCiJhK(O;>v_ws{G**pEgs=!|8R-xzx1%pGO5(bKbPl2M~eS9eoS_ct>`tgZ6OR+HY zuRw2X|0mj0idbE5_MpCnMi!{c)}exhvDqJR3Wx^eV$VJ_sX%pTQi19+vgIEtRkf!? zp(}cwB3z)cC5mvd;~~t8-4CBKCoA~2oz=~~K5fbO>#h}m^?CFurrt}#@Ql37eMJD) z!dg~Dve?c3W=bhq?UP}wS_pY%4C}%VzT;t7>Z^4ghSiWc(Ot+YE~f zdk%w(Qgaw;TEw2im@A@aUwX5@D`r?n7QwK7P=sL>m;dJOr&V-FvFH102^FwK{7Bv8 z&0$b&+qBa?`XX-*gHVfSSRVPW^eo1M*@d3=0dXi3k1FHYR)L|CvTs(TMc@|3%Vw zzJw>~J8{aT6K7ufHm#BLogMa6`ueMN3#VW6m7MZd3&Zi0-;=(xfu5wKzuJ~lew|l5 zxL=K1IQB|onj1yU#)XD+bf}HHKiGm zVhn{}t#iMMrx=63fML1e6=Nv;YF*7oXc%(86=Nve>~Bn`uNXt&SL;q_cM8pYFY?v8 zT!yt<*V<*awp@l~D_XhtPqn!zeQ#B3GjuwtwFR`BYHe;SDAL+oH`UsTZT7cY$E8|Z zSf`_i>5Mk1);6Pks@CR~T(!1++NWx5#lBkCp?xTZbyk~lTbt?DziMBFU#)ZRUnxZ^ zeg9PZD*S4lq1{U;TAkI5Nwu%SuhzMYOtr7VuhxZi{HlEwezh*3-DzKXv%eE<81$UT zTfEtyvGmKHHSFrEb;adBng8Qa5}~EEqCN6Uo}xY87nJ`9^_G0KuJruh=|#-{9ntfD zKhW}@68Z|(FsbS)*zEZm1)Dun%FWcDs=;3>`Oov!;=;}Tc7K$POpPxVW>|9`MXUL~ zTDMMh$karKX$*1ljHIM~yg&1#Fs!+f46FO(e4!2FO=4X8t}?ExjLTtKzov{UOS0Od zEulyeJ*`N%kzEFD4Xy;4C#ghF3MHfpb z)+}bx#qy=G=$gjiagJv=(qf5)FO7{?8EfbXDf&^oJd49)99Ya^p2b`z9Bb$^#PtP^ zSt7@L5wlkpYv{88EVgb_AG_mQ&e@#;&p*iH=>lvKoOMiFbZQp9Rz>%gxn z*%)r6Al52_p}2gsYKhTf%6@@-v$pn`{iz5qPe zJG%)Aj)I2)qEO{R9m{4ps8@l$fL@?4;0*`|^ab%k3k$3T9D;j6wg6P1FPnIw?*#p- zgE=X_g?Wh84gJt_Cmzm4T)pWU`f@5GjFPV5%v#aiFbOg(ek13z~&`73dC31^VLq zR7XLEg1g{JOi9W|sNhxTS>0^)K+%9$%M^<`ZuX$W1t&Yr2e_;rz!=c$1$dqX#1gFa zjKHv-o1kAxqYR*Rnk@qbeMQyYglr-xpSLQrdY%$}Et*d4N3FS-aI7g_jF*~KGn;?6 zz+6Qvf1x7QVi{J+4_1wd2=E3Nmhms%gb!9BiR7)rMe4ETKb&&pzj^hr=Cbmi5<-h@ z7LDHW=|#$adJY5sYp5l)*jD5lbs_U_>Ov;Z)J0HVtkoCt0`+{bK+RUza~Q=LR(1_j z5r##Tyt(op5!&(}bzDmRo6E|7rPna+%QCEj4;BOp8CK3_fm2^Bx95nkgHhP(&FWPB&-cePvU)(IzvrSbEE^H|BvBiY zNcuWXNy*OOJn5r;$h7x-2|I4)RQ7{Wq<`{SVK|plekZR|6?(HoE*GEv`L8p_WHTAk z_iq%2l`X&i2$q!JiBnQ4T7Gx4Kq^Rouxh~?FgtR~e^AmdT7FOcJKAZg{G_$@t0s!Gnzq^U{sg1JE`i;08i+OQB7{QZcxB{ z_@j1HjqSWP(Fx`g(J}3&8e1_&WoRF&$&KmTzD(kB-HI@(DebdLd-x7@KyuLpr7}ZPKy#%8=q1{U`Dz|-< zU{r4UB^cEu9e)W%<)%;j(u`{AZDdsYKH=4~nuk&8HA)B*HltF)7)+F3p?rr;o_vQi zvfvseWaul?v+}G_@2(?`dX=p=d!Hqbe5Wq$HR?*#yP^+LQ6*b-{oSoEw&{y|)CGRF zYnyxZg}m}qt>?*ic;2a<7C*n&J>LhZE@(ZgMQjE*m({cKe2}VpKn;~!$yogMAII1I zSln#sslyBmVZ9)uDy?TV_xyaJ8HSxaDvaxiQob^$jxJF(W~eV?z2RL4uPSUMufnWf zZKo| zrshzTpYopke{jNQdR6aaY%DPzn;0DKi^a#s>ZsPNo6|ErYn_Ey!B3I zI<0i2@)&d6tJ2pD#}CCOhDK8pv5DUCc&c|mc?_ie;_yiP;lzZSNT#zUeQh>1+aWP> zU~nWMQLh~8Ba*=nCSrplv4i_FogL|=Ba_3iL%qG3PBZO4(0?eFijN;iq+$;z#wP|x zM>6-m^d-}n1m}4xAiaF!jx8JZZrQPMLpYOP5sjg?_BKyNBp+j3LIg61%BjHtU%Enl z#4eMR0dA*}l>IiO-YWW1hsP3`WCP~3HSxjB4x@ZB`4^a|aCPt68zW6^;YeGky=J=OVoy(#k1~-+PbW$8(U~8P4h*a zF7F>tPfcEWU=TY+qldpFGp{neC^1DZCABOwId#fpj&iw)wp>EYj=nBa zdRX!jb5p1EFuTSZPuXlz9xE{UZo9ce273=nc4|@H6mNze*5c%h)RZYevoiQ58+A-@ z%n;XXoKg$qa`f;92gZB)1&0L3hdKI$PRWq&VG0hMk|FEOm>~zm^4@G23f@c^iry?4 zO1=$@8Ok6+WwK(YI8%oBWZO;&b^KZwm=Ripy^N-mCLvia6ddRr@8c32@97c|@8}%j z8RkmJ1uRI$u^EGno2=KRK*$wr$l6RG+SYVQ)dL;R;vVGWi|k`YppO}WK4t{@m_0+; znv%%FdW_WI1zQ;{SJ|nOLry?exqW@RjgcLIZJ8>#v2HKba nwlVj80Dh$`K&1^JrQ2U~Fc#of%L+4gd$ST_fDkv_38{JjlvVQ9 literal 83106 zcmeHw32+?Od1f~NlAs8ZASK(9Z7FS9mIf5mocmyj3V4a8hoCSdWlLe?;b73f%n+Cv z%|Q}Ul!ci%k+YqISx*urn**CIM@ppLr;^JH- zGo$B>`La=|Hq2d9mC|f}-Z+&h&KXB97{yxS(~YrDG+svb%`?V@eAZ}~L(iN(dJ30f zX2EFWRP&xQl}t8YoJ$&u&DXO8d7l@^SBFJxZK%XKy=Kc3IyX2cK>S@-Cbb4%D+e4k-Gq%4`Yxr`zlW$PZ^2L0OdiJ5r{Cv8`8mXSW zXk^Y;j9DH3d{tw9cI>JbaRjpX6u`%%BKREf~03JiV*f2F3 zqTW*8M=kKGSXXinPF@?J38Q9c!XCmStXGP>FzWH`%~a;92YxBBBZ(PkJEvOXlzs z3#FO*yzyki+$rfpr*>JMTBAN~4peImbIXj8%`ar;5jrA5pKh3=kx(=ciw6UVNHC;F z!qLVj8+gkHjAA1vjO*c8j8S6lh%*Uf7>MvBZ!$Euq|=$18LU-zjvV{cwU*`iZltbA zO!r4!sVg20=s`hvov2H4Absk3N4M^~aqWsH{CXf7cU!ySiI5)h^CWNT%KcoQs`^yb zvLhw}Q9TiNrK&_Ap@&25RAmmGIezlFMsC>L_0ha>Q3x5@F$hn&2CIIjVEuyra44Zi zIB&=HTU$yrqWgo4njIwj-%#()VtrxSsA#pEq18*}YFn-Gke&#)zX`!WEEv@Ng4JxR z_WeetGQX(RDw$$+wp3XFcTDo129Z_vTr#bwDIC=!ep}bAq>w)v*AoekzHgt+7c=wu z7Y&Vs8s2kT(Rf^sMR-vM0}>3yf?+-EkGRjsNX3Bgn>DoQ`mC6aW~H$(_MHST>8vyy zjN%!J$J;g^TiN3~+}=$*^{5_mql#cS9Kc!=Zl{VI9$b@MH}8YspEasgEu}Hp2|KS1nOoCMIlW-$qz)kA*HRhy&g*tn zfd+{+U(FY*wM;Q<_$YfCQs%f;tyRV}W4>x=vwL2eExxRQB(#^Dgy*>C_v|GdgJTU}((*-($!FtZkIh$W&yUF#UnNl|iBX(6b@1@f$)jfuYENhZ4V_KX z&5?s=4(fA8twkQ!gd{!dkfBMMmi4}0HMdj@Myd_-qmT~Ma$LT>T$P~9h~_|jdoc=8 z3d)a}Qo3HA$<&NArngo-&~ne3GC$xZ24_pfnsKQHEd*0&q|8lu5{Db+du7M##jK0? z57}>{Cge0GB#rkA#M&^Q6y1P2q~_xkJg!}W@`g>AZ;T$C&NGkcSVrmGKla^<8fnEdB z(^PYrvLOj$qvrRrYHqHT=JTX7c%Qx7G!pcYr&)7%jp+UXVq?fu zt45`E0@WXVp`MxdK`CZcIj)_>ZZoc7ogCK^x}R&p`Xc&WBSba%f)Muhu-gV14hI8z zEatxRU{S75S8MrNy~g%P+HhL--f$!q)Puo9lWnjX4aZ{vusW_=uWe7;!e1{`Y*R%g zA2i>UhT3?3krsSfxL)pJ{?Ce90?}$xZk!!TS5+&=!71O2VB=^b9ey&Pd;5*~Ofm-q3CJ9CMt*kDIUOXpM$u^(4P^)u1^u^3 zYHMHz+u%H_jr)UoRP4;oeh0#ds2=xvY$GIfxc3#45sB!Dpo^3r3Pk*nK|>z2@KAMe zVH#{^_OS}2w5*ZNmgeU(<*M=6jDgLvWl9L0f<(^uerMmK5Y&NNc~Hpq`TRn@M)G(& zuMvvIpdxS;$wRS#A2c3!0UAj2v7L$Jr`#W3EC3)aDliIXUt{ESxsYl0;c`2vBqYk z2i=ZQoXZ!DeNxfDbOF1~omni9q*M@!mWCPb#7ggR>BycjZjw=m4uGeuoA=F^E*cfq zN8-`v(P{=_ewYU!VzxBtYn?yeEEE#8e5uG4GXTe-K!TCVla1t{xdR%XGR$0NAOqC0 zxrVv*iG_R>N**M`x})$QNP6=7hs=@r%!`XykXYshlWxh}0cJ6k#j}CueNZZlYuVZ* z#?13le?CNN8ny>PIf8uqNwtFxvzFXSjBYaTDK9`vBMP0FF=n-91XnU?m-a&sHKslO zq=xGckR1xEn7UIh5)8$S1_=1u_$% z>PcheD;gKm)k-#9+M#D01!#pGO2cp3=x};ys%%7Jz ze57DQU9%d+ay6elFU%S29p>4akLvo6c^}PzH2>j`6-)pVOGqJc$z89;$?eC@J@ky{ z4bX54T{Tr{4O37d(+Z+a#}Iu@eIg#6d=aG3C+}%F=M)S@C8!7#PFm;cfisml*XQsV z{FR}?Lz45R-nZ!1>}tWCLIOP~+ZflN;e?iRzKW9Fhh=rVFOc3TP3>v5FOY7C3jDI# z7f1yjuzqoJy)}?MnUn2<3=q@9VHXh~9P)?adQ@P5t<0K@N7>C+N8)_tdS+u$_-ehc z5Ce~3hB~qs?e(;97=`^*$(s3z!U5t_1`AK(Quto{v9S$>gH$!C!n-pIg(H-=M?Rr& zj52RHe^B@cHFJkqg2E*0`$qEtg()_loaO4W>laQ_-u|w>K2)-=X0F^-NMm2k+{YN# zZZ_4vtHV&uug3$fyLtdBn1IKw-n6@DN?E&M+GCQ0DC_XDXS6 zDpl;N&m@vcVQ(FibC43OdVCrkAA9@FV>uX^tsLE;K9iwSLEt*f z(|#B(+ZkprJZ26{GzTf>5Tu=I1HSvofuv%fhQ04Q8wxFCOXWq4{{q2r7C;Ny6OU_C z&z<@3(T4nKUr=bYFp@l~-acQ-%mB6H7BDRx2FCB#gP~Sq2wp4s zGJ5&vjBsNJX1Qaxh2SM)A&*&z`B7;g!N~b2|GcY9)_B$=Tg^5Z8UT!uqnHhCZ zn{o1233xoSoz*;`SQtZb^Ci*WW?6b_iBxQ}Lj?9R#!yx%_NR^ZCcez;uz} zzunn~%EO!s2FpTtcRaL!-=B!X2uQvQHm7%H1T;IJ!C?%-jLSI<0Y6kHun1{2+(3g! zLam+;Z4|~DFqD9`hU@$W0`QxH$CHg33K(mYs#twsM%gkWaWa`qxOO=nBg43W$4k6} zIeujg%to=+x;6yju}B!bb$fU)bPZww^{AG+7lLEv#y!qgE`f=Jl9NAo#~cI$0oazt z;vOV1O0-5US(VbfHj8KJoP}T@h(#wNR@jz`;SvZD)vIC)H{DmEsc2awP;nHloNX4r zpokER9bqIG_0ZUkFhfZ=8NzDDZuQGc!f+G2Mf;n9N@4!z7iPPyZ=Fq{N*;A&{6n=f z|GcB`JDcz?q>1qXixgc5@etw|FO=J94jxU1aO#OXWr#V5(N1m-!oUjW82NA-jXWO- zc3y~o8mXUdBzLfLQFd5~?A2+Qn~t1;rxq3b$Xoa&6?5o9DL>OVnN&~KaFR;l?2w}P z3LnScWR$8X%pz!yeV1$Oy(~6^yyoy!tzxy3+@93TO?dbj_-CoJ4K!28QWKO=C{f*I z{E@A*3br1cnW>yE!!eZxqEgLCH96SYyxgg4H(2Yx^V^(867Mso z6~%uXQLUAAC^q0$%qnjM3XW<&+2F&uhih|=7iUYDVZkng(Wf(f8sg|3p{Y|( zA3Jq$68>M@iHoci_u!5fBMZ=RirUYb=iT!m@~r=Poi_|iOfqmB8*h-E$lGc=!p8V% za}(R~8|G1FlFE%aD|2KbNFKAN*pyN)L5wiG#X(nM%i?CGY${nHl0dmhZGqMNC6jpt zNPMYbzTe2BR(Y?NOJ$#bOlP%PzM_5SG&mvjtp?U(c+$58lqc-n0d3oQZ2|8#mp&ub(|n>rGPO&AiAXd6cgRS@ zMepJuu{(JLRsVd&yXt&Jxa~%wLFiJ$E;<;N1OqCIK+K&@aaWuIyc>)+9Tm(O>~@Uf z@YFL9ifc3ZS@>{Hn5v&GXXYkI>5!u#y)@S~^B%wz`%fR4JhTD$mxk8 zCUlI$q%ulpikgWYLbZ-oXj-J0q zf@SgncvR`d=|xC!kRgm>W*VMo2OwiSb^OFpCL#3Xizi2i=}c!9X4uI7ZV)G>eu_0K zPPf5%$zrI|K`bi{%^4RqV(iEn>P@Cn$tLK9OtDnVLr;$m%%tJd z4G0U83Hj2U&BD8h&VS01JO&p$+`^3-r$#{vtdTMZ=5IKWtm*@nfG1DK7V%J;ACc>u zee@+dQ|Dz}%%V=S_|m19AH_(&p9yKRn;;v$&)mj3L#@++$2gm~dEl{EH+^~G5Xc0i zA1ZtSQl|Oi@NL5h*Z>MDzRY}f|A8Z1Fm;HD&+FDx8`mC9^RuXo%A;c%H1^QeW_3Zz zPznIesX~enDf8(nX2ORPs>D<>$j7&5|DG{Oq0J+!&>T*h<|BLu#L&VY&|KVc8MN?+ zRBd1Kv^xg&z0fx3Ahy2b`7clMw9Mk!VF}?d&Pk606SV<_418q?bVR2B>DS8SBu9wH zDWtIlP3{^foWu35cn~w-vOrMf=%~&${eDi4HH_~YE9?W_=rDp zo`^@=;>9#EAdH#e{?cO_n39FeBEgvKAU`dmf$J3P3HY)yl)doj2T#JX*eYN zh7;QFW4)%vebf!`HI=Nblds{#_bB5#SUzZ-eWV3#I$DpzYdv2(K!3<<{>afo&m7a9 zJfH<_mHY+YqXI^rSPq}G>gnxFM#%Pkm1$pPy3P|s`y1E4yp79;k74{$m+viDTURs$ z*Qk(fQ?qtE7%4==Zuc)_Pb++uHr-Y4XN7B2tiPLeI$^lKn;l}Oko5t;J^%=IlYd~# z^;=v0uLlvcdim34XW z8vEuRWGOnZysV>Yd>5{)XX-eXOk}2`_z#q2QMzMCfKdzb5Hfb)KO&j}+hXB!r0hEj z9|?To4q9Tk(w)u1g@V4%=ZKsZkz8kdSRDCwikZ4Vk;^{N;6vS{!YB3o0e-Ptw=8kx z?LSwmpuRfR`LV(c{aQi)P_X`?V9oDt_P-)XKpxq}@&%b`#n@g!aZGWR^ZeeWrH&O_ zdLoD~ZAIF@Toeu?B?fPT7-s|O(}j~HpWQ34YhgRu3SLY1wr6Rh8N|-jI20dMlz(;f z#8U&RI)PaB8&qy8IHl40O^8r@ga;1dGNHtkt@t~t>`*r2p2|+(AN?lS-{$o@qB40M z7U>i%+Z0jG3sp=%c|~98N98cuX8gP920wWn;J1pr+RCx2WCwgApSYIAts>J~%9TZY zInFc6t1KplsbEsrkDBkLJ=v~o_SFBV&9Hfq5kAp^xXS#ppR(N)%E_q`C3?D0Tl6n3 z=BxE;ig~qx&K2}8MF;{=;Zp0^R)(PifdH)+civae!R4}~W z3P!tD@Hu_1QmU8JRb!4kB-69`OMvjnl0f-gHew9aN(8_^R~-JGws^{Z|F7vdB4@4+ zZuIr1q0T>iLOcA16_vgKh#yWs|7JjW6Cb+0lqN^K^uu-ys6WXwD>3)YYs;? zv*DsW^|y4G^jFg}i!e>dXX!&r+4En*!?J20e1qTEYyT`x!@Xe4R>&iO#;tu$Zw><% zGxTY$QiY5c4COZj<KfAoIr z9t5?D_2G!fMQg~e!OhBbwwN^6D_Z7uDfcMy{g&?+UFE;`B0NOP-&SRS;=b|Ue{@^? z>%aQhU)!@cF5f5FEx7hN)>GW_AEsR4)2`q%;`wqqThGs=6T#`3SafqcQH?Ss3yS>Kg`D`awEk)071n}&9skV%XY}Q2wQ_a;zJ0jK)u#cS zF6=v%nK4R5woXFW#iy0_&Cg$0cpU3T0cKGemXXKd(5(lwU z<;W5%UB%~=a#({a<%KF4aHWO45^`kEXK;4>9MX+6ec53yorjlH8fz$hh%H^JO8dL& zKESVzp0k@(bk~|d_W-R!it_EZT2uM5*B<{tk)PHl`_IpAzWtVDKV;e=5ijrM^=XfI z!z2D>kNDrE_yM#k35(@Zl%K@aZTqDf(QouETUK zr-;84U#lp3r9@{`C^~%@kmaTg57EXvVEQt67(8x8pRQIFowGeU?Sa^*^9%I_M1o|A zJ5Xq6t!1Wh&7oc%;#aon+cs^^LeN%#FOH@&sE*~J8`@vx4J`kN-Hp5?l>6LJ6Nt+z_Dyj;Jmt0*tiWqA>; zJb(bzEz{+CC$Ex>WxDLYynYYL(z_?q<@zeGvOKb|AAi5)ouWL7AjQe@l8=$siDp(7 zYQ5iwSSl~qf5{(N5wc=dT4a`CTqF2LWv%r?c34c5lvCt3Aflysq&$hT6er7Gu-i7%|;~X$7gQftYkZW)s}+K>{zFA znD<*I`a-?VY?>hxG|Zh+-!XaERo7t-kr;|DnVXp7CLJ+U_)*ZTxpivt@aYVF8^i9; zyyJ8#edw9f&!rDPclgB7)3EcOh6UF7hBGL7Vb zIe2ld+(-_YgOF#b$srPW8k0+^xwT@5m>f@l2!8pdM2f3O|ETQZ4)}zr0G9&4r(%%Ru4h=`vLKx459u z!Y_#HFXHObYM?`_r`Oo3GT5!vU|XxZPp1y2PaHX&K7Qoc^ufbtj(_y%ng%@7t?5u( z)9;c^A3At4nLhmV$)`>nKYV5_tt*?mwH|J3edn=LhbGg_qTz`oqWNScmA|*``)t#O2!NE4I+;?^c+_uL%+d{v# z{tWbIpg#ls8Q6#!c+W*C>~2Ku|9tA)?J*Q7a_#a+m(enxyl=fqm2PXiyqEi|tViC< z^!~L!1N|B3&p>|$`ZMs|Gy@G2#{wJKG*e8fcEx0evpEP2#3>haLjR> z%wguNiBrcL=9N2t%0}Di8nQFulnv%l+VPl;0yB-y48Y93;AI+(-urOo1)0|DKfph# zR!-tdV_tafkyo*F3bLlDS9@r_YaT!p?^@ zrUzU+;yzJj|a zU)`XS1o}rbY|zB^JzXv8UH{R3>#yOiIT}quX8|036>>2bj^NyFKfWB;G#6$d*rXjU^0V|IvTL+Xc0a~1Ja0RK?l&zN^Y6Bdt zJnD@w!YP=}-G1oMYk`%)fs{Is8e;#q1U9*eqUzP3%d2SO6AXV=gP)j0B|8zZM1bbf z%GVL(;PLUx95u~tz06V7m3xr3jdcxi+@cwQ?W}JIaE1S+t(OUP#Wy*&ULKVJE`%e7 zOX~>S%aT}?@8Ut~#Jd$bDEsxTm*2wzi|M%R9ofqs3h~DRAz4qw-hsRAL?(W-)v2v`aa}B`$W?%1Z~A^8c2Pa4I^(ZSsK^8}P9g&ZMRvhKS zOYk#z9};*6G|J{f6jAsWuOA12lcDGe(J801l!hSad0;?cL3%lcmE)Ipv9hTXNS#2G zvgJ4tk|xTEm*a7yZeb<|^jfOO_AI@Y6!;Fkl=M??J%+z8D9;V?7neo0{!u5WH(M|N zAWKjy-^MJd=U!rQZ!x=+%p-rPPJC2E&{B|j2W%;sN19I> z(Jnvr>cfcb`h3;v`;pBeliPzQon+)vGmlcw(JMmrgK|MI1MUIu@}t!s#49A%$J?Lz zz1IS%=K7f8>m!y*cwkRgL$OlLitwZ6LtcM+6+}<1l6BG=L;dJqU2} zVO4SxNWoybb$!eWdV0NWePndkwm#;?`uOVGzdmkxldX>$+ZS5aM@+yGnl;)oyw*qL z;p-#vc&(4f)3QDyv-kR#5bI-3u8)ZUzCNzd`gmL!L67A6czlTNDB}7M9zNf?m7D6d z{|Ne6nUoPoKlh8sMd?`P6!rF>QThstptmyhS1d@C5h)P|fOmN11}}kN>Mwa-iv2G< zF9k(j_QIpA_fJ)>ivF?oEB{FC|2eM{rBb3!3jQ^Vr7BbI?SIW`KlKfsw}wyzV^`mthdKX8S4rfT(@@ z{5QwH();*tI>vvki1t^S;}_M_^IzfBx6J>_U+}r2@i*IF#rQ`**QC#7_1e#%ef0*h zJVb1$by+>fb^zRgKU`L)WEzSst5@Y6sApNdMa%(}gB{|sT9#>>>F%6N+d_B2U#50O zWe$+~vU)}40QckI-bojv_{m)y|ugLWGvGl)fP1j`l`&s(m zx2E5a=?}8>!7sG5e^vJHA(nn`Yr0>Shm9@z_rccmb21%+L3#-3>XMupMqhpheOdF$ z=xbD_G5WHeEu$|h=Vq$eqKPeZckKykrxSfy&)tQ-taccES@go_%c2QJU%{i)&J72B zElH|j^mR+#G5WIJ2}WO5JB+@jU5 z?;QUJCWZ5V;9q*;|6X;-$B-vpZ{zVt7lRM*1-QU`B?COY_R}0Xz_oyzKMrV z^i{dM8D?F8t`%@oOocqQ;l$4B2 z^bcF3tn_kpC_f;SV3P>G7H&H^)IwSU(O^iA z$`8#HhQZ)Fz9Q~`PE-z^On@=-D`wSMjJUB zYuE~%q#&-Z4>|$=;!q7TUI}6ben=wLkOeq&(u-;k`Cor~S;MbM){rrQ96E7l4O4d> zI>97y)(|{mSLg(pIctc_96DLq1~!aCC*G>Tv;~%M4Tv2&x%r!rYKgwq3Z0NHNQyZw z(20K+>TjmEK_|=4;en(5F3`yrx?~X0$+MV^)j=msIHd`bxIrfjC-H<%5>og5L5!8u z1<-_j4}y?`J)n~$QYR>%8+4L-9H}1A$+s-%5N2UjnX9)j)_LtS0epYOo^5LI&c3^4fi>Y5xAoHs?D4L#L1XluC zZapI+umCcU<%%cBa-|oLMO}Uv6?J|7s(-D}%B`fRf348UpVqcx1_boXpcR{FE!Iz0 z(VF4JP!-~b_x7EIRyI_$w$@L!X!Zs@uTiwdx_3;@C>O1*^%H@X^%I%9i`FC7`soQh zuh9A_MC%FCq}fGlWdIMKRHul)$)T0hq>R8Dv|Oh35$Qu%DAmee5wyXP1A;__mVX4n z6%L{(fTCCan3td+M>&QOlxld2<%o#8KRKHp`A5CdgjUdg@SmvtKV{WZ`vSS3_65p9 z?WcH#Aqol@fEwi;9HF3q0TgKCJO9*kza{2}jsNzCp!!|;6RUmO_yx+;I({)%)D@00 zQ2X@Su_^D~{^%R__ODH%eSsXcj9;KUZ2S`1!AJ!Rw&w9Gj`3IO)V}Tc^XX;tZ+rfH zf@uEj?Q=wk%|Az^dU^h%+r{`nhERR~EZSEsg*K+o1}Pw;4I;p{y{z7liendLH+vQ6 zF<>iLR<%b#2y{10ca>#o=bS9fz!Xbq&6M4mOk-fm6?w;$-J|l(McIATq2|8*I<;ff z#gyHO>@Lfi!H?Sd?4sg`1 z6)W8XS{aq9XAfxQmaN|cTCwV9s_)=XtLl15miK^GuF7-|XvJzD^<}~7m zh%#KL%>ok`wOJIxsBJ>lvYBeOC}a!WacJe1e432rmgF6yw(IhaQQIv^HH_M|w|Hf!WA)Mkx`QJZyVgI1h0H!8=^sI4sToYbaXm(RxoT2UnZ zF#2kTR;=gm0j*fi*8^I)BHQ0tpcN;5wL>e`_&`JwS~(XdiW25pU7!_(SOAi5 z)(EYXfBTy@`I$j0jLqSuhwJD6z@#|j=V4+Ny-aHuO9_I(k1?Tcrib02m6iK3b$eJH z(zNE4x68D~2Ccj)p%o1G5bF&Bhgkqqazw@)m4T;Z;Ft`w@P8=P!vCSdJI?=|qnF># z|4sZgwoRh19-x&}RB;^#Z2I4@!GUZ1yV*UQUEd3=(nL>U;iyDUa!2qP z^=3*OpBW8U$J%$sRR-E|6{e~s<;&oNx_lOUbFa9{%@8uvzP(0V<;Hv4TvBX&p)k22 zR)tKi=?88Ef(d>M1nKIMg6y0x1YZL|13$#94p({AVRC~SUhffCxkG&6S0t{|RWI1& z3zat+U&x2s!WUu!y!b*)k{4fyQeJ!^N_m;w5U1S@S4m}>xC#>r?YN3|6(TLs*BWt^ zt5E(hz7WzP^oDINDK~eceklW~l{MljE4|?=snxrr*l?9s$xQ{Lad$}}Mwlzdj|;a0 zW_Co#Vbdig)hn*D(!^B~mk%n6hE~a)MF>rgBgk=;M5`ynw=7)c=3{8Pn@b8iwb+TN zs8cVYU`Mn>c~<0LPu1>{QtkyUL1FPzZ#;|;=&LhsrY^rzxQYMki`y|59=Hi9ki~=O zh|^j=Aii*}d#3e#K>dWJ_h6%U*MR?BtSy z3E=A^#@)Nzj8c4kM5!d}7h<~0%}w0d8#g&ek}_aPEMWdtx!FG`^wLOm!%fPE2kaf>?F z8KaR3jE92F@vowN|L@q_51v8$7H))=n&Y={8??kbI7Cz7KK~N8>OTG(j`nZAMC~`n zZ)?Bn_-)UB7305oK+qpHE{U5sodhfmE(Qa)6RDrYwrq3g=&H%KYGFETj+G!}*uFLXXxQR?>dhOe7 zE-6-d58Px~miNF-6r0O{#7&mu+%cMSqI8-}W7NjE54JkCQJXcto9Rh2g(c177P{lO z$qo558O@dD9iz5U+0H$b|E8oGMr~JQPDX83%`Vi&T~eGhXO(8uc2m~HsLi4wMr~JR z8lyI=M~vFe$^O}J6Kkec;gVuK9}nE*rhI-LxQRu79=M71d_8cJvaH_&H<^&>9=OS< zO!sg}@ym3)R}we5z6#vrD)9m&zpN2A8J%f!NwLYtDflr8{!jXbgZ*f8Nd_m-g!6w0wD5nJwqtVIT5uC& z_L7g0#N}fY=XQsPd79PJNj@HvF3C3jk1aBG+(i2Z7BZr*UT~9HwiBS?n<==Rc)6sI z+=zt^*(Ox~JT4@PVc8=)HT-q_IIi-k^}4I{aTR#`^l_Cwt^$t5d{UeNm(5gbzC(Rn zrH`xN6u)`;?ENJB$hnVT1s>$!bxQ8wtx>ur4FhT2{(iT zBLXda<98&g(xx?(2ic^io2Uv?JJ?VaC=$!-#8sBM;3`+U;3`+2!w|dDcdti%JqwQ2 z+g~L`PGg6@j0Tq&hC-9ujO*gBLX5H}uCjW6mG+~)7$&nCTtyj2|GVKTx0rnIJnGBe zimQx%6gAnn`wmBaEo}_0a%HW!$_*h`ul1;}sc*O8Dt5WLdDPbi%GG$$?;NhulRe_A zTQ8GNh@F5)1i}J785olRj;nmF=k>Aa&({;Kg5`|M)tG>%SeuCHDpw;jJ#)6v^u9iN zd-DNVK|6%j_H`}+*KZlrlY5(dALa)zs+}D^tiW66vVmkmR z6oj&@UYBYoR$a!c0lfva4R=~xFOu42s$qq!;CH)sU}z-K!kjZvG0 zBy^>=t8(rcwYmADlw}S^ZPv45xQZgv7`0gxHNtu%(-^f`)WfLF?WnJFGQSJ1vLtDU zQJd9{i}zDm=Ct7|PMT9>j!xcBs`Y$4a24yQHV<57Nj^UhT*a#216MgG%X{D|R{b8h ziba1OxXM*oKZr=;Dz{dFt4M$3wc;wuzwU^uH2J?yb~Rv0T5uIGGA;5k@{rx(#`vSS z<0>0?6t>N-#&*_7GPgLJ~!?|bEG{PZ^t@%+rOW!(VGxxQmdAAl*P`v6QIfJx?O zwKJy<9zIIGEZP%VK6k{kN6X+mB80QSpX(U;|QqAycVwxKU8JJGuEcIMnA?caC7hgQ7N7kBRR z4tceFt!M3GNAv~k;bF!Ifra$qE|KS$%+u9s2bsHS7gx4J@bzD(I=%VJ=bQKo6Jc%m zi#qiN1Y@$=S)*6WL>Kf$xfk^ldkI@Vc)hfFwcO|leOcMC53Sg|TC9(%h&XJ(PDI8V zeOdK~R%~7^$Awpm4Snf!TG)3iua=%Zv~q*V_)het+={+vt?0|mk74ZAQ?q-u^mGOo z3X7k5?qP(wqHAmSYMG!{gz8`8X<*COU&Jfq@M^K4FRS%x!K?4()q=OVtMttr2?YR0 zi?qjS5U<|-==27j28N039ewF#eY796Jw>WFyH(>-JNn`{4eZ=a3;n=(Si%2E=nKag zhLxe_hgOEHkEf`J<$C@h1O?79OhNHc6v`kzilRU%-ub8Ayk>9z_AY4uWu0hWxJQgo zSLBCA5X(DA?F;`IYF`||H9|r8p^@(GYt7ThF#c-^YX9vV|7i0xE}H)gwa>8&HveKy zo#Pik5F5Yz&H=l^Q+LoaCm_A7+G*xKjrL2UlT$8%_UZO^~izU}$TkHYlu z{Iy>MfspQ-X~OwwU=t2~_=ME*GJUwEImfo#dK#FeG2TpROHvEHh3@>a790BF)Q4Mo zZKe-*p3ihe*2VPU>?NZcruT%t1n+A-EzD}4X~%;Pd#0n+Wi=Q^ zdh;VSjOH{4wN1%cW7Kv_-nmel^*k80SroE`+Hv!0v7RkMU#w^4g1%hsKy5cL?UgY^x#%)M-MUvZI-yP_|zORpBNYaAcIBpsHsbwmb^%D_`BfXv5aOjr3B z6>|9)rMOoMVou89!mFi&d~A8OkbErcZ_4ya+xfrit3Y2;O|O;}`YtD43H&;JXhl`# zk)td6lJZ1fR<3j^ttfZMD`u$=%Je~*J}6Vz2ac!j71IZ0at}_vIFS2*sj&~5r2C-E z27@xeXhM(KKpE?_v0x|^lKwJXahXjWyke|xqmkAZn(C!>;x6ZIcKS+-;47)6PF^oy zY#=b>2gU}nG7q#^^&pA#iZ-tol!AD|V<4H}XVWX%KH&VSPCz=U%4Ky7mVABuf!y^@Ri7Xob?MyACUnsy&^KV@Rg~N?zl^;(^pzX*Xi}5 zb;Dh{c)eWd33o~LcAD5qXRntg#BHE|cT zADkfELwM5kG=9-nXZr#I>O6Fos@`;r|Mr}7`~rfa`ImD_E&U09Vl)*3BcWLN6&l3C z85poYFYPZo#((W6seSIgL-Q{nAlAP7^S8C%eEzof)hS{IIsz+}0+*?}Z%B16Q+3~z zs$n2#II>BmZKgY>W1^b3(4AGwFx|O*X2q(Dsk%pH%}_nsz9kPnaapxg*9=&()YDAW z?fm2g2Ue{7Ox3N(Ub+A)S7fVQ9Stm%|GTM2{4*=JWND`AzAD@40IaAJQm@SP-tAs3 zewprpt5|w-4_w94OM7^=jLP~wa22b69$qcyWci-FS~#wdK`@8tQ-tivCgEO4F-F;ws=g?(}J4sItwgnK79jb~Uf}DF1Gt zSIZjsJMeB@`8&+SF}ABB^V`PXt$e(Rt4z@=ZO2t^t>P;ycCQvXT$*FpJqgtP@ zX$LeqdhJwwgK?K|D53k~%G9J%n^PYo~NA{mSGI?;n zzmXfx-6ImVo7<3}J^rM2=J?6y8o8ZgxzXfEZkNwjNJDCwj0_aMx*6E&m5cdWPRnIq zNS7<6+5Ei0KVLIrTo~70s2g=7U0W<02N>*zNIiEkoh_9Y(;{tLn=z`{17|9kY`!>` zG!}icdtS0Kqd-1uyexaMXI#TMbq7$QrWdCdF`;AQno-P5&l@ubrcOQm)bSHX#~Yx! z!YsO&{3CM3e`v`5)1glI72ZU9SEtSMv)qY4&0@ zlEdab>^57f7i*2=b#oiLrEJ-HBRODhV;S*wv-A=3zJ<)CbalQ|tEQ{jN~V_0v5cCz zn-VT&E*Mp-kw$XVe0Q@oo?sN`@3lJLe!7vIFt-%z3+YSQY$K_f zyXR&vrE8hWoKZ_(Fe=r2so40))!$H+ddv77E3TY}?P4B+dMxCRuvMB(MBy7y`JjR< z=1fkYxq0fjGar8XB$jJ)6I;?6g&fk8N+FL6&E2LMS~MDk3Ur(1K=s8&agBN0%L);$?1w#>#zU6t^dF-2d?jb6U_(OUk?q0KaUfZA(bP|K?AA9U^e$MqP?16Agl+YJys)0#ig2HSPz8wn^T`Q G^#2b*JmX{l diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_plus_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_plus_run_1.pkl index e2986ff47c5de6b441e44ad379c558c232e26eaf..44e0c594df6f18325828000d21fed55d86091acd 100644 GIT binary patch delta 837 zcmaFb&U#`AlR^W_RNF}mU=TY+qldpFGp{neC^1DZCABOwId#fpwsN_Nwp>EYj=nBa zdRX!jb5p1EFuTSZPuXl#9xE{UcDuPm273=nc4|@H6mNze*5c%h)RZYevoiQ58+1%? z%n;XXoKg$qa`f;92gZB)1&0L3hdKI$PRWq&VG0hMk|FEOm>~zm^4@G23f@c^iry?4 zO1=$@8Ok6+WwLyycobGsf(e_#zzj8|N2n+@9cX`mYml3>f2d!GYmm{D(xeRW$s0Rm z)d@I-1?CiiE^#KH7M(5y0$Lc@U|RYhT2^&Q)dOu|aSw9xMfNfy(94WKFEfI?%$}j_ z&5@zfBLZ|QNFlOo4ZyC2hb4!*kCAh{hi3>V7+F9Zn5hcDfJ8R618Ax*11KPCx+U$A z0+JJKgs*3CFkv$|pk{3T)NL-H!`CAaP?VpRnUfkE7!u-ImYP>GW%3lL+t@PLGlabv zfUaQ3@N{CBeBi3c=7z<;1Z6V3y%~Wa?u((0F~f86zN=!J=j;&>lFtb2z^RZa!*lb# zt7<}u8G)TR6)^*CbpYGS^WFfzQWl`n1d!70xg3lEAiHrJ%bMZ2{k$Thn-Di#RjM8U DFDm}w literal 82917 zcmeHweRLbwb>|F7Nu>2f$xdP?u{)6y8^77_vX%wXU*dm&3vWyvD)B=YA>Mp)@k!%Hf`3d-lxwTKaE2^ zHE-53x^>stQYxLz&yJf{>ZNx~XUoO8)RiQcvig$AOsbqoCQZ8 zM!Hx{7RC7e^&$CZ@_{6eBAFZSo1HP{`P@HfWXndrP~ka<>EA!`ybrgbny*;~ zO)=0^-%mY=rg$is`zK!U(SlJov|tb57FJ7nQ5o&{_oqs;_Me0cn9=AE7NlX-ZvEm#k=9Vw=lH<|dxSKWqcwZBjE zscfF_(|x+thqmZ0ffgR-br&-RP0;*Li;ieHIg`@FVnoKXnKxUzMHShEUu4}k{&B1C z-SdU%>YVvl&DyQ#BS8IfnM$oXW%6*ChpTS z(OXHWnNcqS%^Yf=wa7>$5RMEpC1#E_WyJ!cQIX|MhSs)ZGBrJoz3SGHW0$%%vOV8{ z)D;bd1Cfv`b;ZJ=z-T1Sgx8L`6bI6!uJ7op`%c`uVzFRgBqT_vY43`~B7xD^TGW;K zxh_?8sjBIX7z1xMI^s@MBf&s4-bz(g@7WWlo~dQ}tUd3|nwO-IVLSohDO2O-cPrK} z6beTI@}&S_Y27;m14e{$}cXz%zKV_DTO2#y*g<`p-)!}#`H0tse4@X16Kxo+I zEl$n8-At9{t{9b4Dqo%{l;*)5ll*5uWK|=#nbvSL5)Fh#+&1$_JQ~A3A9f%6&Y5gJ zHJ5$fG)SlsBez?P#D}AS;i#zE_z*`UqoaY55f?s~hQ6a@LikOa##D7iE{EM{G#bWx zXIRv1YIHOb!78`j!wfar&$sVr>o8-%*hpYxc*JFeV?koqBO>DyJ1W3Oiq@ukO+jVPiIh`>{E=rP?AbBxGijLD$+1VtzxiDl*nQ3Sq%E|dukt8rh zZOKZ3cP13Y#_wBm1tz zQZ1QDJV=E!7Gqk<(6?`>Y2E6@vZrd+xE!ZYw}=5@bSaRZIbSGUxriMa*E|g_GhMk- zGy_wl<(ZkM$AI)S$|91TPuaREU66M<{UQ71Y;C=_S(HdbblYp;UV&( z;`_$$gQuRstyC-glB73I)5|x8cRK1jn%<@O6fn>u0e7y!@lM+& z9xt@(P(_X3Z@noAwej2)+VE+UmkQsS^+-D{GQEgMeN4QRtv;UEq37O#7o-(-ZPM>} zwG0{5!l8X~O#5#4)BAQk_YjTjHohrxgXmy9k=sxGd?()nxdSwUJN*@qi_$2+t6l)P zQ5tf0^S=e!)lI__cc6|V2DBg$8*#`Tw!9dNMTdb?xyxLA^T2nq$>MyIpcs#1TaQFs z#F&xc;gLXWIO^Ws9ugF@fJwkbUn&7mYJQcB0I&$eC1}wga)pBnS$uA;#ScUbc(v~0*PCav!^4od zf^m;VA1Gg$p8}Jaexw8$Eo~;#g}J#@v1~pvZQ@ngv?Sq?a6AwVx==_s4uRah9nFHS zwR0{zpRJHI-pXj8YAhNE$6W;Sa6At2IW+3dYFIm2AV2MX`{MCY5SG+!G_ErmjeyQQ zTHPx~Nh*m-0b*~V1eD#0i-hAN;H*a7bc>N-7|MlM)PwYPTrf*{bIt%VpRF`D8VL?V z9X{%&rip~ZqfiTl-5brlfpH4bW{*;3xy6~r%8&@CLIbcdqa)K!=u2$T-(J19rI|{!4BVDDHIFPpH&Rj6R`Hf z!5Uk!4s<(aem0vo4=6jL&#yVKYrNvR+eEj26Jj+Nf$GLU_P!XzU%>;q3(weFcK zTrx{MM&i-u&}#}}epH-5#BA!)-@JVO87L$w*+O0@W&n;ufdnI!$7yF(0AnA!8=(TptrJlcnEr|=)Sh^*12bjf_uAB=!>xWWd z$VgW%Gczx0{ZTKeY4|M!xHZ&^W+nw=noRlf~@qK94|O#Zq~3 zp3Fq3dD2+<^X8>wxs*=k4<3@)st#viaaD1)C=p9g-D2%z5>{nv)>)}PkR2v5q`;-r z+yzy_PYO1)HDge!Q1dyybxj+(pkOE}K}Dc&(mIz9oh?;` zK1ZzJFZbpiP@Ff7zDc*{R15Bw66j&o$B+RHC$yY%Wz<}ITK2%Z0_m;N)Sl710_km0 zfiLP^fmGrFn-?cH+XC5>HL`t?;dn3*iM#SB@n9GTl{@d_;8FI9-H~`-wVydy6tP<$ z$i=}USdq3YMr%7w97gUSHL_uTB6o=Rl%CvUIOM($e;jN>?l3h?s_?a$h1@YJ+aaHj zdx8qzcK#su9_r>6vjn*b9{cU)19FqRo`U6Cf_06G@T{Uy*jzSu{YUVD+ zxN);7m$!O63XM|8{jE;+xp5EOscm;rmwJTC$<}EA+#3!^&0vFuED{+3tQeBY)fQdJ zo?@z$nlDqkt{S~a6o%|jwYX^Xpxz0>_)c1VwzPaVEb$9-7fmUpx!43@Xd|#Rin*vd z(Kc-ULT-kAjm9>+1lDiBM2L+j>iDGKaF)fJ=}cP=r?Cy|yaDr=F6e3d5Up*ql-9d%qwd>i<8LGXodHFjxls}$SPp_-K;(ELV$5eRQhBPk;l;e`*3T)Fv1 z!=u4?1Qs(P<=&7>rxWl?s z+-lCkYuwGXl$XdMp#lpoX*b=!r6@oih6!v8vdbu~VX^jE90~jApo0koCF=<*3*!9D_ zF)r;Q8-^ST!~W2FbGVymSGL?pzV`USAT^B7Wq$~T;T;zkmRT*gZxA7+IjfO|*4$2!Uci z$6oG*12$%QphQd2v`L^gh&VXv(r`G2P1*ad?O-!WI2giWMw+G9%{B7q(gY!vZd%z$K&qu@=(wGyq9P2j za~QqItsWRxA%>A3k!VzGB-nT%{VC*rtTw)jW1{S`Gx^b}SzC^sg_jmpe9s&BjcZo# z#X@$vc4}NdRY4S$+__#&^XEQ*zvIKyL~aJB&eS)#rryn|DU`MPCMzYom+_tBhP4GZ zKLy_`eWr$Pa%t*6zML(~g&Xgsl0Q}xW$4(_ zwZUzbP*X149GyC@pQmdva$Q#cNhCEl)~1@zt(slmKFKvS`{A0H&Rs&4b0R-ezzR!d z8Eihq@2OdLr7+nfIrCBMtKHLud?i^en@Q$I%C(PJJIHFAjPmf_ry@J)NVbn`&5b}= z{b&bad&s)|RCP*>^v9HE?srsIv>u(f746*2R2ts1mFZ>cE_%*kgU)WJ?=F^R4;pe$ zIa4&#WRO?i2mEjY9Wst_z_N29ZrQW2i=1=xkorRJxmQwW-r^glhB zs`&kQ)&plwoEjf8=xp-zQ%{^eJOSS?;lM?fiu-USs*&~Q5G5Vt?#1x@NIVyOHXu5N z6($)s4i2%s;b1~y}K(zOFr z(e-YDs%^7&fOi~5!y23rT!)>?9S6e+zFUcT)u}ZUTi@4Sv9$%7xE@k%J+KDP*`m(! zgZJ{8%^Iy}csLFw8*XsSy*Dtu4dM0J4z_MKu-OQEUBCRz#h$!l)YXE_DTw265I${E z59Acz5mZQaa>lNH29Qs&yB@ii2m4O&mTL1S;Z? z>$g*=1#);|yo5qHOlSiXPt+RiK!gBe3TX+hYSm&9K5qvy>E}2i(HwZMbz7$ zW(zB1DaRqf@q%w3GM=lNRg(oa;rfKcK;{xxP0Bn3)G^CxIPa+N$Nm|P4?%dO8(lTJqkTekEa5FTTRH>A@(p(B*kDfON;p6q7@w{0oz$q=Q z)VvTE5HtyTC@eDti^`;6G*`$YT-O}RrAn0mNjn`ybr$QxLY`sWn@o}zmrmsi`7HGG z7{GKA9^C-2AeoRK-MKV8n<(~En&dIK-{BHY)QA`bDX>DyAlSYk8d=#7ECF|(LKca$ zqzI4epLzI$6i?@cfLz4@t>OnSzwj_-`U5Pa&1`{e_%>@h4~BZDV8@6{Tn~2aHB5h2 zIs>wR^dq_d04dY@N%*xP`Za)pl0P+`eiSR7$pBY=JCtW!OQVpa{e!shGY za>jQMlaov09W=4WmphN+9m%NV_$~P*Ypdd>-qnuxAjSh+$3=(o^|aMXRcjX1vgJxZ zG~*`*sWBWLy%#Sfi34G7MmS3k8sJLiQ&$Mb%Pv~#TV|r&W>;tvMvPJ zg#f{CiqC7gerc=w?I7Zo&tj3v(t2P$ud`9ut;+EHjjyvXtbGH~&Bxyq6$7a%f|AL?6o~&&QB|cY4gf|ys1Rfv?jH$tiS0`6 z^Q7)O2OkN1;0}6Xc+#Cq!-ayr&exDQB@>0#_^1r{cAAY{pvvXXGx*UqDe*}`e@Gmh z_ALuSefv+ZT(h3HzZg$Bv}2co84~+5|l@lv0_7JwHUuED2+8|8PBg> zHY;S~A^ZnFFo*Y!lwFYyGjh0OOLLmsDU#3bmgu##9&LuNm3!O!=|$6nH&^Y*$bhE( z@bQz6`*eK_sr>8FZfc0pX#d9K*~kuYau|m(ZA9zG-vMoxwiVY@cMSjNH^zVK$Jb?J z>ewgqDOvR?lNt}yF#XgKW2GO}!))v6Z>9_U)X^s{HFb2<gW`-uP{cpv-G zPY2-OfiN`XWSNZhe`R*AFqNA7WS}rJQ#LEwCm59P`?UZfe8N8A0&MR;C5lLBe~kwE zHrv3!nhktDFk32Ai^;M%OD>Yhne1hN_+(36(F{1o*0M^73@KWir?F9PI;8_>V>X|Lilx0jRTikG}pC|!`gN&KzX)^V60@$(Dzw!QAUK|IgzA&G9X-;egzE$E=<1~pn$aaBpG*y zc+EUr^r!!N07m`g$q8V>ph{G&L~KakX;kIv3q4uyh2MPh1|bV;8eL)RZY zAA5U1t#W@jCQH#CvZrUOc8xca`hG>*+#c;NO+DZ0`J$ux_g0+u()QP{`6%scfBoaz zNB-zLKwq5A8Knikylk01FW=V_YeeLq=)Ueqx4hsBp(m$vOKv< zWNYfR^!NYojRGb4Wxi}in^YBLdHX<4{At?%4EYEf;a~v&%>rojXDXFq`QU*AxX4te zfSt}CIGvg{3whorA?)J2N(biVF3vxS{UZm%CH2GBI)#SYIbhuLHneo{&7q4XKHWTC7q z@X7~(zc!4VFV-c#KK%WKV()7YZvN;Ewz6c%$w_6KLmCIT{uss2K2jV@Qov+R# z5hP38fktoEN@@zn4B8cE;>f$c}0oy-fH&hD0k1wXv zoXr=^CzR7A3jNQim>#{cxqZ~6ceb5gGSgLNGbn{qFHE<64<(O1bL#NKiKFtiZ2Ca- zN<5@`PkQm4sw1_psB5ZAdzGrM_AhljFALH(M13iGR{Pp+r2Wq~-YBU0YX7p2vcAe! z^<}blA5N%km9O?YbyQ@m^40j&@q190o;{VX_E&XO^-+X>;`vrrn)WbGDNWT^e2hAd z)r<03^YsCwQhl}mEB?q%Pz`f!Sy-Lp7{foRYwsUwz;dCKoT6R>GFgd7+GD6oX{vrv z)>m@Y65U|OuX+9x$ZM!SA_}Ye&E=b~PosEq8nvM6i`MFYo6F83i_*k?SudyR(FWw+5#Fkte#407Cz8UBY3CEfx`p{{lzF^$$+QP1z6k;g%r5hf#{x|XZah-nD9pT*g@p8Hqg@Dd%i=6;S2zTD5#p*Q!h zaX_cJehY5&j-K5}@7mV2dXKjBzWa&OM<$X_Jbj{Z*1eSMdZTr5 zqjhnkb$O$=anIv@Ilj;LfV#Ndw)EBa#L4i{q%#O2!Oi=xj7O?Pxk zEw=5npF5p6apsI%;q~^~C)T)KuSC=)b|om5YDet%Ov5I&zKj33bA9EzrEl#DxV;{4 zZ4cenx+~CKf$j=)SKxN6z(2&5uzNdN|EJUDUXQ6Lk!z1fzDic*)OGVwsdSsu)wO!h zs&>@1%I_Y#E6`nm?h15Qpt}O!O)F5d5E$4@Cs{G6I~9{{Tzm)(M4Sr>(jh+4D5G-- z$JLn6>SJ$BL>;eLOSeAGMoWARH5eIZgI!A72HME6X>`g5Gy9yEX*5RfN7M^4tvPr| zd{fCSWCj^k%VKa;lwmWZ@pEII!zuLL=yjMqMzLgeJ%LTD>zCpS%k-IN5tWe5yt}Th`^f|ghU)t-*v^7` zCtkWuu>`t78gA3Vc3oX<>fL;Bzs0d*pTa1<8OI_cg>Oo@AOx{uq)Eg*XVrI7vf{4-xrn%5k4k9pgjlcOVTgtH)zGw6(x+l3OY!@H1qo35oz?Slgjkv zucMXz#W%EjsI?=$5A2Dg-jzu0PvDYL7Cwdc?R?+<=KQ7ip%nUWPv0w_Tcd!!7(vS` z=dc<_zGR`vm-~H3zf?Xw_Cn%ePW#&(5C0fHgh#%F+=Uk@8z-8~rc+KMy^btLrwwU% z2!0mN!wEbC1{L#TVnpFzymkTvPKKgPr-IHB2692qKA*;?^l{=T=zd*0D--CJrXHf3sTU8mq#*&#tUidpL_fZ#=mxZ< zAANNcg%vS~mZ_08KYf6n1*XsUQ~N%q1N=ot*a#a>T-wo#ANir}=u+-~zAt+x9B& ze+I9Y*#9u^qqJDGf_L8jcS55#n0}MU^*QZCuXvwndhvn}B+0L4?WTVHtDuj>gi4@? z>}OGd@_GAQ{08O!At%sNm-tgYrN-!0O3i?VR&IzIbo$4lES<`-boytaEE62+djItE zp9XQ-{U5pK_Zs?FZK46yW@G<}KdJZsU-)zf3M9VPasOk#-q8P1&;Ao~PSm_({&KBo zRQ3K>{=DP(kIH_*SnL@W)_1X^7duK~IOAusD}>qu9qwyfV& zB|*K8JlGj7>ys)Eyy~)EBz}qN;VrPNANeZvW48rTM|o8O?4hGAn9$)$<>A$|tY1@C zSeIpeOkM4yt0i@X?Gyd{HT83Ym^SJgoTQ^X-k8h!swxkj5~EaO#>VOxGq_9CQ+ar8 zp&xaHH|ny!_}{6YYbp(P-dHzv^%lBXR#$jsFJn0Jr!RdFi%esK4ePRgear9fiKsj} z>W0dPp9}KuZO&g%`ER5A#Ya5z-_H4uHRs<{_3x+rBZ=nxWtIN`=TA51Usd_Ap+)^m z&H48HFd5L-Pawa(%1mEDVl!w3n7$0PPE23+iZXrORQHPM>%1ztm0G^4^0x6fYSfRW zN-%w$S0$Lf?EB92Wz!JTS5cK@`kGW%Okd0Dis@_diYS=;8VZk zkbjT->0dPPf3y)dG1_2bTKGS3G=l#_=4SZ66%wfhi_c7Gll*%{$_qCC_fr32JNfsD zlz(6RcFDgh@5T7Xh&Z=ljE6PNVBT2pe^{iWl+}*^LzyR3nJ3kWkbkkXP4X`apHyW? zM&Xt>ApfpBOB|oz|D^m|=l_iF!A8dN#s4MJXe+-YW_TaOKyI!;s)RyMYIWMWp6#)Ow5=2GFrQnL6r)DDv!4V zRm#s*D#`ha9-zvlVx@!+CQ&fYE@09Prbgj^5S*n8m~;V?yx9$+!uu*BL{#TC5k#f) z*r0$#MnYkH4A$}GPa8@Zfupd3QsAJ|x6O9Y*~(5$4cLe-un_|cb_+J*8`G~|#%olm z`swZ3=BFD5a#{DtAW7SIw@W*Euo`vnhAr3#6dGRG2s3g`*a+AaNFey(q)9gj-iFk_ zoF%Xklp)<9r`%R-NDPUPP|_9~c}-y>42Y~38^J8Sctb47aqb*t9#~ zzea;f&T+v;mOhL2?fh13P3n{saD>(w9lx?q8G;$>Je^42iF!(ETXWi~m3Ki~5ZruC`r!@~7VK zTj>4b$}ek>;Z}YW)dZH3dgxE|t3-vAfZdN#l=Chq$~7m7qF;ZK5tuqyhMLZwzC;AM z=Ka^l4x*SMBL}LY56~mRvdzY@lk@-k#WvVU0!q~N$j*3Vh3t$+wrPJv7VVFHv~_x6 zCwHdo{MrWgM;Gh_^Y)URv07gHBWiUnJENT@*?9#wcTMb=_D3l@%l)w~I}hZc5+uza z?w!O=R-D+$)j4bvl;5lA0381tVJH8Y69{UPYWy8IO$cZ}(~fZyTRPs z|Mc^pZ|HxV`ghFVz5iP{|AmJ6`<`!{zl5e}{%TEWdiD7$oQqQH^H(@W2kUQq!8w25 zuTuX48ln525YR~T{r>~|{;PXM{mZq&={owK+$;MB8AA8Ig8ubJZT1*3Ktv%ZxU6$H z6+th!3aUKTW*5~J)Gj~^h=Zb%*zFyt1naS{|2g%upi1;p2}9LpZFW%Qu{L{EUG1Qf z=hYQ>$z}b9y26Hu@u@4uNv?l``YEb3)@IxDVQuzR)hk}ajxmGZbj*&?5?k-hXbBq@ z(Ck=`?OWw_Q*E&}+g_@{baTOxjkRuY?q194+&TU9H{*vWvpT9ew2Fm__!X{I@w z9+}$gJQwW5F3HqpQx8+yHAPEIZATb8x!Hi7ET|GL)OJJ7z=hf-Rh}C*$JmKYQA~68 z4BBIJYW)R$wO}XbRlWyyQdId&U#-~5g39;6PHg(~z)r5J`X1Pc-M$BQGN$T-h!l2W zbZ$bUuWk}MS^oMjJLF@=PFM_-wu3@G1}(SR&Pe$fS;W#c+0Jyr|DlYK9#H+(+0LxA z+77R%+XZ7UytMe%5Iez=Ji)`kl02zS1pkK=FYE-hNHXF3 zumS#WCE14m3qFXO&+<~9u`9HAMCd8z@%{>})|LK$d7M~Ia zUq}MF#7)R}oqk|KAei8naPcVt^8)DuKb%1Dzz@YC;o^rB(hKs*Eyqm~u0ACjvZ=v< z2YH+?^)K${lh+hq=)g^wMQrC&^73u-DZv0q7T|>>r7)j(@r8-HUeFUaS#jVd(xzH? zn4oF>JlU)=U$`;cMEfhWq7>RLxJhsgpOWDFP=8Imd8a-l-w%$o2{)0zh?h@^8*bw2 zQ&PuGPAJ@@)u+UPna9lXT#pou!Hv80dgo|J&TN5|N zY6-a*g}vlrlxf=kQ1~gP7*bxWg^Te1=jBtfl!LlYim`6E$k_MF{9Y*0^&@{zxQI~u z_aQ;5-FMKbPeZk5UChv4c*9irzg?ZNrnGFaJC20yIAE-}g@F zcU7BB{R@``>VM_;>itU$gF2Nud^%OQ4DB#&fBI@;|7Xy@bN&J%;Q5Q@o93@D6za4- ze+f}^GXJY}pAM}5^$#`9U&2n*ziN|ua?D?17)|q+7?!s7{BJhQ|N2$xUo_A6UqDpc ze|`Qd_51HWf9YP*$^4fON&0i(BDVe&Sq|R_OlG<6H^G`}8?RtZ^?9Y)-AW~GeKHg` z%XQR)b<_eEnN%8NFoZxfRPDRyYE|i)85e2PRNM8yD>|?Y)>Jp@s0A((RQ<4~+U|#O zktNkDYpQLXHsc~URXx^J7gb5dMQ*Aq2QD(8(%NaN8y^SVwwd?9MOL4exP%8TV*7k} z;3DUh&AkUMazo{N;3Bq9fd?*fP1X0nMeOlm-U=64Q0v4r*NDOeRofZww zvFm%_BKG(_aFMI3ebAS}MXrwELdgscX0q|o<176JF$K*3ZOo_SX84zyagmkX=v?9= zcoBN>e{FFQup~nM#fgxA!IF^t`(0=k^OcU}Ux!c0tIDSY3vh&Q6;6(-lVj?niT}eQ zJ)yEX$3@Vx;Qvtll&ao2|Ceys#kBYok8RASg#1Rld`igQMByUAIgHmKT=D1~e*|vw zkuGkMPE`se@qJ?o;??yx!8aW^%GV#~eg~|a6*9B@(M^1JIf+jRBI?z2A?d_P4mDkO zag%haT=5_2;wD|(1Y!2(=)3e2{1tLP!3N?~JS6u|yh09e4>*JS5ys#WUT*W@wKm_v zofn{MB!*Ajjk*Qs8i~b2fno8b-!=VBdIWAV2IjlP-=ud;ANW3gm$r$(Re!N2HXMaZ zw4ty5j1xC`g<^3z^o9=Hq$L)YD=z*}o_24FkALCN~COCEbCCEhVhd4(s7I36n6~j?4 zdB);eb-_)Ryl@jc7T32<;Rdw#W!Ho2$4$oQSz!9ym_JGJy$D@q$Knb%#p06tA0B~L zdqRmDBTMXm6ow7Y-I}o&un{)lCaBe7PsqZ%M=XYyT5uDl4bNDF*wX01W}`t;XYvW@ zgZP9LHhesv@|BL|=EVE3lw(xsHrf+nwbtDKu*OaMpLZm}DQ{~+!j0#o zUK*3Sseh|A7S~P?hrlg*uz&p{;U+>Y-UqFFLck(B=v1iCHN8h$c}?OLNR>KvN)-^s z4xCB|L44Sqmecd6JNy~Y|0|CWZc*=FwMqRey?tZ<662vxCB{KRk!3YVWbJ?Y62)q? z=l{z06K?RIIDzIbJ!ibr8?YpY!} z*;iFb)?_#0Ca<8ujW4(5%B^KAD%4fZHsU;TIs>)}*cBv&6*EN;TdhKva zEH1l!)@z4bVsQmkeaw4V4|6Q8CABh4b7S=0u&HfKs@hC#tLn-HH?jL+YO`nVg3^^3 zH(6CPW||ACE5=RiHEpCe-ClpjOQGOqjP^^v;++RywOpsh5t(ofR$?E|Ge#H-1)x^*$sm2>wy1bp|{0u zrY`@w*v(+NM=@RImXE2ECfo!o@r25Hl21_Ugi0ZZjH|E7|Dl%P|Guf@Ur+vTB~|DD zSXgmHVg>(q)4)bX^2ICb^Cx)(w-1j5el7ka2^Du8{v-=);3oRF`u-9e-51fC?Z(fo z%0IQASr*y_jU7`q4X})L9aFlFDUE$W?x_yo*!y}PE zbkyYwC?oM$G!Pfi%bEa7uK-xiy8$d)#`L9k;O*I|W6J+qBa)HxZR~QcYoBleEb9sA zB^8VivRnWba5!E73z(?&0W2@6kDYL^S9JGsi-OVJFk5}>1U%zAdG0vbfP(S0junh` z2Ga##$#^rESgpr-fGE?%U?Qc7!9?LT8BEj?3??bi*9_?8_}EEHK(7S_ux#{WCx3;B z*#L{n$4(jmmezn?H#-7YIt=LL_OTNOz*7IP$%6f{6PZnSr{TjU@NXf?x6`O@Q$R15 zkDaXPn4$m{7snK?D*%?nqv*`zV<)e>2lS!|xcm}3(2@jLmKp#S`(r09jwv@?0G4Gh zfQ1Q)1eDwKv6CD0EKvK`HwLgQjX439mXDomRKUgqDE2=*t-E>Mk#egH6JKiC|J(&^ z@BI%gkm88p6w>qX2`S3>q)EVj&1L^{7qA@w%d5&T5epzhKrFzT0yYZY+Wl{3r@jAq zi`eagT`xBPEUh0qx$#v9-~wPk$gag+Qn1S)SWwTa4_M&T4j6?~6)*|~getvH$EolJ z*+Hl3V;9hSxBuy5ztPZtq_KbdV;}W4(Y)04_tL5Q*hgdk>SHHs_iu1OMs|$B{Etkb ze+duu(WKSKMsSKpiOex+dIHL!{ypZeJ~py;|HZmv2=)I_&;BX)p6uT-e@FlF2GRPj zbUy#9SDowcyGHA;0;XYlj`_=bO3l~jFQF{D|MtgTP_6U%FZ}`tgeJzyZV|A{R^++> zER8B}jUIlgjfGy;7u3~Oy0X>B+vv(x2eYzUIHuUD=zc0;sM-v$3@F7ktGutOD;Jgb zHI>IIZ`+{-yXLYk98)G$308UA{j90-w#UpW?`1VU7l1_s?6O;CmG@OuzES1f_&n;i z0@=ec#a3Vo$AK1r<*HIZdpM>HsDMTujw#EkJ}bZ%TL2cjeO7=UX#rU5`oeLb1z>S1 zz?Eall3E$2xkfl|Qsv>XSZ?~5&Vb5eYCEs4wo*wO_}E5Q!Z9VNN-)hWDJo!Uv)f{7 zYXn&AwwT(Asvf&RHBy^!OlbsI7F2l`IPa<&5kxn~m>FQ1RC!Eo3+jrgO*p1Bf_nDs znA&XWY@{~bzJDH$DR#b~uNHvihN2G-#}u1BJseX8RDD5TEdWbVkPw)l{Mo<7ONNJLPk*@h0uNdZ2^|VQoGukc8eTa&UbN_F7DFB zU2+G&@mP;Z2Zb)~(#2i)qaw*J?sA)P7r-kDU1y0L?LWs_DH zZ~V>TD%YNG!BtxMO6sY_am9hFfD3{MgCBUFH62$_M&K%_MTR?E3z7+b+@w?;dl~Ma z*vl3ceE4JxY_Llr;@^P|qlaX79_y5lMfL|YEl zUtd?iFZPAHjkrqDma8|0tE?XW;q926*GJ^r$klTHqkDuWdIN#5QLD*tkwtZ}|E*Be ziH;2y-S7u^S`Vskt z51~wcx))Ce#N=LW3viVkq}La)NgooV+I$C{s(@Qtl($5$j#Qyy@1s)zvFyO99dL_? ztL;yBdbL^>KawL{ zK{VeqfBQo)XkK{hG|fLz?_Z6zgZ>>Kf}#F@u4(>qO=Lp12$F@K4H(W-2y|FJr* zg#LZ6QU8wl%OGY=_rKo1!k}pV>-S$mEHr(cEwh_|lg?u--Be0qR%I8JvUjVdE!V$C zvW>26H7_f%-6DFes!m$#lbfo(2X125_rOhVsQMnb ziS2mcft%QlHJG--O>8n@++>pY6}r(rDq%0V3vSX#Y?Eqvnb?*U@ob}31a4yA4ko$t zss!UEb{-R(rt+BBY=UB9n^bu_sb#w#CN}Y*lSbUcF3-5hs;ccmZ1#wl*n(<4p18>k zRl>zFWkKaN5}R({e-GTm&iBGi)cyCsP3-&cftxI-`X0E+HI?syn=Gq*58TAg2N5aU zWO*I9$$6W<+Zb+g^Ix>ZP3i%?ZoOm5y7{|~J`&&2hfZF)h5Q{$y|;+J1E=-`2!T1~ zC)LTCxXEVuJF**;0bOrAh+dtz$?`e^daXDYxFb12~up16kN zkHAefkL0zGNORYpTW;3^^Zbg52ddImpKd!gpfY_;w>JaO#c znPU@&4+d+QzRX=RW2dzp8OEcJ8D~$NdZw1yJ(w97-;vql_vexjktbrl+?TfkJ6*bz ztz?W$`nhDWRG7)mnc@pF)8@q?Jriqu4iO0I5BM8kInP>I%Zm4GtM*J~cIGP9K^){nX

mhCv)K_QWAqqVgB4ovCti-kdL# zu3XHTmugnr8OBaF?5`K_(|}T?QtFCW0Xe`y<<6Mab5R1D$DPe2(dmp?@(%_YI>R+% z(PANJ0RvO1vKg3}ugKAsGpVAfnto|lrn}u7q#K{Pf8u#x<{g&2rQPl3w!*m4a3mfL z#Nw^)bJ1`VUwo>6*J%>ZquOzDGth|S&LSl$PsE@)VLDdSS=qFJ^(sf`aZf$A3jaap0yJ#7jSCUN*RbNi z#(70~A#z`X&Ib*ov1n=qt*w*KoPF}CQ`oPqExf1Iav9{0Yq=~Aw02vjY1gRbN>I96 zzVh?6@t$g~icGBeKrAvG8o`>6hQpyiG_GC>4SnSP2K{FGa;rFq#&y+r?pu^bE$99R z&Rkl?yCwpBKQt^~H;paR{cmX5*J;Zjs$e`c8W@$YxTbYs%4%r2XB{nvg7J|+FzP;L zu^LwIOv%Km@?xv?Xk6B4G(3#eMQl#-Sgbk3EgcULEyss3#NdeQvdSB_9a_OgYET2* z&G$VTj|C&c!-4iUr{0PhpK3)=4_XNgM~3m#y04eK$Mt5^ooH<)9*qXVp|-14Z$^=d POEd7-jSQ=1%)tK#T}mVg diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_sync_run_0.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1100/profile_sync_run_0.pkl index ccf76d979105d478602e44eb21939efc5131dbd2..ab7ae141819c7a2861597edb3cfcd79a12c4caf5 100644 GIT binary patch delta 1121 zcmai!&ui0Q7{}kYP188{qiZLb&TidSW_MAkLQf*ytX<49+osz=kf2@l0oN+4Llni% z)07C4WO&$4U7a4pOOYJrNzkK+e}D%MqTZ$luRd?rb^+TpkU-$`eBbZ$A>{oVVf`i7 zUF6mxloEDbHbchFs?P{*_@ABW7<5LJbEeMaujI0(-ji7>+`gI97Um0D;ch--R@9CA zE@Pk+=?AyY-N-KHO_}JfLI$$)CMH3*ajUc%1#x`Ye1NZl-lZlrX+}+}+FW#c)`S7w zp(ai6%M=Fj9+G(o$PO5mIT%rj6h_%2XlR?_rKTxr`xN4=rRd(p>~%ap5lfw)yf{0P zj-|rp(p3r z*$tzz071PEyE4$;_GGnf!=(_P4o_-RQkrSx*h@V%fFkXwE10SfCi2~j!)I+G4^$%* zNmbit2(^q#cq`(e-9~QP-wAH;$plX$AXQ4+pL>sjEK_u<)LQZlf+BI}%Qf#1Bv)EO z4upRE{N9Vu=N7nBQUG8CK9RHbNpFFy= z*6-0Rwl!1JoI-!r1%rOvpF)9F|7+SX*gOjjTJ8^f3Y$|*q4*UQhSn0^>d%4f3Tlp{ OOAB3&c@H+I?D2n=`B26H literal 137026 zcmeIb3wT@CbtZZaMM>1lqUqFeoKAA0_yH<11QL8fja=wuS)nMClGZgKR9*k`{V>+HSuzt&!RzWv8H8m2+dvkRe0XE~F* zc($0F3a3*ug;cS$P`P`wn4iwfq{frEv#BHJQn~WNrxvb}D@S{~!;|@3IdgU{KUWHMhl{D?Oy>DisGFh01-IX=|3

8WvvU2&dbt0cjabwf> zk38RoPM^yyRIE~JW;*OiAEFXeR$LV%_=Ps~GDk}!7AkMUfXx+iDlzg2?M@cYmiAb? zcRzQ2s-&v_VBPAM<}zij|M2taw^g=NfuUlmG&fULB|b2GD*cXT%1J+PvpG~As#{Yu zFPCCc8@}OJ)k^(wnoVYMJWg*hD;>y-MhV~6!`$zx#UK`H|F|lMx|0LRPjyCQI77Xk z!Pryp?^J1ZUW!`M_YQxw((&%u{M6h`>QTfILJ3m^I8J$C?o6evR9>iTo=Qz-W|K2; z9oDW-EmS)DW3hpuSgdboC^{7G?-^M5_yWG{wp4B*{b)5b)*I;!_xBBP=-ip~1~nt3 zH>EeHJJMVA+m6cCDrqmIcXp>cPo(b-h0I&u zVAz}gPF^pvy?MeBz`i%yiG1X*AszNSf)vAc{=?fSb6Y1L5e9BChm7cTnF@ zcyOR0-EEe&pha}vIr$cl%uUa>cO=mbM#Hflf1(>42=@;*COW-zw`RH;MxaG>;O1!U z^~`k9NKY6`Eia>K! zOetJQ7R#CBOh`u)PI!-%EKa8PGrK~r?(Wl(XNk9MExmW((i@Eq+I*dhAp0Y|k%91F zZ%+f3FI~8mOKyt<>us-Y!K`}6ks#=-uP5Bor|8T}f_;7AULO*yw4XY5;+chXN9FFP zGO6==+ie{MAdp^gO}YJ%K`iD&ebL$}7eTT9aCATgIX5-gjetmy`Oe(j?3q;2DyLJ{ zT)t59WLQkx> zCmb12nYnwYm%d+FQ2d&(V>**#e8D2Lz|(-$jHYyf+F6>0ZEFd+RQYKd_Cv%gj5JgV_UD;!m%EfLgHB(Ai)4M(} zoqNFoZCD?0Cw^hq9_zUVBO^yn9kAYwPDN!C^m6#XsRQA&sj>%$vNVD;?ZS?Td6azbG1H%}JOwl) zA?*q6)|i@jR2vxDjoND-9MIlWJ~3CAN|sXzbZfb^-}BBzRNmnS`w(c6x=;q{#@OCi zWn+fmpM}c(QeiMXK5!RsjW1J2yamzFt3l1WclZTxR@;(psISEi-GOTn&3Xe-zr`Zj zGZ+c?#R6v;e1UUkO65#>uFR7nCKvB|5ykWu9vtlTn;82Cd&5J$flHBN4%>_wS7q+N z*M{F#`Mw0u@R^JBh0|B9))Pzu@2Urm)67GtI4!3d6$epoFtdj-BUXBSvkW=rl06pX zuz#1X(=Pdr->CCLb{Op*9?Kr5=C{6nNC^m;t_Ax4L2{ze-v0h@Eb9M#4n@Mz!NBjc z(lHAuMkZ01^{`ljF)(NYQ3s19h>MtefYH5S@4@l%Xgw zMZGEqH)9B7s~7y6YKtFZxDCrQ@@3x8Q)lo}k#MZvuODK)y|Hj_oow&JT3VW$1!qN^ z2G|F18fR0P>5DaGM0>-rKL0ZM00;L5SN$%Y=z?o@7qcjXV2tkGL!Erf=Qc9B_{JTGs!|J^{%NDrcQ6)#D*Y>>Gxrv z2fzWx!bAN^O5-=IJ7zMonKE(ZwF?kZN2B4sh)V5h0)qNt;T|32n=y0dn$qf@(X4xq*;cqsTA+5r(X`a?-> zJ>L&f<0FCe%P-@eNCbPB;EdJAbC@N zW4(jnh~KP&Vlj}>!N9y7oWwold?AlbXW3%jns*8gK(W%_#6BpMJDbU+_6l>({Bu`j z=OktpV#M{PWTDbmj||^5;`Ut|PHGGbl9)4!ut~zhPpHrI{$W5`?g3iLD3&R^KcVTx_fY!ql*r)PjdZDuA-LshzHbe*pbeWBV z9~k-D_R3~G6Ju*oE?(S68yXbNX+qdF?Xf1y7nlc9DF!xHwiafwN8#yvp>k>}HEmTr z)Yi(nun)VhZtIapE&P6vCHAsYn#+-3ONoXHv!pD*IfX7@t4K0@cX1!x$O5U^S9oJ{2QDXi8}Eeb+6Np#j9P!0OYjg=itkk$N8={=Gz zFo7zC&nIV|lNce|RwA!yi+t7Q))^NvYiEU3^lGR7d3)te)CZ#4p#}&`A+u++>$>1# zQ0(2LBAm;WT{N^aDNv(_LaH0dg^H#p-U}m7R}(|kT{QYgNLn~wd8f%s(IU3z(3%0Q z65>+Yf2uf_a*)I~+p`CVxqPeMGkSKGcJ;xAZ_;e|0V!jTg?&Bt_A@1-_lC`~E8gPD zwVFz`cUyaMf4=^X+mjFU0E!Gp^MQu01o#zbWq-Am}&hBiS10 z85-z=_FFLKd=GR<^E2mCn)7tX_#BOo4FYBI+u{xl0h$WV-LX@A(@DLNgN#RSd6nSo z_b@5if-|*6_XTMv|;7w0JIVYG#KgOmt+0VdDH?ZKaj7} z=5F^EQ@JUKKwU^2!0ae#O!#1{kX>Pt?bShkd2eR`5dwH@u01F@5(6ZJnnUd@H3aRD z9!SOl(O}0c^>%`MZ~}x1VEMI2GYt$skSTSPL$uYP0WSwlNbDUZEsx3u;4nDH62cJ$ zIUj>C_&_9%C~3@6`mU}~y(vH%qgs?X?Tq$!lmzuIB z(*(4c2OY&CLN-dp_Bo2(P3|^T(1_8-B~><{qZevR+rXu98@R7VtPPdT=Tph&sEL04 z^=?jZ5Jhwk&#U}Cjvk-MOkIe$^Z!Ar7e& zH=ReH=`#(`0;(aIt;q|SQi2Zl&_Sl1Q~6vuF;_|@X7Yf1N(-N;Y$pi?p>X*E&V6)O zcF-@umMpozsi!TF>fkU2iF8iPovH1g1KeC94;+*_Fml-4M5dr^c9f8luzt|??yfv! zQz+ybC@&RG&Xx0LQ@JosHie5*xnvYal0fk7Cr6Hq*ln(~Lk4nw;geeoqy4XOQtVZu zvN>N!oXMaqbPQ}`=?u;xEeyxug1$wPT$@Yz>FH9cyfAFdc^WiL1Hetmo~3SXKl#My zs9jig27aI1Xc(XPnvtD_`-O$drc!2h7OdQGCuHib&Z)`ksV7Ez-Fdy|lF=5?IfW|i zfH)(l6h6f%oPvR4<2iWsG*QpC!zbN2Q!V>mF{^ne@riMk)C*Kfwdo(AMyn6oQRg}M zD!Q_p{cCo!gPZz-JK1|a*4F+WsYUi92*fOKN&S43f2#GUsR9o{2fFQ0;`p&s-UhyA zy!62jd$Vqs=MTH<_sI^!c;yvvh8yghe#BYHYVG`cNl^)^*yFC=dw!#><3o0qu69?m ztZA^f^V&CWZS&70+pwYDye=8rZBkp;Kx{gC{E34Hj=S6W$xWc0Kijm?uBGa|O1t{l zK*A+OdGR{U_93-vHKc~QR()PSERKIFT5#V}W_~RUY^ASxF_o5gvy|#WWh?Oum>F5s z>)nOo*?pE~Pfiw6lOzVH@)MzC3F>)=`TXl?bqw}dNMndcN);8n|A$25^N=geBxlb| zC2<}v5qfeoSq_E3HiS5n6Gz7nj6m&1>CTXRWEb94GLj(eA+LR0ysBOZ zp3tHXtAZi>A(2Y=9+r{>;)R#mE8Az1&tHVFnPpoGtfxjNI3b=WUOe6Nte?p0Z`+$g z8M(Ee!U{}cxrNFR7KbT0Q}HZ`C(vP`aqe8IFV;rH!;wmnV5#!~)T!*k-ngU$b6L*$ z>lK#8+gNSo`%@X@D(~S!z7UFZhdEb$7_w{Rc54r=g)CdTfj4!)_m07rWM757PlR+F z{P!r3=R5W&rhSTjG!KT>eTqK4DLa9g0_x@MsS7C_9Nz#{-i6@@D;pr4V*aWFhq#kG z*LPPsMB;~#N;`ce3(qbL8_9G>_9^57#%!)!m;+<>EZC+@PCMsttRBh^)dRdk(E(_$ zK;yjz1alwYO@I1Up5|z&z51!n-EpLiBN)CAWDNeXz5(cw_11!?Zn~*#0?G+RFNaJG z6W~oA`u-5Krh!9DW8T*~;Hr>N&mOC%H|A?wp>@!VPTJiuB*ri_=x8V!>5D-3*#q!U zNZ%l|qK5`+5p)lQZA&Duzl0X#EX*O?%312eAhR7Hv(r7jgU^E29_eWj(H%uZyaX#! z>FW(ghP-Nqu}HLUpq~yx4b-Xvc&KPo8ETjS0&XP%&4Yl#nx8GDisj=d;K=*ua5Od< zR>g(D-h;yIH>Hr%GDUGmnWA7a8o;5~o*Jc14vG!LpgbEWjLU9{D(-IJOs#2CD^U@2kme2-3xHZ;i?> za_f)uKx8XYKqhE!|y{TTe8GV>nJ(uATyr z+rR)cUt_g=tmGE0$*niytAbB;>y7k7(_J5c^l7O>xZrbiuotsP_AlD+APH!j));IZ z{Bs-xROrmw4))xx^GlDLmCXsI2e43iRO>e}RC16t_V-v9s;2~5Af+OAhjBc3u9!+d zr4&Mja%##3J#JT3pEk2!r5)z5t|c#Uy6!Mrmc&`o)NH{`Ve~2hWXH$ekDN-cf3%ach(@qv)|@0!L4AW^9Q6uX>8oo@_61tIT&Fs- zAA(ow2q&LX0}t&v((QZ1dyOFN?V5`oH?x03b5U!Ozr`j3T9Z6&&Trl%+s)^2$4&D6 zy*)UG*sBi&xVA|B&`*MTXz;nX?7yzrI6yHoiZg&d8?OF=7z_pAxNy*m9uHtJ^>nC;w~C*b|xb4$v3vgE)1LP$>=6!K|&? z&`@6w7~(;nIR)d4!^_g6enV4H23-VBk zNxRf((wjAW&^k_f^EL}w$4N(8W(U@8EU<+n**A!YJ(3_3hg~BralyUAEpfpu6&JYe zr^g~#l6zs5so#I?ryXe>!LVbsF8Hk0E!z)}m#x*>P|Nm%ZRDtBBZ%*Y3SJ-XFx~u? z*rz4iZ)xqV>+Uswv5((kjT7NRT2;%l?eE3qJAPW_vQht1%|;!-13&~H92Q(b6j+vA z<>Lp#yzI%1k`Sn)?=$n%4VVP<)oL}~rEb6BlN;gJI*B(oDDB09bE~o?6+a^ztYUWm z<~vBmtYu_leMUA~3r=gnX)QRnaKQm7tLwC<@uZ zc2EvNJ<>XG@{|X7?;C_7H$(w`8Pf%wu{uVByVR93v-xwWS)A~-$drj2h=(}82Z**G z*gY@^2@Jx6t38uX!WM|OTkXwwC;}<1PVHU*2ZnHFV<=c&P}#05H1qMXnhHba1mn>@ zk{p~^gL&nM&wXinnXYuw&8fJb%o5w%o;pL3!0#?Lzzn#gjqT(FkKs-jsDOcw>5Iw; zxu@Dl57~?BLQ5^lMAnc8;s{G%H{VlAXQsa*qwq*m*$IAmdo7SREpno=M`SVr})^lW`PMQV4E* zXd9N>AV`bU5mt4=P&JH1!&0?2r0&fdVnQ5%@XZ@SRWN6P=Qb)GsSAaq$Jh~3L0>OP z_iBtCQ4o&P!XkMc1(Ca9ph5MnkJUHM0YcAkzqT9a`eHD(Kzxa5B#=gMSdBh%U=L(V z-qRD9p88-0Po;Ke3^tU^$*E5AqmFG<>$8Qqsm!dGq``SZN7ip*f@xM~^~asWMr6F1mK;Cht6?$`#a2r1l2ScGv`ceZ^{-hzfk z%rW7~hX|o5>`K*>;}(Od)CF~sx2IeBhG?@8(T9S#x21r7=w^`J+Q4ex zLnN*9_NQu?y@$wu$VX?n4@zDT!`9IjsoaI@o?-VPU&17aD;eA!(+{m0$Tod*?k565 zvjQu=L*?w-A^oUx6najM9v6u;U|Sq{!WpK(jPK%WsV2#&My`yDs$Gj0lH)rb@cUx= zp?!cYTd;CfMcsl1A%?QU?#>t%hH1m+BT$Bp1m8^;+!vih*K~bUF+XEXgJj&59su={ zRkT`NxIP474~zpyLEVx4pWb~Yl3+&ab*d2R&c(rgQWnEU6qs8(*l%aM>Gr!`Xb<^x zC1 zjwGVJi9RusW}&)lG|Fk)DrVcZa{9j6*Z2F_H%!??=#R1s#Fn!LIfzY!J%*rB;cD=L>w_!Gj_D{9pE&8o7nbyboggm+LW?wdKHVl(&Fxr6IQei*s2g!b#*9_eK z)hv#}YSPC&#jx!%DxiD#!>WAWioCO$%X^WFZmT1)Hhktye4HXX>VJaX$LSAM#+i*W zycSRgM?r4Ga@7Df^5!u$<;{3SOym@0lE!~=1s$28D`wU>Kg?1Ma zpUr1KG=*_M2*+<~wlQw%x|NsCx2&+-VOCg(3b4<`@fxNACx*pA01>oqfAI9vB9QY_ z`cmzGc;`Gp1(4rk5%MLFJ}3i3HGDmHrn1wvfvt?EE>yZ;FIT7jr&ZF(XWdm5s104~Yu4nCT6&GFP_3Hut`hher1>~ z^dsZlybg{So$0$np-QX?bG5MgO6I!vmgWit+|Rf-J^6mVX}+@!=0hg)g^QWoLgk-}@7xfEL{SC{+2Gn_upNfSB@r|DCrNnB5toWwPU!>Nm*X}$)U zD*XXvIRlAQr_q)3Faxq5mb1gTGZ(QD$6@?bE_ntwChmtl+sBR_Kf=~%o4PYnT8HjT zkRkHI#f6c-*pTk4e1ePBCQkCjux_ece>kbFkLI$zm~i(Hjfb**~A^#w~{L zw4P5D^SE|#Qm}Z~oXyXsfZL&)aZR_CPT~rcd~W679 zwhzCx0(*JLPTR|?fO%jAJ?`P4YKAiE^2+qR!;>&Fohm8s)}@ZB1hGRZtfrXg|;Rc|C^ujZa#-QX7j;^%ps@fi$Fdbb_mE=P{9FF z2ygXC&2>+1^X)XbWsidPMmpQFkK^}_oS?G9RKuY*%r<+O&}e21m~rs&B_IvkW8tQT zv-%ECHclNu>JKV1h1GC;1)A&&db)&VaP@J93Sqf*HlaRvPE`ma>pj-)0`6nLO|CjJ zEd7@D7bGdY@vW&;g52Ahnt|2;Jor23s$5in;c1u*PUolaf#IU2)040$OqMKNo@cbD znurc5o8VL9Y@r*oLtzG2g@lkdtqbQeld1jc$7z&%{!YLXFSTdCO_@;64D+5Ac8(aj{*#Vlm$=V_D$of76FO#e- zm}fibL)?JLY~kR0anAn3M~*)Rp=HD8Cx9t=pAAMcDZQ!gw0C`94lc8yTWI*Y7_K5N zw=sV0Yjkf5->|}d>WUP)_=fH^p=(L#IuSs8xcY-4;JOaur&|A8eeedR0Vee3|L7?I zQeb{={2kq+_S$Q~8T{+`#}G(hIX~fY>O$EF7xQ#`f)Os2i{<1Q17;KWKd#kL4|&j| ze`yx0_dxxgqsjyo*Tvbt^%5g&C;s~tF|YG^bA$AB@dI7Bg?oV}P!U`NS5BUV(t?v& zch1PH=Aq~P!GR%Y5y8?&6-#r3z~GCmHB-6IMoKh8N|)i$27RHjQSnS`<=T|;M!^6xr1mN@w2$!8LWo;h^<$VtF0XP}Ds z+(M=8>pJ}PBbV+SG}J8|;V$;49!jz77O zU7;FN6|#Q>*RYxWWA*bb`q`BI6Z&b(ew%*Uv#;O>g=YU$|NeXY`jpyJr___{EEN}P zH7K>Wrqr&JW4Q6_@X5rn!%rs;96ELEsUz!ZaC@Voqcuf;pA>!Yz=`3+p(jo}cKq0( zQ|l?+*wm=>zM9f^9vwe8k~sS0F?Xxm$=i3JO*PP_8fa4w^kyy@_a5>%zX{aE`PQYc zj-$tW4<#PQHEW8r+Ueb0rj=#l@-6d4Rpy9cItry6>lV6&3mrLj^5i;8?NEJOuT&!H z6lSHFJUBn2v%&n;MDSnQb8{1sTTyO#58QXlb8KbU>VY>-4+zKk#;I|urmY^R_P`(i zJK3C6{cfi#sI(ye+gX=-dCN{u;%pN~z^}{$LG6(Ig2J%}sQXAd$)|4QR$!|KT0PL} zfmRQ+df*Q4fdx86o0?28Fljo0$vP@hP!b_!Bhr*|sYod;g1)v&2P**KhTDb8(9t*LG6M1WD3M$p$GCp!OlC9+B$OzWu>^OQeRNqfO(MY^A%T@5XkA zsq3s*`2TJce95KQ1NnVu_vs5^DvtGxE|7XrF?k;ErDyk8XMxxxXA+eC{_-49*^tAB z?jXnIQm){t2P-2~UoF-}IFZk#)HUQ>q0d8qXMh!*l%4}>Lj~`#tw;A;y^dD>kvgo?lgf}TM5vmyncH8WXnu!T8vF8H>3djN0SXyG=F#pQ zZb{MaXeqjul@AU^$@;Xz^nwUe24M{VmxahFbtU}h)MAs9W2a8}%1yGrjST}%pE3q) zWM9ME!KJ$m5W>eg9s0(9d20UEv4_}i^TCPlt|%u$ z=U=2?s$JF3c!~UIr>_0T$??cRpC-sZi7(+8sNW*55WI-Ce>HyW(mlq<=8=*pR8iQ< z+u;Nyx3w8#7M%21bQm#w6&76P_}0HbM-L+)j#p#zF(M>O=b%20l}8Y|*_cOg44__)R|aE%S1(_MlHa`OrTx zZx9vQqcZ*gYUCUh1nOrjYe!RqYtE+-)NsJer9niqOStmxfD+7WME#TjdI=@|PJiooz#QgTkI_9^{FZ29nao>36H#9eB zenXcuoZk>f&2PvjXnsSS#`9Zuo1WjE#fNarpcQKDFUZ+WPW70pKMsdlT8&ln`e#8O zD|I`4g3beE&}s zmp@qK5d4_)U&K008B_V@^{)4#B$YkYV(i;0PjWiSSJkBQSN@12bgaAl)h|2CAHP85 z`}V>LJ|$`YAF3KtORwAIk5$_*ebq$!<3Hyr|2yvXuQ=NO@2llI+V3b|bdJug9v1_FG@k<)gGkbNnjGH?Pr}NS%$Q zESjCPs?Zx2(nZsfFeFo`+&Wt8mebOV}oQ8e=i{^yHz`BT> zr4Ha37tMKj!^G++2i&70COC~nvmi0CVxpY?Lgmk4^Ed@Rh1sE&TjYtN&T zscvtfSXaLPnNNL2!sypDB>a8~A3x#^UzPB;Qg~O+8{YW=o&Wc9_)mDlMe$=`w;G-FT+J&_INOTElaO4ec5B{Ltk+j z)y-UwQ7XqxU!xL(>1$Qqn7(cZg)n_BN?2q1nx=9lBsJ4lK~gh)*>z_6vZ;XS%cex8 zFZ+98`kIiIEBg8%m2*SF5Z+Jw~EBZ1{ zd(uxxIMdhlkT=|ta7ABZ-f){f6n#DK4L2k`)7SXtyy15KXw|CO^?L|63qyDj-mi8~ z;A+1?n;zPKbp2mQx&r=hO92Ljo>_8mP{uYrH{|Co$i8CUp)czJh zjrqTr=KscMe;dUAO>D;eOZ0W+=Rq3v_qSsyUI|c@)fL+G;gi5KmVxs%gZ4AYhrW~C z{hhJh5-tAfLS<71#_V96@x`o(scd-tkoeNw)q{0?ZP^awb|-`3Wdg$=s|SXc-d`>! zX3qtH;pYqGVmE+jz{UT&F7XzSOHdHshC&_c*b|LRzLj@;)|e$IzhrVhX7m5|p{vB-{NB;}x4POEM*VO(AcmJORc^4{lUUkv5!AFMKS0B`M2iFP^`rA~F9sd@>>`T&C20+LE8~2;|)q~mHAERl5+5w=u z4N3-p?hygigFyEW>iH!AbiXX&0igS;fSv)M`=XQ|0J>X}KiVyzyA885%~>vLvuTK_ z&5r2v zmh=IjyPZGiOF;K)YXIF>--Co;C)W$QPdtK^-3_`k-mR6&Abr(J7sBX+*-+Xmyjcdl z;elU}(glKC`)`NR1^5>r3{bp5oEoJIu7n44=lwT!EicvEe>*_;+o*KGte^tj8H!g* zWwgt?6i1tk&EH&?(gnKYC?%&JrQHQP6#s`H8fx|#7Dr>yeLEJyX7=9-bYCH*GMCat zC+Q;)eKiBRPx7jZ%B`-@rZfO_r!d7(#?6K7eoSCD4Y*HqwE*|w7U13j+|&1uJWmEW zmAy~4-Pr=%`A7nbw+KL2vWCcT1_mK_vhPg*?y=r}TuEGOrw6xu!5nt9Nx;2L0q(2d zu>^2u^;Oz+Z-|?tr!l2g0rv^KH3X0l$M-QK|MjEJ1>Xa>Yb{GkS_8Q21t05yTJR|n zRwb~Nc(7pI9N<3YqeaeAc4U6FX2EA2-SudZzu1J7{hRdxALoMqq%8Qx%S~vJqqmMo zZ#^cCpama2sup~t@+|lW8j<8)DSH)g@77x67etHP*0P+i0e1#z752^<8fNEv^@6`n zz&$d9j06>})fh47k7HLZ;XYdAth>{YyjMQgK#P23?ONnxwq|)1T=&%???(Lt0r!|F zkoanm2LkS6wOZsXR8rEYwF2&@;nE_%L4-toPB=Shk&k+@?0KR>dsO_fY{Y;v3=}I| zM+D5}DR?&=Fmt&{Kzo_^UdnErTI9>wsd|7mkfn8Lk>exupWm9@(D==7vZ1+JGru_m zinnn?fu{!@Tth{PEWaPsZ9aemt8bB zAu*WR?D$M=m!-}?A{VQI7N#~!!U6&9Rf)kgXSbl9_{6lwKLF6aEaMXZXxr%n0PVN{ z&H;e-qU6u?b)`m&+)f_=XkU}`0f4qmzo0Jxw3pTZXkR9{6a3$L0qxZv`VGv7L0aS_ z*}X-8Hdqo66f|eRzj#g&!sm6?B7gA~YLPqnzo*b$akTQ*X_2FE6rc^q?cCo60osvG znB<7QngO(@cx6TNS667$rvdFWoK7ur(l8f5+j<{jQ!u7I(h|a7*P{UMPu#4A`D#nU z+-L#bi5B4fj}Y+w@l#tch2i-g+y`o_n)mfYhD4#9Z+mov4C1y&+o(!?Rr3uB`8L3Q z)ga&Pant%iG}A`D7n68vh%_ON?_*B<+mAUp1F!uQ97+afKu=T1H~qn*75PkoSY-c2+l@0Fi!gnVOUysG9J z#tyQd!X}FjeuFAnRVyc^#-nP!g5z?x9{E0wOyg8`1N$kWf%3FypnTW1pF(O2+J4G} z3;CWPWRdz@knd}~+C%eaO=)pc<&2e_J$PD`yO4r+!vQlpox2}1*nWyd>@a1wPUJiC zz8Np_?NsG~O@kohn6TKA2<~9jmS!s2R@$p4L)jqdo^%=6S>|P&2RejtXGi-^IPYq z(Up3mvg)JCJ#m%hCv0Te0aKdetE9^P+w9Q+4-KqtC#M2o4ReDr_FF0ioPv^)Q)%^X zC#Q~U05GcmnAWcXVzSG>a)gj)L0(akMw3y#26m}@8+nGWAiu8iwN|&$MERY+>@5HK z&l2)1$iJOEw_lKd_Ab9muzh3NjYiDLHSY_?F;@jC|X9F%}t@ zupJcBMgSR$v@KzPEA&_8Hl!TJBCp9CW0997XT~D!8Zj0*FEJVUcB7C=mHURIW-M}9 z;@3sKm)xqhHOO~d!UK@+MF|f;zB^ygE|J82Pp&FQ&Os86BoJn`)TaIwg#$?V7y# zAm4VF1Npw@LcR-tf zkNNO_=zxPfT;RzLiQ_Ok5OhR*9uvn=aj5+*;&_m6kPK%BM96(G)Bi{IHVsfJ9)H*?aRdA6k;HwHg+lmjYp+QN+X+jmqPZhk5K}i~Wb<|~@s^BlJ&9(_obu|VhtMl91 zRKd|DK^vkq&2Najp83sTP}~cNo2xEByjB(b#yJ}hg>{kZHwX|{aApVKOtF_iPBCcR zF4$?s+5mduyT)t7YEkfmgl>qT-`^8?0r+fudbNh8&snKFk8_wp-zvgQH6^j6A-KSLO zpKwE{{nDpYQnmfkSLCYc0SicmQ$Rq6C&sD6BrLV=zRyByS8+E(<7!f%e)qu|aH1`u>2p9fKiC zOLAt2(vX}PqP#3`K7jbDgn=OhME{_!tpgArl^6_B+T}1rIWMJRIdy>R3{e^q#t`MA zyfH*+!`-Zw>u1}ed<*UzYFyK-_QxmNf>&jSqppqVZ+Gva80R z_>!a#Pz8@mcmN=NO~L~J@lFXxy9E$mlAc7>93mE`wzz~bwb{MQ)Mi7#Ol=bqa}(uK zkTKrO8r+|xIO*>fVd5q2LR&s_$&IVQ3bc@F8~m?0rdbt+;Y&D z0OD&=1viLSrTxb`4T`UR{OiQYIstL!ZRR1vH0Ej1xNQm%;Cj>^0BlQ*LWG0=18e5E zFblhat2O&u|D%Ax2E|`omnwL}{jDD$j>aA2ITMaU>_EE}AP%1+67-ljj!GN_h$Bw( z`&&d-{2z)~!~V9)|J6X^YXZdS+$+9rG+>o)kqQ9BNkd9Si<=Wo0pep(-bG<*^aA3b z4O@-eT>UU2h=+OQ+knz<5nyh$0P_}L-U7@QT7Y@Dm@3W9lyT=N*-06n%jIyZEnVk& zTWqKF^$qlg`xGqh0nGb*W8r}&HOM#C0n9O1!`74rPt0pcOrZ_YDZ~kSOye8%0CPeC z?i>yBTW323>ku)@&1sN-vuVJ5%m*;X+U{W}u_0ztQyp+JUSKxsP`??!-m)pp7l z0ZJ~weEB8;^UIAi$QuCW^UVR~jc(vvc^duT0nAy_fAAY?(ID3tJ2sLIt#bMv-L_L4 zfVtRCVYr*x0)B3uRW!&Q#>&lk7ht~Z1+BtAKpK+sQv2yyCdn6Ao(DHt%X zFA#jO34!2CO#tTp8bsV(-V-vLQRxFL$3fqy`3tH!XY{NE$MUTdX$+OO@AP@w{t!+EpqH>w)k)c-vMSPNi}>~)_~RjTdRy6V(^ z0r$zN!S%JZ-*E#emH%l1?5pFitFoQSueRS&e#7>gqdXa7gU#rqvi4n1Y1fTWUK6~J zF-jZF++-MwRnrfQQCfoVbx@eH_B}7D84>T47(N)KjU6&ZIV$N`MSMx#7^7Tvp_Ai2 z*1qkU_$ck#uHsBfnh}-qYG~*A5Tf~(R!qY}E;?ijhh~JPmOoHC?2=;LJ5Sudg zZ4U>N+=P$`6PrymOl+${G}~O|FtK$?Iv-;51H=mwp8;{3pnQmJUScw_*|lY2D@e;7 zfcUETF(AGysp}D2Lk;n34FK_F2@e3omn1v@5FeHBb!v!Tmh=IDctOHJL;{E}I+Y|dFwt7q&RRa5*O7_^}ME7-9kTnof`j2-YfDh|cp!AJ3Th(kloY+Vq4 zhdArt@5bn|Eqz9?=7#kG85SUcOB1$HR(y?pucSdvF54)un<9Ytd{cmUu!i_6f`8NP zOV&KPM?4fDo_rABZJnnDn5WK#lQT2<$z(ZS3@0Y@g^T&Qa;SB?=Rzp^hgidh|0w$( z@l)wZWf=E7mnLembNBA&&QF!3I$ypkeNW{rhz3Y^I_Hc-u~n>lhhIp)xw0(@@Rw_%Z=Jk(mkwoc@8seKFO7GRF6I$u}7d?4Bvj`-c~**7p04adaTczT0e z?OEA+VC3+=lZQtR?29adl`1XzzeYMnZE;+ci?&hB*Qh|%^&xkKRunBw8z ztZdfv*FxnX7Et)iY_sR~XiNC^-Nc;uE0rC|QerkWn=f8Gmr0#ps0`|liF1;Pu^J(C z=3+UOEEba&l`ten@1^c;`O=;?vtOl|W7wHZ0)@vaW3@oh+mz>z|;Hco^T|sqq7cj*J{Rb-;SJ)nnz0R)Ss*A2@X&d^S}M zxxY_KCWnB}zH8*Xnf)>BR4Q+ERv8x-CsDt0ekOw&N)?)$N9Hk)gw;rtN~vP`II=kM z{<-8#C~0T72lbk^-&Fpssb=^Fu58#&Lbs5bQB=o7M&4qkdnylepT3S$%}`%&PoytA zR4bc9NN*%oweHl^R6}zN%{7>E^i&gjOLeO0Y|B25@452NXWy~Bj zlR1+(TbSEpWoDCS)yJ!NiU?AFkfWewXR#6==1QC{gjH>xC03)HnN5WXVOq+|nS3sh zOU|bBWKakrYpm>raB}VfDy||E1F^p#N$CyW|FlIVV40>R+DgroQslucVfN-?c9|xr z(`rS>?I2)MPES6Uy7(-rUQeV{pv@} zhpOH4**~Jf5}_wYljTt8i9#uS^4N*tJr=pp%SXo#jDYQ-mhRtG%H*Egg}0gH?3t;g zHCeu}hrIS#nAs9&G9=L6jqC9cNlE+TrriUq?zG!g%hsBH?{I5Pcdf_| z44+EBqw+RQwmd~(a&m60Wb^gNowcUNB9XqHp&?nh9r`vD(mOa1?z8o6zJf(hb`aSK z%!uiZ>>yC-?8^u^I0~fr!$*!k)@GV522->hG&MGx^C>;k;!Sn!==0|?p^t=2z z(s2^0KuE{azXkmYBhc#cq$_Cu0%5#(zr4Sl+FyQU%r+y<|3Kjue!&eM9evU;KKjMv>{Pmx z3_q7D=D@Q2*ncp(emYF}#E*UKi{a$d6d2!9qErCG@iS*<@@JAWpAYA!r%S1_@zdcF zpp%ZD3j?D8fB0M`H}&%>ir#tsa`yEcGY^!NF=j!EQw_l8YITz}_r?N6%qcN<&n z?@Q>Le&s(zl13XQg!T?APJZ*>{h2TQ`nlA%-r0V2Oz7EJCCq#FBl#Js$Mp|7-bf_U znL=W6E;E&wo<=kKho+LfJ<-(QnL#^su!Gt|&yxQB)tlenFD-o*-W~EG`9*nM{o4N; z#l8AjBl=0FBFBzL6A2W=|72}7$_OVS{-_yRWqwOP`+J-J|Y9IRO zrcFbe|LzlCGmLY8rY62?aQ{^6{~NFmWX*V7(<21Y{6KerAN_e6uj+7!t@#P--@p0$ z*Yf1Ns>AhOz@g(_#zn)J!pt0iT+}a2w$>rIHa=Y)_ch zUvLVU*|}MGf@Fz5F=$~gC(qzFjeM0${mPQ#U>g)4qj`Yb+-xF=OeG$@(<1M9KLgAH z%#Xv9S%Y(vlM;#FnRh8o76~Mg0JND{2rdy>5EJ1HDyAmqm;utB64qdSOdNjZ#DS4x zhwNcnw}a@5cuS$bAbwN|@J=(7mgX=?FY}uGPUwU*uTfdDZp(Z}O0ECzYp>-cz07O! zE95ERl3r(P(Dz1pOSsH~^6O16^_SnbBQ5bOl3wOr`IUJWQTXwGj}OC_{3N~2UwQZ} zmD(`k;?Tc)@k{CLc|rD7;vpGJI!^`qby}O}k_PLJlKqRUE7E_$Ha&{;$?bf+anB-<{36asa7+4_CxN`mFZGYN ziz%yX*s#+xq$3iY(ps#WWeELVfS>dooc8lwy>Nfh_E8QgSG+I5;g`Os8H4XgME&bc zzmWYE)bx{^jCpi}mHjGShRwM%Rq28ghDiTF`q7HIZ35Uue;+sjja|4=4TM03VWS%V z4w<@8jqGo8G1=e2Pxg1|&ujQ|P%r~em0#D_&&dG2uBEmLlO7O$Ro+{_tsZFgK&uB@ zJ@ESUz)xKA<7R5hZZ1}Bx?An@y{bphIwky-;4(q&m3Tqn9=5qAnzUPZKY6c-*z#!g zK&uB@J<#fbRu8-ZdSF3a&)epNAl9Z!-C0#|>}$(*V4<#DzV!-@HHwX*(ybL7p%Phc zM#C}L2CbGXtFu6l&+mvxVRn*a>ihNwt3*>u9Sus)ESiUUo>ISdhdFfP-i?AUIn`-M zM>4eg^o1}LXF(=nU9d7atC&2G_tLX_EG^(Cx%B?>TmgEb4j6)b}87k)Bszm{jF0(ujeVESfnQsjShx;6)f%o%n=fcIso&4i<`~tw+J`aX!#;e zip#~#_GLh3+_lD!eIDmDl>u{e`NIg}6Q&>h@E1O_f}2iFzRtG2#<9`fOq#}jyXrW$ z-1V*(k+OPhxt>`X$Hk^NE*oT)2FGqmGQy{ANu<<4*E9#&apx>?&K-}#j_1`inWb?Y zTVB`2rY{CxY{h4l)jbuCW6SKUF-!BZ99u>Q9OM?kaflu0fWzW*L>!Na!YobIj;9&7kTni5w`=WEbNNR>D@0V+pCQtyOH`({52@+gFf13q z;^tU#_n2vs0MYFS4|&YRU2rndrW@uzcCFF%%0;+!Xroyu zaGtMXrb_5wLTl{nnRUDxKM?t+Aw)Z-?R z3HmYPMD(DK^_tzf@u7d!fr7Q2;_+YqTfh0uLK!t?865K)I$sSv9O|A5e3o}{o#Eq| z-{7O>H^foY0Ut#k{`1>QGQS0zWfEW?X8XO_X=?3Ux=K7CIG zQSH^5`3=og^BbDm{QQQ1RXw;ERS&v-a0Pd+prYHUaJdIUis7>S{+Nr^dnizw>u0F^YBR@l)v5ey zGwpjAP^g&os@9dNBKL4KF8{{QJKI10Ih3z$S%J#zuA}nh9tikW+rJ|BV8FN9{xy_u z{2OQaUH`Yc{mx-_lrLt@P^)VB_B|x<^$xRN*R!8l__w-zye^v8ANZeN_>5_r{6Y8O z1_@eEXk^>yE%LB_L$XD4N!~!7j$0g{%d=>@%_7;xh_Dvm!sIBGV@Yaswc{oMu%IYa z!T_%eK$p1;)5MT3*^nmEYf8OW3x!3 zQX@8tWWRx3L^&s@9J{BWBIRg7U9(7?Vm64)Zgu6651MK}W?aIdW?xC2Fm`qYx5 zw~p!SvV=kJYtbej!@ME=&-68(M?Te1%((XWGkuMJA~2lkYy5NG@MX!L>1+Jgyy2H6 zoat-)+urbN63+BB{#V}ch|ssDum9z#e?iitRjZ&cgqx$Jvn*KV8Qzf1lhWM0HXCgrw%kJoK0`6mH_hLH zby55svaI3nR%jFLch6#5oAFYU_fXuT{p&ZIHoK+op}^-_2ix;nT0Y=iylnByJ_+S}oi zz%!O%TBb8c$%np^-2I)g>OB;%DzjHF-mFYx@$NU2OruOPjmPTAG+Jh_{(+jkYRN0G zuCz=M@wcCBnIa0M-+*Qd?&^=l2Kq=|5$lbHqkMU`M_hpjRikIw`r-;xDfch=i7VRT z<|3Uu6ZFa~Xdy*?ypYyizsFpix=NV|oaYf&5JnMXMc>QrC4qf{-)MO4EZ80wocF77 z_}n@x`VMi$ErRp>*geMZYObBF-V#z}_s}@S?xAmj*geE)$nGId5S)jc=}-`hggort z$~GV!ktGU@we|3CdP`fc_K5PY!^)~@S%#b4>#PX7_r8YW1oLw3KE=GTA2|l@UChp1 z2yRkbvGS!x;)%*{6rIeA`F&ItVLWgW}(&CyJCC7PHdhbtCPDx^erH?F*tVV zfMHmG+X#}z<`|E@1q7bTqqTY7)s%&RpTB8-|7uO}O_@>Wx)-k_1#1(peBdUG5 zMqGiO_@2&h-FkkzAoE*yTkZT7u`p)N?QHy_3Bm+W2LII>afM+negpwZoM0N}mA`~I zW9;|YgT#nhvap?;a!ZJ5jQyTUVZv7-+a2VTTS5>)L;nWm*7}cmxhl>;`CX3?oG-U@ zAR{F$@RV=gQh`FX=mF)+jUwb!$iDwMOjm`>YuCV@+wH$T>n`8Xel6KBD0|gG?bK*} zqY3#|+b@09MEeWw_J5Je*L{jk*OCotzqE-;l3Pmb2;Bc#!bG)dzWt7C#nJ!Q|2v5@ z9R05(OWgmec}=xn>e@v6m$vHu2blsce;9DJxh&8q$Y;^C0a--!3NUQomjxIFNw5-$v;4xVVAGrX0+4U0@IN60pg5BDIcl)#To8fL@eoZrZ$@rS)8#X z{may5kIp8_*`^Stws9YE#)PEyp*BOpe8gpzg!xdL5@#&B#2Imk&(yXmZ%l2Y(mJNL zf`s|pA`+LFOl?Y>VN(>-+zm<1)HW~iJ>rac8J_@ghCRLk;tYFy1H>5x8Q%bLMnu8` z#2Im^Uw}AcRKf$q8J8u0(3gla3boq=C(SzD=BVcX%q!~?XUtD+ts~AL894U07K*Il z{~Y_<3+v+lz!0rXoDqCWoKs!p&8RXl-G;FiaR!)>~9fSi8I!^zl~cc%E|wAc4852R-ECu zMFcm6XmLi(Eh1x%TSRV=IAf_L&S+hG?Y`B6PjIB~xLZ70;tViGEpbLmoH0{YU+5cI zoYB+M+v5~x*m91Zo>)W&%$;c<&1h57j8%s;gF#nQUUD1MOp9b@<}- zm1dyZj-&q=QT1?p=vuA3Qk7;1+E*jZSeYRH&HxZ$n3jdVPHyj-NHf-Z_@aR{BSMUM zlhTZ_FE@~8jNQJ&7sR4^4qvQP4`0yD=AWg8yc4n1uA%F z@lK51{NIo5Md?bY;XZ@0;yQz2UM`@j*oB+d=OCdnJNxh|k_LTbX0@&i3i0)0F24gV z-VJ=Lv9v4hJc3bQD^K#Dw|BEYM)|Kp$kwmk~!v6Hxy3K|<3-n&K{B=ZWlf1*v>nnu53@O@ObKyfjh%a`kWq zwf_Q16O^!~gBq=xNA1^w6>9&=?`UZboVu>$tC4>=*Ufy`o@6lWmv^GGRVTT7u zGwkpHX~s2~e*&Z#%Mu6*8Q@Jk z+ALsF8fvp3)!L*Pa`^To(Pp_S+AI!f#;w=Zdqw!aTO`du-*^sZpeG*Vc@>VM;&@yf zG{S3?W+0!y{q5TLzsox@tYX| zzdq;RU3ZqR-yqd$OWM(rcC@4&tve@M_eekA1`}`kR&bb#n>hs|c_ZqId%A&HIhCN{ z{*hmxQ)JED_xQl@^S*aDH!rKW`5f+;J146`jc8BIDeaJJ8+v-8L%4uRZme%0?%1Gj zW&pD!wx(=xM|<2fevp??ZDDHh=ENP#@Av|(#{q{oR;q_LMBD*p#{aDD`o$e!UX-|l zB`Cpgps<5ig&hQ6c)#%)9#(u;9pYdI*P`6=qQHU3vm5JIFX`jdBb7 z-l?{xUTu7IBUu%8#4WVjA?yGmeq$}RrsDgM-UFQ1KLfBPTT?3y#U0Op(exbN(5Ewk zY)$E4Z0w;v)OP3_^?N1SZtM)uGEzFBg{v{#%lZ=i3Q7k41?I+XXm+i-qcam;TM z7Kqh7zfHK{7_Y4<9Nu^U+hHYc*nqkjJ)S?kM z1sW$`h0@o*+?l^L^YSm-;trI5vIHLzRr_MQWb0`r-&O6SDmBSPQDtN zH-J*sf6N?Q)BZ7>(B!(vrv!qBtd;f>CRK`K-e7I(O<1KH*H zh&vXg4lM4l4}IXs+oGw&9itM10qA9UV*q+yN@a1!WeM}K4rIq<0Q!ccbBH@^h?)Ut zyEohbwEKH7m&MjjfVks^ga?Q_RwX=0+#%rs;tu13di(>#9itM?aCEJ>!%iO{?pT!a z(QXlUEJ@#?Xs@laQ3+#lhuzCeZJiS4Bkr(AZZns|hd1o_Omp)xwp%EMC3*P>*K7)5 zYKu!c7I!Sk8&jJ)ys`A4rd+1Eh{RxOv&&&>D@dt6)HWeuEbg#lI>a4zzcJ0(t*b|E z=4I*s0C9&sJ^|tmd;9|qZ`kw~B<_&%1H>J17k$;*qMDcR0C9)?{a^@0+_Ag{afd}r z0joo;Pu$V@o*G+J4*t(^cw?RXA9y=Z6&^Sgiw}p(IKkJJ;Ik%MR99tx3y0d@`inc( zwZDBanEzu=udX^DoeYoDb(%i>9~ihp(i4Zp(UAW`D&p|D{~Gdth`kxdQI(#$D~@WVzHj{80`aoSVkgEqkmBxVk(!gBw;t-Ql@>A&eVP;NKli zUk1?yB{(+cW zW8LuhN4t`TSPpr}n5?lIM>0kvjU&e{M{Si z4i&!$2jPYO$3K{94X|AbHt{Vi26;*NMSpon6UMBN18s9y`VkU`M;`LZTJb{UQ#EFV zzB$Mlqq)uVi%9jjq`Db?u_@c7j=MKHEtKlu7|r-U)BIvH$3J{FI*sn$Fm|Cp-tRQN zd!w;2tJ`?|10SI0_(%P_H^wX^Q1TFVs?Cy*F)O)ygK1a^1M3;H`mT>WjI^_$)p05CjG7hY>{c zoBw{;aktP*fp>2pq?!FNYV`mL;z^94A9fJx!;W>$Z*`Bi(SG=9jWH|KqWSp^ebaD$ zLx#t>5r{k@4$ppgWe4`2ktGV1<88I-e~$Uhun^NZzb&8Fcq9B*l`$*V@sAsSiTO{- zL)sya*iP~QZJ-I#1byaUJ2@d*Bb!)eyGTCT$tibJRO6Db+)V-Cu>NDlt9MhN{H_ry zUm^b;C{&+g@RYAbGv4y`JsK$Az5=9i`HMdb{=C}$e@60%YWw9#301|n{rb#B2Q}JJ zzK&}&E`Q0@{_#KXv|k&jqVnZ#3^Y$wklU{$KUBW9V6~m9B6njnF8{{QIop5zF32Ms zLq#*SNnTb4PUV4UUuAcO5SzF z53%Dj%}vPYFtyqA$kev{p#BQ@_{TMg=|gSv66TPHxQ~ByN(`npyBwxAyUr{RvFqIA z@ejKnuYUpZkjqlOrZ4Z^ z9EQ+efIK83=`nyJ59wTkJmhALSy?;k9skh#TMLz|cl;L&i0AkRd_Z7$z@*%A{_iE( z-(D5|Z{7Rb7uT}Cb;v{5sf;xfUUEs{5X{MW(%1siL2 z@P82tW9H=l=I_$`+a}~8D~`K4aKuB7f24VrG>D9A?&dJ#&%v8EQJD7pj(;e5$jUDT z#aa18PzdtFbBp99R!d&eIzG}mKGHfq(mFoUIzBQtQ?~Y7bXI13E|*Ic!^OE=hy_&S zr$mC0;Y_-(vXgxgOlNFLT3UGT@JG{wm7R0BOqt?`XOo#62bCA*&ZK`J{b;2sFBur> z4-XCa$x8cbFfv-&mYn_Ge>j%^3n zqVU8(esU#29HcUrHDRIhPJVLvitv+7T>cT+htzI}-JE~4#t$Cyn1y%3*`&Orp}DM; zrq7SOL}x{)6250eNH}?rkCNEuNBFA?JU?uapMBEUy4Kg!CeNA#^h`Jy(Vu8qId1IA*yN?*0R=0aZBnCs& zOY-K1s9oQix%h&{iv#2#5eW~Fhg_EM0C~t|nO_3rA(o^MkcTWwdd913&xhFQ1LPrg z{n2cZhuB??rZ1XyM>4V5WWvN|%SxEoI;BII*zDohIy zGVDxjtMcYUY|9enkcYTQt{^d39%7fn@{ns%BOhX$moO$a`^|Aa#QyY{4%^ zWBz}A=R=n6*x0Q z5C{jK1fG=|b6DNNAI8mz8N_HT5BZItIAfm*3gNQBafZo?*C&HwE&TZJVGX7yyB{;@ zCvNuA^p@bnXbDbQf|Hiuq$N0M2~J)|!O75IG#v32oWy#2!%;E3mEO?(k%bo)Lg`JF ztp`R9?>l*T-Y(s$c$fMS&V<(|*C zun(#{qz+N3!)y6sI5C+oT+GjvL$l}hXnSxt=*IG^ET{WQ_S2MZu$U^(6>}?N5sd>;oB>3noV9vlxFhf zQld0jOqM6p9MP(DQNa1+xm3w6WMR0oa&NUX6_CoE&E!&)^<6Vl$OPwL6PaA%xibsH z@yh1h+-%~)2iF2u9DU;7Fyyx1_o5mbO($4m~CRgc)9l9TS z!;${}e(q52i0s3FVvs=xFd|n6raLMd`RMIJc00m`jqDEmP`_=QDqk$57P299bEU2H z{KD}3T=qT$qPsi$W3hpuSZpXd0F#S@1A|nX4senJ>+-KSo;X<;_IJd>MB_obz!*_1?;wk`MiV7=QMhz?Pi%}&ipRBlQw<`?GY z#7sIb=9jZ6TI9D@>JJ8e&ELkg@hWI%`XuF@wsqE&V#Vj(dyln{e~Q9%rijSffnQ1U;MG98@p z3w))PCvUHKu_Zea4@N7E5i~~77}1Pbpi8hqw=%eU`5v^EOIb%E{UPx#ulHcJn|}lk zIUw?kLW9YeI$PE0! z7s6>I;}8q{EAAV)t~R*HpW3`S;8o{M(&Okn1DY%PTV*aR4)lvY3iawroJ`t!q2UF` z6d1yI;G#aSts>uy2h##h;Dd}!S>Jo6o}|cXlN;wToin3-I_NI=M%g^(b4I&E`_Sy zmE7E=bEWiDIGdR%W=iFy>TP4C!gOvXGoGG1mpOhuGgn#qMyfs45?-)h9GP7fx{%QmNkk)?TXl-roC{DrnD81EQI7R<(d)Z~1*QkYFWl_||-W~c%qAFS@?n3<_^s(fi~a;Z9;whmcmR28|Yp}y$Qeru|7 zshBy$f6L+ATqPFm4Nn&4D!Fs>h52%*H(bi3XL3(xLcLt2dk)T*O9y9iXAc%HRkDS- zShz2GaDFaVA!-LF)8!1ef3%~zCpDR#nK^^UqZy=}TB_ceo3>^-9O|`lWoxcbu}&4{ zGThkgU87HTpws8)ma0}cGcz4_$@xtayb1E~+3GGXl z&XxCD`}RF`VXCaEe{aL;m*;a8uK&o>*|%1A5y4O?Q=XrxD2n%toXOtZ3Z3jd*IPpM zzJ@hL^X4+lY9rVDs#D!^$fOY>)|9p%bWb^BCi zGB=x^LFkwceR8SV-JeJdB;tM1Xy4F4IGTtr{m2r&?2gRbQuYByY=3_wJlNOIvGZrM zThxe<-Im>+?aJ=dce|@QYplJL-P4=xK9#*K6w3DCgU#N)+3yt1{9yJ?H|7HaSC4jN z-_pXjg4f>qck)`X?e#N$_JBV}XCtF;HnV*j=Fx0CkTtV|K^&R=p?c;T8Vtvyin+2! z>Z1~gaDQ{Y+b(lKo9WzR@(o~_>mF@yMy4Bx#=}GYOg9j%@neI*Zo8hk+asMv5NI9*so8{rxJ+S$uvgAQBY5XKsG> zY^G#YvKec>P%OKP#-b@48}uu>KNcGb$NRmbxfJ~anRIF9l2s|C=gQNC(kvE(1a4RW zV?0ghkM+mHeR^uB@1g#|!AN+ZZ@}}B3Z324xjDucEJ6!h4d@>nf($gEa@Q9;&=-rM zZG(XY?g?o4K^v-8vLjv45SC&KZB%Iqq)A8m;xysuO$yn|opQEz1j&`cES$0q{= z(V=i6xKPIe^>)BplbLea>a&i_Pfz3J&h=pu%|C!Q{v%6zy;?oc02tDU*+DSGLGT1G z^S{wq-I1!{QTa@mFfYP`Y+Nk#RM?4!S-@&?<=kAklAfE)geZ9m_{x5(Tq*ThnVE9N zn%?`t>A7btaEJB5`pnPl-ETb=CLB#c1wEr3X0@k0navOokRmiJbv{jK-uT(vxfIAe zwck3MnFPXCPR*u^ggpr3r78tRctVdpQVdhd(7q=whS4}Ce9_9yS&U5V?|tV?x$SMq|QqrE#K8>o$gIWVa`%rr=g#+B1Dx~I%Q|U@3g>J2s54rA{MD=bz;D=y~%*6_D zH%9l8)vY-KfR?IvNrfTy_yAr&Hoin1@n$4LuLd>ij*({|Tiueb2m9Ir(M>ZO?GYF( z9vO;;^|0p}xdt#I#`}XuuAcJz*>WXUnXmAmh_OYFi?>SQ1A~J@VTbT<=YxRvOVBs^D=kF`aX&e+YZfYuT&g%*FJHy_;*mbAu6*RHSl=K-q`_FQe03WMh&gBj zFlAmSL2u#`vSN^H2LaeSge)0nV$lIa1&$`wTQD!9R2B5ycJ`BC)}6pXT1WDGgEm z(Qv=k`S^ADZWhMJ>)xcJ;DNqy|DdXtyWqiCIMJZSJ*uwIds>wOWZpsvdSs_kAQnsX z^@n4=)sEu|1D*)(59poIA4*;8{%+#&ek{#=KgxJ~us=MQ2rgKyHb+TUh#J=Fa8jG6zK-XOX$5x@Qt&3d!Dj9kNu7HxfSf`c!0ZuhPut z@9Drqe6jk5nZkukiR(yG%?zlfA(Y2e00Os*WXRoxA%-h+g*mlufwC0qD;O#~ur#uz zdMg%0MVLBFL29Z@W|yiv-Zh&mV-198;FbAdhykPjy|W6OpGRP<1}ddX_tP2%q&ZIr z3#I+mWaT1@J}Sq+*6PmUES4oY)0wHvv{iecR#Vo+`?28awGKRB;rBzVrkBOh{2Xbs zls>pPOBMoLQs^R9horc#C-r4wp zwX?!1d9Bm`ytDcS>I1RoP%j8mAWLPm(7ISBm)HwP#WzWv3J#tPMYtw z@laEs3SAE9G_Y;{LJjvvTxbCm^%#!~h?oT}uH=alG4^Z0n zJaIDvx^2+rH8VcD3FnEy$N)4yzVpOjbRgWPqkPsxHHAq{imP9PMgb|ru|9UdA+=d( zuzxTdX@uXvJR&_?rn3Fl<^2(?LkG2fy?*(O)!YzJm|&>+cG!>>X3l4{@ad!5!vs0l z2lc&Q&-9~VSU(3A@0dZ~a9S_spxV*PUZpSl0}PC|zDz9>zM+NO*IxV8mYFL`f7Tug z|JNN0ql1w|3}3w8Xa#eGL^v`O_{CSdZ3*{aDKj?(&1W4NhXJV%BjJF;lYu1oLn zO4Tjs=p`?;EliEuzSCn6{C88m zD537|V{+e%{^4d*#&eF0m8JY`NL1aarnVDv(*^XI-pl~=9}mmyOkd2EQ?#Rpb};Rk zD$G?<^W{uxrU2Ndy!46cF49pD5?3fCxk(cAKl6P=j1p`mr5`(fK#nBKERT-CiiB&gRe-+W)n+d=?v!mPUpVf|*5{ zLfgxQ>FIK&vNU4NyBai2FMykpKS$l%dHRvDF`HO^2G4$Mt6_ZNt44kn;b)er+se7w zSwQn6-O#1ib&jX3#~vAr)|d63MWZ94OA1vx0dq!BDSV1k*#3gR*7JDO*Frrzj-9SA znQ9sTwplAXjZch=q+TE@wWfcN8m&HTN1f;KRPmMF>|eK=9o*Ds>a)G);~kyvky_+G zib#wCi|XfN{8OukrwaTEy3lP$QzuWHaX0W4P2jO=;R|u4xg-V=Z|dz@BHbut#&O{?^Qe1#|9QIE6y8k&}<)8yXs*zjJ4|X`cVn| zLot5)o(l6D5n!wF)>Tzn+s&G)i`AVZFJNS33$J$-OXu#lw0Ls5n3*J1Kuw+qrOU9& zJH{JdpHTZ>pM*w+WTZ?{J>X@lv5j$I_Kh2x3F{^odg=`z;EM zjX(0x_~B7l+$ggd(vj@Noyta9r2X{hekQM~7s5lB&xaLZ=zmD9(z~BEC4rdXVrO;N zO#10dP&l)GYl+R%XzM0a6s1c~^gZdPw)*d#ts#xv+D~EzCe_?h^*F1ZOn3-tNe)W_pP{RvotY3$PppVsqDbsJz#7G{X6}b@PdvKV2r>t_?PAG0NClp{KvL7B8bRf~% zFd5z_2w0B<8gAyd!L-W>mm|kOA{vhR81o`30W(k;51s6)_j=^_cuRwoF)T;DLuh~B z0DQ892ZK#v^LKhm8yJkm!vosL)j1&d_YKARV6U#tzy0cVx2LqB7yz@(!*wTdrBDsEL9)SHckwc9AT^d z{no|W=0H|Rsl>fu>N`lN4BrEU6j#68ugQT zm_@r*w`;a1_DwowyV*oXOt?`@LxVA|TAXHqL6`_>t+vm62NR)yEuC6RjUwW(GKxCI z21nEYR?w}O+2o(0xvOqNXa2)@)ZV?xd(?(E(mNc(DyMGu?#zooILJ;>i+ z&j9T~p4R5q?~CpFOStI<`Di3MI2gv_f#vzSMGZ#i!6;m$1lkehf6lXTh{Gi&cG`J& zHuguM4H(4gDR3Wb{>z@yC=;wX9g7;hQMVsEQ{fFIa3@3lH#}t}2C=cy&&I1i76CjG z?i&i+C7S=5yEJ6Mv}B~!CDYXkHB_Tw-nP;bw*gsXD?`fECDAza1a^X>dUa*s4p$ikA22{ zdRIAeUV58txV=EvFw5|#!5&${H4UenH1Q=$h!3y0|!FU7IPH;N(P_LHXiBk z2k4@|Xdl_kuQc)`HQ-;`z!0I(!GnNLO9$bQug)KUa5MP`Nra=priuA~DoiIB5;@qV zuJ(Bev4J>tcQ>e-U6@TDGAA5V`q-u8tK6R_^ad|$d zA2zZ10b3vjVm_TXgzf#{mSD$b`Ty*0_5kJ|_<8Wzd*43Qn`RS{K7h$XKDuRmgHf1G z^y&BRSK1y=Y0)TnI^s7LpfvPGJQ6&M-RUkZ8Yh<^kRAM5N*H*5@V23v*?Io$RIa^h zSKCQ@)OOPA^?cCYPkQ}M3)=fhN85e}Htry>4I|m-hKNIwAh(7+qitQm9V2aB!41_F zxV@%BT!;<87j2)9bc`r?Y=VtrkX+)Jt&2Xhb=&y^jKVkSZm8}2!S-_0b`n$}b!+ha zaFZG4w{<>k#eUmjZ&N3(`RjaqCTrR$KIlJL$Jv)=;AEZuqGzEV=?5Y)81(_45?I7T z-w_Qi?FDyfi6}hj18?$yn-K-H7alo+1d04_`jm#2M*Uj~=n`;GU}-h6r0VBkgYC=i z+I|z+m$f}?Y|g_*d%|f?IPD4N22MDk@7Xzpu=OJ6zJpq~R;T5K$6x5ZphXIrKB~%- z@)6iZT8B@csn>%fpn8I`-4`m4K?REAYXTLQ)gE<_%xvL&W)|CfE%IYh55&jgv;)Ua zCkO2&IpYpCs;=&yDWu^DL_4l_7o33hwom7-?~ncnXu_L#V1MjVE}D7on7hLL5jY{o zv80^YRW6S6fl@Yb(yw;Yd8s&+%o69@t~x`joPYw|PXvIX4Z5*ln{b?SnS)bVxHMNj z$lcW*f+p9tCgf_{ij|=R6pXQ8vx3{p+1zx+3Ptw!T5)T>NH(>uilaZ^E-dEV1TX-- z7CCL#FW5(&Qgs#|i(2fhq^@|BF%ap8k8!W=2T|~djeZG!m%FED(m0=3yL@*QjJn~Y z@o{qott*^|H#`fd8=k7+B^s`(wI_9V*)YU`zizK|2+cJ%9)uGOoW6HuE*8@sPe;fY zM`sRd&9amE1|kEn+7U!QMO0x9lwSJoE}l4MBQU<4__>iRsL5=74A zT|I$qAfzSsvh_e@kPab$$};d%yoG9gt~fuHo2_SQ$_q=BsE_y!OAGWAkhegUYW4Pg z`z+NT>8Y|6qBFgFU9}#H;1C3e_?{0P#e)}Zg{dD;-vXBi8Jt8|f_GPUv9R{Vo)n)? zrEg#WI^u!g9WK>7*s*{GYkQnwFqOHe&hK{h3vIihqY8W~-j^U(?2V2ExSg0<7w*s3 zC4@#qmHi7RQ#Qc(D0c zb+;an0J85e1;Er4udh>ptf!Bdy}{FSyYqcQe&=y^QoP)*8sB9Qp#tQg{UZu z%Anj2{6 zuRCI>HEh`$7!dvs*FqE98|l0Q-?2Fc(E+R;?AR^e3HiIO-7%P|L89;*G!cc<kWqcq$f6#l=OM zh3c})D5qzugzeeN`SeY`Pv6K-!&FR!{wS|N>^EzWqu4>%gNXVOcvDdBzq$^?tC#lR zHp1@E{-JieMZX@0X>l$meBXGzDmkl^172ea{P2SVoPT=mU74Z;!CjE$u z40|pk0=*+2QS^N)^7dLO??oxPt&Ysv?U_6CNlNUf|0%j3r$1B~XEDm{T0k8fgzDYO z)gsu%o6FIV!{d;2E|Ik>`6EmD)712jl1mJ7mH}UBaMbo*dX|45{(d6=IEUYlzo+s~ z&|mSO_4;#Sq@3Pd%&(#u)f?rCDF00=?R)ElXk+e#@_$M-zTv)v^8bm7eI0rbYU4)q z;@Ue9>A074x$Rm2+g{GJ=-Ss`QSm**jUc1pbjOI^2w1o3g9#!wZPbjv^Ih0Ere{9! z7L~sR&+@zQXQYzfjUP;l`8|{_-_3XV9{j*JlnLKZ)!1g_u>2cT2}U}92Z!eKcjC9^ zMrwe}@##EZ?*@)~UoE^1w`LQ0{jPx8#x*z8xaNEC83+j02XlSkV?JO_9P18pBH`V{ zQ_oyC@~HW?2bP=61Is`>LHmdNaB7MI-V)avz?=*6HiN)VsGqO7@4ro+#09jGQbSaA z^C(3DRna>(=g(I6*bcCj=hUTY4?Mh1o7{?o@s58oeI%62(Wkz42_WjfQ3Ja_XnIBIlfZwrO0 zi57g-;#HEb?t|s|Vi5;49?VXDfKQk2?tt%*$wKi`Zf>di6MRq=qU3Bmbxt6fx!?N0 zd}cnw?#b2Zes~zpUf`@L9fz&-nersgr%O-boWqgKrO-4V|4fzspz`8@Osdo9n0fdB zIRv-a;kmPyunxyg{LEbXERIY(1ZTDno;Z1&UD395Xr#0b-I*dUaK0h2vW~D)Na*9NssL^A{({@h{Gc#4S$9g$H)H8?J#d@(dlw%>LY|7e9FMnRlS2KTOrmpMul&k+)RgD-XqKUwKu? z4s4*uJseccP)?m*nZ09V5+0^Al}B;n!Mo|uN)B@m_uj9(^`=XcS@Ozwc>JN`XHsV# zNgW4rm9G% zCvsBl<>s!@U+id6?!W#K@A^x)Y1SSrboJvLXzUUN67AUN#M?eq%wQty)e3EBa9il4 z!7cv~cyF}3BmXddZ^{8GKSDJe>cD7okQt3e#()tAA72XEu>BT}X*j13@np}`5#;`m zVpF&b$5)`)zNm*wxCK`qXQ&vKV`o$9gXdhuFpA!9?JMFi1{~$86T{7K`A|`^(j6b0 zN)+Va*31m72JnEtbFRuIg%>^nf5F+p6h1JV()7e6+zFEx%eseW^+Pof9aRp&r^eYm zH%5oz49*D&p=_EL&gUjGht!YLBlp7104FYX=KnoGLT}b1+KuyM-i99c?R)CNlsZ7F z?Y#W^!+E)k@;K^^JX+Kg>@qRHD;5l1`(??UAE2R|Tpf~(tPewQ3(4AjdA^%I#4Q-i zmX5p^+w4Db{N#hsSvI{s14zkzZ82KO>CSbt-Rr}0aFPujLBnUoa20W~jq&SWr9)fz zcolx8&PSnBZ|G1HI+KLX5&^=8b3Z5n&gw8e-~Qk3gV!++Fr&BrM-Kr|0`qn4f6^gp zue=gmz|ReT9Fc^T^A)aSE>?_isX!hdjc~bAs-({v@S4E?ai)&C=mA~&mqxKh7tHTD zsSHrVnwr;J+V<1zpaYn&hWbALz6#911jnir^HuO8OjZ7Mw_gb3|q<4?VoY z+z@6raKz#n_$h|Gx`+{5^-dcpaV{;fKwqlvk?COcsP9xz?Id3ks1jC>{dnr=<3~>(KMlC$EKCufTB>%OPU19-i<6U3 zVpKb^EXH|VX0-#2AK8Wp)1_MA@QxUXZU zQzwqSFLn6nnG=s4-&BJ;n~{!rNWW1?A31z#Bz5$WQxBdzarDe)s2kgwQIC75?|Eqa z$Y|=JM^DtZx|1IJzGzduXj8ptQ@`l#Oc{qB@_T**xQoke$X{I#os1q$J&ZHf6l-6Q@sal4_Ug<3?18sWTXrW(tt}jP54mR}0Z! zu*c>W61S7w@E*AHhL_kbu-yZ%pB@m2^Yv5Xc1_zoQ0sw@{=6(sYR_KHP!P4C=dWg6 z8kH?8JxOzm1Ok3#90+QMlou3_H9*5l@<}-j6SpJVJ<#rfb`P|BpxpyEc@HdAv5Ph{ znPOnlbOMtNtO#HvLdHg9DOG18rK||z9n~&20K)OMOV!Ia-q^`wC*j&g6)>b3_&NG#u3@dc1a(MT_2~;vSQ``yFhrc&aemi z`_R58E{2I5+ZkOX^P*Dv0`ALC?zheXu}ROQsQ5#bd7!c(#}hh$94AZFMb{p%F+%Ou zVq1h$g}ID6gPbe$1=#Nlu%VN(b3ko~@P12}sAI>EJo=FJz#%K@ zXw^5>UY)*7j%*=9wXCfi{Q|Z1P3CFrOSfikXJZ8zWCXcKyJMt%PO_iv>dj_Fm59gS zLjwnbc(p0<7*0)u({YZ0qJgy^-CA#QI(g=dui_;Cdss1G_bG$GM*bDt9a*`}Fn-d! zoTVXu?LK2ihw&oxPs4RHz}oZ1j>WgKR?58gD58=^N3ydcY4TrhV5=-VbSdV_t{9^=3?PVAz7S)>Q;*LeNxJm9r+$~HR&vtR5?2g6V z61bHEh$6KYwvOyrRQNCgXqbA^!vUnZodaCt8&$9yrQczY*6;Zpi*Mq z%@T7L2iEo?wd#iCoe1vbGIEP==ig^{EZ!r5dnIt61l}QmkOX=qu#W@Z-m&;jMdel$ zy`O_g>Z1d-r(qR*3{QtS(+fKm4|4#`JtBdl5;(>IwEMU`c~AlmN#J1#oRGkKBrqa@ zlM)z}z$pnl!hz&&@J(ce!o3}R{CjanM^rzE@$cG`#s`cY1}P!e{?rcQ4wq^DCH{h` z%qs4_>cOz+2 zrcN$5;ritJ(GN#{tBS&&-_ddOxBjok7G6%igP(0bGVxs0x#1?Nmuflc$bfN z(pPInLeJd+6wGVH{Zt${55rl9e?Zs9`(v&=_CuYZqsO=LreaE*;-@EZ``wKk2S=={4;~P4_HNK%IHZi^-R}14?LXU4*_xP6l z2lx<97&Jri_;#XGO;+?~O#>enzG<3Tjdk;-4DrV)c|hNarQ>;ce2HV|lUn;02f5I1 ztMvH#%<)Ak;1BshCmtmKNaa9q?cb{$^z=Wd9Q?teu;9zIzKD64>PPg=@t3K1|5|{R zsToxKn#w^$zsbQadaA0qi-Lcu(zZZ<;+LHCU%IsW?$55Rsxna_M=$8&UGyF8PyRd3 z#`Ifne|KH`U-~p<>ahWpj*X6}rPjx@=BJ9I`E&*W?b| zvuql2x3w-MB)?_zD#=4s%67WDa@QWviAZ`d{IXe;6cGO$W$mQ9YxnBhn0l7Y6-kM& zW7)LiZg*Wi44-Jv0MWTBxiOM1n+uX2!|AfQDtC|>QA?>223ALnAU&cVBo2~>;|{~^ zve`3Ebh@Rd?x1IO>tJ@}=qXIh%V>2u~U`&}`AElb=sDtSWQ<950mf6Hy4V@V3;FZ;bSe_fOG%wKly zru?-o`Ix_Y28qsU1OBqpGk@82X8u~1vY5Z@mNS2?O5GKIo#yt`@t55?=C2h=@8Yj+ zI_THtuklZ!oI3tmmiMRl>uc`#qU2Zn^*`M45y`Lk>tK(oe4Bq2e@(jM>-nqaxxjeE zUw`0^pAh~;tJcjP<957{g(2LC?B`t*)V1HBMGvn(?n2fz$bTD>0!^dxX`&_T+n?B5 zFaKfC)`IloT9Ag&5-k5AQptZ1NnG+DqJrcS8l8Atc!Q#yHMKAjD}YT4S%o^8eaZDrIMOGAAp8mC{{|nK%#*gJ22rk z(mh{ZG#XMSaJ}$FhUf zJ@aslOdS@UP3Pt~%09k9?-xtN;}L8NQyw+h!$K{h4-Q$y+iO07@krbu@Fs*hNZVA9 z_5|cAfwUQ4qjh)3usLxNqo*La<^o}cO(7(t@x4sz%^Am(zvzOr*Rb|pQ$CVC?}2}X>0%^0F=*Vv|q^+Q8ka9rU_Op8U+6`&5bfXZ@6JALBgrJ!7`gS6Lbkai*%(*BkWX#-E+Wi0&t{to=N@PF^Z->-e$ z=&+0TPdJL7&1U-BL^StntQ9<59!-)W?lS7Ys8(FP<2qAHRgVuc}JEVr=IJ zKWG~C&DgB-qUncLB-yyZs6W`Rj(&5ggv)pW^)9&I}Q;&YMlS5%5N`tZ#kY+T^iBeg`h^1 z4HH3?pp$|MeeFVUO`-3*D5&9K17maJ%e?xpfNc>Zqv+oX$XUuHqPk3~ia@^g^ff$9 zRnbZU3bsx^B9JfBfAJ>>@>UuJYQIL5UG3KixcxQy(pMX6|N1XF+y7!WAm^k&{&oGY z5p{R}>$F7F(SF@mE%g7`zjo5^xI&P7?fpC2uZw5;-%xLih}P)KyQ2PYoqqQoP2VJ_ z@<<0k$HM~cfqjIfzLn@)B^i!* zk3mpFQb4u_U@dVBg4!t=WFM2b-ITIh@-YZ%NF2lK9rqKR33sRe>xS5`EDFxvxt+L+%})aYO1063>umhX-=EA$_!a+3aA*eO>yTd9I$@ zEQw=ovy~W4x$UyNILZbff+*5CZpZBo{f{D;12y8cF*6Vk`;k-&%ql>CP@qw?gG z1RmjlkNoE`DN8nz|Hgg_<GT)yh0FsvWB-O zYk0fBuX$H^21^5;rm-f39we474zK(9#x} zS8;sfLCv#8CfUe@2&0*6d_%jNu4j;@>G*~;uJsIPsr8Iwd=os|vz~F_*vd&w|V;)>e@ekAK}@x_Dh?HzJm6-{eR%se!-rp z{kpGOX#dqOIop5b3oiP#{?~A|tN+(jRjB`+^jm-bkwY{;fcDXsVO$loZY#wt2;R1h?jjP$2;()mV{GvXbhHAU-Hrax`{3E#66a$yzAAByEsisuZI{dl%3_kS|F20Vd>Ql3($c2hX%Y3%oq0{mf7sQsDHHPb|N1G6pMqy!ahZ^lVprqYB1l8n z@Yplp+4bPr$lgf)dtT(fU=wnOJ;Q6WzTJfUhgKft7l(ja-y(2ao;)alha^BRmY2t` zy8K7$TV+Dd>)SeehJ}BMO3?bY1rzeM49~>Kuuf*;*t1TT2hUDA@NDb-NI?s!?fAv! zT*~0Q6yE)b>oqT5Z=07JZM-|x#=HL)#=Adtd+bZ&ARv0qIMCZ${xrIoLUDj=dYLvw?5pG2Ar;ny5qk zE_341j8iZOHZ&(-h%U@#kW6tUfvGn4or2_bif+cl74VHHv=A2p3ixK-IV4G3YkN?- zOVKU9Q=Y%6fFTF)T?2-U7sSvUjXT0ELck>$0KVCTV1Qf%V{*4OHFSJgVDDg9C~afi zZ04ve&Y1xIn34a6zt;hLUw$w03xw(wa!unltfwU22l!sUThVWfqb7f<3DBDK0lpo+ zQ)(c$FYwK_=`^b7bQH9Gr>v0-)D-x3_)cN88|xoHoE*dg48Zq^MP>eB_P7kq1^#B? zSA87GFh~aYPWi44eAf)k%_1?14SWw1*HNGMyTkf{7SGHG$*J~RX#{*%>|`4F zj!YwK)A7h0b}oL*VBmX!tUIW<%>v&~9&QAD*9^Hmz&8&@SZL$Ir+YmMAAGSlZsFsz zk>**CAcND_uByYw*Ge-p8z~|8e0tWrC%mPZiVZ1;ep0rcTrGb;D(9d@2MOV z)F3T_0+LfugR}u^sTQt+kfSU2zYk1-C~+v-0g7*>yXAN2eB0>m>V5hSgTBMp2&6pt2w~uRSyC_*X#<&T zS{so#h9Xzxj)Ct5xnn5O5~>VE8WP7KqyoO}^bAGX=^2W&!9<24Z8{7^PDsfNd@su# z=19Q$(hm$m_Au~0CMg(-v`xM1p~(6-VqW{O?*9PbdsX5CfbXjk9{_yY0zK10} z^PJ6(KHOFmu3>IlmpkURWx4ZFvnb%Z-jMsUq-Sokx!Q-@?DjCX6{WV!ZL4y}-1gGA z*LTX8q+o7am%DmyGp|Yi2Lay_9{_x>O1$DP5AZ!E{1E_rMvlJ&0eYW{Av{s zN8bd=3+NIhFCa~jynr-K%#OzJs)#iraKa73|_9fl~54W*z4WVP5=)l9_prtt;&SSBqs;#+WlZJRWK2-nl&0z?RG^FqQ$}m35mhkZ4^-$8+m-r3Z*!p#!@of`$ zI9ll%-_QYSJ%p&B^$^mi^$^kojc-UZijL%2Wi#WO!rY*_G^oF zYCnkz(tZbhh0-&94){>wGT zGHCxxYeZi)uZ!r5FZ00mi!X5`*wp(sJ_q)oj>Vh<16m))CN}Pdj#@TttbBubc(>qn zj7=&$ydI&nQ|u%NiD>IK6v`e=- zgRyfnx2;NgA3WTSV?5mE9>&A%mNU00L-2YX%$T&U5x2SYEnY)#`~3&t;R}K{EB^A} z;WmE+;NcU(KLL1nMB){Hc@4oOJ^&BDCf^5`2=nWPeTiSz^516h@Zz6+8r<6g9=vJs1LZ4@S%TLut=j3DP%3&ha2oFc!9HTS@0tCr_ zNb|6y36lSihL$Z{PLTYUycI)}THgj+d^zMlHzIDX|1%J16%RkBL>zE38GCCyypb<) zGxB~E-y9F`4@&d0Bd>Y&BaXt%D}Mzf{RZLXRvRyGhfA-4{dmni%1|`w z^iy8*s4_G(7#>tWdQDk(s&YKhj^y&as;qOQcqm+1ZF=7s46N5FXtj5fZ}3M%Vj^lQO7Aa2v7QU z%Xre7El)JEA4g9#l_$`VP2~xsX)aIf^2ifeIh)RQl>5Uq3v!R6^0EJdmqGHxtK>N4 zaudfX-FGYcjXAujqjIu*_2A{o6FE65Zxt{1wI8pYy?H`gm#{RYu+kG^Izositvu12 z7J`9zxdSN|7v@W3JR3hk*Y06yenl`U;M9(^p30OkZQWL|++g@1k0@PTz6%CDH#g!t1q7 zdMoW0Y?d96$~58C{_WRg5+(okJLtFG{%gNrCX{jMljYrlDoxZVYb+rS$`lswpvn-XosuC+8?bZ$;@#4IhA8b64FGX-SR9uG;Ne%r zu|xnKep%v`{jLWOw;fvq;NfGEKL8K69cKjK;dcFk@Ng*~Z$sf9>Qur{N-EwBOF-ggIcs;W%OWm2-6dry}o-xbW?Pq4Q)7LYb*)4c+fPJ{le*t*7 z?WjEf55FewHvkWJIo|O&`nK~kf4$_f50A+A5rBse3w`jH;Nf+91HF4jk zFHa=Iop`-@d4B|c#?^tJ^JlYLU5ITFQl1Z0J=}RF_F-Q01-d_36-e=~2jmC`;XE7-d zoz9d(z2Q1yxSvdyG3aJ+$m>+)QZW-gn=WU<)3X&_+j2Ht%t+Cnp-*@OU*VbY!$*&g z9zS!~dY9E_6-ri$ZjK#3b2xl1Qwi07ot6v@0Uv$O=mj(XlUS!z-{Pz?4k}KgewD&Z z4mFf2v^I(?U=#_fH&HHUN|lo+;`j&V(=(y8UEqGyYudh3<+lf#k!v`tVHbI93aRl# zbxdUR&1SamHQ&e*i}dxy;!s0l42?Aya`aG>cyn#2>F&rs zjPJSn6ErJY*~t&2ilxGI4)dn^J~H@(KQNz}&!p%}IV2Cm*$XVaq;wo+&rEsp&>0-u zo|`*2lDQO`-YbMZXlI(roX<^Wp4m%mSUFTKO@{S{yx+>qrO(b}rVfpbKl0#-lgIbd z^p+n%%|?eha7M&IyA0PbgF~1(Y9@C!b*?zS-^$IV}BNfZ&}{tzcY%g$jQKE{=J zq8L`Sd6EQ;N^Uk2Du!tyujC4Isk!uQMh^zXFp9>!UJR$_FQVcqF^La{ijtM?@cmC) zL;;gDP0&_mrktS%+!7XUF6CEgka|MR=QtMxBFYn!Ph~DWiE7n7JgXm~n-F!hFk8eM znNt45z@x>P^d#zAI!E13U3y6UsPRy>dm;Z#B9;n0I+m`4LXQ;7;nOEhjqJB5gkFAV z{O~A*9%|{Kz2)58Q+shYlb$_0m9{1;7x&Yn`z?%YDKvQw!zdm;5qVPg8afB@IBlL= z_fCK@BU|$O@smG*KlJXf8`T*7@16OBxUb$-DP6jsUOF{2U75^=<_pFBM29~$RpU?Q z|CDNcoYV3fQ!k81Yt?olo0=(H$docA%>5Tixr$~~jOhA>VbsUWQ0w&So_e^0kMLx<-?~_nL&JDUVKs^Nh9N}H zmoh2+`DLc;PJ6X#*`Cwy7-`Sxbu;okBWJRASKq4HmWK!oPR?bOEWHlgWOI67Y+xXk zkeQoT4|+~V%wS(M92?^Ei(qZ@WlVaqg2+!`M9g;O2Z2fFUqHl>G2p}>IezlN4$~aQ zH_X=-V;$Pw+BK|$W6I3MIE?#Yoc*x_f4hxajcvH6yu$l6FTveAOD4LdTI(1 zaXD2kLg4rp=Vl6L(=%TP7pAAnnTqk#;WB`euAdD9p@Dq(RBmqS=T#ED^7@tVxl&=i zm?~$^&1U8*sp;Is%v6dp8ULE;fs7pq?Bm>A*Oxudna0nAXVP=$<{{ZV|41=2cl4xn z^h@@m)cH)Q45{r(hw(yqHhnQQJ(I4aE)+^nY4RXw{dAZloqusW6^o|gzZNc*a_7^P zj7{B9wVw@_&lO9MZXxM)eo5VwA%D%NN5&Vz;P2*zO<>!X)QywC7sD6Q=QGo#0=kS^Z~aQR))+|V zlXxSAQfjs^l`(!^zr6~&GE@0g+?|`8JAfbhkJozOz`5y*eSMLLyk_YuoQ)B95r6h< z0k`Py^I@Hm*!#9E+l*oToe%5hWH)k~vD1+6OZc09v;WRtV%$qvwsM!oTL%h(!+-p zjfC;G^ARy&~Kq%^J&NmyVuHAwD0X44u^(*dJ zDBp2;pZDtL^8Wc2e-iq+bNr-7N;~=2uh`=!r!$OypwsJH`{zQy^D6<*|A?MW9{J+6Z{`cw6EBJFn zmg24&zoxe@yV&(LCEg~+pGdnK+C9+jfp!nHd*HS2fvpKYVa8jJ)>3)%-RLLZt9l5m z;%<+KR|3np>J}9`7TV<)yp^DzR_dXC>Gt?z7ZCYl~xQK(;YBs$+I;J%=rBFhZJWgNu_@O zp%BhpV=9x9~u+CG~*Y$A@)3|qm@S@Y2hD;3BGyGKH)oa77jR#G za=)eZ`=pjWRGBZrOw{p&3<$BcWrl%j9bJ3C29mW~WgkhFlFEjVEhN7HL)-zj5)}Kt z5E0&Q)i#1$Naret=nt7-A3J{J(TA)D4p~u0tG>_nki1&mKgS-TSRZ1rxyZ)L1CDGq^WP%aT!ZDya7o}fve_K2o8#AhhPQ^&TGKRU z@RxU&e&#c=&#u8ojmZbuc6tI^?X{%2dhtWFXH*|))$`Y%N6tNXus|w@g+?xE8oF@D zZroeBq6m;)gio&^g;K*U%lPzQkrfIDZdqQVBPvc9WKEj9=5|T5LcXzysm874W=v&H+>bKEkCHyZca_wK z_0RA?F4K5Bf>i#MSkv;?ezwWM%xmvQYaItseCi`hKbusJwcc>4`!9451U&~c(+6Z9 zMB#9(Wgpf|hf~0>+^6xGCB(57wqCPSrLV4#V=Xcv^FNq*%|b%O{Bkh!+DSYbCRO2f zajfM%g2F9@Ar@U7L#FjweP!~1LO63tvCgFT^PzVv<&P2ZbL?@QK-@(r2Y#OqZ4CS7hB|MucG z5Z2?{_<=p~&n6w?8~p0SC4&VT($V;a{9Q88tH*cca>;-vYJ5XLygjT-V;~Pk*@eO_4`uLV~ zxMcWi_xRTR=cv3mS}m7ga1`d9-zNGjh?0Tu z{ElvckU@U`%bchkfdTsCe@EX(jlO*Z1&GQK5D3Od9L?#EeW|Yf zr@GO8IRXP^ZlL`R`fG9oMRWSY_3i&>pf7I1&|{ANmo`!JR0TUJSbNQzqkoMyHf#Qa zC#;u2-(-V3dIkc^=5PoVp}P*cvt9cj+p=l9O9J;ed}hItXW6vf3&BjteL!>1Ew_Qr z*gJH|_@Ghpefka}C`y(%pev|@#6judpaZ^Kt~py`mq^``58@w=)c8T7bM;Y2Z>I%^b`!F9EVV|OQfq(FIeR)qdNLB zi{fG*R$|NMOOK-*iKAcBl6ZEB^wJ02@wSV9c8S#UId}Z3xLjhFNIfg=_;rbAms>sm z*&TmX;@?X38}ICK)z7X!tke*HpF94l)E|uje?5$Nb5;6+`D;vijrnU>;+Vhe9%cSA zB#!xOLhhKqY*$#@iB8WGL?q zWx4a=ugg*+=C4&r+4KmC@sAK4`)l*zuXQQe&0lha1oPLGpFugqS8|!x%gsgIsYv z@||P-n(PK;KbO(3+4>d2qguay-DmyEGVlT86f4k+o^$Rph zV}1Ej7_8A46iH6YOe}uQiPMrri_%8FXwAIeR%FDYP*)5pMMm%u zAzEKyD&zhYhrWW(@}+Y>JBH26v;o3CC1iPq=G10@A=3C>ruF#`Ipn<#m%f5P676Eu zBk%d?D;mgqh*EeSK&Lu9?<>QIy3QkKUvqs$LwOH9eZ%lP^o`P2pxc7<6-YD6Z`a22 z5YTvD3oi86QY7TDzGBUzuNeDnE$NTki!JCY{$&$b zJoz{zNteDtD<4Q-p@Se;(^v3^b>z3uNMC(LO@~1~W0k(*gjZjY(17$B9pgdkf3(R$ z5Q|r=Pu3`?nBPZVaboeXVOYrEIRXR^Z5aK10;9idADLa#SF8o;E7l$aWnbU4D4H&F zk@OXW(ADcJ8f%$UUd~_C?{7o;iWghgSMcE`*P*ZI-t)9WUvb2xuh8S0g{FAMwYe|tlK_B~o2XgEK1r@Yz zmezo$)v}#9MgMP=q3Z$RZW!BRd-2*+0O?Q{FTp>lwdVE+`9BT5j|$8Wbl-=R;T_CHxq-`W4Q z_9sQ30!mu)vMsyY=J?2Qqi6gARxF+>tgxM~c5oWtRJ|~FSS{}xh zk3@-%C1tS&V^!|z5oY=F)URnDSVF;?ZY}37*aRj3XX>#$|~|dzZ~c)@Rs#&OEn3@{Fx7v-_O6 z&F*F9wutno54YJIvYqI3us)-n=j;?d`Z7y6#7AFNlsM+Lb-81GS%;54!{#C8wqdC= zbDPZ*%x%}CEFXQwy5wVh*&^#RZ0!#7+=QfHZZo78>$%Mw{s=mmdYAcY(WB2WBwq2C zN1suzFYEYkZheNW4^jN((Pz~2*Ab6C!>H#kk3PfJmoR@Vdh{6!k{?X8ZZ5JuV|4@i zjIj}9r1e3^=JXj0&wma5EBcH@w?3n0uhK~VgWRFyKS&}$@*mPD5g%#l*0*0>L6pZ{ z#ZRB%u~+ev{~Y#so+Ct_do_;KcIq?K5tvQnKQ#9kUK*RS9GAd@5_pIMNb|5f30mJG z%}GgfJ?mSC{I}+@S6RP=?~nQGU~_%O6o09x)A}0Os$s~*rB6w$=`+@_RKy33sA2O4 z=`&W^`i%CG*Y!tw@D7gbO?QMxTb}{JsIAXv>oaC5>I;2c>oekg1L1ykuzCIVi&%dI z4hh9wQRD3w9ZH?C?o?;or>Dp*!{+i6m?>>_#?_~AM|huimxcYLw%r10>n?Nn-A;9e zd;1099ziF@`KdE(aSx&h))Tn!1PfOcaEW_JgH?m^m0_g0PIX3-&XZOr#x)Z6p1nSC zj|IE^Y_qad0-Add;Pq@TAL2qt3Yen_Ap!W&6dlk2+)J0nn@m%c&OnGA9U`C2_Kq?HA3|8GhR@ny51zTPhOzYjrs7e>8S!(7cE4JescV+hE*Q9Ciz zrU-MP;ag)8{Kp*A@2~OjW5PSUgVF}JUksBDkA^otdpsh_rh2ssxk6^-WAhGmhG+YQ z8s9ABYP6ii*Vwu`1Bo}boPAEVmjy3p9pl@LS7&^^v3VX3b;>+Xk8i}aF9dJDzyQ^B zIg3tG;~NqOEoYG?XnaF1rLLt(XanjDue#RyGrW7f`ds&HzkoVpT*FPBz@E)3v!qT? z+KMhb(8&HS3My>{Aa^7A`&?otf|8b=iaG;N6|(Q|Rk4nP&6V1T-=N>~qKm%LmJofd zEnxbhPOvN3MO9I|Dt1v-wCcrx=-v1-BfpF^)IX~IcMzOkYroc3Fnu}k9h7|AuXj~! zkiJ8mLG6Ev)ETOI-2b{w)P7MXA?RqoJ{^72?O&;>b3p&4uhzF;YpY!CPyRve{cF_+ zb-F$sor0pyYTo~=!yT&s!G6zQQrIxJDxGZQLKn-Rs*X+ixL~C=nk8xa2=PQR)mX zQi=F}>Wr4$3tX@K_gt`h0haXC`qtl^r8W5v&2`Ct=$i-8#k^#FNCHa!L*gL$54nQm zKjc#KUvh)%TaP+pbpzWo)~0#pLIc)m3x=pO+}ktOuv1^u84EK=O)ISJ`1RQVUw2~p z<~>rax78hObw^v>(LQsseU9|g9q{p{PX&jsxS2EXk~gXjxTh1CRZwXf?i&3?+DF#P zd5`yuJnegibL+~A>o4J^J99D~8-TjQ?{o&l;M@i|u|D5e-LXX<%>bEOyiM7=H#&#S ziFfl9s?dG&I(Kh$|E)*e!J>{92jRBCrS5_){wCmi@*bk{K6&UM3q=(Zqn5K&Fl9Y~|Z!IuRL3|4lWt=Trag+UnjE@2?%^H4@0%x8?C52nt7Y@+r6Q;XSjB6uoo$MQGV(Uzh}vJ z;?*k5+sE5fV|B-xg}iDll+a5dvJhWX`?^Kl;n=+aO@^pD5{q53+|-m1D|`0Yf$nYPs&-mJP~nRxv=)E(=`KI~R^_%C-o z>W+rXU3_4zs&gIQrX1rNSHk6upr!GxkvjLe*G8QSmb+Qh9p9DlE!dY{y5aaX{O91W z2I>yuUJW;QLN{Pu`5@E{TAR^@2O9a`g`n1a7$$;W=Mp;+)W~}mJypp2E_$j^`o@Pl zqh1()jo$#SsMHOh-?K{e1@1>d3QZ^aTARZ3|4og)!2L*|RU<@S)Ga_M8(-$|uRG~q zxrfvZs(5a{ZWGa0XuhlcYYO@2_UpP*S7_Cc(VV`u;-uemiPR0cOd{H>{kqec{-)Z$ z{8cCYm%c&mSK1tIzk|M`{k8sgv|sAFvHrh2kI|prKSR$--BAqTrO`76pl#q46x7KpzV~~D5VYN0*P^`Lhx~my5p+kW&k?!4o#KS9m5jG0Ccy+F#x?RaSTA) zz%{Eox+Tt`?zk+^7=X6v_}J60N=gQx$Aqeny2A$P9qNt=Nzdwz006rFdoa7jdOkqi zu`atG0@NM$?hge={}Z=2DSLNCfV#sjKS14K@4jF-y2rD79)@A>{|CJ8b?5P={;R9q zncakVlR4BTy-{`l&DSti2C6&Y7SUx_geB~aR(C*HY-#sKOX`jrCjUWL+{YdL?Z)OV z3JH?hm(OdTQAvDCo&>CKvpYJX zE0iYD5#!%vZGG$6y8-#C`)PcC#9u9Vqgr$9-k{|mJ_%fF9(4y<%jw-4-Lpv1K;2>X z93=(Z?*yf}LFy2rr|r#Zq<#3td=ck%9cM{-}hG%unO&mi^X699^EC!4WH= z-tc^(7{(DOc-|Y%%$4U$89JpNr>s;mQz7ZJS8I>T`^-FfpKb5{Xz%{`zS#XS6p4jn zj@=(MZ&pKtk#JP`xoYaos#EDhEQdaXAzGSaw+@@Dck>*p^&#ssxN8bXB8~554!`jk zhrH<7{!#O0CFU^@ayFt5fl#CLA;_m7VLYibk8y1OaNva-Gml{`IdJ>Ob2}FA;nqL9 z5xfw&l)U)$mgU9PPTxRJD0vY*(NtbU8YM3x*L%1Tc+$E)WNkNOv&af%4Ah7?z0-TV zy+nRj%Zr=c{?WvnRX4pEl1E$7hx}ZVO-^e+jMlomS^4Wjc&i8XijGDjn|QN=kQ==H z(ED+V?^2bzS=O4d9dY`jD2rS&28YCfh%{H>(d>y-t0I zb$r3C4{_>yYwO{*K1BIpZee>{llAb1#y2Fsp7pR}d<)iy)K1?>{y(G~=>(3ZFLU@W z5Yo$6zwIBZe`D)I;LmCN6Qqw&+Keus`FbD6E(Gn<8Sqr0NFU+aI(AVlBM{tfj1*ylBWVuq6K9}9x5`shO@B(9Uv z+lVe23QSrM+HG}l+v#pX;_z|%>q9I_!3gz&pwNs^Uz0n0Sq@WwMyMl_FHj${BDsC^ zA?uRcXZy!xiSyBi*!|$phm1-3MhJEN_h_y((TBM3YL7RoqT~)et(Xs%+FM;Ll*6o8K0k<%ap>U`MD4)sOXte2}VY8*?4si zKSea{u_O(OlZ z!$0hZc%(nl>h6)P4YVcLr%66~w5lXRlkuiiCCFalq0czv0Xw#qOH%tV-p>iOo6$f4PRDt}vD5ya8 z+Gi9PGOe1kD$wuwuS8!dW4egGQpPZST_zQ;yjgV;eXZG`piWCcr8o$rT8l6jzEs!# zVbT`V+P@~wsz6knWg+Nje~rFY98mjhXHhjm0U`79uQ=(Ce+u-qGK$(SWm3?${knqO z|C?(6wO?}5f9e0MZ@=^X*XZj#Mb!UNR|?kXYsDG0*<7H>oEXfZ4zXcZ2B?c9p->9c zWZ8ANhYb!hKy9boMk#Grcstz*ET;FtkwyDV0R*bp-SLq}Px4w;aA3{YQ_ zI|is%Ny4JqGBmBcVJ%1s2B_`y3{YQ{da*iWSyKA=Cz_Cy3{V>(O~;`Qu_OhnL+q04 z0cvEcU8e0UDqwfWus91+XthThvMlid+K_c|W)+|fS(W$zZHOV|2WUesOMHMfWI~)J zq0OQVu{(xYZdf{!nJpq+$joNPvF^=&am;KBl5!iBd|6)HcDhsAkYP!|EN8!3W;UA+ zGh4Uh_F*=gc>=W|^(<%8VP@-=I{0WqR)i`uo82DPhS+`N(1zH$A!a$d1t(Z@3>e8Gs>H-_Fta$m%b>JLIcegio5aPX@)Pa?C4_Vs}YzAc3V%{UL@$y7GrG zl78ZP4^3~YO^mkMq^&k-t4-Q!leXIARacwXn@MoSNPjpoP}7^#VxoQh;kb%{vSmx} zN0**i3T3xdcOD)+cK_*PqlfR0EM>d0J9WhF>Me+{4m@C;IdSUorR<*GZ1AdL66>JqeQFDp+PYRKg;SG-;-$iTB{X|}zxD=)ZEmc+%5%Oi=RZyP21}XB zd}&T4vGZhR3#CivbD0aFvzIEFbg7iSq*P#YbYJf6m7Q_z9c*`uEM(tQ4Yzl&eLw7A z>mQ8whX?$Imj0nAcCg9Nk{`mSC!c_v<;haIGMVLsR<(yBE~L+A$~KXuk?!gpHEJp%Gj}dGm!YC>otZ)**aw@+ z&840?yEHOf-99%zo4PnTxin%{d(KT?OjXjQbD2u&e5O>+73P-SbM@yV&r$@hdOTSROM1Jvy=~^ zqpKa|rdu5`FGJXDmi}$A#ce8 z(8kB%%rVla_iV+xA zjG(Y$%~1BP=KzM42r#U`x{*T50M&F3cON6?cn{AIh_OHp%vc3r6d)VR0W=m81z=-A z+Hf1oX$&^k*E2Ynu)%ClgE#L@G#Aj~>k$Yj%1_J8NevDR32`k;%`2HQc?vXo*)rHO zguNMnPGiXM+^ki8fkP(4+nW)X0DLi2FalNlX?eyWpAp!BQxOwT(W^;dMV&YmF#{Fd PSir`~oe>CCl&S{+md);0 literal 45072 zcmeHQd2Afld7mXIiPS}ka%|I1ow4G?OPLz(eW<2OiK1nCMM^wm#8uSsaCerwtG%?d zE0I20nQ77_n6z%Z1^P#erbSb?4U#BHlO|1zxJd&9DAGho{xHy>a0}#*wrQKTXknm0 z+~4<(|(^idDA{ zoT-%3*}S=66j#jYOJ=cJf4M&TV*LeV-^k+*B%?slnUnfMdMi` zKb|b5%yHSYrU$cVl`2|!tv>s?{dT{8xnh*j=(-)aoULZmWVxo7dE>ns?eV6In$DV_ z>!V}pA!_-d(WCV$`t#@}m6}#{eA!4|FpH`2Dqi$t&8n6P`USI6H1pJmM9$vLQbflN z8tR0)#M{WCy#7&E%|qigt1_O?E{~Vjs+m$TsQClqwPLnPrN@(oWwJia_Sl1Z(#YqR z5It)mXR2=Bn@y_)_Us*1vzA&cRn@st(PSOS+&BAd4+gSUtlO$(=F?hRc`wz#L9wQw zz~_4~ax9y!*X;)|3AIX*7X}>fVWYBQjj4wZU$~sIc=O-0Y4fdGw#wR{csBEZy_YKV zR!pmwuksonN-Sj_+)h22hi(>v{oYMmitZInwtNylc514<`8*YjY>~~=h{x^+ESe?k zmxtN7^B!aVss9VSA{tJXNO62He2FZLein02qd&;Y3RqsA=kD2ep@K&XrBp3%K88$k zrD)UwmZ@5=E!#a-wQlc9naOOy$m8jRczU^R4+X-pP|P3qhhpJCP>VqD zZotgli3^!~>?fKwPD@Q*V@yr27ANvXVL4@}`cd`l>9S^Jix<3DrV|Oa6=#a+l6Q1; z^qlWJ?KUj&*`Cb(+t@f*1KZy;Z`B&U`CX7X;=H^wzS$8^#(&)lHWTW;PBYPN%QN%y ztt%}a^lQFYjIFfH@s_l>4{H?bb*ojlOX7QX{iQWRE}gjg7Ob9|#=>{x`iuL*AuSy4 zvi`zRE$HuDe|F!}%-qxUOuv2LsjPWfobBpKkR6$N+X9b={Xs42i#RUua4?_+!#u48 z%@XXvVy&=jR@7?7RBNTO)d@Kq(4x_>1LUwT6bfnnpucnL5af57MkT+dRx3u)N|!1H z98N?)sUY||Isi}+pXT>*P^~*O?2BQ9LJ@~^f~wh<&K8Y)_E}RU;-#(mC>!;o^So>u zT)!_81Xkz*cW=c6`It1-{6v#=I8&I50Ec{$O34nf8%Qs7^tSk7FT> zV=XXxS}r9(&kIW=tYTbRxC&O9rtD3(MIRolfI1 zb=gdUe7E$1Q6?ftt~k9~Vx*?Gac&j3^qb)60otOUJYov)V=ytAdzKx1dHMXeHctdhA^llP%V|{S9umosG2&4w`xtay|b$92c0O=t9L)gBkJ(l_IUJ5eL5RnA$+kfVN8$m^AMOJ6UL2oRtpJLOhzw8;ATn0W zYAZ;&Egp+Fu9T=RPRvy|xQrHG=rWN}@CRT!oZm7BpQ-2EQAnv8plNvjey-ioaD8BRAk@ZKa5-?tu8Gpw8w@%z zn&D_97RQe20+;icT&Cs7V}_$qzh4VB+)3? zBBVPqnN1|a!D^{o0)JUm84b-igC<#_9o`1tV-{DkMRQyd;fzimwg;2ICPZ)xCZukM zHX}C-B&iJE;^zI55Fur?AMlSi!vDDe<~0F~WVrMGo65Ywrg)RE82I1v;sk%+&3HM+;dCtP+w*M){rD@AVNU!Z0uB|7vCJ zC~+^Smlg1WMH*9+)m2XHd}nrOhrPR802d-kETzn}x)H&dD0THH_^eU&$YUfYeV(Za zBu`W;lJtzrqsxU+@CXc&cNI(_oy1Lbo!`NPdg&~h->C37m@$^ITp%?B!6tUhTQo20 zRwbzyk4}hWS%xKHbYym}vG~qxZHK*&twEWwZZ8S`N7SaHG1_G#e?jK(5|ar)X_aC* z1KXSz67uHO^6X-uU+uH+qOMET5T6kocZ>ofc52m`UF0j}WA-7M%e;x@*R~`!IK;Xw z77$~00kfrl-VHVxGbY!tBx~h6Au7K3z<)#QhQ&$Z6l-FsQZwZ;_=CR8ckTNH8`#Dy zHuJfIL|Pw{pvP43>)_e*7Om?od(&vIOKD%;7*vad*xY&y+``DP>%Lhhrc!;B+`@!S`2n%Kcj52H>vZD&#L%LL?xPoCUY=|t z(Rc(pMZe>)#6n@1j0AbFHaQ^o7tBJbvZg{u$Fvw5T6umreuCq}#Z3>#Rv2_JHcngO zg;))`sKG1GVV4@VZa^pH%AL2ebh`!C6L%st3>&ZzzCnL?%B7Uf>x{*)Z_NY+0WELRJQrirh7=Umfti?J_ zrg$1u6#;)2y~nH>E!7wt1eO^alo7ll<(AjFF{ zX@6K+3~A7bIa6dqWbd0b@~TwVw6{1&N|LCUqBhXMU?8Buiq{$Rpp}Ji5=tNoT|>|j zKt2y8fzVxbfDVwcS)fCM!axTis8*Pfw$r9|zqwjAiYZvsG$|cfu?UuGSj>9c_`+5U z3L9bc)dBK=kxHrg5+t!sggS>E2ch0+$-&a1K4=@eDFydcK;B*=%Pi8}b+j24AYI|c zgE1}c@3P$XP@7lEwN$pyu^vKk7!w8r>(Dk5Wa9?AlP-wj;Nin69}FX9sa|r?7;SGe zRNPSZ32LGPdXx+#yFlLq&k{2UJssq29Z;d0gv5@o({`v}V$kL;Yxr)auB_TW}TOWUkmUuJO|qLN+cYIO=h$U zVBje*w%srW*?~%Xf)2Q32Z~m+vTd^%iA8~0yX})9T6SbRS1IMyG^Vr@G&Q-?Zj;R) z2>>G94w9jUaCW>x1JZ#xV3FX%(r{{Z3@9)TGpjD6v6mgJum*)LsKssrt%^Z$jib+T zkdcN36YK)ENkDxcX(ZwAMDUq8<@EjM;=aQD=X|Jc(DqEj4@CM1Jmmd>}}s6Cj%lL0Q9lKlm#9dSO8 zi`YI2Nj=N#GTtY>Vi&L)+JW3)2~<{9d(p(>y~4 z#w^6+^L1THDbv;!L_IDc;{_@68*}uEB7H>C5ZP#+7Jz;HJvjIsf?%P5Afvwx!qL z{e0Qp!H$5sJT$j+(HTs4z87~ly|PekorsKS{Oa6jBp z4cNaMK+yftumCA`AMKI)c`CcR=`6B$m1@;;4G8^w-QJlkipLr}e?iqqcSLGwfJg+t zlbcD4J7VUw4N>#WJ0Q9?^G+TIcc2_I6!$^h5_BSYL!^lT&o4~lT1ej3rBl9jOUL!A zxWoJ^0{##r8jwrG;vH%;mKO5`wLnC2&Gz61kt?n>7zSZQcOP&9suMF#ojD6qxSGnQ zVO#r{?eQ^Y#@+!c9Mm%?(hU!4dpFSN(Zv(9$Bz1d5u51tJ>&&JHhK)tl6S?dGDObQ zc6=KoA}B3Tz_5_4mCG=-9M2>#GAFSEJ@zh8QRTHP?gID?rgKA*G z9BrTx&7!dkw#+|e?=YOYQAn|#{iqj5BifB)iXZ zNtox6J6@9LZ5V&z6-MrG;scdmC2u!qL*XZCc_Hz`%##-xf71C?0p9e(&n-ia`vNJ< zX^jxWr$4v)!o%q4`xy~_why%b{q`Q#7^3s2G$loRA;=*+q=naYHpWNjY|PC8b6P7ZKs)f&sueLip#=td_`S z5y{W7DK5!L@s)?eh57J%0h`DQA{i!l&@4^4;^O`55gyg!NwaWby=$K4@?n6Y*^a6*~kH55EQGR9m z)Z;xK&m&R~afp{GH0NB&wKxck`eT_K0^GHkD&VWCOe+p9st1Rgvj!<;o(y z9OD^fD2p}5Oq8SQw^N_2S5~|5!)l6+i;VD!8bm1b%XZ3o=TS}$iEPt$LaK}YVq(17 z!@m&~$WYGNE|mE!(o}@tvQM5lt0*7%3>eCcWq<>r3;e9#{0C+DlN$N=yz~`qrBbSu zb<12KTNOQ>T?KbWOrr87mL0hoiQwH*3)aM#e z`Xz90(B*!-NBJ`Da$D8O!a~1Xs$3A&qe9Oo+2t<3KmAp$T*+QyPLG|f`jlp^l;Iyn zT&nUp{$j!3FfXEB(YQAH52v*YsV{5f9*66HSXFTUWL>4YzMw(-X6dOlaGcpBU1yTK z@byn=mzrU-muA?Xg=gL+GhHF&9raoLinh@O2tw%cn^J{%Qd9Y~81gCx(5(Ii-d2*u zBM9gZb8+OzN_y2F@cHH3$Px0$4T1@D_|dcv_hOdC{*v*`??pwqqV)Ao()+)Bu}AsPpMT)_;O{K_AHkU0ak3U2#M zmvkUt9o(Q^vZPGb8i&kuJfFW*I0CepgOHECs3{nurHxo99ZaW^%c;;Zt~v`v&E-@$ zwH%HH4AUPlf}!;CC_CFAC&Y=P3dAW|yn_1)yqRjXY#kjR$4jQRtU(SpzF?%xQjwiZ z;L>n|P)qeHkoOT>xS$2pKp4iP*V7U?g3ckEj~-xQNiT?OgiCJ)9#=G6i)$(xPBI*P zinda$X&J-HDB?fGS1XEEDUse%fdY25%5u}WhiEfjDPihTDNLNAEnAjC!$q`!LMLg} zSVouu2>!%FWou-wpLlxi*zC-d^Qc^ZER7e@Hu7}rMh{8I?aQ3X(jJoK8|Q?`_>3q> z`-}dWH`x6ZTxG0A4d|EZ{Q_K-^+*wU;(480bx;=`oMm& z!(yN${VegC9KRuv{V{|TC(D;bc}f4jL=)WfYpY(FMOsUF$uEeC+Vi)+FQ7cdwUaTj z{&-sf`6|P6DBC_nh*GW&(0@f)<$(g$kL451YOKE`;$`1ORu*^#;;%JF;#t-!qYHR& zD*wBp{Fuxr!uIlH;{Xjy{i8dTX8>Had#lDuJ)tx!{+d}?e&r0~i~T;_KpOJF1hUDN zkc&!15*hFdQK@}zLp3QxcubcAq8ZmgOHO)r%8`qv`_(H&G|Jw=jFDh<;m!R3JIEe5 zGds0t&|MbIwt4#(=k?=f7oXOro}M~2y$Jp0GGvw)a9i}^Jgh%flgWC*WB1WzlJwAn z?kD;X81)3M9>2U&rq|vQG@@h_NE|_ZR`J*a6?4VnmD$61MlENi7apHFeRghXdclVa ztaCp^qZ%fH`;cK~ZgFW*f9lw&v-R9RQ;Vqyxqrn|!jtC>sde#2Ywg*p)<8$C!;AA%`l%C(`pk*<>Bpv)W}ce9p$_+MRdukV z>UYShj~|;$=u@ZX9zQiRwRA(Z_ia^ssH675lMBaZ^^<33T6^6`(T)@C;S=rQ6Ya_I ziQd6#PW0#4Jl}=Y#o%tbzWPs|3QXxw%*ej>(R*v1YK|=~$2$9YowUd{DkX@URyx2c zot;@+6eE1SweD;kW7gHST4_<6z)SfOgl19@)rQBe>-B}@cC>e0#of&A2)HBQj=)cC z1U~(=B%gn3O}H%;zumF&+r9s{Wl_06-4Sp{z#Rd11l$pDN8o2-1nP7Tf|=BrXwuUx zn%uez%yDqX5hkHIWUsoeO6HVPM&s*5pz9C@NlWv80E^qnEyR zL)C43?xtc~CO5hBW9kYdHb$P=P)t;7kji?SV#v{!Zk=dO-iTo4n;WnELS(vRUZ1>7@sDz63XR;O>Pce^+jO$dYi8&W80N(|imGoyp?-(vqXn zB=?V`NKm){SjpW33BvJLp?x~gQ_l1c9USmnnPg|f!1@C%N+pl-4aSzvD?RglJ=gGm zr^CawJ}0FY0rb2CV=(+~wETQl8CZW)xr;zea%G-;C$Ng=_4)0!Z~me~yBl3X2j&-) zfu7^vumSXmfu5;v{NxvRU%hq@i!*B0)p`Di)GP0!WU5>w(<6(cmWVidY{b1Wu)bHh zgmE2N->0ZZd2L{Q7{TD=$ofI$3ZAa*MmN295RcGD_w?{sQjPYbzaRBxXicseFB5fC&zLE8Q7LKkDh<9#T;(7fD%X(3Hx{q{^*S?7&zs#aMSC6xh zhL_a1*Stu2lWF5U>SQB{&BRr%A8nZlmZRg}wO`$0*gXpAtQA%l!EXD_f2`+PFM*iv zb$)2e-~KA?>^#0lSkl1yi%hvavwnpXrB^&xpV~_aTMgVu+YFAZlL{9K|8+7Mps;nG zM%pHLRLs%H`rQiMFEYA*4+}@u`BaarkBImCSQr=kaAy6Gfd6h5j;=o>-rpm__ll4Y z!um~R!@W(`WM1-2CZ&KvLqQY-5RvJLv++Bqz^2N>9EK0LmuMDg+lUATekzHqzThjSw! zY7#fj55C&J;fELQ$*F*LCv$>T;shVxj1#mjR;zm9QWs9}av7gW1s4XMsfizQxxD62 zpYl6wTrQ7!NV;6!t>^M$0o)7N{7LE!gE@TR1q@)r$!s3zNZ69?WEOZW9G!=Abgwsa zbkB@2Z0T}z7>2hINB1Bo*~wR5R6joT?Yri7d0+elf0)0b%s+^8lQVlSsxPzTd0m-* zUIF{)Im7t}I#_Rx`*S%y&eFRapUd$%iY^#d?*8w#b96o3>!uWs%khn@yFwBnpV^j> zq-8n^vaBwTLK<@O%1`OP})W2{S1L(jV09+Rrv25v7z-;HSc zhDj5*2ZbL#;2_+ia+`B|J+0iH7y7$vuQKtNrv(+$Y5ocN^5r(+fFNN`Lpv2IJ_Yzx(K|(SLXpSAR!(Lzgsds1CP9n%=bj zu6S$KZx{5c+;(nn($(L&+?>?5Z)?l6EY&T!xo5VkzxyRdpb+pT{hj9-ZYDrt$6YNI zixkVZpvw)ub=+VqjPKMt z>;C9l$wBVw2X62eU=ZJG={bcMe2r}wpPY|J{r+IY7x($JKvaAME_V{?FW{qH2a+|t zj9YxIY$2ty4};Yc_*5l(Wml=!@m<_K?3J=5Yjnfm9+r{ct4`VPqKn(DdPj~TFo>O^(ZgSonOB)!l$fHIl3JFToI0gPIG`v$Ei)$-NCCxj zC+bTGFgyCXOzC0COUwnzGrCUMJkM_{-{gg{<`Nm~JuKO&MR`-a8G2ZYlQU9NrT|UN z;G6s{b^=p|_+-X-Vb?H6pU^28(mhPUfm1SMy%{s)fLPv}B}2iRDMJxREBP`oW+;OQ zl^&s@)b!xM_yE@+H)sD)zYy0T!zranV3Q;q2pGbI+Ylp&A?5M1^#lxI#%+i(*bvRe zDYZZ^a`cD>2Rg_5xCF;LIr@3WJNx^&`FJ{q5b~)a3vLTcAg(M(kkSCUlEpp9$rm}q z7=a8W+D0_2cs7yYaAQ_n<4tErXyN{7`yoV>)9V{RY%mf8sm?N9O0W<*; z<|4o_2WvwLbpwb~5)wr;!A`+v5F6B>%{vm!1sM4ze=L*b$zabA_GSQD$&lf>S*`p6 zhfIdIHzP16eW6ZegR5W!s`%0Jj6*&numh(eCZM8clfa5PaVla4D!Q_Ojgvbg5UMCu F4**$Z-*x~1 literal 45065 zcmeHwdyE|Sec$Zek-X!ZM_JZMt~{pgc#k^H?d*G5Q9bf_qUq(Gcn=Cu7&Zj^FX#~Hh_y0C(vW>&3EG34xD zYnr)IZ6j-K^>QD~m+VHx+%i}$XOC5Fw6;*_y0v9@rk&BI)#@xcBdzsLIPA>K7VA}O z_9gRWvof2j=dD@UHP?exv+GT*vDI1p=3!^txY;xt=yb=4-Ym6>YOc{X8ocx2UVp-c znr5&r7{>IBdV+d>V)}Hah4DPSL!*}6nq4<@*R5K9wuKiX*S1^rs&UUE_UZQDJ?WHS@XIw%weql-6e(TdiWf7Sr_TY`a!!QR&&7XB z@Tyo>P~cZaFmo)MVRW1)u?X#EjTc5e;ge=_!=6!3p1giDZ}aYde#h?H?NW>NKl^g= zN#`In7;aj2yVBw<|6A>g>EuRkKuM>ogT| z#!(lo63)vP*u3)*WBqCTE4(3^PL@b^063u=IYfrO(gqIa{g?XMw7e9>#p03vO z?TYmbGRd8yQ4d(AR;Ru0jM%M?voCMuN>#Ihr}N_JO`PFqA{|dhqX_h5LQ5r+ouBLA zypCA4PVpIcaw4A5lF0;1Zm$ub>fh83fk%VfWlK7=AeSUn-K; z5=sC4m(q1Dni$-F&e+<;r58HIap&0cCF`a@?&<~59mS4kgJ<+)MvG*kej7Zar&Hau zz70-wAE~vg>sC{36)m-0Z`cDhMv1h1<9geD`2XhMs}I2sukz@b8^W>!isTPjg6p6;V;EThGv z9LIR;P3bf=AN)hzKiITDKjtiTybZ8IfOW0uK$yip{JD<6na-&l$WB3z z&tfCYVlOa)wuSmif8p0o%&6BjqS84*GP!slCnlO$~!%4R5NYn#yRTndzr0cI(=-Z1Eou zQIW+c)>h8WFDx#somHR4jH0s!y_`S0c2?W4S{|BM71V5@4>S@*E9?DK$l2Gl*kSWc@lS#?Px6WfM#VmaJWx`PW@1_@G2~cgK*%NhkEJtS zg*>HXG^$?WDy~N|S|XG2p*q2AB_djK7;SirXu}dD6F{0bn~>oQKoRsNqNV)_LePU5 zEfX0enL!tiI|qQ8Q7uQa{d96N5d|Zt`x4^GL@bg7+!{i+JqGk?x2vG2h{yot03u_< zDiyZ8lY(k8T1xjN;}S7YxmX8Z@aG(6ux*Em6ETp%{uL)Ox|WXn(}y5sL23-`Z!{Ix z5`zEm;DSUVlhUG*A=N%1xRatAu2dQGR#v`(wn3 zFvtt~zt!A2P23CGWd*`uk!I9fYm*Z@-(5Vq$JyVgf(sEP*78g*d|xW^`A#pE8AWUYKBNX0iF_;0A)usKPbVxL)Swk^30{@GaZ zd(IPr4fHUJZa()2k=AEr)iWyib@1#JoA&h`N7LS0E=g5W3IB=LPe+$^eiS3At2J7+QqVl(?lDoz{<=-ZZ zHd_821m%B&e-gozf0ydr4Q?rahjKnxoKpUMYU6&Qk@6p~UO!OWQT`8X4h6*B8B>)1 zi1G%Q;I8Ddfy^cEYpoIJok>WSGPFX;y8jcnC|K%0z&q1l#YaJFQ2e5DyO#6&;5u zO1*N!611eRN-jnH&ZK-4QkR z*Fk8pi))D!Vl5b=g|NiJ0WECbg+?lnI^W6A?LOE}JUp~;Pw1D{Gf7D2dJ^Iu{7Of4 z2&8>Qp{|qu$f}UKKmjYHQ%t2@{<|ZJ($#81r12Ipryss{I^cDgq$AKO-NBIc$Mk&b z$!JP*TL-z0eiZWlVT8gVCVM{z>tpro>RO+8JRZyFWbu;_`Y1O6!qVb8%w+sY1ZR>P z9aZZ$tSZDjT4Qn?{zi|}G0SM-y9S};ZfvTekvcnrBOVMSh?Vx%Sbo{_Pk6W7!vzCV` zja$n@#gee-f{~ox{KitCqoc#N@i8->S1WahV0~$Hs+&}1L$*bXtkZ^3(axbJDBBxk zl0{0pfj*}aNiF5x#o|O$n;BwfhA5^EP?t9v?R=>!wu8GM)UJZn@vK^0hl)`bs-Jwqo?~2Pr~|V*g77DQ9p$AbsMk)DsS(+B%rI+(r$?F>UJ`%Yms5< z;={}?F9(+v0A@H1cn(W~4uqdg66OkNDd2-4dT4F|DlG{H>IJI_80mWmq24A#f+67k zDD6D5nrqf8Y5}V{ST!(7+I?1fb3w+jT672}>m2QhWoMf-Ap&&tPE0ZaIkAtjJRXl= zf)g0fkjXg65G%|;p#7ewiB0EgPKZn#eJK4j`MdWZdk)4^2 zv*-L8&JGp)();*j6=&>5y_D}PWkX9XIF(G7|JEp^p8I>g7|1IeN5-8o@_p$z`_0Xg zZSX@l?HtKtMUA#?84SMKov%6v$?$^ojld7&9CnC8(#tY~47iC+pdL87keSL-dwq#D z{bgxU`%&>8yCS)r`0mx-!e)kPy*Hixw9XKIF%!|J8_kW=s@PSl4J$`l#@;>%n>NgQ z&olqLtK3cR8g#t0v8^VPn3Vqo)+u`ndTQ=X1J@!WeC4v)3Wu=+wAG7C*%_6dF0VXy zVdd;1?De?a9qC?A;7vH;KsP%>QKwn+ynA6p!u>|$4MU?$>e}fU;!B2TCw_X&Iao1Y z-h#ynQWo6UJo*<#Kxr(3H{2gx0Bc)LtSlO3Qt9;iQ^+q@{t+85##By@+ zQ=ve<9muyef`(jt2~WU#_p1V42Hi&Z#0R$$J^^lpEt6y*LS5(H%k@&N)$iO}-0O^< zT|7_J&6u@m<$xIA0?^5(oKfh&nfvcJ98paezMBBh0_--X$kPBv6z%} zJ4bp%uDH-(5`-1qdcX;&LC83N`3gwkR=!k#QSCEMD8gW>vjJO*dUtzuC*N?z1XW)~zPC@oOHFpX?C8nCgPE#_Wg-eJc^ zoPD688e1jY0`T)p-G(G-?ZyrMK$p#^FSV_<#fWfjsfS3dc#{=1L>io%EIS8_GkKds zxWIfisMD`7YKRKq54S*&n;6U@&cOIjtF3PVwSh*oYUVnu9?yV=eD>m{1x5kx%I;H+ z4%2Cv)jS*7UyK4pewCAB6=D@2C%_E{Y^c(ZF=ucM>Wt79-hiKGn0jM2o931`7hK0* zw%{tRepY?iYSv*unUm5Tkb-bv1G9-~hD(iFG@(PS*Wf-`K|ZtD(ukVf)jbt_ei#w2 zI*%C!k?A?JR=QW#7cM05wex zg%}(6fSXaDg)t(xY7~4)`Cn7TA1Oyt{$=U{IoaJuKBWAsROx+ZLdw5R)dsAG=Q5Pw zN(X`{#Aa{?mpNE52ztL*kE41N)D~QsB5*r`?!x8h&`U~2Agzx`e0!I;F*8NDvAhJ( zwRmi#{2aoE62FyKsE6qhAgkFfj)d&l4`Bgz8<5b?sE}oCxZz&}rC(*Gfw@UaA%JN^ zU|R@tIBX9$H0gN2ZZ!hlBJ=%*hN^LIPeU`?FzFYW$xZuALl&hsW<*adNE?3U42}qf zCYXq5i^KS<;P!9@5il3j|5pj%UqiS0BG*ME$7Azcl9T5<56Fc%?}t%0krhNToZ*36 znsO_E`yWX76u^DoORxgC{~^KsSR@|PQ@G89tEHJDiG`Xbv^7->CAg0_*DTZ%#aYZ$E!n zLNKSKl?nVmq8w88;+@LQ;g5cE?0+wOPgEwuagk2ZvP}`y7tHP9uQF7Q@^>o7lBi+& zJ<5Y_I3nI<=qksG$PR=>J`py=t0L1q(j?D2sLHOq8SMchjD1S2nxy=dC=O z7a0*2Er?L&m;IFOE~A{B64|HSgwz)OrNw-C!+$3lkfB_&eJJw{q^StOW?#5`MNxkC zm%&jMZ4)dA-Q8#Z!hcdGzpjyI&l|s~Z8YochGAP9WUOKoN}FKMh)Yy{on=R^ZX%es zjoR3^2I5)vf2F|?cB2if{x6?vShe{}>ijpmQN|6hZ_wrby%FWxxW;YMAQKDYX1#e` zG>-;DUuW04{L_Ws)*8*y4d(7R*sEXE?2QII!-!2)zR6!~cp6qj)LRFKRd3b+Z?D-QR#u-VLkJB<&rI zS^Z6|Hv~vR=-QillbBLV`6V&sEli-*`YpU|z;~)TxcaChpYH<=V&@3dsY4 z1@!sRvQLg;mBsmz>kjCl92><>ljl>ErR{o5*{{fRA+`;JCG<>xkK^xkMY*MnjnC2h zfBV*m^5x(7*;ix#XyyM={q#rmQ<~q8^>{&{8CMLWSZd&|$Wq?OnE7aAJyOuEL?l;; z_?7$samVhI_sHBW%0iv}vG~3HyNdEpg1?XH9CUGp*v|*Vo_@$GdL$iBS?l>kem#+jnwB0lWAVcJG{bEW6awR@ z0&t2RZ{TKvaIw{D*r#V_@ltHBYY@cEu9$hNUSpUEY#Qzk>T6yF@IIB);#yRVCZf^! z2kD8NLI04g#|WUXq!&asf~EHZjw>2&#I+O+m<)(d(Kc#rt!UasMf|7uR#VZMby8d^ z(705h_))tI+iY<$HFPvx@{{oT|^5ggh^ZGI>I8V;7>eM?hNdW^Diu&UA#E& zKPw+Rmgb9S8;Ko<(L=Ik?>48hbcSU4E}jq>UlaxDTv1zuH|ZzO+Yvnf<@@i~WqEn7 zWGI(crpxlI68xLOAJr|>&pooRpBGnu0kIS(%S$>z zhI6b`@1OU57D-gT3zsB)FC)4Yi#j|VNQkmD2KJK!7851uX9?Hj{2dk9pFv1*vV237 zm-PP|w7{;Pr+H-&X?^7-zaSd&=J&p@pghHS$r#yw##2DP%J3@6dZ!3c%G(q4Ur{!B zpn&yb`9!z6`!9!hId+ki1z`dB+wMp_%XVdS6%T&p|FI}PBQuK7Tb_&@pknF#?Y+v2 zs9beMTjoY5tGG3PZdR6=R!YWLPbDL`xh3hc`s+*TicxW`^J87zq>$k;We$jD+ypH- z>BV_pE}E`ZZ`RN$XAiSRg4sp5{3kd;&cx-#`Bjr{v2Yvb9bR2F&Rto3!I*zx{?fuK z6rAf2T3*LJ(W}ca|JcmsI@ypjM%PHv!w80-9YbJtvbc5p=0<~FN9$0Dl2ssa1f4}C zTwYjtcK*35OKS@&5!_u}{&RGyYa@6F87?lZuB{r+pS^UYQ~s~i zW9mZrzvC$zD*pu!{s#qP<^M^+Ncn&9w;v)vqviiC!vBNNr&WE2R#)$~)sZ{38XahL za&>v$xO9HixOo02jkEJ>7oT6as{xPRsp;52)1Qz{pF6vhHRhjNdiK)A`L(-hee6!H z#|K(Ja$)7%qH*EM#r{!`QMB(uhxkH=_(F%ue4+QSmb2q!w$6`Wcd>eR++X7tE=A{! zFJ6>m9i#XDHq|oQT)u6N^EPRd^%|ubcWiWmHM)3lbydvp2W@q4{~WWSdRnDTZ2>M- z>JXYqL6irMeW2H`SYELAfrB!}jzzr`G^tnJX zxrZxP$9d!ayB7?&70AvA_X0R1w08C{;P#j12yV(N53|2S@53_~IX2AIttl^T(wSb$3qBqoV?-t-0T%%lS zouPkphx7S`b5|~?&zw=Cu2KClw=~qP60;$e?tZ)b{0*G*0*_Z1&jvR--ed}82@010E9Fl@gmCU1sGp9FG>YR# zk57bd%`xOKvHfJ9Rw>l@_lz%HRz{Y`MsDN(UZ02CV}4pMsxb7mf5?*Vw*2j)GO_)> z@(5LRnQQapJL_-g*7EN9xBQY%zq>=i0G3yjiIH=^=b+jzPK?a|-j9B2|J%1e$>Pkm z{q{0{MCz@dq-1JcB-0~{qn?O3dhEu%H?e(Cxq*3|+CHSHNcq9U_9TKMb5q;Lm0Nha zy&uC2<3T(^Aw4p}V@WqUj*`b%$>qn8aSl<+zNziA3dNzaPf&!id=P2-*z!$nkF#)k zdqTW((~{8FPO+>nDKCtX^6~cXqsZT2QK2`^v5=;h^tiXfNP3^C<3q|^H;Jvp+gw4~ zw-PMJz`xsn^A6J~bpr?yFl3yuFa*$hzFzfM!01)dgbG`0PxLN|;|Z-0`7Q`>y0 zr?#iW`{OLkh;w*x`-G_fvn-t6{+xLKya<0ngnSyO#4eiJhWQMBQ`;v+_>`!BMuewC zsEPLGL_TaP{HC}0ZkXDh7xCvsxFEu3MR-AkUlid*5q?R8SrJ|mA?fI;Uu4UST{elS zpQESle*UnUp4xtn+Etdnq`ahDRM<9P5$=DXTYP-TG3J*e^h2BN+F?iGw(Y-dyF1$5 zn9sA^zdlJU>Vc;#|Aj#g@3~q2dT?a6=hzZ3IE<$cYu`TXM}|#`{1>`>hx7ApoM|)e zNFUt3iVgQ9tM+yTkMAofyu3$OZs5Kz+%NZ)@)T;y-aSM3&UnugFELV9QQO1J3bu$9 ze0C>R(0-}aGO9O*u!1)m_)03+FfdInhA3e3++RKAx7Y-19`lh5*t`e8=4E2|dZFuH zqprnVIugO>on!Jw$?`J5kuWCP%S`b4IJyw$=-zd6bT7^@Xc=&Hn1+uLNB0>}vUBge zuD<%-Uw&d~-}2{`<(~n%BoXqNT?tA0Hq-;=_(GlX6_7&r%t;4>`6nyMUW}aNOLj-RkfD zIwMeM-@Ej8p%=NC0LdNq$ur>&Ik)%Kd)4g)+#Xb;0k;=$djYri;oKY?)6CKCj zFCh6fe6{OXu5C1Mi?3a(<_-2`uuc}=s${S1D)kONjC+8+Qnp;1Za6%^GV*)XDf=UI zal2iqn4cca6Jn{G~*H^+Cr^l$$-q_m*`bx%T{TX)%SGBC4I{;@hX O8Rhg|1$S0j+W!YjG9okp diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_gemm_run_0.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_gemm_run_0.pkl index 96202b24ae407ed861a74de6a1ebf566d9b0150b..998115a4e4d7c96b9777b1297b13b7c7bb896058 100644 GIT binary patch delta 761 zcmex6nfd%{MuP^HsYgl~z#w*tMh|~UW?p4_QDTZ-N@`hVa_W>G;eew2w9K4TAO#f9 zov1G%&FtvwGNp$lFEJM=&r=j{Yy?7vX7NT-HXGQx3Qj&8U@not-ouieT9h}%o1urb zI5{IVWeU)O48F+{ffJZA#3!!}lu*x*?qLcJoRT5y&6ptv#PZ%O84BJ^8Hzw!$(MmK zLm5P}oirG$}(|vvEo-&=igy-r&G^Pru-h;P_xipU^1;OkzSZY4U_17LWsG z2FVg~05g&{He;{@CW{0s5Yoniq|L(w)!qE=K~BE$?xCLXj?N*TVXh#Tvw*~qLxB+( z3XH%|U<8E%YlgBn2QU!c2TRr?1ptS;kCAh{hi3@H1Rw`yf&wrckxk$Lng9;R9+9Hd zbg;Gn*C01%|4_dW*B}E__i=*deLaJNv6w54W-uGn;LW!}%muXgdIX?pB{(o7#I-Cn zuVl*PDbN^W%V5tC_GSP&jUmHx^YM%c;xZZD-i*LF^~F%Z2vl*Yd6~F;Mqmd{MNB|N aKj)SkRSwt zGyqysPLetejwp(Pk2n^c_HCMWizsn=Y}&rW?y=qDv9jniZQ8U-_Pt)O%T2QT+P>_* zZN0z$H!~mr4kX#Jozy1C!8hN1U%%h~`!HGa^#A^^jDH@@JJjavNb1yZIy&f#$CBf* zbSAHEIGi3E8cD{EL{r1D2cC?jviU3dom2Tw!Fk|tL_Z!)?im;xjO{T}R`L={WyaFZ@l*MOFRfQCkrV0YIMSR~Jts!8ad}{TA~LR} z-LaA!$56@{;VFnjcDBnqDB(MH-k;ARQp*Q3*)&r9p?tRh&FI+RL^AfFyjuUk(UDAs zTV(U+@)P~4n91hVb%U{ikC>t@B%Ez~r zSo__I-{}i@x%EVUT+#|PUKd{%x5TZwt}(to-n28`+#BEEh_@i4@zyt+zxbwZA%0i9 zP0K@kYei}TL8n(wO?*eWo!1+5dc2<7q*S#=BBRml=tMFS$-gs+_9}|g<n~w@dUoquIPODGvG3CcIjzR;4)La=JseOL6?6cPOPnx6|jUlG32p=?<<=sakXF zK<~%$af`a);gQ%0`55n_@F5`r!N>>-TEk+#-Wi^G&%C|m^=_oCM%O4QrteL&ug0eP|)i_ z*8Jw-Hl$-1UQUHtadn-FGk>pdsu*Tj0~d z7i-k|$jW?>h&gG>k@w0a^T2Uu>2xHI49O!H;xpN3Y9Qty=Ml_D?J{PBopLOhiOEAd zJ~5Q~l#FH}f1=#;r*^c<$DK5d?WB1G!?aCMTQURj7)>A%nvEh)MrkrV(mygBK{UtP z<^I?JX1EMS%yA^F*y1ADu?RH^NBGdVlUzD>J#x~C%<+g%$|EV+k%=XT+IQZU%w!+k zX~g&(O$^=0#jzuuT@M_5;8>^pL6j&G8zIg9&SRa<;aJwO6WNqgV_CU(EET(7Mjj{9 zDS0U6*cD9=XWHdmyN;irHzVA;1$A9I#v5y1eKp2*nm29K z{6v2yJCdEq@{WiWUD+2Em*0)CPcfUg6ju;^%I9{Q_QjHZvkskH3)@&9pyhSnquL`_ z1|?5X!=lEdjbLbd_uXn$L=B#nLo|Hr%ON!51|j~7s>d`uM@jo_m&yCy9eRRx7;We6 zYm@O`emmP@{8zjC;{W08Z)0djnDKpn^Wih(!t&5>-V9U=R$1AR$oOc*H?+&2=-WHBh5NRMP=t6Htk?IQoCGl&vWyqH0%On5d7JqT7C zM#?*y;&-{JubZaWuXr&P`m`nVs&zmchKJ0=C}s>AL@_dA+8&OL44tYh6u;tgJH0_I zn2JL2D*=zwYJ41pVm1`dY0#M1R1I*c7+<9GzeSW95PQ0|D#= z6pvXm@CV(B+v)e1ti7?2SP87z*md}WewTvbgO45+2@7J#f>o6|TF-}#YGvpTxx82p z`pl}y9}0zhNMx1quz_ZcYE`3zLT;zqZx(F8<#M}FZB>G8N@IE*h(!j*lF8_Jrm~6x z?f_PLu7Fvk2izf4QP5pwy|XQIYP5eWIWllxMHKf9#xTZJw1t4ni-oNDE+pXbyF5<6 zQl%F+CPzj`vb0(=nIz!X9l*A$q$-QGjI8;v^D%E?Y9n8-@eO;iGn9H&g4lwXG?0MT zR=+hCW9e08?sfSU%xh+whJfPrU_dk5-caBT$7AVKEGYva z7|B+aFuJJU8PEr=@BGmWA)=9oCu@OM4roYH$nW&%9ilSaAciGxu-ahB zi?~}$v)T405C{fP6dtoxJnRs!;;RyNGX~3umg0!n#sgzKH7b{Bp9zG}{R8^up|Uti zRu8S&vGFm$Cs~;<-}s_*Wz+6Zl{;8+$5O*1sn{N4{l}MREo##M26|fS>FeLT>bup- zV~0sXcI?y;nZ#!>SawgS8pZ0JWdkSF05x;uy7i!doWc1^wFtBj@JYPl9DjC>U`qU%tM;%y> zx61?BliEV(>cMZ;&?1>nnpm2`5nos%T|DGf&YNph>-Z=Z$hwbXgRvob#Xwu%%O~&0 z7GS5m`$IC+PnIp$CsG7jaCgq}(Vb2`B*#hYSFks5mL%=bTD55+H8M1Ud=?|zhkXKf zI6g|-0ScRT3XW9lL?n|Qh@|e{t6Lj>xD`szaLxfU6xKwo+Q{3e;V`cr)0gIYY!M{J zc_NxTZn$vJt_A@O$z-bStxDZiq*imz_f=O9zEY!Zq^ujjo>p~Osqn=bt@KZt?_{)= zbWzM6*HTrA=LC z@3Hhm%xE>gR1;rNx9U5TihWG!7OaU@;hjd%?J~BK*iI%h)b4Man__y`7Ud=T+FNY= z0)bFS!AjO_l^ejo8vrO~KJS){t}R;S(^P5HTgl+6HO4)ONsMl)_m+d>>ToK?#>8m~ ztWJ~0Es^8Y(Eb%Na zLwyjq_289wp4_etPHzgudUqfO7=ZjPY+R}gPC?+&KJ&q81&>^rn<|#siYw&vI9}l-qZtai%4R0$!_L6v({@}HOKt3>15U3FgRBa>6<{)g*?eaAQ*J|{i~+~*a12Plw1w=YfO$s2eI)n0sVMF2&FXP zPIxv8{5l9e#u-$!?5t`cme|ShXlf9wW_cg+hFoBWP**ER zZLt!F$q6~DVW}IUgM)H%42VPdYOF~k3A{mgM?sVXT+9b39&|pB-pf~|wK0tq%#%co zuhd$JUpD~jKxj4Oi5MZ*5_Gz%jcPVZ@$mS>;K*oY-}8Aex)kLI%1hSg^?@A;1*

x;kJkAu$j`nWVCixhwB0eud_yfG%rS-bnlkNHjE(DrAo)i(_z}tHKSTvO*?n zKa_J2wOpOsX{MG(@(by)q&$RL)q*Z|VGwnGFtuh=HMlm&uDPlW7|lgcXiu8TqjKQz zhulaQM$pyO7xG|;f^>n-x2-DW22N{0AP1OdIkoI1ub}FXZPQ>CY#o&0&J1joq?Z zi`KyM|nz%j>D@f<0+U)B5uiBae1J-nzxahjmAk-J;{-OKIYc@?^AA09umw(rbzS z=PL9*(rX1KlqDia7CaHX^rnTjk}^s+GRP7S;-$N;G*88Z6<>7Uw}@M;5_d@^cv_;A zmhv*|gHbW2)h`M1axR~s{AiIB-6XmxH1dDmsDZP2t;Vd{=Jr5pXbde*mtnT3HIODj ztF=Z^pAl_B+o?7Uj-|4ZiA*fQ*D9I(Q)&arAP`lBr54GAHv~>z;&sdonuOKNy94d&Chw9L+i$81I}% z5A=4o%k*~m$f1XhbRL9Mm?qstwA~KqG;c(Iwv*BQJbEp84j4Y-deo^!3^J0KzMbuS zYYocvZ6zACd|gp%`2s4hKEP>MjmAt56IX_!aU9{rNGU_31E(BI7s*rZloua=1CE)| z*kUc@-^WZ+BK??$tndgzH84Eq_}I9^wbRK%br@vZ-NL&8cg6-7YgRe}V6s}ly1-9! zzb?p)C52EtGje!rB$X{cGZL>;rOt!RB`R-0t0_Vw3l~_h&*(k(C zqZj}z2H6Q4wT9{>k8>*BnB}^~7{Yk;?Oi{p>&7Os;}aN*9>tg)RqIAly3WB-gm&zWlZx>j;Is&X5lJl&*_GVxr0bT$t?>%2WCdoHxL3 zHV}eXQoywu5#n}4K6s23yxO#~7Sj(PwO~Y~Q%c&%x4Ox79S&o~mK_`!g80@pRdDe| ztXhkuC>D8`N>+~Fs8+Pk`;YEF*m=JT&9r3kY9r}b63E~Un51qGvSysiaj2m?7~3&= zVCsUn%*6OOKjCL9b`?$PmQGV>Lt!Dz{-$t(1biNptj3TOvEN)eD>6& zGhpb9pWt32x*cS#W0?U+85pOp979~BmlFL6KCF=^JUF#QQtRLy_d?*qnd(1}X!a zJVIjTj|@O4gU$yG(2NNQA!s3$icxpw6kW8q4$5H zAJFcnoO*H61|{-|lb?D&a{3FD?8Lv&?7daB@x)N7WDhhP4|c^V{VIA&H`iwCF2~#M2b; zyJ=04xJo&C%NG=h&r`7Pfn`K9#4~Xu3nvIYI5P!ke+5A(u?hjfaj%whenZ3YZ*QgH zIPp&?*n=Cy#6MF3|1=J%iT^_>42p^WODWt_9GL<8nT?JZC?Y^d+huHhhD$q}OAFwW zTF1xM09{${yyN;XkLABM@i@kvh7=?}|B7*vWCh2a2p#8s#+}g#$Pem{>A2F~all^gGcz{ z9D^A@#O6TQG@sF2wO3P0*W;6GFDlO3dTxzltSb=3!ddXX7Px+9h1q9;fRHc?;D%?;J4Gb;WI zu#=%;Bk|DTFk`2$Va*)RM6tG_QxKWUH(ARUoFoK%<;*oFm;}f`Wg-~@a({U^Io2Od ze$hELG?a;D*%$ay*z#*mhy-JMa2&Agmoz8(g8DWASbp39nr|DxZ#y-vHxnBs*@DQ> z$Vn`nX$9Z$TRg|eXvDC@!g@Gm`OQ`4sdD~HBosjeQm>!qSm}$@^=*b9i!I2YV>En9z2my(uwRxeG6!=}|czWbXeq>=)0@<%QGsEMM-lpAz z^(9S{!Lfm)X2ibe)JU{1mxky(KKL~!$+6=IIZpu2aS+)}c@Tcx3CJ@O89arAj116` z*n#7}kLoLphHFZr`7%W4pNtKq2{ENmd^ge zBXeeDMt?W@8>z;#g}d!w9lI>t?3vu9YhV2Z=fD5E$NGQM=wFm58k!9M+)L?KY00xv z4XS}fBJq*&$iT$NU}VS>3ix8)=%8B}^x&&Bj5qP#q5Ekw{9;bH)u)p+b(}#aEXF`^ zE64v9>^j*$BAGwRoE6V7nf)V%Z%kDW|0$E|QC|=KDQoz@iXQJiL07lU3zn6Ub9;<_3H;rn(iFZ(}w znXx~DdHz#Q_65BEzwf|OY}VAee?{El1e+q2y(paH>h$CpQft`cuL|V1hSgn0*qBS} zFrnA|9=+RGovepNZuk`6q`TVycde|>rN_bTYs zkL>DwVe(~({T9;y6+ADlJ@2tHFxD*J*2fbYH4J==@yJ+?UC>|jWf;?!VeFs&_4P4w zAJpxQVd-(hBUzSwdO+sH()SbiDfjg3mvOrXf~M|Wd( zO8`FO8hOy`_YXybz9H|>;6VSNuixzo`U0{3L1nNXr$nPMw`WTX91;|M}~)6}wgxJ+}qNjb9r zW4)aR4|EyD@TOHNKU!-Wi<-)Hy0I=dA~4#&p(nr8rwso_KQq+Fb%)elC@y0kV)VZz znEmtXua6o2jecgR4&A?DZ}``Z*=D>@*oM8)FAcTQzma}J-44GrHyZwpallZGaR5&E zhw@PRc|SB{X86bKqkSsYdAcdof!UqsP9{i{b;k{QFmVvJv&(GUoSAjf2I3M{So*lGb}{P zK8F8Lg$F}xsE@!`vFuAia5ZR}le!tLzsa4Ef1~{kz|6?I?vyLPqWhmHHN?9SuVFlj z7c>9&=y*n#;ZRpA{S#jcE+e0-V~-S_wOMin4|_a`}TyQ0vpN;fxlU+Jbk)a`=HG4|W|2_a1xThzpk!B_c>v zQJkg`4i5AlJ$5wmaA!|ApBSJNQxXy}yg_^abN>`ld9bGz7mt730@=*YfFT$2_0M$mZ*=u9b@e~#>L2RrU+Ajg z|AwyrMuq*7uD`CUhW>x)`v0k`e^;UY4_$AlH+B8@b@gv`_3w4nNT-qhzt+u-aKEMN zjrd>DNBH-ws!v|8yhl}8UtzpgRlVBZv&#FQ$Lx>Kr}Yvr)X(VpYO3L`np(}DVP4Ij zVQ#pqrW)?5sfNFA==u1%t{VDEf5t-FP>scGHPvueO|9n7Ft6s%FgM&)Qw?|3RKuTP zZ!F{ueWj`w&ziDU0&69(Rsw4!uvP+VC9qZk@A?wRt2lli8;EddQYeKcZy|R|3>XCJ zwVygDIR>D~ohl-v-btPxM@RDN+*>cuze57S3qK@(Xs_%k$=si4f|u^t z2+7epjFU`vb{{?t1^pX8JZ6Yo2HYMxb6#?NQW42_dwlq18T}d@fZQB;Z`LvUqkYGY znS%C-N6=<)f|di2EO8mSeUlJM+8~a{EzPa@x9X?2m6004D}TthMSV={s}Yy* zU1##LRAcOoDhEHob(CAZS^rBztbY1+wh`e9KO|{TR@8Hi@WUf_mcG6xOwwJ<5;D+t zgw>1t{y;^rAFda>{@{_sGrzlZ7dLz6&*y)(PkV*!cX7tLaAv%^S;0!7W>;vA}5!9xI*(ID;N%T65Q8=(Tjpr<$89WzZ0lh_q0ZNCyVAMW|=Nz63cxK@Q z`U3Q5E%XJGo+;Rz$MY0oxrA7tUxdB@JqiaFtP6#M$AV{l7Ye5f7BE2R&=-u_1w8BU z)Z=M_6X*-jqqWc%OnOAv)Znppq4#v5_jIB6z!mf;4p>NdYVp|cG{ORU^d9CAq1o{H>Xf5;wlb+MCIg4iov0Owf&@Vz?fF8vGi%C4^@La$%3k&Gcd*}-$ zC=VD<;W>}zDLj|p1o{H>Xf5;wlc_nas{1xD_t!oP$&9685*W0yj(Zu)HM1OJ)(x!J zDs)US)|GLKv<9(G&1s=x@|$Y@3m!q= zB%7LBdA(75UE0kjLm=*j1sTa!2OD#9PV|g_v?_gg)Ex{-Vi+epmoA_#raHDPZ zRcIWuxqElCef|2SYnQAdLvh<+ykxunVosDTU$@>X&c14sdcT{y@ttPty*n&lzhp%K z)*a#{95Hz{w_`{1*BgwePsB?-O6e z^WEHQHvC_*e$)2iJh}L$4gZ&}N;%l$`R)(CoP*U5@P+?ZfAHlmfB6SL(7&e+|0OTy zr41-&?Uick@Lv@x?Z(RM;hQU_E3b#YTQr?3zV@NvN;drMm0G25ELdA=zj5duUU7R- zf!1FauProPMg3S?70bex*r6*7j=u6*vrVuG3r)KPYjdql7`vAH%4=NwUJy0d+ZSGgZCZE&58J*#kc@_*1LUsj@9=GuIx%Y^{g{AGknu1PB}q7Mp~kd97`sX#r*0Dq<|ah&D?v3k!#+e427|9{D2l z7z1d~pchh6F|g!@*JwCINww|8;Bja>BEDu5XYYFr3Ez%tn$69NxyxvN3?Mmn@`Xn~Kb(lzv0KD^F*g5SM{lv!pM_M&tA62iISaeaw3&D;pi zHnmFVw=>8JioNz#o3wy1W*COG94ct$24Y8QFao!--F$?zc42I1*DfJzHnwn&WWC~Q z<|T$|m~9odqwKCNU=W)@@gmSy7g{houCQe(~~&n{fKK^+f$()gaW zy@uv<4TU$~;c;ABxU`VF@d3xRsX48huVTQOV{VaMU>JeLKcYx#f~|pZJYSGv92> zU1rbkdrd|?xPo8CZ1W{CH_xtNpnHAcqtjjD%YP;1uCV!U+FoCJh}mnzkJ@X+Z`Dak zjifM15hdjeQ!LU{^MB_JGh8bv4U&DGghbC1sM?o8U~aPch{p!eMqvP@ZE-1GAq-P=dV{ z-^{*_DRYQIP^<`+Dd*upv_Hj^X{LyR-NuxSlD(cOnzEhD7_*OqmtztxRcT3KQ&~Wl9ZG>hLYtHzQurz7<-*UN0zh zqWyDBxrZsLXy1->ikM^V&oX5awxY6tTnLKD>`w{G-AK8hTojZx!G2aynwheV**6GE z3vw+e8<93a*~IKum{Kc*w=nxvrpz@PBYXMKVPDi)cTRvRP851?3(|*(ND>N#PeH`(;VlA}N>93`Ipi zE+nN@3ZFrG&??37IkX&6`5dYP;Yi_m$-Z4uo|BZTXj-CjMN$?dMH0i4lKr%(oRgGi z5x%IL6_sg8Ig4;4`*}e*Cn}$n!Y@k7^OAC244;=KrUYe53ZIrP*gq%rwNmf=n#6>| zPTJJ+brr`Tz5*fn5ppH@ zKRv|-fDcShk|64&z%Fvp)!P9Bn=25C&C?_zOTYn%WRqHsLu{@9Fm~z~#OCQujBfVZ zIeiybH&1I&#^&kGx_%2+LpsiMV0woh{`_OR#Q40x~f&g=4nkr zdGqvrdid?Sx?5MBdVC$azfP|1oZhGFyL5HGu0Eiv59;bey82;VJ)o-}(baBU?a|eP zx~hRyOD$+S9b{C*Yu>4-H%}j;cv;^^*c0pka2~Udz_fh9MU9yYq2trk?u#9uGK{4s z%$seoMOS*L&>QWy!nmk?tL%ljy*$jy{?YdZWbM|N*;?0lZ*~pdUT)hznkA7O-WK0p zdISw^8xM2I#k*4gnjhtHJsNK5#a*Kp5?27qcfWZ)ZSt;_dt4MTK|*Rj%}iQW)Of51 zZ^fZyp2%h+qfb`h&`ylwho(VZfo3Y9kbWxpvBWP?b*#~2#wCw$?w6)ZNU&f4mtvcp zbiqyi_+6WjaVb*bw@~o`jkqjD4 z2(UsK|7B^YLVy(v{>#!(LV&e{@3N>m1GWeTY>^BabO!9D=Pp>lw3P5*1<+tqH#E|& zV6b79hAMPe;VB9oY?!41^p`=0wS)b#s4yTtY!Q6eA{jK8@L`2AI?U2gg%2wjbQsD9 z(dc|wJLoTqsuN<1AjB5QpuvO?E0pnJC?5o3LWmU%J`Cjp7dj!<4*tuc!hjgDMKEHE zWYA#3h!x5RF%$`i9}`BbU=U&`61dPAv33w(7FDOj7VAMEk%1WrRw(1ZP$US%gaj)X z92klOE_4#C9h{d%)mgAbuwaX1&|t!X70O636v@7uFMaJ^?8q8SxUd4Ku&Kiur?U)3 z4CRA3bxN!q446gLIkCm35GNTlm~di+GD-~P1OCW_6D!mcjRZ;zu7biOEn&kh6xlGHV}mq=L5vZ{Vor=T7s)ON zUAzv6ubdW})2(~qzev~gS=VSWAnf>m%V@DVofdPWoL302xs}&9K#MJXTIa&1ZfN}2 z?ln&AJ-~_mgt@S(!#X9m#)-YHoS4Riy|XOXOCoztfHcQzwy7IhEmv}4eFupXdm4;W zt@t#^FeZgTjDa4TW?}GPAQmQx^&$odl-D%pvMgY$Bv3R&g@G;uKL(mBPox^H zhH1=F4QInZI)Nl<06&9iiqk1z$-sbtHv^>xS`35|_%YB;;Kx8T-39i8^GL0T7i1A< z$-sJnUjoYn76k+v7&)r|;)@6|Fe(;-7%)(6V74;LyOufoB7k1-=W6-z88|a13IHlW8EVK=joLM1_Iu0`~=u?t-WhCm{6f zoZ2ETD<=>Iw@CoEHg#Ad6cvzd&okoH@Rj z_!|M_oU%>`Zv;h0{8ku@SF>mbAEtu!Vc`oB@nLg9_->I1HV}I+2+FgJcruWY&k5mq z1TKNW0tW_GEc^`T)pBBZ4g{5`fa?NJ23{=(`ihvaF!;6wfjGJ6IUfcd3tU$lC(J;J zf$Pd6ULpLPX#XsVQw%ptc2H=bzU~I+gy=X=wowYtO7?retAUr|JlPh`!?j3Z@LQnD zz=1ta+*lYy=2j6LSa^;TVBosyh~)~4B9V({81Y^p&%kwo4+F2Z1)MAL$O$m;qoA+O zh)OLBKPTBgOH5dJEAd`-@L`k0frUZ7f+0I6hNmSv$S}~W;K|Ml#D_`5lZBs0VkG6X z6rK{pQ_{p)N%@=aPK7n$JF2$!N%ZY3keu@keeHL6UL^t)jw8f-HMA!e67xUf(>u#v)H^1oj=V$h;lB%~Y9?Py7y`{2!Yp z`}`VB#(#KVjV8-x_709kCdP4vBQD^~Mza}%o%%O^ifn}@^Wyin%YHAl#LA89{)5Kv z^xlRh6E&LbaR>p?i7^LGj4i#w32mWl4uM8^{3m>3Y=pDx)lZDgy;Ud1mL7h?C&pGz zk+C?PB9l>I`YAGcEuI+rFF(B}#&%TDWbL5E44RCNj1_6JcFJTLw>-G@;2B%V@G>6_#xA)+|{8=fsF5TLw#(G&v@=SawP*S%nr` zyfrOW&}p${&|*mwR&2416-!nj#a`l6!ELCq0=P1w#+E^iB~6&I#WH3rS%nx|yfraa z(222S5MxObUTm?97fV*5#TIW(ixt3=5iPb1S}ZAoTmlPLdNgdYj1@~(A;lJNO^Ox3 zk`XDk3{os35q%V5EhCM4Km83~rG!htPT=fK>- zEuiU$2`hjaBPMJaOjy!{2wN;8!je^Zu*F;RU!nVg&jM?eq$49k3owY=Ph&`>%$wH>EPAU7L4xN zDJfxcABId#G!cMU>G7wfzh`iMOop#XCNxiyGqzF2QCOM30?KentZxs2e1KM-uH*0? zgJe1IHh^#hcM%8&Fb_zcMN$Cr0nY=H*8r$Q$KD8U1F|#?`E?Ei0^S1-0|*fE-zrD= zAg2xp5K`bk-5@Oum=Ne15H(<9fNpXOc%B^yAK*7gkVEQy8_+PoW*iq3B@$SNC_do8 zDMoM&#Os0E&2iKX7$hJ)2&n@@1M&kT66g@XG=PNw;ea&)ma7v8O`H@-BwZ5100jXD z0z_m4YLe7Da6v$fK$(CYKEv#Q6(N-lEDo3=1myt-0=ENXXyrg4&_qCqb%;&`;z$Dh zz#5x5G`ES7M75R6onIidD!f2K_Q384I0QNeG_egC7s3Gaws3qA&?98Bw-WkC681m` z0sV0hQ2|~!$7S(>2SVZ;AR-_^V1Pg$HwlCd0tN&;_?)0T4PkneFc)(NvIh})fPV8_ z6dtJHvnWqMp+HYXLh67S0`~(>sKJB4^dKKUCqm|%q}zc9LJl5SUyfwlNdg}5*E1ZK zQ#l?8Idvd}ZIGDfQu3RHFfhR9M98%hc({=y>q*|iYRlA6O!b;q`b(b;q6>L9{AxV_!WWdk#s&F!4}jD8Ux9`E0Bx_ns-SGZ1X3n1ZcIypyE; z8diZr0dD}SxGhX^JBjzyTho3`8#fG%_Wk493y-f!ysyD{Z>VeVCU@KZ@fF-mg3s+G z-XG%t0l6)aco!bm4$jc*Kqu8#eKeTwsUKGw{&D^z)5h&mYtryPRkua0NyG2jb^OF& z5pR5R((o&nMd4@O15W?SF9es^cs+{K_|=EofP9jEEmSGUw|x!dd&3}~4(QRycFSv_ zetJQ^B{PtZ?u8;*cmnwr4Ow^s`Py|^cmnwr4Ow^s`ATKsdkk54!uX1pLG_q`e9L7Z zUr!Z~Z`lC(k|mPxJ%%JaVSL5Apn6O|zU4BIucr#gw`_oX$r91`9zc3r^qnxi;#E*R zCLrH(8OYaD1>{>cB;S)Ig6}lP_fwAvytfQ^&)vBLh~uJh|5HyDjBnY%`1(HaMxnjUcLeP%mqL5r)9$P%Xzzx0 zfLsGymRgy8l07|5mqsWxA|Yr1)Bt+{P&>n=?0}#FzyfktBSC~OydFYP9Pb0h2YEbA zmJc9ctrXS-`GA7e;lc?>WpUIGLU(}VEWp}0(g*MkD3|~wj^ASej1LleK>Z*u2jM-4 z*a43NkO$Nc=w2HK5CMb%83ghLL=OlcfIonP5c30u2>cEx9^gL!g+S8)4nkUP8>6c! z0O0`_dw~)B2S5-A;5rBp@_Q_%ae)mNqys<+RB@gogOC^mFb8B1&>_G~AqFD$7VFZc?>~Bs8m$AZIr8%2`X18fJx?F9n+2=W82m&cV7Tuczy+1(;tYXJlhnA$cW3~@lfaGHc4 zMDBp@Z6_H%y72j=q)K41Z1L@guRK zx;{CGK=7Le_(}B0@&0^whq^8`F&fc-Q?_Mz=p=rnB0U_-M)cn^&wu3i{zPCCxHrGF z5L7FEp5p$a`ww>B@5(343PW*vJ;9a2NOa|+D@#D9bxZrsN$ za=h$n2eYTfWBEio^!Rn#kK;#dYbFwZ2ptu?M+y3ZUT+Ah%jNV1w1PK}A#n_mYrzXf z@QH`XjKWU*JG@m1JLK^@gLWy0H%c diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_gemm_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_gemm_run_1.pkl index 98e382ecb8e5ac9e6f0d1a7d318a84819eb446b8..6a8f76943badc12258d0949b1218632b80358675 100644 GIT binary patch delta 723 zcmbO|h57w!Mui5JsZUB7z#w*tMh|~UW?p4_QDTZ-N@`hVa_W@H|Lo)@+Hy%VJNmjz z>0!xB%uSur!&4M*Yy?7vX7NT-Hf!0t3Qpb|U@not-ouieT9h}%o1urbI5{IVWeU*H z48F-iffF1v#5EhI)B?F2J-oqz@t%IcA;Ixsjy|DNGNgN$f&-^y$a*to$N{muH%o?s zH&ccpkXG_#V9Zbk5h{}zgT$FK#3#!JNvIRh!i3Nw>}52iGzrOaq2NI0cpsPGcu$vz zct__D&oEa)E?`D7j?EZs++@CB1wyW1LDpsh(bf|zRS$GLi+hlhFS3srfj(vg`j`>q zW7Z61Z;lL=9+9HdbdbXM0M{TlXa7*Y5Z53BRNrv8`xrULdw7O`{J;X@z)V#Dh9k16 z96(bc;TRVpX^#{joM0n-J%fV@o52P(WAl{|a{)%a$vqjeJQ?g6!rlx(+ZZxDHy_NH zATE>P?ac@bb6;qLv%ys`0#%%8UM4P|5!iuK5ff0+=Xq@cKt-K66)^)9{oK?hzztWF Gss{j&B+usn literal 51739 zcmeHwdvqJuede7Rk^%uzph&@{B+D8zFHoi+0K|}h6J<$Jwxf_DMM>~B$F*RH1RoGc zA^>PfNxUU#$(Fz|D3OTm*cP1TY??l3+D@8o+8#gJ?rC#+xRNM0X}Z~?^pD-{IlDZG z|7d%%d$#p{zdJJ^01hO@vC||biNT%wzVn;={J!74n5y`4-Cr~Q_ej>M)}=5ka*cON0kT$A`NkXVya342~w_vGAD?52aQ{#^R|nh;%Y?CYfzkr9>o^?N!Cp z38dVzcVuELviI}hFN9-z2POt1dySG6+(gEc6A4%RO!m-gn^jBbR3aQlnX{_*)M#o% z9*9qd;#%3AYsGO6jk`j;1ffuKi`+y7Z)(0Tn?k0RPbE_cl={=zZUL6DiNVQO0YPDywc9j0}v9g=4ropx-S(US7rT^SAo^es5dAhRQ zMvRPSNA6bZ^4_iPfEV7}ck;xDq{(!oYGl)hWyGr6YDYGY*qcY{dPlZ6N9ysMBMtBM z@kZ|G7DnzIY1AHJWJhsfJRZg6_VdDwG{I}_FF< z^Lg=N70s`xX`DdX_VLNF6On|R8i~l06Y=D_lzqN7m)GCAmhxJgQ@l#M%V#2EO8Blw zI1xJ|rxM}u&9;aIG+ z@NK?UpUdqv6-!$?9?a9KJ=i+2)FdM4Tn8fZiOHdLGHGjXYj?TbrZQ>ww);@r^(}>D zQq5mvSH7fc0%-Sn+Ff47Ov>%P*7kBKuJvqc14fpCNHQsVcC!913nIXqe86?tqlaxh>OM=d7oS`0GxCcjzjY3kUWaMJ(&uR4@8_4 zJcNO$MaEFjEJtF=h&>VPecYVv?bA5#!;|*Z40F)LR2H1!M?bQLOORpeAOg++U?mu+@u}=9@kSGcpBFll!W1X(yNXpra z$CSq>QgZLac;r4A&p4SFmxsokyTgg$WQ)9e_sLUqXQX?Vpl(V;cumc!4<#nYL&kGC zcWVvXWjL%osB<@-LXU;fb#Njy86OO%A|Z%3mE2ci=UJ)u?43*|_Qpm}?2VsEjZBQU zx;$PC`Qxd`=@eY)y>|Djm80Y1G`4Ry3O+eLP+0WAe8Im-0@;ROkZ!rh%MXPqjOOcl zCb#197WPbqmEMOAyi5zMPfnglrbbhfDc%TCq1Uy;cE#s)dD=|7jds5mZM01rgG-xt zL9^L}2Ck)TObk%*x_7FM5T-w|GgPmrE@_cFY~o0?ey{ zYQg*}H5!VKl?)sKHw_N$g|#+6)?#9z_+6g%G6T{U8umsb$rSp8Q;E@3q_oxsJZ`Vw zg{jM0U%!^N*Y69sJOOPSD6O@s4LuK%)q0WQb{6mk+-{em6pSEiskZt&ei!D)=7Ulr zdWYoX7zPaLL(wr}#2$`}4xK3#N~_PSxP0weGR5<)K&wyjxY~8Ubp=P)mmCSlv6!KG z@>;==SevU&pQRP02>As3e)yU94pAmBR+L!4?Q6rNyUnc)y(Nh$ZnqyjjmfMV!?Iao zimy$OA5G^cC19&%e|z{yx%AzL`3#uexS4$yeFeS44}8d+gHlW9cpOMDEDf- zG(ive%A~4b@?PfU+Y=aE2O^(_5_V$~5BI*yz}3iY997#|Vn6w7Rf< zG<%VOc0cB?9_=NstF*Ozvc?zc>ly>n^}9Vt*R+BJ{EFMO))EbZcccS=ms0bOWtz=4Ecj(g2OdykDzB zZyC}`94c6J7)A4=&b3k}Z(i(nbeZVYOp`8{J~X5z;uFA4QZk>u@k#00dflliw=w08 zj1P~FNA?=?KR!jPSM3An>1nQ~&wsP3?`BhvCX<3RHEU2z^taGkc2BBXV-u$$30_8; z)5MT#7?VJsc7q9CNtVu12|9-`PfU$YjO+8|JGv!xGv?oMAXkq1(zwVBT4LvL5mB$nc@id<+w0J;t%Y$dJ6|pv~{))AwNs&@Auyv<&l0Mbq`kal$KjI9GhE*`=rC zJdO1V)&{PEqCHZn+9$_Hheq+7`3(1CoxlUe$7ne~Y12xhhttD%7p?=mw&vy*f-)_-u`4`lroTGFnY~T5Y128jGNMmQT&{rrW>zKkNn4PBK zokr3vGM189PR5c{?{E1u`SNbe$_v)DY4VA`w zE9hLc%D6W=h2BkVy=mvTKAn=jG5Q20@^kfs(I;t6n(zIh=P5;6Ip3nsi_TKO+v)D2 z84CHqc5~57l*jwovqiJK)E{Vf7JZh>Q11k8-g!lzr?4BH(|ba(ek~va>YNm;vvu&d zc&!M(_JGS<9+uMQV{2nm@ig1-$EpHgykfTG@BrxdxB`m#s#e_+4=2K7NlMy$Ipa}4 zgSp%uv-KDz*~k}poA#<|1J>XZvBx6%fNFx5Lo8SkVTE4+P2?A+NIgK4ZPUaHCd8Y^ zB4ZPYGcwR~4l(BW6{pr+e^qOKM{n9)7pFm8b-w?n*&xp$r03u?Muv^~K=M5R#vd?B zS5SWxKcPpw{_qx8;ME3&Pm*E9fYwJ;?>3L**$;c^==$#^Cd6omS>2J`C@E6Zk zY)mlu^5Pc;`}(OXZ7=9%3u}~z&`{IPO4Wd!{b7(%okx!q_1^8^P<@!2dQEt6?iT=> ztRu~(HC3(G7|*eZ$0B1`tjMKoDiUaengSF9)?<|92XGMRpLq`(y%&>~LckLd+Nnha z+SlV3EyOn==JU1|5a3bDhym*Q^6GDLry>)x0h z9m>GT5U}szNXQfNmed8sqk!NBO0?E#<%}kWIdSWsTNPs}x($cpB)N=u*Tr2%r{_R+q z0pBXqK!#%B6c{h9?4`;2J!RA`PA_+DR`TDK)UI4H*^063UV z#!#eIQu8rm9LNuVwyRaw0a{n_Y7>~lJVxBO$%{i=a0I~D&C2X+L$Buzm@l1G8&o(z`Q;*?A|SF!Uzv0WhW_+HJ8d63iwpn;$1=Y()%F0CV-)?R7nd15e=EA8a`W^%IMyixEdr%A7&$Eu zrq0Am$T+~qes`;aT4mBHp_cpc3c<>h5eijiJ#F*)Oev6}hzY;d?r|$#sJyh78%s7r z8xEk| zq0b7Gr3kntj=_4aOSi2JYH#lKXi&Sy14;vwn7*c2*Hr4L=8*zJVj?CF;jNY=jhE*G zw-+er=8T$$q={g}fNuT@>+-PB-UJDxr1PNY0gw-7a{*Awyyoanw64@OoIHX`4p7fx zcDbFRD$rXn;bF+ngF(HmO#3dF437~u4~0xhpX;>k_kaI0^u)vMj! z!ZF|(^R%rC=^0y#O`bsA)S5)7)k_~w8%D|kE}HA85)gJMs5;deL%ozVZs6Vr z--T@VX0E!qIq%7vWLB*_a12e8B7XK={B<+6;<1U*!EA50(3^s?XVf7v=8QJtySt5& zh|0Jvbaj}zHgMM+2&-BSrxN*Gx@)^-wGve+47F2X2e%8Mh_xT%H6^tR#93$3(wPw=*``%cU|#i?#(mf&w5CfNEZkFlj+mm!-Id4 zN>m^NpQ&pg@7mlgYdN1o8KVt|ty(%yk0x~~cUMrzC$oC#y5LWi!m|E5%8_S#0C(MD z^fP)Odfa^A{7qruj`Cu(Py$+!_t9;E{o^HeAK5j52}OnQlLs$EFWqUWt(A<*jR$1W z2XNC}RrpN#g5_`Y*f+^rER%P^L-4YMsVv1Z>xWZ5r}Z~cit=POO3$NZk`EK@rqn3@ zQ^p%Om(|{wRa@a693PrMjnnm)Eouc6N>FR9VZ6_fHlj7F_Q8qqRA@383GwMlGJ8R- zAuR+Vt1#Unb?{m;iS*;z2~Z5LMmkWpCjFG&$rDDobWOp3t2Oj(Egepk?*cw~Sms}jS`Urhnk|7 z@ZP$opxPb_kDV9{L$y8R3?2@roKEz2uA>KgyIW+sJA927xM zRGkLhc9-xgjXdWUUOrzd_9>W7+`QQ0pg`3jzbF&(yG0I9jE<*@cZ-ZvsZ!^m1Ed~a z5jhGDuMLm`$;TgkyIG^qG7aui@|onN3Eb&lEn^{ZVYnWVgzYC^1)6Y z)J+qUsrV$iqDRnWht;alaoy(RC78rU6-;vTx?$~}R=3iMrKH*PzIaE+@8l8Y_kTx+ zK9R)klhy!AgC!21NEX7Em|$Xw4fU7&Yb&?)D+U$hjhn8**i1fVIV z&xTd|+DuH}c~p=7khUjj{oeW-*L66E`C4jlbO;JucdLS%C#6wrY z^S+}84t3t=MlCIvy4pxRmJ~90{Ux>AL#z%va-3@5HhOmS9T>QvE;AXAL+@tq$iU~h zvicTL-2`2e_?gl1tUAZ(1$Y@64w{k&y3-c<^OKRu2p>B&?F=}&Mo#glA>9wU){*1@ zv(@TZ^BJb2F5-x1lBCU1sj{BhS;Tk`21_LU(_Q-hn1XN=7q3eI(V9))$ zNWGwSuQ3|xv1oSb&unP141_f1s70qq9>dlV zClxE4NQBRnhJtGR7b4BjNBM~Sg-Buo+A{+N*nzGciuG7|LEg~4mPf+ijwZ&jr6Y!L z;Y7+sqtR_`Ez<-Fs(32g@su)E`&HUj3C}q(8fB_mW zp&KrwfaJgbSM>TyVZ7@qj=aG{7-~c*b6y2#kYKu%$ zGv|Tg*@LbKl|cQ#=tvtK@NAg5K{LNaegK*%no5g4uZVt`68?3XQA97(6TR=#iRf1; z*$=}c0_Xr>5W~u(7$d1iqv-d@Kyml=a3N*SmR4{Vavy(!RJ6m26yNk8&^+2KlBaIPh+XFDcGw zeagd_zKu)_qNfB)@i28TkDypZjwf_St>5g@6fFmj@ToaEGroh(iL_}vqp@nAW)#lG zr#AW-O@CAy8gH^}`Ox()!x^(&& z{Wbz&yG1>PmC#(W`{gZ+{nq_G4~T-$QK4;|XA=8++TD@=>(K8Wt<`RNVRSG*tH!sT z)v_wsDQ*WY`s?7|Yvxb�=A-`;)Vgr|wi_bWpPI|XWJZy0%cQ*M@>79L%olZc!!+I*iT^zOWSCe;_1 zm|W2|h~%Xktp4jR(gI#MyW|3s041noC`mZ(?+nK#PK0CMa7_#iB_k>Jbv_ri{DBKP z!B`)h1S-y#9a&znHqEfe@-mqzy{Bg3R%5E>dijkz;T;XD3_KQSIOWO!m` zJv?st_BwY8-uQbisAmmNVif${zIbH3t4Hp7E$fpGbqUD0PTq#IQi#zeTCU@OJO#zK!^UZ z$WVd+Q%XhtW7pb)VHHa|s3#J%j)<_A^}3XTSR<)Ffo*tTd=Ct|P|5b}86Gtyzhvv>%L(GlD%@UMbjC;LYf^QXyk;#nrM ze?<3w)jH+V2VV!c%X)Bv$xds5noRbR@FX{9rp}UC!KQvsptu#R>Jrk%SXzYvz3R{C-o~nAJuM2uhj_al zZVSRSuqwBn2luZmPUE-rG^lJ8m#2^TXhc;4`5``=s`@b=S8oebZ%XWsQ2tkOy|i(C z$Yfy5S$?4RCpM}W_%r$=V>)(TztN{*OrM6afBM&VCn&sM_cywwM~#T&S@72bE+>+> zkI+xKr{{nSB@^rxm7RNI6WDj12*tyx5gJ=VNgO*cf{`~7A4?Km7SbV4gcvv(#-71p z_(wvtopu=T;Lt<}<75bH-_S(Tz&(?DflKa<#U2~mgV8Mt{ES=V!Pd66p|IaK)H*ac zaAMGRg48?OBPRxx!4puX3r9TOaI0_VL^GdT0fNJHLDt7Ny~M-Vbn6^RrQ*r^_U?sc zWb%Xy5ZB%#;laqnIGJ>+SxQnMM3YbrJH@3oWyCkynPEPv2c+ggc^T^vqy5?8_RsIWJ7L5(+L>WG_4tOr z5np#^+i*i^8~#SSG|bZYM)?hM7vj>`Xv8=A0mC%<0R-WH^bCc6pMZtjjQALRw7-&h zp6<#zj2qqD3zOm-(TsT`e<^UEqQVval@x&HXB5{6mS1=39htHIGrB87ewc^wKTIkz|DAeJt#MRSC2ng=!knGU&DC}H)ip7>Uc(%5l}Zv;}c&C zG9#O=VvizmS(Q@ZVX&x$=x!mj!ZrE~pay$-yrffGaIQk37Y?$>ABv$i=IOvf+5~Hz ztZFxqokLxws16-Fm>5T))Jo3d6S*9H2rWphK76R_XqYw&7pk#rKH4AJA3XZGP}k?W zdhS08PVoc~+mkrg?`S`m?9&4SP+d|h=pZS&5%Kul6)?itZc&v^4adnUVdYL)R?tE& zdx!~Ybs{pH)G|}+agSUM-+$zRuD)RJvHOp>aXL{nghJ)DX=)MRVDHgmM?(*F_5`!h z0V**SAsWFQl&6L?W0;JJ=m;5NbX2oNVIb4!N!^UWG|AO-lU$F!*IdP0$-Dp(-$phFIv!Ev=G%^bS2NZ+Y;sPc`IrcPxq$v%kp54w=49S zgT}K~ki9rhIm#=SX`U7>PpV{VnNo^3&9s_ldg$QMqq>BD)?BNKWy~M7Bv-1`5%f!d z(6ajpsQh$T>$X0nn;zYC>84LN_vq$s-MmXTKdze})6KdPvq`sqR5xw9>CjChU!xrD zy1P>1-=^D*bnemZPTjn>#5DX3cOzW09&VRzzNPo@A6io%yx#hdnzBB__^_J#YyZ$1 z?*FaLg8zdq0mJ+)-CoW#!j&`2#WUQ?#WUQEaOF%RTshN-_lEu)uj{5^FO6qRv<=gk z%$75aaOKQ$@eKEJ@eFq(TshMSSI#uz8UDsZ-msUNx_CB>4GC;WU_$~M64;Qyh6FYw z@M|xDEbYLL41_o|DHKAJH`xb33>Y-(wVydC*ax5~o+={L-btY!TSv0$#rI#Mf1M`3 zQ5apLAA0~&H&^M`^iRW7P<}8WDUnCZX`$jVoY1c)ZKC`3X=fOjDd$5)6Ngb6YW%u_ zzOH0i>{qZ0?Niaqgg(v`DX>HFlb?D|Ij$Xhp9r6#?LUvS$hvkDp{M&&lR#*l1unEd zjm|#cS~9x#ngiEjU9)PZPty69e5Zq^!Tt^W2F)b)VzhCkeoaXhC8EsN4qt4+>6N3Y zee^{~XCAnJfAB&1)B9v^!NdKHrg`a!jFKX))7Z&$dk5lUQ_!#Y-Z8tba~bg9ynCFn zuwLsMK5f{V?>CN)(UA8#9kV~$f9#kkXditTbp{(~IRMF`mtfmJ1*N1dVtm9>*HA4? zb?|ra|E$;sd70G_4TI~9ZhojH@>B^nl?U}!u|5(3v z3*V{VAaBZD_oiF!Ca<~CEq9~R{j_?fhMj_3x6RZt^7%paOg$srrfo9~?CdmBvf?o{ za+2SIkAj8|za?tUf|y%)%>G6M?0^&7v~8x7ksA_okOS+lf!`*sO1f=E(+z8$sYbR8 z(KSs6;hjA^=;ztzDv0~){|KvL9XBjBRUJ2CmW~@en>%hS!eJ5iW!P6?j|p(!)N!Mw z8kZgJu&=_t3i~qbJtEvIa9I(y2JWyg!@dZ+MTJ9)gsT#l4OcBZV7I7nu&8jcsBmhT z!F3i_8kdS7urI=HQQ=@w;nMR2e9qyTMJ`Vx7uc6!UxeMF!l7jf*OR!;<2rxp=c1uiSD8U%rT5q1kkM|{DhM}ki!E*o;GMJ})}!@dYR z!~u^PTxW5majEct-J-$)@}Ms`wLF3A9IjbhPa_EIi?CZ%IK1}iSHOf%zk20J7uc6!UxXd9fyWcL&f%KH^)x(Sx2SM{aOew8 zEmOFj#C0B520>t7gdLTIzTna`4WFlQT|h2#$OZOg*cV}kIN-s$AP!s>T${TfjxKn> z0rH?PIJF44HsPwqWk(R$7hy+bp)a`fi14YvW$i-a=|bb_LgPUw*dY#hNVqC-*>Kgu z19mhX`hpAO0p}T9XK|%*sR#o5BJ8Lv^aYomC*X4q*DP{*8o9u}4Et-}`R#i!m4NHC z`)yuVR^i(+h3iRN=W%6_7VKzA^aU3P2hP*Dp2Bqj*BpYtz6d)i3w^<5Wt@4ahe0jW zQvNBF1+Avh{p^&b(*#OG@I`VZaKkBC=s z{V09ihR>Y!ZQD!p2r>PRCNS9XnY%2d5f0ane)38hL4JZSe6IfFl~-Q*$xrm}>GE+3R>~ z+h4^S*(ulryG_XL5v+BUHen){OTT)(k-tUz%R;)Ya#omFvc7g*M)kCbyK+0l`YnoF z%XT8w5CXp@?vpZhR8G9SEtLoMv&*b+mf0lhH$`g)D)TPuMbUb(N$!WHQ0B6jvCm$w z>%(OedhPY>drm8>oE5E1wp8?4za{qN(ic(VFJAXwv7>_cXWwqLTN}RDh-d-@I{dWM zn`V`Lvr@>~$$IO_Fc!*Os+c&hst#Q#~LekvOXK) zSOvMUxi5`KX+ieiyKDMYO0un&Sf<)}Q za#p6Gds^z-DP-WtY=Tv)#3ysycFoymTXFWyiuICgX-3NtE>j9QygdrVmeNeI9EWgo zed+zE__GWmO3#b*iuHcf6TQ4=S!{}9nokAac>}e6thl@HQp|>xzMDh{c zb&&jg7er?huhgq~18<@KMHhc{7Ee7Z&F>d6+|1$jZD)F(z47}< z;@xj{iEn=GZ5ukc1@@!qS2ba*xR?jKe2bpVZpOZfvBr+FFH0! ziYO>m@Ddb3P%@%Jkdzw9AuwgLq?~6CNm8bm;~WcGm@>_jI=C=JWsX`&*^G2>!<2gF zuuFK!Eu2pc336Hw4gl4 z9F-`BsHB;rK~lED%9J~pV-r)d%u&r0K~!o4$J2tsB!?sf?`Do!rd(tW8&hgUr4ivp z<(!~wXO5jr$uP$*G*D4FFF5YR1B#9>qs)TCf$|8BO@i{2piGI5udvA%nd1^uo6_lK)IHcfRl4BMxOA20;91DVSP72;3Ikrf_tmOEL zsMMnj2rnu-#Nak5xD(GM1z(gLcZy1bsBD*%OUOuaJS9yoh{~5GNb)bUpkd$3g@HxqGSqkQ)$!EpWj;~0;m!!$(CC9VUcO5TCf~;wx<}rv@i|uUX z+$~@Q;j1MvXimKLx1x$G5+%_uncu7hHN7*op9$`JxqtP{G*EKM0p8iL;**x=6-TpD%{J3tCYBiO^shilX!0o$r^Ikpu7Tw&Vn=U=S4n15a zH=Aen>-H|)JfNHR>*fQx`JiroMmG=Y=4W-YTQ_@j^N?=#a&x7A9bWT`%6i*OA1^}x zXW8f3LEt=Qe}b$%Cqh)1g%JM!Bw=rSk-=|D66sHvSKEAzE(u?1HR^AvyJ3I7{DqEB z@gz&zNB`IHXg9~qHk!tVvuW`9A{^G*ecAT$*1rlA@Rs=Y%EPE=ck?t?+`Kup!}8NC zjz_~Oy*O(0d~^YzeD{0j)28lRJIF;5qt7F?UuGsPC2Bm@12^N)lAlkdLSv7W;m}US z@k`SnuRt>uP)NU&{8;pB^g1@^G2@KK_x4lMYuj0H@HKvywcynB0xu6vAlHwG#wL(I zg5dQ{AXR+pOc50(XjIr&3#qV2TX-|wpu!-A4-XZ#a?ctSW~tG+uw`&z%izKmVP6Cn zW~tH1upW?KJs`gpVP6Cp21LK(#&ldKT;LHaV8vQMf>~6Z7Fz}_woDE+CbZaM5i4e? zDMO1b8myS5rhpb}0SjhPbzW>4yx1~1)R^#Mi$%1UrKSuowrJ2|mYM=ytOYcfMb(M1 zWe{V_1L<-2T7I0q{RcFJN!Gx;z}{mf+}L6fH3s=0MH6mp(V)g4AB51ku@+Ea7F8$5mO+j! zlS7ROIks5DjX^$0#DpAMG`KOy2O)HFtOZ<{MTG;gW6NO2mdT;UgdJNfBF7*fkVhu$ z*kUztNFc{hV-P}T$67#!p%H2H*fQv`Wpb!7p~n`B*fEF%iI~u1iv~LeksyRlkF|gR zLnG4ov1Kr0%j8gFLWwOFF=7x25;38~7B5hOV8jfHZKaOA_8sD9`=fM#hV@sJ6Ek?d z4~`rA&N^;vpoklD=LxatSM#_19zMA)<;L6^!?W^nq_D=0`EG+9TOm-L2t$E!YQ+jZ zjA$}&B%A^46hzPqv!bs993hig(-YVIK)$;`EB4ynW}G4eP38ty!8kuQD_ybehc9Q! z{N!|%5N+X_MRUodW$_C5G4V&U;$?u=R#9$jf;BUDgE+A{;PoP#BUUT}J9sXHv^Ies zvx0#DNw)Gye`cpfih&o?I5B|KbMs)x=F&}I+`y2LKM`Row)Dym-E@x+r_6{D1Kx}e zQD*+MPKBk{D6?09Hsf;@R2XQnm*@*0vT}Y56q!M&eaB$Qh#I>kOBP?F#@<=Gjjyp} z+gPn+!@tgwNe$BD;2knHwhx5a_HW^Z5J6xEJ@GPVvo~_qSBbwOTESq=mWVV1qcSV( z%3bEH*@}B7QtVp->uG(>E8D~+J8p?P+XJ2iL{x^jGiw)HvV-xU4=5EdXm-#%ADv;d z9rT{Tb6LOX^>3GCR`IlCudi49V0<#TdaYo-`ab$maCz73z~9;NlGx!V$h|T%-vRc> z+E59ui@BZPb?^aQgBVunS|gaXZ(VOBwg_wvt}Aw=6-qZ$B7|I5L89md*#$!F5|L>j z)sTw2Rz&%URii+MuYk${aSl>Vl0f$$RAV!8xFqTXt_?I32zdLC8;Fc&^MWi_;DLdi zgF$O*JPcN-*D8X-01rj6<--!_ury%NW-wAWc1p+X@P3(?N=mW!bqRzPVz}Lq2p$GB z7&QWWJ;)H^%iu#{`ylZjPrswRS>o0*Z}g?R@FuT;nioK$;FAF>1$NC2cFjI39w*U2 z`tXL};e`wI7%sF$jUFSS4Iaogm)?(Dt~K^@rmYVYPzG!oOM}jv6~?W^vt8jl8#&UG z(H8`L{0)eU+%Afo`Z?Yg+9ATe1cBMa-j`94@E#!C9YStisBBnqg8>TV`iNsiJp-9T zv?JZ23hhGGv4bK)MpTI)!380+1QzYG1ojP;P+9^DFTJ6$ZL@D%W}$`vQcMEjVzGf9 zLfSb{l=Qm1asw>%q#{%da=BexK7!)Tf%?K5;Dps4Y6}YzUReVo?V6Qx)-G^*Aag+> zr6t9mX<)h8>+U|Q^f7n!5_Mj0^x+#E*@9T#A?_r8nJE0ZcR_tYj%*%+vk5PY_Ib$1 zwt^DQihc8t`2yW=&Wj2)D}o>7CJ3)wZfSM}%_x&wn!ol*c@NQhSHZxc=y(q=QDaPt zij0<;zUZz4Q`fKp7CW~z9oN{XS>ZL2j%Bzk&C(+aFOUFpR*+;;?j<7IX5XgNcM4an z;K*#^*E%Wb%qu&@3~2C6T0C@XM4W*N%e-N|{z>Pov~)EGK8J@5*7<8r6dY<9 zwl}WMieKv@U5;0lX2q9@8(aGQw}~y=54P;O?J^3!#+Ct11$zLt4D`~sMQIYm82F?q z_T?GyTOznIqQN$U6$2TDW+XVKSx_Z135*)iFd)pphkGM{bBqW! zFk#@mo)CyI17!tn3=7}&5XD%v>gy>puQ|35pSUGK)iwR0*eOn32fg> zAaa=FSt7s`&}PK6iNv~riUaSp4atI~(>N+d{MH?Ua$Y3<5A>TxwtWRzaHa|j9LPLS zYjt==(0Cx97-%X*L`hI`Ml={Gq6;EXeOrVeXfqILVA{@tY$FyBVw)g* zNx6&jWFWo3Y=QRzrF0%N5ph=_x|FmSw1R>X6cAlGK>^DJCKAM0qZqshwo)P8FUJ5?N zxiN!G`>w{O2^U>DY@(~{Zct^K%I*eL_E}yrw#{r%W$y>o%Y|w3wLz7E9o}G(XWm}| z`?gqQUK>APs!Yg6{{&F}&rOqkb%Q3m1kI4{4Vo;K+&4H8nvCNJM;ySJ3a63=JM}BR zi>!bP^D6l5Sd$+Q33&XiUi{vv@!P_;pvgpi#~74=XvdfnJH}St;)FJ5zY9kr;jO>A zJx07RQo;Sty<=>g1AygsjIsA?$Jok4AJ~qu@BZ@c7;7q_$yz{*88jJf8OzgTEuh5= znvC|0%CKY0H)qEdv1^Rju~o2R zF`XQH?YZ;79BQyptVQ2Cwp>J!#mX>b%Qt7p7O{1V7_wC`WHA$hY`KUaibTTP~uJx#maDF%Qxr77IkiH z6`YtE8@61;hQ-Q|Vaqos!xq7g5gE1$GAw4og)J9xVKGo)uYKnpC@I#M5Ms+WC&U(Y zLTnX;Sj>bETQ1_mVrA&CtZ=o$&pV?Z+d@}353%$FxV@jCPR5(dD7r0pAR;ZPKUJ3Mn_)pN`65p)Oa4GQgCbM#?z&{{xfPa9K0sI4c2iOc~9N<78e?ad5@@*v*chXe9L(_@}3I!Yv=nhwa z2iyfn513yg3IiCA!+y|{-_8Ic5$Sv>M<^Tn8RdB9K3zM^Li|IQKk<009#M z6a<0@lxs#LMS8$^!1;g+0{;Vg1mJBR!U5FBK_)D>awbwL5j z_#%)tpmada1ZcTKzg<_12MP#~57^r-Ay|jRMF${5z<9vkfCvJjhtfOnzj;H^{TX0# zBI)pN5upYT9e7fAhra#12rYNzIbej)iYKU#FhbyeKmmaTUIbPr2I~lNBQm-tkU~h zV=}p0nIU!dmGUPA2+UKId`TKB7v3L@Vu5XF#FHlewwVo$_!&(Z|F*D zmdf&y2@1Gt0|opLqJS%AC?FjiMauGo0xlcM@`M7mRBOuegaR%b%JPH)7And27)tU4 z0_M+->M=n9SBp@A3-0rN*k^_Zy0uNL8dJ!SB}RRjNv6)4E} z0PW)n@&y0oPmby_!T(l^@V}k{{O>gmkJJ<#AGKOk@jZq{Jb{3V#&JQNC#R|0HJ)KR2Ng6uw1OhGs1ms6X^_ZZ5t3@ba zPZ<<&)j$Db1sd@^hDJPrfQtYDv55jlN|~U5t3@baPZ<<&)j$DPjxx_h!rfOs@w1|U zJ8lOB?EeX&fX8HDhf}4~NVu;26X6}di(~foF$Vy@8IIdkh`|~$$Z!+{P#F%q0q_G( zmVwe65HhZ`2gD8t86Y|!dC=H{wjYo=po7o`OYI5dvid zK)9WQozU$AJPF_ph$cWs0G7b{ax4g>a~Dqj004%-9K8d`mj%>HCv6bi2)#g{gHZJY z)OQDj4<$<=TR=|*Qm})n9~AXo1_46(PaxQDfx~zJ9ibNpCBM4>%W(z4Z35~Be&B@^1nc6?+j(`mB7U^&f==K3t1jt8fe+1w`JCG1cDE0y8J4+xXfs0T& zd{LyMEP(t0NCx-^l|g`lP+o+>AI=d0hzR%)upgj4r~?A?YvgBRKzk239{@#Qet`70 z(J2co?d`7SOzg|be@Q3=hM)^QV z83>>te~A~#u>bU@ux_A$LoPZY9=`xRF%V8oBwWVv`Ob|qGd7NtPi0g1)y9%vhxIG; z>#*i$X0*|n85%+e81NsUAX(?cguWV<`mP2-beA^lfp>xYL<<993)!;w@-|Hbp{XaDSP1U6YB1=Wflu(IV=@mGuj2RlTiZRYZm-|(YWL+eH%)Sg9z*6@@`90k^dWMi zw4?tXcV*J{c|5LwyVbyhZ=SaJv!$(g+gxt{&C*sYh7u7-fD6Vbd8q_i-EJQafAP2G zQERhOj^-s`l+sA2PM0LoQQV5FRr`?z^H&vJOU8IlMl!|=X?1%PH-6;a-=+&K;`;xw Ch0=Ne diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_handwritten_run_0.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_handwritten_run_0.pkl index 592c7a657215d5f1351fcf2191655fda2c981c08..267839c7237aa52a4f0d6cec3c625f6dccafaf2e 100644 GIT binary patch delta 752 zcmaF6o%zZhMuP^Hsh)8RU=TY+qldpFGp{neC^1DZCABOwIdw{pa6nOhT4qivkOGS5 zPSls+V|Mg)nbO0OmzWEb7n;0LM|kssm_MwOpC*|zWw1|HOcr6v;G1lfJb@`geDe8Z ziTVub9;V>HDH*cfj2Uu3Ebq;dq2SGwp$Mdvd>I%sltF|_k5Ex+dT?NTfNPMOvwx^x zh-;AHl+vUOam~glwNo@QIC^-2vYvjyA;Iy%jy|DN2$;(RHFxs-6mgJKcBjZX5YWPm z+bPCir%X0WRUqUa7TkuIp!$d3J;=#7-aXVa-qAV4Gt3p_BNmVta#%3}!-^3YR*ayq zV$D$YX6DFHnJkzlS&tMl9PU0w&hZ|eArK>g9GDRbz~DwUf&*v-B)CO@!41}i6yOG^ z4&(&O`+5ckV=-48&0sdD!JGf2*|Rh9P5xLc%ag&LA?(cnbOJ+$=VrB%4_q=C-rkJB yNcDwADjQq{BT&Ve7B>a?jKB_@ikN_k&aGO;4^-5NQxP*zk--5rA#S*$R6PKOKHLWY literal 56301 zcmeHweRNyLb>{;JN+3mw6y*<5c4SzVZBV8me31aOrbCI8EeRqmij-@wB0qt^BLN6s z#)m}8c0$5V6EiQYY9ndZr^{v1ZjS9_T{Y=Bo6|p3^`y;F(^L`LH2z53s%f{`_H6gK zr<-o}G$-EQoq6x!n*=3UrDf0-=Dm6I?%cU^=gyrwckY`peBmcQ64>AIoLO#9hho!{ zNngMclp+Z!naatV4kzOip@?+E7n_v!oswed+{?MH+1zs|zAhl03i+j+Y&bf)?+B8Z zFDm7NI=Oi)>GOwTlU`{$U%Dj_N+lw`X_1wZ8>MI>J&l)6O4F%am)ww)(z#(-pB~Sd z&2smyU_2`AdfIo!7un^H2c%spY6TZkEEP{$64SY%b?arLcsl7z0O_1;KOIU3P5wki zOmO7Qd3?+hF^kAh5XG);({{pqd)Mw<8tri}Q>m0n@A5?hyVAH2{h3rc9u-eY$(R%& z1iTkPJ9-lI#R8|3p>$e`<>Wm+Q=e&!W64@;wps1YZc`vVosjz2Z^{yir5*Mzi$5Mq zhbA-eOv>D4NlLy*=!|6UVrcEyl}ROcMMC4d64U8mJm#?2?7K2Cssu>+eJP32U`Q{w ziGE)sGKTCS2_*wLc}r-*6lK@uE>kFFipA5W;do494GwM{I->{SGr-l9k|Gn9lK0J2 z0|&)W!3*EegVL;+DCXpC=(J2S#$N^;^G;uKGSzL`x%1@dK#HS(@3QEpGNClX-+Lyw zO>UtI%}FVhiKKarJG^7TyKAW@xZ_$O$oDOaDVi6P7`u7jtAwh&d_P5fp&09@PMvH7 zEb1kUulpI@d5gg<)czy9A|fYyNO_zXR&R)?A41;~_1pNn3M|eo!P~q)kc|&T> zdKhI?MzIhAtV}wW8JG2`bWUC$ko=*jFM_K<)1+uCEXTb=A-W<1!yMM7|0aDC7iY*y}C<>owJ=Ynlr!S>zl~L{ONc+lCmUx$#lpUF)PIi zbdqk9FX<2Vv3Y~8y1I^8kCT#7mx60g3VIw4OtD~VnN1+%8b44Q7<&J;GvKYt%rWiBt*99-Gx->)`{X2w``x-DRZAERLRXk^~9w)`?JzY1AgtkBhN(V>+?)*b6A3>Np;&+hy^19OVO_ zV74SBtY-X@X*@GgLI{t|>ag_oRv?6@#|E;ytAK1`oimI%!gIMTHixSs ziUn%eW&#?Tm^Rr=dovRg=)#x{5-RvFpUx>Nnf%}~iyviGxC?#03*FAt*f$MwlbBy~ zg(VBE>uABFt^Ld=Ern}76V@W3R4A58`(l2{OvOd4AG%Gcbh67NMN*P!V*4`_vFA+S z0@E|aFF&`v+jP=GYelXKcX)p)w?{%T@Qz5!J@J%}ma9j`Lz5!vIN5C)m;6{lro^Z( zL5j7?U1B=UmM`X`k0mTr(!BH7lm#tgwN9BrF%wg}ySpBUq|(Q`lyaz*MB0!98PpER#~YWSM1B7Dy9COCwq&sm=IVFd-j-t}pV&n0G1* z#?rQ#cLH%mt--=I5EnCvfG;hHAZI$&S8~sw%Xe2=(2zzhO{KA(WGmMZxiLfwo1DB= z#T64tg@qiH_VuK+ccK^wTCGL5dB2OM*4+H!vb$Usr>lZ6_4K%*f4izk zsg6`;Je3ZmGifIKAWTZdpR3nqaofu#F1OWcb=WMup2||XKwPbdOyzJ3mW%)0=|G`wO=dp!Wg6nYYA4^Ro9Jxoeapi05_ zI4w?3nPP^fczQtyXBE0`WMZr;P^?6Id-X9LJ6@~; z(gj&vMbvLgO-IM$k&yp^q!h^bCD9*`M0|;q^guv@NGv9X%LU=?a8%$5x7`jVb5&9Q zTO*-pC{1c~=~Uvjy6w>S@jC0JB3fX6!5T%2nfJ;6H11J z+wFm!)yCPh7^cVOu=F^q5S6e0yljJ_eEkPpi9c=160!s?CV!`guZPNTZO2ek_C3V1 z^%+S9G8tFuE|1%3!Q!Xla4VSkH>cx?I2M0tQ_3Hd0!)=JFr~<@8dfRnO<2Eosg`o4 z=iVl_`5_}nVJ66Uo|B!+=?&W}pt8M-Zxw~#rpYuQ-x7(RmXZuhQb@SY4!HaRT5?I1 z=2G&Rnbjr~kMVT|EcaLpVdw1OoVP*Vhy_dndv5_KJZXP0CpSG54W+PV!DF)7{A|e3 zD8tmrWGVFbbaHw(EjUmwd%=uVO}ELPp5kiD&xQsY<>o{bs|e+dv4AvT%4hI(k7;T* zmPTEs2Ol;eeU|MOsmqv5j5gW$wnQRI8&wRFc?yd-nozxPzeXYX1SU<6X zi74$x5o}sPnPbvvF_rX-vEAG@Re%#c0(I2leB z2>z>lhq87nS>b8eLNWdaD8|@m^Z>2l=ZG&tm!H&pgcB@6YedQWxd>?%JsZ*RlM4>GGm36zh1P0 z^4RS?*q*Cmtk{IzSzk0oU@IIFgDO2WG2O;R>*{FYE%cO`AK z$o;OqYjaIbg=8U)3v0V*AQ$^iX-7AI0x+^Kg0@&{>v|ZU?o`IpXf<4F% z3o2}!Jyq;MEzHmcA7Lz8i><_X5xdqNSGh3=BFqVMq_Z-n1>$B0HZ@^YffFx5XZy3| zE-lcz<&Fy31?;aK{lLOD3MwWdE*|PC)I;jEl zHhN8R1AKsUvM!VZyzpgum9oR{;41th{HWEi*PHeP0?E+?JP!zC9l`bLh6Dp4(&ZI~ z%P?wSRdcYykf}jle_HaLq$(@n-^WV-BWmsP&TwF#1_dIaz?3yt1pKFj9u-uY|G(7x zF5t_spcg00Uu1MLuzKFcJ7%xm3m1({EESrJNdc2T=)-2TZf^lKP$2wo1pXKJQK|9Q zbgft0uV?L-8zO;}f@V4w{tk-B&3vXh5SxgDVv1*~u`WkEO*- zN)jV+7*tZZ7v&bR+>=Qwo`Cmmm)uGzF)6d?!+*UA+Z%AHrM-<|*)T3)RCtBVcoF3o z0roNi5Tn5Neb3NfF>zUu7OjZ6kj477>@N8}jaOm)N5zuGmr2JbrI-cI>Xu|6=5xZs z9IS5`eeAGJYrky3PQmHiT;pej+4F+DKAsTALue3r^&3(F_y&V58&fdJS#gz-*NbV) zED|~B8$=jr%iDBk1GX24R!mggDyk&1>s9Ip*vU|W*NFK z2-{af-Tl=;S3vjcHKBW`I?OV3+ZQ`lL*2vGLG#(@@xmFE%|-`j^yW{h6CEZ}K7B%4 z#%)V0Ozx53AtAzrm)8@Rum*mm5!^HcWN>MU_0ULM7|a2FK`N$((95 zKl{rcw%oDXpw*?!*}O{DFq&1_g&lP&NEB85iRx9qTs$KloYn8ts8-ah;)yU&9pZ16 zK+JCKuR^HsBBNIks=?}jf4dm)oN*^e)B3T8tB@xADnU1eUt^V2B1@!tU8Q8n>bq+p zOS(G5GP2w`xC*jNRR>%~mTr(`<6V2Nnk>&&ud9?S&+9FKYTt*fv$kKk#!K)_AZ)$bD zf`O*2jqapNz5-$Lxrt<(I662wRwCXSZxLSjVT&f)3gu|AzWk$A33m zQcq3oajvGGQ)o}@f4oytPfh*v&}!<*qCKC~ZP->!x0>2BxSDzv(4H^$JX2CnP3<|f zntEPGd%imIwUT;jYR}ke>iG`ZbNM2_r z>;6zTTf0qiD6$M}zBq=td7wn-H0oxAKgd_Y4Zsylh(n25pqGlY#@~AX%NlffT`Clf z+In-|_7pSy%7#cBJ3Y+KJd|Dr{;3iP)+h+)G_VC$X;#bvEmH$p# z;c{;cZqgK02O9;&UefW~?%bF9($=!YVT=|e1%&wJDO{TNu!Lffc79{cU= z8EQLb+l>>WUESn0T*dY7GX}XO;yW{qvlPs^JI4-GQV<<@?k1;?*^ZaFZ5tZblDFQ8 zM!BWXtvV+kFZSwIV+@%P3&{jmVsa73QN3kdyVZAgo0z{h*hPC!@xXqqaqYPDorRF_ z{gf+Sa1(9wp4(wo*Z+~5^GAep z@}nR)Q7jxv?^gYNaMpl>Qm`?r)t7bXhXKnEvXKKwucgPzj~S0t_I?0K5R6!)`iU_fomz*ar6njo}S|f<5$1I+0;c0qa6Bew9HuI~?dz z!)Cc1HvA!aTLnkEqOe)P#Py7oRX>kk>6q^t^XzhYdT|WX0;j3cvyL9{GtMj6lt?g$ z-PWc#eW(E#JCrDmT(>D@bGZS+9=uD7!}o zhxY8YB9csjJJtp|K~KlR$XkUm=#X(eg>#cVHE3>%f6A^@CXvATzFk58(=3k2UOl`^ z{PDze2vG=sjUAT2BlVmWKWc_%cAK8gNEwOw=I}FO$g~7cvsXps8Ny;nDL*DpHTtA^ zf*t!Oq@Q8u&gdolL)1zND1zy_4_ccgHa-pE4@FLj`NnbHuJ7=X#~wLwXdlz#+6i<3xB(#|gDL2PD{wr9xH=J(LqI4C=}|Ss z)#UVHy7gW17Ez=#i+*1$9t%Ou1p)yPN4KLn8OkpmH9Y2zq_}S$#Y(_=L)_xTG`x-y zl0QxB1T+m%gHq;DlzN^5kMJvzM=3N<-2cotqF6phC-RvO6H)P*spsxTOFzL>>HDz^ z^KO+lFl4B93V*1cQezaZhNd~BoPNB``&IY@N$I2T_xm`Vdt}#)!d<)hp+t&^g$9l7 zIr8YfF>&lManF%~{l^CRdSN@be*19?l~E&R9n{Os?4;c&2iw9h;c&TbwRPx}9$^DF zaTIBlz=|rN8Y!A)UBRGUAA{rM*zze;sjDOD67-qvO0Qk+PFYqY7d-XH z$Gs}qCNz?43!A`tLwoh%yOFL*`4rwk7!K$mY<4j|BN+!yt0ZtR_Qc6VrrQJ;#YuiN zlrJ!j;O%EQm-5xR!u1J7T4K4&$EG=9QI9OMD7wnQ--HFAu@Fwc{}6|PmBJL0sV||v zN_Xt|E!Ci&{$*jGkimvITkFK+W|BAn!J-l!?s_;)+Xmi{#ewY*vS*4nx!Pi=Cfo)NaaI3v7zp1g?3-Iv^RXNA|!liw>jsFFiz zR(Sn9d1{hNCAmO8FMRDh`4*BBAUW;5AbkBidBc&r8u^r|{$AvWMXpn-lM{I$W#KcV zdiRj)O%~2Fs#^>BoMho8qWW_XJmy>h(ve35b78x?7~~ z=BT?Zv>T$sW(C_)Ap5`YU+VwrG$`N!Fx_hZ6cJ=p-b??2&Jw-~bgu|Wh{p}gKA?=%F$4l}tZ=i7)5Hml#e@Cg`>y4l@m%GNa{w4*nN%Q&O5t@yg^5T%(r5`;I&^@YvDe zv3*CZaOw+x6-aUaF$#E#0tbdi$3{gM{g39tuM@_ELip>r^6J9>jVEtVVhH~ZCHn9= ze)qdbP-*xZO8QNt6>7CDQ|su})~a8rR(n~k9it-y;-SG&@xb5{;+}!A1CQ^!ss=Z# zRJEh5>RZ*S_wE_?iUW@gKXT~6z}QvQZdj>yXIbrSj~>}OBtCldKyj-LlwGl-b-bf> zyrXqt-qDS$W}L8N{d^0!i@{x%zl@I_vJZ&+52$T5(0y^8rZ8i!igg-!oy27MN(th! zl{T?ThYpO6DimIAt?P1bMkratT?KF~byvD!JfBeKXLiljMz6jy*QU3cQ zbz=P}6QSPr4=n{gHC(LSDQe}1wmr2XUag(g@@s|rVJWA!Q%!TSkr_ytb`ICKymr2f zQ5{<*wDrb6NLH$s(^_+>1TTz99)DpeZm@$5>q{h zVaC25?F;VitG+2nMt-Fj!saR)gNiY}_!+O6nbw~|#aR_q_PsZlx!Ly*sJi?I5D*)H z%-zSkO_NZBd=UgVJtOs{Gg$1H3v#ew$^s)4KF((_Q-A)J$J?ZcpWHi+A@qCUr--ml z?D`?9Da`BGej}9dk1#?g8`bNA{GvkXzQKKak3MR8xX)zg&AI~qEDvw@8AILp4u?Y8 zsHx=oQ2&s_wR~rA%{Y!I)($(1L{=CPNPA%O^jfs9?r^h3yJGMd9T^*|Xz>Vt1tZcc z@YS0TehYVduQUt7CVe7kY~S3Zo9SnBSW|YJYGY#ey0Z^5Rd+; zaCRv7_bTaqN_xMNnw7LmNp~vg0}A|ZC4EpyElPd;O1V8O?aJ;|?gx}~P)YYG=_5+| zsFLni(gR9*P)WT?dPqr!lysP-OZS8H{Yuu2t$M_PsvqN6JSsdbGzkX$SLsi8IL2(C z)yn9s6tGATu9`p3{TZvaDf_-~O9cdUXAd&O)aOf|s=)r*$iO%liL6xMF`-Gn_uDcm z+ux)g`1bt2iEl66&hnmn)jK-EFY$Ed33^O8DUa!rRKN$rU)p#hS98%oo?4 z;7m;P??LvG8TGs`XeJ!p*?#3E-haA96^L?w`J3$diu|FQ;iFCMXr1u!BEamX{6}|o zy~=+}4b<}^4f=`~<*%|FUAy7`LKdS@vqkbzCvYOHL{t)vg+1Pxtsoo=`mnQvA(j=6 zV1v%=M^rd=l=1qFlFOZnWYk?Dh8HkC4hSt(#G{})_naah4;U&H(p}icW}K`x89pztZ&dv% z6OZg)fqxT&%kyvhe^uqHHp`Y@EI~DW0z9~lwRy>k%l8ErZhoGV*M;zrK!jENZ1^1r zTJO~xIj%J4JqITh+rdZ2yzwbAZ#=S`c_a07IxR*|RWWazPT+HpFfCyHUpHM`^P#}H z-ylp^n!k1ZOYiVTFE@VN@Gre<7785YvM8~ISwwR6fz1lnZq0)u+zHXII5+S^TSZn1 z9k){Sn+mKHC%Tnct!|~LTPZrTAFP3-Zl$PODVVqs3ak`McVjWu|I2PY{>|WDw*Qy2 zix)p1ef}N+_L14eKjD`%xcqMZ@=w*vk;BkU`OeRK z!|dO=CL{%SluNfxomQa6tU19d%n2r;b=avFhUWl-{X5r$9ucsOd0^+7kZGltP=^8#1uo6u0bYJ;gK(+&vT*6C<}1SdJ;FS$=R1_k zvrY5O=Y-D*^C;JGwrK;N@~ruVg_8TN#gAXPEG*0c%EH3S!rW!yC0za%Pyp=ZF9Ck; zoUri9*(TIF$M5D>xHY1dh1tdLGjc2qpzAKHT{nATDS!Fr`OCk^U;aS7TYwW`8?PVKXFc(|77fC0d$-1unJEw z{^9Hbsy9<9;*NRzYi1cGywhdepgv!Tqfq@*_}d^Xq;L>sDF`%HOv}S7Pkj!cAk5J8 zVMK^UmGjg~VEN5-L*rncbxJ}16kO7PM^|b5UT~*MKINf$JA2|y)@cRZrM;x|4|NIe zApC0mv#@LjZEO|<2$1uYWQRs{g`qjagBG~QYq+%<(I+NjE9KBr$|Z`hr^v!JiHtk) z^tlLzhqxZo2@4v_b7E<9gZu#UjD)3^Ois*yj-(xC1&9iw&O)A&DzWBEh*#-$l|-98 z@fM^1mW~1m+=_;Rdat0fua1CBrqo0|G?^srXUV$N1I@%ZzyEomTn6NMYg zBrV_?as|J0$XO`5#b}%FF`#QIl$pD1oV#G;ziisYotlDxq<3VwYB6j25hI%M703^$ z8sEyYd5L82OVHc+-5liS9G`S(vbltR1-I(_!%Fd|YH7*>RJFt~nsS*6<81T{d2^ID4w?o?%2dAQsy=)yFWGAa=P7C@?E68-*-pvVkdx*>z?Lqwpwgszn`v!=ghB&`c*OkL`P zN|WmT`B~EyMjGQ2x6TMXF9^DsgNn_*vWE^m)Ln3}GIcK~6+JQTb2ZOJ{Am;MTo#n3s;A>wlg(-MrskTUuluV49b3F8nb(zh)1xepB58tnL9;KfH#BdvP(7pzz~jD(y?B`bzG( zhaY+Wdq=`LB=T0}d}96Z8lq6+0OZb{Cr<}bj84IuC`h>G=w)*|5X(Cs^jLdfciSvh zHHPO39$mGK3C6iu$zLPvU6(Bu5=7-rE_XaqaWBgwTtz0YGvot61ARFyK;5s)U4NdPAySC zr$&c|JF{!-5bsCD0(SY!Oxj)bIW^&r>*v(gAphPp7O-=_UEBP=bc`vm#LNZf)an+n zHL85xEEX``i664)7^N0gRfSxld1U$X`~MzC%?Z7m-N?E`M*{lM(6wx2)xD=Hvasq* z`+zZ>`{uzIxP|q_YdvGT*jBf&)-9}c3v1oNdR^_KH=~8s;dWab&I%S*CoHVJ+P68D zKV#d#Evy1!glHDlyO@FsqkZw8&RVyyX76D%>B{b4X=nCcCB08c>yf~+s|!MQ9naX- zEv)%SV08>nLtq?94u*++X7qelu8Db%~S`$#b?iQtahlVMMB%@B0;I*~8Ks73p&1J0Ak` zJt(5Pu)fHGU)e|cnl1Q3N8N&7x8T<;_;m|@-GWaaB8vJ#F&3yajc*XL&zx$m&^_>x zcDXB9@Oxc778f_|;}f8p8cU;O-in;y!*Z+5>FBZBZ7vJqYW6TV;Z8hyE@zh8{D}-c zSCo`e_|3%dSyDl6PKBZY^>#*X!)J)ZabN7DRzxSa_N99F6*>#M8c0s0CkD!bgVi!B%g(yfG3; zu`dpZp_q7bJm>9~H%^vh3G(__CMr&zPSN8w0wPW%ULeH$87g%PVVjD?(?HIj^rihl_P``}OioOR zX*lysY4MblOoigH+`&KmfexRLJtf5g@gyjb8`8JSP55q8$S<+m!x?!4eLE?Q(kw&x zw!AJ7Cmtb!wFi<>F^RWPJ-(b2Pv*Qc^7>#1%>*ziK+R7DqCSzg19Z^Kw`!SFlhIKA z2@YLoPoB<58HL~~1=h)}^w^&e`9u0d?d{ZBUO@CmPKp!$cr=ko<7>4in$SbQJR%a; zurJsWM@^j88b1_ik(>R#gl{|)38klV2WP%A5R^CMbL@-<1JZbAQcNV{iQLeejdB}n zO(B>s&dW`iSO`c_qvx93-~aW$-1^}C-&Z9l;=$vvT6#V9 zg25BM6Q{-i;4MSRsKfVn8`(#_a^WB28NQ|SE)p7Rje+!ZLdu2Tg$Ecu`g;c7_BCX} ze~+gaq;2@9vCZvv*sLCp#o^+gHLe2RFp-o1rwDv89FMEF*KV`eTpl*u@l{jH{|AoN B2QvTw diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_handwritten_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_handwritten_run_1.pkl index e837c4a607b7d4fb42421294595eccec9c79cad1..b3a7aca797ba749ca76bb7092a160edaabca3eee 100644 GIT binary patch delta 767 zcmaF6jrq!MMuP^HsV8F@z#w*tMh|~UW?p4_QDTZ-N@`hVa_W>G;eew2w9K4TAO#f9 zov1IN$n5CrGNp$lFEJM=FO*zbT#}y~pOKiCQeKof2d!GYmniT(xeP=&BiIUQ#3L-dU%1do_@h0!STV4KA}?xn9Bq; zck=ucagbAXr^q@G(87${DaK%@Og2kZAmkqw+=iH-`iI{=$jLX}J=8Pa(K*C3%oXG# z7LXWnSTO>_iV+xAjG(Y$%~1Ac=EzW)ESM%)j}$T-?mkA&@gANb5F>ybm=OxV;6^rr z184*!xJ7`$4c3Mf;0CA;>n1G7;&IN+3mv6#Xa4j{GdkHYn3i0Eiz_YpRqe*^(gAqDZ-RW%&sN9tl7o z0OLmz{gaTeli18F*R_$K_33gIcTbM(WF0lhIh)g-s>;c3kGGrSO1s_0NjJM{w{6;< z?H=cJlg*xH-sv znM(SDN=S<)v}7u$Z#7+K3%602aNiCfl(aq_J zoXw{9>CV&PbVv>) zGHQY&Z!6$qn~W(cLqS!$d*rQz`PS~8xis3-w?w5{D!s!$9o&&dL=9w8>G-sIQcK3P zC?Vi`2ehM{kUthYoeZbbS}dpU_R9nEILDH;*5PK%r z+Afqa7)2ojSebM#GohPP>72ecs0G5){wQL5MeJ3);&i#3c6-0Q&*f42Jnr15a~Rxa zaCB%_KGWfF_af8FGBXpQCN2{~YeH*7)=;Z>TCcYj7&{kg?+$g0gf`l2p-v3h&`no5 zQqjnFLbqL`@dc(HHivGnW2m6l>mMf9i)*i*??Vq%XXuc9xXToBte8bZu9}<~^44O= z&_63-uD*W7>E_H8+Ebk4bU2khd+%~=w^oSI2G3PWK?mlKePFhAo3a&mW=R0p*j@cfuMp`a`T{|{L-Qx2GjclVkEJH#$!X|8q}$6_iEqdIh9U!%UU$0$&*{3nT$OrgAL?oN?(3%YmaTktucBpCIMhU@tWtXG<8{ z(MJ*rm9%X?c1A(VSgmK|a7<>ZcTe~I(Ny|)x2SITAJ#x;^`QylM|KbG8{Rj*TYeDa z1g0uI?A<-STba_*WlJqtERrTmmPfQuGMe$zP(nWfeP88`v27QN#`3n=wgYjY*+kTabcD0^nm^}DJqYDg*9&ZM!XWUJRvy*W&ao1DJcz!h^zmBk$N_O+z9 zccB;vTB}93`u-Rb);fRbLbfr*thy=2X&T0SqQI!Xe<`C6 z$r-Zt(CjFzJ)utD--iCL?yL&VRwOujWO{pGtH5XLM_3p{B$FihMk6K!jiF8kFEX-x zC@Z8@H`KlMKCj~L=bB8(`o`;WyA_v{Yg83O-8v0hTv$y^7pqJTd#|$>OLVa)DguWb zmlH$Z!^OQRJ|@dc7{i>!{K2O_(H8nQ7=rq&+|-fc7w@L4hg<55>&stG*ix_;puRi z)adf5#9?=P>~5vMw@UxHJx;gctudA8>zO8fq-v+RJ-zg1#Ug@gb30)Ms6mWQjv;LG zayky3HJ-$NN5OK1*S*d@#Vy8CNylMz;>1*2gW1+&YXk(~I|^k(!qaPaVGUAcNO*dk zu!f0sOGVG|C7_ROQdBMh9au+!mE6LXR*Gvc>~9`n`K^H4fuSb$KGcc@8c7C>e~%dd zWne)9kE=Ql6t7g$@kAWUzqFhRgtQ>j<%`TIy0?yH3VRbK@EwM^oGH4u>g@pt3DTGe zGV*h}YbnKHYZX+scJuwB$PZ{X4eB>XtI}3ZKO3{$gyS*3 z(17_Kt08Qk?aKL@^bJ_SB(MP&gvOH&gmQYzgVW&@7A?poyUkCBEzLU2olKZQflnuA zcG8Li^|BXiSl0B&f%F-!xBPT?Z?oQ-n8q?fyfGfsCgnl`U--ypc4BSREkCeJM))k- zGBVaNnHcS~@x6(}H0@R~NVYRr#nGHvRM&4cGkfo380hCK+=B%aE0~z3Eh&OcODJ1R zJFTXY0X4Rh+oy_fqAXCyD9(4UemeYSv%a2jU;+5JSip$dR3M|A_D4?|C2X|Z1E5Kn zQu&IrxGc6yim~Oz1)+ap(Ql@v8{3M!M=*miU5eI2XG*k*{1T6I%Cq`yf)mQubva#d z4O^80P)Va~h4m~*Vlb|umKx6_kT_hdlCez-jwC}zYG6E>(Tu*oWC{IDzg;Ze%9eO} zY@r$d9W-NX!?C@+8aU^%YUsl!PZwy{So&7is8^XjjM>j^?^m!GEnoUlPA~Mf8cW}z zX{wVCH&Op^&LdPXLjBq_PcJUwxLuyz?O% zLL0V^>*kTL3T3_5i{)OGwXetBk3BZA4OX$z=``eeLAfek?%DgjZmdnQ%&#(u(Au=G zzs5RpBX(&0(cn#M$&MPjJyNsXN&wt{QBAnMC3oSgra0COMVy0 zy7h26nFr`=_Q`q^+<T!a zDkz%&KaJ;Z;LEU}7pE#;WOTBydVYX+Y(BFO9vYcgDm)d_f^s0_$6mB)PZ2dxAo5QH z{ulT$sPW$^ZZ_I)X6@ITqQR6vGo6e45sK)oe4;uKn~Z~E!Z+1goo8D8XTm9!JhaIx zy*(I@rPWMIQ=@U1Q&PE?^fofwlR+z|q2TMuGeL?%}9Bx0i{OBtPtV15GmtPRe58}h)nyi@}?Z-IkaP+X0*b~RN{f1@tc+iJsf0QC}b zw4L4Xg&G(~+(d^`bUSK;u0VHA`g~pJ-dr1I1-f(6owW#D4c$$(L03Wd&AQONqc+S6 zbl0D4UkP>P+Mp}Y%}QUW3*BwCVOF4JHWhU4iZk(wFK& zcW-T&73jVoZCwd<_tyqp1>LXJh3=u+Fe}jQINP}r>K?8Qn$Jd$7SE_`Haa+Kwtdo= z=rED;=@Z%_pHWL@q47GvmrZsDvzs2=Wq^--kxdnmFHtb4H=jannCb>gDtT;~O4gta zy<>aJ=2WZg*p!up25BO15Og{6Caa_xS)#S; zDkn?U+*1!((zPL0kmZiODYu9(@$Z0s$ueP!G+P&z-^#;0SCtqQfNw$b*Vobf3wglHHh=xmUG+5o zo!Ss9u)Gt^uLsNTiG?{=h5Gdl+{yLdt582=M>r#`|BT;)l*mhJh+m2vS`mIP3yLl(ygxc>|IGci)ham z+|QKNQ&)Qqt)!l}(4Mc1ezmNgy4o|ol6w9K?K%JH+wUx;TV3sWVkPzbC$#6?FZ;^s zsjEG~CF-eI8=JmknycR?IUHRAHeVdW+&ok!bec`G(jOKo;RfI;CdA=HJh$QXlDmtG5<*+;aL9TxD&EB-aPE_DL~2RqMXI~%Kw67U;m9X;FxgoT=GOg} zaG8$!rze7b+<5XCa(jhSGCP@TgGpq-R=|o^x|iKsbc_|`cNmqOOD)PNKAKvTjAQf1 zQ3>s$0(oq=vpm#x%&`*(M!S2+XSjyr-DfR&Th#yb431JT*X|rUPf7Raz;8D>bIftP z!fD&myqbLVPBiOn#ZJ{Z{dlQQw{gvojO&nOI1-bCI8N#<>DX<&vr}f?-e8yfd&2+v z&F0nP(DyQggzqPL@BF;i|A=rlEcM{#9-ckqW-t2OaNkZO>4-8p_#1~m;J!_?E%;6$-VIK9(w_rXa64oYBSGpap>p+e_`2Oiyh=m1VB?^A#M!1#W3 z=@{FZQHL9&Yuvl)IRru+qVhvV>fD80Zgv?g*ZmW;R@!lIqg0&oJsY?bqE)0?o< z%{IcVxI4{{k>kfczZJI&;D!PC06d4ibhz!Ma>ua=?hjcb>i`8i=$UjP!(0N^gkwCG z!F_f(&9zH6(Cc%#J@7g=(5rE4Rw!{TugYqi#jke6chz}y;m$QWhFdXKoZeoS!-X@6 z#<@AYJ@4^hT)@~VZfWF-O*xnAO_20(#7PIklQ^)sOE=jW1M1E2tAg9pNU;N!-U?5Y zonw24cki_0BAFsztaWsHo=$|3s|sV#Ve4AD%}wvGLv0hzDLYb`L;^?qc7y^?u{%We znBiIyh$m*kxP|bavvU&2GLBgBQ)XypkNi|d%V^9ohaV3^q7pjIUR6aNuEfw%0nD7n z?US}icI2Oseuf=6qnGdxw^q_+BABZOpt32ki5WPCdu)V_ zKbiE;l$XL?B~NQmF6F!Br?q4p>O{csABLKWn@*qrzy}DoF~}i5+<@aTT&fdAIot=O zkP=nby_%dpOtHRG-=wN^TruE}#baUUxIiGN;@I|dPKV}8rwoq;qABi}NB1P)s39J4 zS{g1#2`!MOMFN_JD}z$D@HF*2T{^;JD!--BHhJGO6S&3lIXaEc9GHlT&zyPgKD6`+ zrbpk0HJER+zK$V7wbS*7#sM`(;aX_g!s6iLt-i0o8AwYXg|pwU(UC_MXB6(<$OS5S|YHb;;KfvO|zk0P_K_e@505WOwY#!mdjidxE@~eIH>J$;NB*sNM6z; z%hKr(w(xPkV$miwlW2>`;Jo2I=Ez+LSEYE0Y$FVZ%#bxZ7@LuZ1EW=wHyC>AR3g(O z!##0|9}4A*iz9gZS(J_8|t*|BpggK!kaY7_=aWo{3 zOCMnXAS1XP0CmvpK$3re2mG)#Z~xczb)*4`Q|&qZ9!`Y_Fp^3oRD;aOS94&=gEVZTz$zkcTRfqJo&tm zb1FHLW~H~zlb0sBQ8P*uOQR6nmgJW#CVTWooIvwaBERhJd^vsx!jD zrCvr}JN{LzLWtqpmU*8nX;}K@=GRCkesJ6Qgx-|)PqCvVg&XCIXN3HLH9PK;veW%t z;`%Oj431Abl+ou@@)tPhoAUM&p_tN9Fj}VLu6tuL8Xl@T5z*%yH1OhnI zq+93$REn6mdGYuUvqvd`M&KG)1QkzQg%JfdeQOn_dM;;n_ z>lx{LywF+bm+j)_!ZS|S*~_hMeXemAK5dkK78~*X{#-iT(zT3 zyrWILqfHUs(ao%89Is>jd^5OlO=Uyt?$cMjCv6eKIZD26L@#m1np#+T;tnwf3=NmQIQP-WkHgPogw`+%w|eEa|x z1CYJ@c#k{)3uHl<Xxju_SsYS?EX{6Qn9j`>9m3!-Hab3D)#4F( z9V5~w@zt9Yc@Ix}F0@M0Msp%$?YODMG&{)Vu$Jr=!^p&*O@kh$>W)h0QH%K!e%Dlq zT(Z>ChfOE?{|5nT$*o~1PU8=XcLUyyZWxfQKD+rqNCvg;TsUD-DA zyk3MGL`Y^TN$Sc1E()NMR>n}-y^~YM5u`R21U8uEbPwi5zj*++$+L;B78`M z4~uZW2oH$xpa^{;JS4(l5st9%()}R)ph&vDRgdmM)sJv29+sYxS|khptM+Gl|7T!T zsb(3|;lH^${=9gBRojyNK)RX5N`Fk}?q`S@`IkObh5hp*1LI&cx?F)rq!#m@@9L;$ ze~WqOy9@uM{@~JWEbX~Bd}E_Lirm@9D4Wm`*%W1Im>rEYissXP-h!dxhrzWgyIx8l z=e?F}r$jdAuIx?H3$u8u6@=$(ifsIu&c<2TLW99#aTYe3_uJ@<&SC5 zYVg;kChFPICUezD<*O{k^uFVN5d)XYKa>0%%U_OuB>akdv@2Uh_!V_mS6PQIDg4mi zUD=PR@awE#>+2+|`iHNd+``&?$&Tm;k{1s@&*^Kz_y8d8Onfo& zLkL>ml^ZcGv=&?gr-W_b<72+~6qzp`TFQKpdMce(r%%-|Uz|?hQ;#qmV13^(Oc)rM;cZ@N+_>%^ayKjxZhS{rDgKOEKBLz|ABhs@pdwum{la;HpUEn* zLYTM}LdHavw?drg5wlvu3em7abY(wW14zRP@o}|6T)GP@uEAgSnDK8G|FVO>oI88? z|NZUrcT1!9OLHeid2|-hBZcUg5xw+lP(Zo;V(!E^OT9FRvPX>qqsPhhu2mYmi+-gM z_U~K^f}$r%rAL=3U!cyUdC4x#OR}_n#ATL7<^hBKJJ*8X-O{9PX#-_0PywD}N9l9j zco#_oWJ3YSxfUeasW4y*Kq&C;JTma|lk22+TQ5rQKG}L%TDV(UKzyN7L|<-MXgw!= zPFk3gA7xZP`OcSH)}a*7T3B2xd){1{{n|xoaUReX7hjd;FG{Z<`a3`cbu1kP`1y0v z;%hIrp!RwGw6M&h6}2qR$-mC~Yh0R>A2YgcPJX-){f$EOSU!3=AAQk`Ca^w!xdpA? zN-e!;U1+_4e>_BhC-H!e{<-<06~V&J3oQ7yS(2^)g7eA3Cu6TlpxQ#GU3!dh4`&w8 zzLiQ5Z!F+nD@!QjoNntnBY!!5LiJDLZ=JN5Hc6WnzHJ8GQLXc`1e#EPf#Qqk;q&Vx zM5%ul->w;D?Pfi@fHLA~k-B_Ek9jf`kL*OF_7~%TKV5u%RRovW$YSfV?$H zsDnh*TkjUANJQiRXmrZf-z+LJQkBve37mkpgKQ?r3F^IL29fqJEWVe$U}fw;G~9`_ zcNP}sFBX!{??=JK#hVz7LD3Yi89 zjNpn_RJlOC1fIW%9%vveuuduJpQ0#@_;b?vJi`|~8G=t_F7+t-hq{7y41V+dvM4%c z?(&nY>)zX1DMB{1N)qJ9`BKr*1tWfGd?F-o_Y`?XbK}F;X{Ufh!Z1;y_Vckc`tB-D3>o z!sJ9_^g6KD++{!O3@CGg2WxTN)vy^4|77>RK z!S6g$7E2!?1{d5Gt{;|_nZIbAzhLFRY{tct5rTj;A!fN&@q17?pj>=A%Vr{yx3Acx zxw8|D0FUw1dGLPG6h$*q(W5aNu~Lx6MWrbU=thYVV0n>EgI3aizsWS=H?j~GXerC1 zb@WgcF6FW_9{6Ue`64=-76cz88Dyc&nkaAjUx%_QjV za5a_or&9xE&)kEL9Ql2t5fcJ^vp8SaIDCdnq;Y_8`}UKkgDFO*&<%9Mcixf9)#p_l zRSuuI-F-!ua4zHFW#%4UvpA zo%^yOUn`H7j>tA~1L|d{26+SOolHN4O}_Nc)G(lS zW$$J*>CSFrVORDZ5#B4p`$X8ds4H8agU&TPBHJ*a7B1>)7*MgzQ{y7Rh5?n`3RrSW zVdJ8%U*+oS`drl2Frc!@`9_<(8wS*diy$#}SHpnXl^quPH@|@Pnz{%!45+J?#cl`# zs>%K-ana^n$wk(t_gC*lm^9jMbnE=3@BPZH^Oqj2$Hn==FG>vb)m@yg+CYDKW5Yn- zFwi#)^bG@j!$42p0h;!QW9%N$G(G`X@|jV0zujB)5=)QWZdbhA_ZeT{+z7E^}zCSAIZLB^c zVZIsP-;S6>s16O$0ZI)ce*HHo)TJ-%EN zPv(5H`r1$!%>*ziK+R7Dr~N8#2k2ndZ^V6T@##b+jjzR?Xh9DF^QcN(!@eL(95s15-}vEZo8B7mC;SuP zXgEESJ2?CLP)J`_NO3YA3~CdZDK(LdCvwBzZ`RvcYl^{qVL@-n#KJ&|8a?0A_b0#g zUp7B*-=B!i7vH)q$23rcX{Kz{lzd7oQaXD29E!u?bry}D$Q?NL1qkn1%Elc&fZNQz zs+EiUSLESiHa|i@gRME3o=IrA$dBOSg|F^DjgRYEGLgSQE{18l)8%s79nLF<=%02;#L9^~YU z99)dR;9>*@7b7URSTmHpIWklxzv+~;Lke>acON6?cn{AIkY`vx9GD3TzyL=!fdgm) z*f*0CyCm(AjNk+t;p-V3OxO%Igc+in-*hPnF!D{7StQGo!JZ-P%>cBKA;WWX;Gzv2 zG8x|9jKEm*g+>4yTm>Ug1^=ctPWg<$4xEaZfQnWgI>rH1)QM9OGf+|UrDq)6a7C$l E06YKS<^TWy literal 60241 zcmeHwYj7M_c3#f_Bmq7_>WS4#?&Yqe0R=W@rsv5LWq=?hQA2>lgNhWI+iJ}8V5T`Q z&VvM`RYGD{b~e@PFxJ@KO=4S>)UM)k*(+@-&aRUxDZA@hr&6gBEB=vGrBbR?@++w= z|8N{7cA|XeKIVlU03=9>vU-Zd?Y@t5&%J#g=bU@axz`4N{n`Jnu;0aIz!|CMORLLO zGo$6KLdB}qn$E#<)p9mpu;$IuvUTc;RjN1NY>r)TzK-lWGuD-S+G;w3=NC@RBPf|g ztC{mTyBDiwI$v5&S*z{bdo%f3rC_caESEE67&)_+GYpiwWUbbkW6nU;syAmHe|@PL z2sq;txpL8(xNN>^7ADf=j5Q&#>v*tAwQ^OftTt!f-4Bpgt7ZjwH=Xd+d_AY8D-EN< z@ej073S>)~!LTrlv2pben7^~_l6OCGRqL5#jsI1m=<&v(2!V?XkOdu!HX3b(fGUIm+ z7-_U%5z#Xia%P&&!}+XQWY2*yHD6OpZtpRiFa@xp);IAT_pYvbyXBbTmbY8?H?H$=bI$k!SEsaJDfa`sY% zfvQz&6zaUjCsT{LuWY5B+>_gd;5@M*rf6QtVuLUBXKtuE8-J&wnJ=;LH0pDP0E@m7 z#=?`V-+7BMCaC@MydvsOmPm2DGxStG_n>ot7ZD%=JZUub+s@F_#d4-mu%2l;2juwB zs9u(*-fS#6{#w21?8;c_e97;mi6H`ANO+*sms2-0d;t4GpPBhW6aB_jGNI|w z_2bGJTs%Gdax*vN9DFfvT@~zx`W)D(T(h@=L>QZEP zUEhF2NCy=wK=o9Uj6{Q4BpG6jTi1Yn*?h??|0K+M>P?Q$75PD8g==G zi6FkM7ESWB-Y#fU@y`8_2&AoAP1V&CjcgWD0<{zZC1pS2QN? z!vw|yeVsT zD%31Bd-%0%>2(!DLw&8g@au=i)k_))9yq&nXl9SkIaEvMEMk}p5+4{>Op<`iFXfjF zRC8%uU9!>;Nz{y@Ss~U;jyj`WW(;uP{Mm{|IRi&7tY~N)t9V7tmsG~MjgLK5sMRly z$r`^+98e0aSe&1lJ~eY{aY}t!4XHrZpqG0C-t*3?a=zroAE(Qu zy0ub=c!Ehf=A0dQ;**=s7bN11Qrbl>0G9A>b;trb#K0WGU~M|j2s9uksR#H79#>Z& z)nOeL7^4rT@yufymXk)+GBB0OX>>+LFxPhv)wItUM7NtwCnfOYau|*furo3ItYW#k zdIbwM-dQtPo=knUVrfex{>c_;8jt{|mNP4s92j@2f7g7@&U(3!Ct1QHooG`_fFn<9 z_1zC>+>at3CQ-jpJeqnK{JPHCVAg6@wSETRPhD=9g#e^wEs*2tg{T%BSL5_2sRg-+ z%;VheMb0W74{C|98v%jMClu1+3HQwh>vChMR?pWPb+%Q~p3}1lCmxdkht%=IkR>&!@Q?*b{`oy}l9H z5yJ8_wGG%wLfP8_?ttiZ+ou=zpzs5$bTFyK^nR;!LIcy|&VmgUaRQV#D#hM~UynrN zS|sYat3)C}@V&7<+}A;39`iU$0Y!baic_a`n@S`CSxrzJZ1k{TF+C9lPn~o}cMs@x zfaNrJHzS>hfl=0CabB_qYCIGUYhgDDBQbUlgSM_UieROQ0|8qB4rJNNXIFbW1z?gb zhJRfZ*eUx~v7c>q{Q-xgAxIfq8OV4t5<$Dd?me=X_>g}1#v;L>78ae^i*FKDhN6AI zK2}>TE`jt+KUD?imbQ#^xlk}GHS4L2g{_l!%!oY6N+d8+Knl1}2}wO1jB1fcpTV-P zkT2%z#HhPaA1E6RX+d#7){TEsj{^T_Y`yR$N1t>bLUzr_Fl58C99y~JU?IW1&ePP z(c-SsoFo>2IOG=YWS;#oHjf@06JgMWvAElKOX6EX;w2W7UV()xuxJ#sBI34GB$J_F zOw+^t1VF5%DsOi0GLgiQR6+}fJ65&UfJ-LBSe|tE1`DBEig#v*(CvU6ugRpY05rA%;$b-9dNI%lFmGl`_O_b(M(|n2@A{ zc6Op$5CK`G<$TGSkn#v7GC1TMNMr3J*@KYBG@ZyswDw^aL=KN}l@2@R2hrJZ?k|+D zT2+Q45%C4oYC;+i;Sb|tC@ z*H;*|&rALLL6YXM4GJ<8i!`WRaLYN~8EoQ8&+K%9|D0Sr+#8qSJ z(PvbIUn4yh2AI%kl;{A9D;rdbV~~w7NP!h0ROm+V;tqCno6V#3tvXLYti_o3N|Dq| z2s{b40wwFJQLCnn(y`+rS(af*=vTwe$0TV^*!PIwI;DpP1dBQ)>ljxd2!$}TP(#Vho7F#-n2K5p9NBrj{JXtX(>9sv~8d?0|raiFWzB$fz4THHs-cd%ks&0>u} z_Yw3ZlL=_=g;Mle z2&p+VH2K+r)X*Fo=+MyQg2A_iVDOXIxAf$UOtX^jnx1Biv_th4eor}aL;K-7pj&hA zDMu!tuz~aEmW8rIA<)16(OZ9H6{M=4wU~1+h*>ntAD^|Mc3_#1hnR?tA zz$p*2+aH1fxXF}^$DAQ~wu2OB5L{2K2~Pkie@d|c;m9KwbiJ4^S5{U23x&to18Pv8 zKB}I3dGYzPv+5%kf<@R4Loc_|fDQA-W=b)go%xb{^O+4;_2b-p(h(=)aXlGJz`h|Y z)gtUP_9kgJY-jP38?u)X*g5=3Jz?h&!0CW}3E`MsU~hH_0iipP^4%)B_GMNUnA$b| z1T5_a9@{3Rz%m|!ZF4f=E@c+l8z-Q0txhd0b}^#yXc(#nkY2m6b1W^c!)TpwxdU1|M!QciH z>se5O8w*9QHp@cU4_(d9_!FdrwdBjk%AZ$jBb zDEjnb3{e3ze>-hj+MN{F^^g|s)5!;Fc^uI~dx~p2t;0DE)iMYoT)Q})z;uoD{d~1t zK3i7?aLNSczO`ZU%kBneIBTV zxTvh5PH4o-e=td)vc%E0)d^LQs7 zg~n9`zAIcOhz@PAsFsYmQ@hSV*l+WWb(Pc*>{vJgH8)?^L2 zEfFb{t;0E%KuZM`-U~Mg6*AQ8N})S@2N+^3E_bH8ntvQpS35*7aCiizVF#8Y=a(ao##{7&;}EpfygIX|Jo` zJ{rpccE90_??^lDdi6IzkB2Zj6JkwS2Yo->Wk9>5T*JZyh0NZ5!3@NLy4m5^lf<(wb`ov1!63m&gE(c6?_5npw>^ z;aq61bbUm|X^20I)Hj={{mdWB{#GKJMonkO$wgR#QNb5J#xJEfgICJ=OmjBno2|ox zi+y2GQ3Cc?@i!GC5Ox-iov{zN#@@$bP2_ck&ef}}T2lK`slcABc7 z1iMVIEBGU8X8?w$GMVZ^1;($`5FbHK`BDSD=nb(dS5SH^rOd9wKEje&U971y>HwJ= zVo7mrY?5Wa&Y%r&Xm5a4wO>YzGc>i5uZfOJ4N}fuZt^@7uB%#Qu&O92I<7#E9q+Hw zJICB7ihm1Hy}&jo=65UBD&KmryTHEDxhm0I(g&R)_3CyhQ(1e9eTg`_yi^uyt$YZ!3gGl;b2XB$f#(_fa> zzn`jX=~krZPGsvkszRNgsi>|vyJ?<7r=6L$K3%CUA5+DuvQV+oq|Mh}2LdoP9am2> zZ)q2})3l3Fs-%U@s&WC)4>2@S*Pv8dFpEnW6CRt4!1;4#JrKZL*A`CCrp8rzIyZmz zx%sIXSe9|~Fj8|oj5l74R7b}t>KLn@qZdHrh2TYvR}2+S(u^D%XUdMBsLBaz5|$W@(Ab5CEwOEJeN@(!tPnA)To>17kbew&`VL#? zCcFdW0aYGPX_ay%5FFDO)Rrx(XZ8kr22a^wJNhb~5}|bMcLzjs+f^dG_fB=dcNA+IMe0ZkEzkNRSmZ5-p{lC-=6Esdr!a}kLh5`aJspkWs8R)J_5_;rt;G@$nK=y zE6=s#J?7dmj$sgnfKk^wmPyHevowh8f>M<09B4DW!U^#qkl*B~n3LJ-4@IWWod;`N z&*ZbP3Y~O(K{gVd9pKf$O@oJNuWin5oNpXkI5{(QEQo`Y4(@&rxpN?!DR#s{ZX#xs z5prniB)7mwfIJ1y1Z%fOr2@0LiCp?J^L=#C5C1fD%94lU-M_~)vydchk2u{6>U~_j z+^`xJW81j(6e6|URaVpxX>i?R)zYy4ksg=>S>}U+kbaHP%~S~g;klWdAVID9I1IA1 z($Xq;IdBqI$y|aV+i`Fj&z?SWit!0sGUb!4Lw6cxF~eH+KL+5Y^_#3(;X?=RnnhEk zeOX$#<{1}?Wz4Y+VPjUS=4x*)I9Phs8iSeGm(^FTY8iHIX(<8*(*XBO5J;h$$t_jR zgt}e11P5US0`%Fa`v=~S-hyVzS_9@q`^I2S=A@URLJuozfEgmC_^-WgP$6B36WAT zYUqgo{9uW$WT!)c!^aMfffsE%#zJ*)bj@4%DKMk&ir$T&Z60{41GpEnV zwq}2y_T5|F((E5lF>gcL$eXdZp`BpP&Ah`v?=TP)C%@&(^};Xj0wbcA7tu*Ip+|5l z8yXJ$phnvv7+Tw~!d}QUjW^ z<0>?cmW5v{#wrC-`fIE-q~)YTBBYoVA$EkC4VSqGDw_1)#;*PYLq+bRD<+o#;zAYM4M|{wSPoHzY8#~z)gFQ7Q+hfLK?OgbWLDa zJVMjRzN+J*t`9xR|uZm3XDOVBkl?2ZyLs_iVXQCWczm@uAy|UW*|5nei zevuIYQG*C&eu<~7cMj!bS4o_<5>j3CKOy?7H*`e24COc5g)&!=rXqw!!*l1(E6UgZ z0EhRvnu*Ji$jNi<@Be4z(04UD$$Rq~f1ttQ1CDHJMvYYWzqec{FPVk6wQ@FFv+Bxs z+2Q=q?`d!r3O$8O(8T|Jo<$t-ZvtRwy8uQu1@I$nxms>ijGDDfmX$^}zk zkhK^MZ6)HUf4MaD&)3CM_J@C0gNL2v2I$d$dKOas=`-r|yR9hW3J&{W9Q12`<%hU& zcf}y1J>zP*dP!6d2m{|`S6uxQSbzLWty0ZjVIGgy+{Ed70Hj^Y{7@s`wzv$9RR{0SthuPq_s=w__}7fgDwGuRX}Z`ced)*eSS{!S z@A8*U=--E1-78kMN)`gtZuLi6yBW}(p{u~kRZ?THl;0O4r;bjv>VJf{<#g#N0{V|3 zb@b?Rb|n-JhU5^Fz2p<)6T{H;NAu&*07k1=A6^otYA?kryA?UV z<@};h{{I4=2Pc&gEP{TD`%nM%SN0_S&=bTbyrgk|nc!-qwVNwNO)FK}S;8%P`y6WOP{@t!iJ{5hgN zQdH&7QT(GdMF}d(&)%c7oyv}%Qyckj$IoTE6y^Q*d^`L*2e0k?xuRSX>DS&HQ2y{e zMR^DRf5-d(8R!GKXZ(p64|~Lyl;cTE4`oF>2r@vC+tUC1e;=2rNI;|uFv>YuP{5PH z8JLh*)nlXur=B@;QiW<~Pz$Poi9#6{QdEtKS1)k?8O3RH&B7ebEo zB@?dSmyvE6bXUW&k%z64ft8D{TQkZvssC1+z}e?Sp>U;m6zh!*g(~)ghfp-S$dBcw4Fl0u0jJ#+&;a;w zindg%Db!uGkt(za)y*Y@Ie_3#JfsD3D~qIY^5xm7nbXtVO~0)->Mx>g9%FA_0 z?%S;h30OktK_Dv1QXAM$A}l&e($8{v64BVq`R^G76er79M0rX7-=GiN@$0ExnL%1t z`2^1_%lGE*eV<2mio*un^^@i4nyk*Be3jt^B=&X@<$q_Gms;V00@jb^lkJz|FO7KF zc9E3@0kQBmIwSEc>y^<(Jh+wrEm1xsGm5acJh_sAHf!_m?o_S;xathl&1G2Hb_%sS zSy^t7F7%ajILCnZfKZ3szvP)IDTa%BrB?{&;g%Y>5ZyApSx}oz=YZs-XQo}bXlIZZ z93bWFU?zg(_s0Gej96#*+|2ZXN!K@YTF&lYm@`hCUwGM=etG)LsRihgFG0oUQq%D- z%)umkC7p%^volEd(b9t-%})&?Fq_q zn3`TZ{o<+HYVg2jRqN}jen3`zVrn*JOrM>7_RQ(&#oMZVaI@MY>uNvn-290de`uf0KlBb(b85(D-}!!wE(UkQ@ip|^neepn{AtCEYc1<}K|TI?xci{^>ePSLZF-OB5w zS9+JdTPgk7&+E29efC@BwZQ9v&qojZ@Mn^I{%p1PlU{YVTi=~7FX72>v#+<+e4D}X z(!Cz=dcf-euLryy@Or@OfzNsmG#xmmwbBL?P5L@TlN!5`w;FK7hJ8gGd&!}jSHQw{X3F$_}~{;=HbMN9fOUC z1GI&?#YNW>G~1z5C54{`DE5!>cH%7@MIQ85azi8c4g0Q5G8$oc?Xj*CMxXk}OeQg> z_~!=wH}Q9;%frpVegbCSpZz*Zy4~_~mz3ePkCpp@oNtn!?8pX{6Z&dr+-vA8q;;>Z7Pjz1#x+Dnv7wTonWWO0Ovh@;0= z+(*M}dzIfnzmBf$Q&gnd*o(X}as;sERcicseEB5fBt3?5w@VxMtrZJ33lYh2C9_u)~N^|JEvARR>C z{9_dPTP(`=4#43@-AiZsHv{ak=k%Rr>G$_jCTPN zb<8>bn*;Ra<_6p!M81B3)K>HbpAz7&GsT9}YuBiwuK8|cNC9!vjvMXyf}?A6p0D7$ zkp2jT-QQ`n=L3(4Z#260phAwm$JQQV;piIw)T3*o;{6LOOp1AUdhM`)|78}At$jtj zKQ6*2L`Zrdga;o*qiay5V(&*p_>_P@F2bWC)I@!gA|D?GV{0eG`?Ls8itv;OpB3SA zB79zir$zXJ2vZ_FBSO;GAUsL09i!|Jx_*+LTI2bXs(N(oEY+*by`WrHPAhB}u!zn- zzDeF**((tRHuXCQ~yy64B- zcfF~xHp#k%t+#mkq{ipHJ~ddU_+H`jzwi7A{*{k1Zo2;8ynuoB7=w8;$fgtR2Ol+Y zHx=B~;p&0EWB(LHOX~KS|F?$P#GhGn_}-Tv zn%#BtymE6Eq!Z`ouip@{A?5nW|6&>Dt|`}Vv>!icJ$`$ivU`O7?NLf!4gBiOvy1>E zDZuq}AMv7Jfj9$_0gO!L`UgDj=4(j)-z~@=v>(s?pLT5Paqe%zdihb*;>W-5|IYPu zf5|g_834s2@EiH7R?!=+$GH_0-J?AJtJg>VFH24wn)JVSQQ3V14dO93UsdMf3Irj( z5wd7#b7Z{|36?1S%za}Mp->;GgoNQqCp_uI+T9V1c+v?>GAZ5gq!S;Hu6cqxAv4_4 z)L={WU6xMxn)Y67h})J-TpRWz6Dd#GpizM(uaH!5=QW<+X3xzfJc1iDo|{Y0%_Xd6 z`2{;$aBg*rk_mXe!evfO{J`$2Zc8TgU?il4xd+TmB@=!wnRp*3%OsfyK(FBD`%J9Q z&HyCWM4|8fpKk43ptDfN^S1R0N{q$ruUGKa+}7~hRIlLXi+4@0;7>lsdIiC)>J_}@ zu2=BQJ994UQ#`!_(r574la@-ume`YR>lKvl%=qnCQo9}{=iL?A@v_x)-#eaOfycQ^ zZJ#Yo4K~U)<=j8`%dPFtH^0gV5CYnvSKxbvt0zg3cE`x(kDRmr=5}=s-uHA46pynv zJkB1I)p^;sN#y|zSo58!l;x2LadMFWd^=Pd_A0g7Z+hSABet>iK zA9ix~zuE{V%`T@ZcT+!6aRepITdC@6e5&m$_wricH6YuniBLD~583IeI zb<3<)%~dYd5k7wSosh7>O8e{xt%|8mFyhvwz?jxm8Qz(=l-mK;S1XoA-rBWnu`aN! z<;)7dX7euIJm@(r@f?=aaXD7L#BR}DcXuAHcZnzBa#iWpA&SKkx~_#aob*q-yLATg!I>&0q0=4VN`HsO)X!{7;J(yQ$x;f z_R6l@YNoC^d)O;wOE<`!{~nf+-@irK@24xiYK3wg7o(-CW<8x_8CB;HCC~+0xXngX z(oBsw54LOL304Vr;o!mmzq7xP0TB4Q!gV;trKM(S(%DsN6b*5|z#-hA1&?KPzm~jX ztog#f{!^dQfV{P{7rIv7KD`mv!yUb0FG3OvC~5exkbUXwJoobA^Jiz9xglo? z4(agvl1D%txFbU=NYkzkV)^mcUTvlZ8g>JT=>3D?Xd;pbNA-9l86=&rurx>Oti)bM z#knCnhk!%(N$9psaRl1FgeO`FH~+YT&JrS>p#5709oLhw1Z``PmJwYS*6TnI z+zRM;C;?G8Fbps)xd`Ar!Y) z_6!aA)xO^PYI02Uq8JZ{f;w*PNVcY{TR*npNEFw@(J_&=n6P*3#YcRPHXQjqT;Sl= YCM}xWaLBab$RXnfM~9MQ3y!7ze`^0iV*mgE diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_plus_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_plus_run_1.pkl index 9dc3195fcb480d879cb10ad3ade7e8b03115b0e8..bc28d68d1a58d2593ba02852d63bb35a87bc380a 100644 GIT binary patch delta 752 zcmX?njrm(Iqd^19)YondU=TY+qldpFGp{neC^1DZCABOwIdw{pa6nOhT4qivkOGS5 zPSlqWVs`X(nbO0OmzWEbXLgM@p0ask`BuKkTiVQ-GT0}7XcJ+|;G4|ZK7lDieDb_@ ziTVub9;V>HDH*cfj2Uu3Ebq;dq2SGwp$Mdvd>I%sltF|_k5Ex+dT?NTfNPMOvwx^x zh-;AHl+vUOam~glwNo@QIC^-2vYvjyA;Iy%jy|DN2$;(RHFt7ehd9V7T^+Ix1hg>Y zc8W3BDU(?{6$tr<1-Bt4sQ%%14|4L2cMtW9cXST%408qfhy^5u99E3Luwn#;6(cCD zSTmHpnK?34CO_zutVaqN4tF0T=Xej#5Qq^#4$KGzU~nTF!2vV^65Jxd;09|$3UC8d z2XcbteLaJNv6w54W-uGn;LWGI%mobF=HB4IDBV-rkJB yNcDwADjQq{BTxnNrZ!IbjKB_@ikN_kW*s`l0aVn9QxP*zQOTue9Nch4sd@mY$=%)n literal 60227 zcmeHwYmgk*bzaZx0@%fK0gx0|6zLW$lC$8Vcis<#pm(tVNpP?r_5p@07;5&;^z2My zc4jc$1MHGw0<1(kaJ9NSV7%dDg%TII4ODiud^Qbj~k*-oWW6;%8qsYS*Qy*YRLh>}p-Btu|-g-w%-2>P8KCH{Iy9l2cT3wT52f z_=nmkg$fl-XISX^*tmLxkUuhZyy-MU#Y5Tm3A0kA6?HV*FlGYBAG~K#M0pYX{qu7>lR0 zL_EWYt$3;@EfLltVV>5X5Zzt6ZshZrt?nH``t)@-6Z8W~U#W~16ExIIU-490OC{5r zOmyxpxeM{>>l0nRAISMD5zc7wFsGqj`btEiS|YX?eHFju)0IzGz3XBkmeewtK6Dks zoaQlaxq}O5&b`(w4!H+kDVf&AQzag!7x@g@u0_+lEE~`J zB=WIvEYSz%-nt1+IA^Mhje_Wp&SI&gM=xVh!WT$H`}D>z`-0Q!QH(_*;W)PEDKD6j zNGz$&B%eD9J?n96Q!+M)(w@i&>1H=ip$j-B*v&y;QF(wMCA zo5c5I(Tat+sp-=*rx&KwXVr)bWOaHuHMKCMEtyUabFK=uy3oZ?6KgHuej(uQs+){Z zo9-)M6?NIJ&`~Z|&}2lrp`oK_g(w9{NWQ8!YI(ykb#$*|pXhmKRk>gE;)!$Bies)g z5J@oU#;m)eM7(j+{k%lHQOSA80>B8~uMU|QzkB zcxTOEdGgL`&D0i2%l5`tFBB z_oK*%LDY|mN3)NDOOLTO7`AQJowERc`bxtnhae$qfgD#aCbaOlnxa1$EzHGYKEM7T z@>7{iM$5!K7c?wB_;@Mr#Rv0pW6^d>PQzi#B&|6;i*GCv1+$indIL`+Q(C0Yo}^=C z+ryRXuS83X(sb0u{o9;6k36^O&B8f6>t6CD}nY{t)i&!Szs3*u! z#{x615`+^+1Wrfp_Za>{zyN~OIHHHeZv({0n=aXCwtO>&4L0<^0IAQszS_Z5P zq3kaK_dwvf?GuX!P~?GG8mwtD)@PPZfj5cv;T(sSaq3gjYsf!;Or;i$v&ndnJ8HIxHsp#m$;CA_$HGH$PB!@1Yof-i1Y#b zguS}Fh>d6NnL7BkoT=xk<+4$;&1do^7S7&|Nz+$Lr1{}hci+cjv2a3*2qb#hUw2=* zv|MtCNB7u#plmd(Wjq@Yk0o(Rk<56UL@^fr&{vcF%5N3ql~X zvQ(;=6H+3^{2C?mIxDS=9*UUP@k+$(= z)M`K;5a$ozMSH3Y_4Z(>0Lg_@s#drJ2S@6VG(olHxn_32-4D@F4azZjunA7C*mQ@V zT`t*>1|b^Fj%t6Jegc{i$vZx(`twNTnhES+PlAoC`@VR=fppoZPd$iY@g%Y zE(Fp->)8Qj65tIo8P6?yCV|7cOqi$jZh$&=DTny{NSk3fqjhXj_%KKldHr>Ldk2`lEV}+c9ZLGLX#Y6!?iH=P*7YQ$ zHz~1p_XTE1!uZp}-DC=RtAs1OOUa(AyJ!VnB=VWBR# z!ER?iR}WC3RmAiz+J~8Zr}Hjfq2@Q->2$-OT~2H31Qw8|>Y|2_+*EalEPQUWZr_DQ zjekPgvO(Hh%_$e~S0#T{y8Bfr^Z=faCc<$X7vV_LbA5@YVp^(?)@KL5`|!LFnnOjC zA1+80&9Q+F6-_P|d{4*)KYoQvU(CoL372-HQfQIxi_;3MoR# zz{+89lJ>$@0WZ)ASVCYG@hAI~RYCw)1J-4PV^*2HSrr6???5(ir}$b|SXrQI+58D; zS`9pQp_zz6%A6ERBfWf=yN@YwoP^S~IyJx0$%w|nU}F;zG%>tR<$$Lp5*pIIbu12X ziH!G0+1iF7m- zhKB*6!@CadUalv>^nAM1Bp6Dm2=ooS`+^KwU^*j7p~G&Rlw?N81w+Yz!imuK>BTq( ztqv{RX?ep{8K~NLJQ~q3fV{i;fL+2NEp({3&eJ-i;~0_9YFf7)V~{gM81a4|&@Pq= zjv5M&k6{}|<^#fDh>H>kKq3QqLFZPlos^6~Z$wnPuABE2$_D&92nDd-&Pb-?VJ$7R z-a2FmRAVv|jX`Io4_S;rmjikfQ0C~<<*8&J{W9?TDg#L*`1&sN`XP z(Q&>*7-guTDQ{xSpo^e9(}yY!!f>1SYqzMT{pm9)SUY7>>y&3$y&w#XOm7VrcV8VQ z7FWrfh_uSq;Y?DRaNHYi2D*1>lsCEUA)J?LjeKc&9jQS(9>Y1>anjWyhjcg|jcffr z+u@@}Ro)jy-d0289%YQ9+z9~9T`v$rPo%?aNPC&e)7aYszb*bS~>3-^Tk1$mn6?zKLW6YZ?Dzjbb?GgY<Wp|A({i7Qh#M_pSIu>Sy2oES5r#@$ ze7!LwK}|*A%mat4-n2YIG)Kl>^=ery;Dh#(M=FYLNcI68CFQ+nKhUj%)`>dxM(?bV zPDLTajYrn$E&}UJDucBoEYj9ZM0?qK4HHJF+H~m>Q_*xdN;<5bVAJ6QhG)tf;|}JV zEcQreh~P8(ztfLh%!HS9+9F{ym0XVTC&aN`Q}KX1a9WSS#fo%k#0=1Ay>|zomj$Ou zA-KMg&X}MchWM9}`c5;upE+UK-%4bosOj!FwE#;nD)`bz_+=G$@M^V`Z=TBr&N*<~ zVqF|ml#umh{LLl_gjK*}XY9kCv4>f#fxPa}d8gj0CA%-Hx;sDu2JF8Ag(j-8a#RH+ zSXF{u!yj2Y128<5&)4T`Fnpzk1PF3AkR9knZ-`a7g3@DI<=i^#<1Crg#hN;!4v@Ja zW)#oHW?1$c4B7yP_BOVv)@z7yho)9aw&=L*Am#jxCeK6Rx~f$MtB#VQGNSfKw0)tNNSTzLmZyDh5-kDf=F%tTw{@A`Ww>v z_Y)N@-H8z0jc;8;RVef`1=SUIH;r>>wKLP!XKVGPnm6+?lO^diT`nX2Q*s&K;E_>Q}S zE&fgSG}C0|I-vC>(kvv4+&R{zge8a(`nvF{C6+F(jmo-`6(U}hYvNiA@=si5Wgt(Z zn(zux0#x~~)vC2ncuZqZTehga*&7TRd{u+(Xsh@tgwmbgJ&??8SB3C_JJi9x5^N~s zcain3Jougy+to~q{n|td_G=qU!FE7yH$ZZ*CpOu2u9JO*u=@&{>%?E+WFkp0yIx8S zT`v8Y7HwPAV7p%ZeDnY9`MZ4Z1l+JR0wH;rNyWnHR2W8ukkoj{rMighUb?&TJvu&E z9vwLuWRmc{Z)jgnOXP^$O3!hp_)9_r!dkz5Y;S`uqj(#!kLefMMvQ z8wfK{?hY{PV5h;vv}H){Zk%r%pFcG-bv%rNln(ZO5BYN-lPPw@LVhASngU zNnk?`rU}Mwjam&>a}&kf73TctU=R*!=#-@dZ@d4HX=WiwnjZ0uFShUF>XnAsFd5&* zji(T)6|b?Px=4fX9@Eaj{71T74i%Ua3PSo#wsEFH_z!o@6`b=7NW%H_S0uiT$ejhFPz|tSu)6z~CC-p9w-KbTj#-DjHC?t5)C{ ztc-j{-O-5Y*^)PD-am}jQ{6{(ow&K2QK?o+5aa`aybgQ{RS-K;B^}(3H6Xs z&K*B-iu0#k+~Osz_0`7Jr*-b~%B(*v<&!{i%IV)=Y1%dS$&`z^L--hiz`Fryj(9a$N z+ES?>iNGs+BmxHpX{c1-Ooi=g__0$vDT%|f@cILyJK5)C_tG}U(y7{4+I`U01N)Av9a7jicaVxUUC3u`N8?~Zv_IKI8fH4m+2Efay$VlC341JaoUMH~+u zlV9Rx&H6sAyLY^#SwEy={(`oVJ7a%AJIUOe`KN*YX&`o-{Aw@HTfY2zi)g>RfKIZ7 zBEcOkXz?gK2GF>Z zuBI6cI5!iS$@Bk-nk-6h%#j$?AYGZU`vnlu=Gs=RACu7UB8)3=y&j~+u)@2TgY5;~ z4cHaAC?dJ+cv?7)n`4?U5a_b|AVPpmqI?q5pWs2KvdeWY#QkMe67t2e-4`X^*UJ%n z*J8eFvG$#x{tbq7phs%5d_rcTF}7Av9Fv|E``)$L63ZSCve#HV0SgPwgMFoxWCE5t z;zZjzM=aW5afmH+NqbMRrCr>sw9*V<>1v+5xnEKK{^_&N2Lpjg#IoOj@^b~w9$UXj z@${c|^M`W?CY7`@jK2qzeacR}Q`t%Ur{5&|+YUbzz+^Zi(kWWjDWbXq0!%*{j__A1 zht{^^AK~$e41?lbh8^V?D2YHwr$$jh_+kN#RY^c{^3^4|H{pJ*`pfFB!Mw@G#XyG!NjqEUWNs}>5j z=_ub}XY)gUq`_4v)D$j56aSBS7IDJA34o#P0vOp8z>l@1dbLs0ZF7n2D)mBX1!wuB zQlR`PYcU$yO2kS3Qf260t&6AZkA6>sgPo-YwxfUeGKBimXVvNVTT%K|ob|&t=pP4_ zAK{kW6`f4>^lR1nWl=pK419-OarG}?{qb+KTD^3Y`8!^Z*B@#2QVm{RNvmP#eg0y@ z)o__dDer0A!s)vJq+QPcNF&d-xCD(=2j9=Exu`GjFEyz6+j@Q#Itry6UF?**{1bev z7W9Gl`AaAC@58C?RkKhh3ju1k`eUu#45-e~RbbURX)&0}9}CKH(21t=XLwu6RgNK` z{~)Pj$Ce5!k!Uy~Nlf-qK+q@1(DO&*D3t#{kLSTjWdxI8kmCN^|M<;4>HqrO?|<*e(X<>-5-r#Scg?4Gx00K;LgG8JAa`l*G2mE4+fM! z{XkKEfd7Bs|Nj*9fm}2GQqaR5@g*faiQ%EFhzGF^P~@`ofBxS`RVoq^=>m*$UKSMa zWN;QHBxe0MDZ!~{&z@3Y*$pLOH8fGK;!29TUNf8`DMINsu1hLn=BwA1ZPK68g}M-O zWV+mh=*Am2}KpblsX>wWa!-J%OXoiE{bs@-fUe7WApui<-yv2n8b@ zFT@J@++sey7zx8h(p=0Z@{5U7)G#AaBNi_#jP7cHa;Ekcg5h_DC{{E3G&L2hM|)K9&3ZffStba&J5>W%t~_BL{!>$e}0 zz$LxQcPdL`NS2r5N`|kCj5Iu`E_^ZeljH3G-u~}LA5~>}Ij&lv$S>1nc@eFQ;(=hx zbU6-XD9J;nOZ;W{D=15AmQ0u9UWT$fvanx!ZQ$=uBcM20Uaqe)oaEVBe|z62kV56< zyd>A{R)hpBA@m@S5M`+i>?aWx9VPeAa(WWcSj_qFIRq3Z%hyDCx&OaOAGqV!Q@t{S zw65}Lo>`Xf&ENYzhwK!G1-R=c%hNSkoj>_1!;47l?IOzmgJE83g$D{)KbB9nU(#O= z@v`kAD+@wm;%{_D;#t-!qnGjERsQ!y`H0LY!rt=aNe0TS&EMLoTnBK~9dL{#SlV{# zSa-6r%qU%o!Ev>E8jj97Z;9$k!VOZ=C4~EEOAA~`ZkgIFcAHK2faIiSraif6caRty zAm#30CW7Sl#`;ZAtUG*uW_sSB%Nsf^XZO#~>L)MEzot*WHhuQ=JXFaSq2Y76=?3R# zVUoR)%fW)#9i;nc=^=>bX9p1&&1}#exVBWI*8y-!ROKLTrJFNKz#XoeOVE_1%G^VE zMlI)0&pki=@`ZB?r{}`(h;H2kQY{1CeaLX;-2B44{>s$Z3r(v@7!wNC@8T&NuzrsR z-=tvB`W6L2>!0wqe~JK=TK`OhzmL$P*2o66&fjgV!Od!ouB-LX{Oq)T_SC$7=G3eD z)bzrcS5Dtmg9kRNI<~Irhh^0#r_N>d>6gzvfA-Au!d=xqxLNJ-b+sROVeaIN{=$Vb z-K`#^XwMHF;2%1`KXkz2A9@F?IXh&r@B9#`i^1KHzJ^{n8=cl)JR{pWNblWssum+I z&pL;AokV2qN(thIl@7B?XU@#eiypqyT6cE$F>7j1twhu&<|T+MnkQQ;MbAWbC$HZ# zd#AlSDgEis>$O0A`dj6ngm_pH0!`tEgk2~UQbeZ8&b+YFAM z?)QM-1AY(qJ>d6%-vfRReA;`U>B1|mnbVnQGSDfS+(6xyPQ2hqfxJNQI|POM2W}Uc z2)aYe7!aPYo9@kf@5o`D0|$wWaOA)&7B}?dU@@&*BM2RCtDn{_un)nl0x4}AKfy2i z@$$()`nn<;?BIl72BBBmoz`J<_$(@3>vCEEMc2?#_z9ru+I|7*#x*i%yfm%~4HA-= zo^Tovl!iKD$n_ohR%W^hBDo#Gw07IC{1RZgiI{u8V;aQ1CFN04?tu>U(aSAcv_muB zyidmJz88S|)Um!LWs7>Z??i6<5FQv0T&=USbTk?xFI(#!1jONKI}(*fmN;=@$6zDk z0BwGDVZrkR&2s5fN#UmfiuDt`oqQKZkq3jd;?T(9;lTAtwnrFVd!p-vG4MBk&SVm^ zN^o{CcpHCrdOX}7>?dFjjC`9V-EH~VvNF8(k@66b3n(WAiu|3u1V-k!hTrTJ54>AV zLIY;!l;PmXpSl41;&5>Kr~9V9b^G?`Slrov{geMV%O8q-wv_Ip3`@hrQ=Uv(qwO~k93otH@Dv3y~)xy{r-W;R2Lvo z$Lu$Pw+HCU%?@}!hjUvPAd&hr&~ z7t$Z0u=_iW_I%(m@r_2;9#P2A_t@I!SU9@IKlSL^sCfT83o~LIo>@C0;QuNM$JV|i z-k%iVQz9fi5W<5~(dZgfso48b5k4c}kBjh_2sKgPq{zpqU~G*Kw$Zg|5r0aAr$zX@ z2wxE4iy}ND!k0vt72#PClD-DvNqX&|vP07=-SIvuQL0Raz#0#Ffw2f zoqqy9=j}v}L9ZNv^ES;*Y^bn@&cb+}b-c;8MbSO^dtW8}_QuQidzBuTe3pS6Uh5to zv)}QD#@Zz78aChJ>Eoi$2YqO;P6^!PAFv*ASR!G}#; zO$BdtxO(6pSU=lIN;7xO{J%TYwzW+Jt^WXM4>H^D5ZnL!Mr^-*#c}lIt9{u1Yc;11 z*YV`d$Y<$&UlL^d49irXrT1C-is@UBvUwHOagFjeEJ*LfITHBxR08+>d9(Csy1hwW zaApmG>FT;&v5R{TaPIv&^ov{E`|IOu<@C9Cbi-$dd%yifa4nNRyD^#EcmC0HyKYY? zvkh#WI6r^m7LT3XquiKGu?(}pbmtR(Wl` zJm7|s;UzBso_GXk<+y}*<9hp1`8(~{*5e7~aa0z>zwiISjaxkW_M1TT*;eLT?Z?Rz zGWPZnAi76+@wZIAr4zn%V(tD2Mttc62APy@_|l1wM%R47osb!BX=t!1`aVl10!?c#wrbm!Ok5xK zB@TK3k8?wELW}Bgq-I6FTX$^*&n_cbdSvie9oTPgZSnGpHXuG zpR@NZX?{5^X&9URNptqIWM{{@{($-whLU{$@I$=bBhLTlKIiXq{#BpzpYS>VY^k6w z%uP+7rZar?Sv5j8eESFbKIdP*qsyx9>_9&jOGULPXYYGC|9Bi%TuG}jn=8qO&hk~^4ToIYaDzEk_|M#Z zhOM_9>ona|r$-zCIMmJ%T3mHZqh2>wxmZVd_~92q!U8K7u;Mf=W;?-%JC{OZT32Ow zXYx{R1?a5SOpScCYlUS;U~3nR8oy)nK3+TMyDRbCl{mN>t5jiE=(aA-lkQ3qkz_cX z*3x`d>6&nm6p29!nUG3EbS;LQbJ(>4)&gRRlkTBNG?Ph1BC$j)gS+|lng1S^k>9>W*&m_{zU*?< z!L?|)y5Z!CETifkq6E4=3wPOwN}Aab_mOsOJi)BsCLCNB5OnvK^8f-*SGWvEzr5JY zPP)4)jb&ZjE^r92Uf|LO?yZBTrLE)2a_*{t4P|G_wN@ ztARvx|G{V?4I8y|0@vh(W4N|KE)iXoSWBq5IAj$OaOeRE-Lfc-KwFpblAT(VH^rNwhqX94 zBQ<3T(A*5Z$=eo9V9F4md~mU_YnY=?=#&iU9;V>HDH*cfj2Uu3Ebq;dq2SGwp$Mdv zd>I%sltF|_k5Ex+dT?NTfNPMOvwx^xh-;AHl+q-yNfHhO3}M1;h!MmP^ChzN1Po!u zZHO`05Y5IZwLmX&^oRxrI>-CC1jjo$`gzAY`}?{1cshp=@~I*VZVOBxt~6LGr2%v$ zi+hlhFLH=60z-@u7-Ec|5M#|y_U6b?ncTZnGBQIP?kEm-A0y{@4^OZ=SU?<@2@1e4 zM>c^2XaXe6MSx)r)`k@71`wwxEECZLI|ZLXY*2$XS1dCZ(BkV62q?-=%gjj)4h#u# zElbTSnKF3_)PZan>>0w|3_zzaWO#0VxqAbLOoq2NBQS=2F;p-DRm?kij8i@%umh(e zCZMAH8^<_+iaK#BVg@R5c=n7#389D?$vTc6HrIG#qj-}k8C+Y780Yb6X7F^eF*GnR zGALy5f?X$oq+A#nj3ODL8J{j0X zMvd_S7tj?T|6o@P^t4j~@8pFi!HVy3ZNKi!n8XAP0_<9VL9pG)m$8i(8V1-^0mESX Qr9_}AZb%>iRi)|y03l32^Z)<= literal 97578 zcmeHw3ve9gec#>zB*8aG+LR!r@|u#R0|hSb{a{2H;6oC11Vs>#B8Op4hr2!8E$^j! zkboXG2+MISZ&Jm+xSq+>)^x^?Oq$qE$C;#^Msi+GJI#zZX{Ju6$tY=)xa~AmJ8?&8 zCuV>D@3FUcw+DbANP+T#!~1r>-S7WB_WwQp@1?XqjlEwHs)GBpE)+3 zuaxxh=hH8wi{qI}Rv(v5vt8)rdZngS7h6-`++z-O zc6)revX){^NTo){)I-$rL!(Dq4fH3zLZy1WF`h1E#~ZjvWt#OyrIfm)*UEa48jyU^ z+|8a8v-MPcv7Bj{6KVCBI?J2Ly6OuA<6~;Ju~^lQvERB@C^tgEQ7uy`HwyF3O0(`8 z)oOaWSa?DAjj|>^Fy5@!#*2ly@#%e= z_LPn{vn})PLQXBQYu~6^sH^2lLp@t5>#Vc+d!}CSV$hrAmZ{eDVovKS@1q(xDAp7d z_;oKvnZ1@uwaokQ!J4%)FAO-o!|B?5eM~)k_|lbZoj3m@D>lF0EHqgAlP~1&Gj~yi zzM5Wd78|_A2a>b-53i-3`~x=&!F+JVmZE!QouOLt9jB(6D}Sd_x=?1{X~bg=0Tz8F z%&mvmxbq$ZS*ZW#c||mw>>=gx!SE*wH2MX6dm8<{ysUu5u_b>n`AKu=ky0hwEb5Qq z9f>F!HGsX-Xf@|dZ@tknw`KKAp_DG->ZG{(e9Ih;#AD$^AQ%b90};(1j$v$H{ z#dg%>OOL6!h4Qgnvz%#ED#f~1P1hQQbkQeXlPBRZHC@Z(kFnzlU5$=j@L!~}t-JMZ z!mT$HN@&4&h@I;?5i%M~#I#r>vC5GzU){(p*G0jV$g2%lRW~0BMnj>HmI!b>>q5a$ zD5m+Ny(nl7&YnK|u~vS_-1kgDzaq$P^>Hu+@~zG#H=6J#fW|Q=3{M1OT0FoX*@+rs z3s@uozPH>g&FM9@k=NB`rCRR+9ZqOrQNFX~aep+d1*5^9$s)cp`|?ykAu5G~nCW*F0gh)uK)l zA%94V$DJCR2nC{AILHZC2cS0U^ZwSWR7()6=8+Y+ zLV{g34r~|)LNFTtj|R={sWvmp&^2Os5i`hQiuo>Sc4n9gMopn!DAyb5az^*j>nSi- z#?*SFHmd5yx~}F9y_hS%tO7OE7ds2Td}vI)q!AO1^9lA(c|GR-dM2+E7a&Efu+-%= zvGUH(73Nc@=F*rtr)R)ktEWooDlr};)}H+_dMENJK)t9U^zlvT!- z7#n@KSZ`b$l{NkoqUMw6#q9aX6Q`z5%}%P1U{KN66y2PhoSoF>^+p#PN)->nFoj z&W;l90Z;x5<}+Z1q-1}5hXqSbX0$u#JB-$gBRDvF*-ENe&88cA3d7o{AM3hjRhS=k z;`$KJLSJZr>Bd-l)8>`}aerFoJ+i?#^Ehz3z}EN{4a5ju!>IP_*1_b9<|j*060@ayf7tdo6jN`N0&J;l)Lo;WVrHc7jFV;gxvfwDJ&BoOV;e6bDidc#M-+ z5zU|@NS|#j8B+E zJU>GboKgGEPjjdQAx0rpEp;9dqv1d>tVKf(Fr(pcl;(Ud!7mS4`{;;RfII;E=9L;G zCtV9lG#tVCH4t@RVMQaMU`WIHuHS-U?!@*Cxa@G))#gYj63}8HUY4y>^2i9VQLKWT zT68#SnRjC=tv5?JSkXZPrw1G~=Ji5uvGeE&I59NOR)ajy;z=BfWZ!1*VT)W}uwlqB zoKCdSNHiMMLMxtVap>r5tvr09O`$l>-?rwqA4Q`vEgJ5B0>7`mSemO83z>&&IG<(o zRHjlarmJ=R;jE5Tvuo(1{&?J?NIJHNXp~OtQGfppXm_ztDm3Us-n;jpG)5*Mq&wY! zLw+qNP}J$8?q-Mb^BrH(8UPFq7zha`1dm1tkAnR_=6>E*2tUu*WB|=h zgs~xW5S`7A1%gp68tjjYyCHnWcyO6*_tc36!hR4b4t$SjED!-z80`<3lblH=3w9zo z0U;640!~boXe{UtuLyh?^HRu}Qx3Es7L-6pi#m=sV1pJ+^liB95OJVUsa8O4HdJ=3 zW<-LGm1suRKmzII`9fJAm*;YJblz|7&0tZXGq|8fT4s2q1IQtVh8!B@;(6l%FE--^ z^8>}o6}`sVNGF>jYE9#C9_AM~cz0Fl>mEWM<5?CeWlq09VhTD7+6s@hk^|-*5Jpw# zJ7jTkYGm>)bNeHuLLIaa9)oFVOyDRm_4kA3Hn9kUDrnRej}k3|s#zHy2&OSL(^z1~ zA6_E5#oSRXfjnU=cgvj3>N&NY!Ko^B;V1~NQT529D$jH%`%C!xKOxSA~gX9 z$+rM1k)-#X)qQdc6Sd_E=!aG3aS&+i-D-){0SG*iXuh(3B~`CwQstwZOKWQuJ;E@_ z*O}%3@6a!8F?TbBY{Q=wi4IfdIfR$4zm(#WOcOk8|H<-WP9`wH+ zG(SLNAeB1!2f{AMPG&@KEp(hsEJBha&THl&`m{wI*w}?3Z5py=3VdQ@Fkp6c(bp!R zF-8)l<)K&J)8fT(5tKRfWq_;1ZK)rdtu=KUO8nuVF-hmjdj+}JMO0eN#|eC^IDt>f zI>uCx^dRetb;9>myJa=pJ&+3pmGGTXuQ5X_ZdJW-x^^nVM?luSFBuQhem7u@Au*2N$F}_# z;{VH<0~;)KK!XgZgoIZ@>Yphj9LUe?9Y(KpEEk;f(5ln!g z)vw{U?ehaOf|7%LN6@@nX!h^0Dq2D_P9yx0IN{!Y5tw^mZg3%*yAb~9E`(UBVnK+) z9oc1x1k@lx{TY?R*70_{rkAr2e|E6q;^7EX*WyAKqk|O}k3o4H{I342$z6n;vC3PUOerY~EHy5L5MAs_9_K(wt=A0nO)j6bLXCcExZO2dgCtSN5AAr;l2{4Q-w zNaU@Nf`>T(Mhr|?e6UWCrZISi{HO5r^R46#rXIh?ddLhrTIQCMvoQ0Zf=^z@Us5p# zFINiL*4d=zYy%4Q#(@Dv@fip4o7_VnjQd!IEqjP%k09F|I@hRK0FzsisyP75k(TKx zv;fbzmx@xh@ey1Z58_8w!-{X8%w}sdRp@6>i#-H4=}9h8J4Tbr*`5XsvYH=YP`$|^ zb9b{`FU*(qteVLar_FQRRxMsILR4+oR_$TB_VS9R6fdjE9Kb-gdTSeCwXr_1v6@l` z%xzcn^d;(|(|A3?3LHljqsb)>?^77@Vj;WW?}YzRY8SmIhWn(vAI12vZd1W|Tfs)l z*oT+Q9eis$UCvc7X2P5S3P3$5vm?DwsHe!LhisVkW-H}Js#(`l#R_;n_10(1T_m9( zHe96&laEnzH>Je4WReRQJZ*m8p-)?Gpm*ho!Io$Tx)!=RnkD?U47 zebmYxH6OGv6cRktr)%kEqcX3TH5fH%wQM;ZhBXopesE^$)Rfg{a}Yw1E3H?zE2{E$ zFcN!LF}GE!sks9BLKa|K>T@tbY9%Ld1zn3YU2UsZa=E(RXeHHVSBG-+0a#azc^c-y znWxU3vnn%+$a{5*qI~xa#VFzW<(9d%UMQ6yBTo)Pp58GyJ+Pj6>RhlB)|1PM*DoOX z%t7!t{b&@XVipE4xY%+TS%EdQ(|dBJ6Ed|j{5?+_b{bQRfuvEOT2|fHIK z9zQ=h1vMb9%R@q&L%8F`NUSwRSw~s*ym>xkLaATl6+<9Ja-PvKCc5d%#9SUUcNNnw zEJ6a#1a2*+H%A7gkeSpLF9a?+$VdYwxP4F>sl%w6cZB?4-hKc*J|RWg*dYNhJ)k7^96 zu+7{*b*r@*Zv-t}UB#QSH{NtiuQ=2c!So7pg07ox3db~P@ks8D$1>_aSHU5>)2Khc z)p!mo1+?7<^#vVv>;q6OZY4ir4nSna&U{0#DNnP*-l#bw#gKSn4$>@XU2G+lbbiQq z22gNVY&NP*92PI)C|4+p%O16%%&p+NLi5Zf=)(edlQ!rdhZZ+@BRQ=5LMNS9;tPza zn23(99%xH+HN7_*iiLxTNCKRDyQa5bjfY`|83n)JT0HxCI@p;?f#!u)W(nRIHgc8v z1kh|Z(Ck7W7`q5uI}&h_h-T5IK_VNbz(O);lidi(OBm}H#4Djluk@ySnQA~a6ugOz z#GfVLk0y?@b-mU&g9=VP--MxVTI1FEaJmPAS$AAX$aRdd$sA)48eu64hkH%w!l77< zS(5iv_B6Ia)r@iQEp&}TO}9ho+CtZO1iG}5{@T&TUJ1-rBplV^qKN(Y84kf#4@$fJ z#R10MJ=+#^y5hXc@$g}rlM zrF-LW544~^NK%Mi`ENKJgk*Uojawts3oQ!mVkkB_v@IBo`BBkYgt88Xj6p0S#t#NJ zkxbSrlo(wzM4MS6=gr)hnbEDXrh39-ZcA~!gqHcJP`P4^lH<&XU`$8Gm_H zgROG2rl+9g3b98+&sr>xTh-KOJ?0=ILCvISC)?pf>)}N$4ck_|RH-dqhCMj$SvS}_ zOhTuT*h0&dB!8!#Pgl8aF6e?=)#xi8&e^XPr)q}pl`O~hlWP1XGrBL5vLtCV7yi9vnhmEQ6 zab&p$rOajyN9ZmCb4P>o8)VVms+H&ok1+rV=B8idjUj5KyTp@$PrB|>Z?H37t4GjH zx5S5ou^5>~+V}&uRTzp|E6`j2yoYXp!2^822t`&#Z&|?LqbUy0oqY+UZ%bh95~Nt= zMA1Y7(8q##A>Wo9)J~1xmcZmr776;HeH(USgG54pXt(44~gITkh>QJtC9kHMCEDViW=A0K?@3Ef|Z!W$CI!Y?rd8m2K!|= zTv{EPh4ZjWhRaL`jtewW!SD!0*8og5Ef$M{f#i?b&c;G19$uyV;F0u4ww`WcV!@&w zim|=O7WW4*Xe;nrj2>w4>V;YF_dE169`J)t6&85tgt}ko{si$rkldRIA!x@ZhzA2$ zhcQ$8euAEf5CO4A`>{r^fX9C+do6*XQ}7xhJQUbJZv?)Mv8+dhI|ipt|91(@sy_iQ z$ku|<1r5vJN}fcFzw3n-C-1ioM*y@9-oUXL^tSG z#T_)L%v(dpE*1#{p;V6TfH$S1&w&uRYV!9#3>aIX_h;X-BWMcs*+DxV-zfnv7Vsn2 zce?HEffh<&b4fUyEMO``tFhcfoWAbeUC^M*5rPvA41CW7A!uPt5C>HWF@iRd{eFUL zk?h>Gbw4+4y;&~|u5IhhJ5X?KTc=zf2siGu(B&anflQ=xlzy%j_fI|Rad}9tm|#Q2 z1l?}jX-kF_*-2cDM?_S_`ky3hyLHuJw{{&-Kp*@@T_3rQC@haluEQc0qN@M?u*r1t zU4f4)@Nw1k))fNIS>WTaSqDHrOKQB6i4YDz(LC1Qz})zTB-AAXG7R|yia3$!LGqcu z0<7PZup~<-0CjfQX^0|SEeqm=As;!h5WeF84OSl^2lg-yB_VkK4EAqZ`%q#!x%*(M zWux0Rk!qRi?qhxKKHTlb-EQ3NW&^hy-|Y{ZD8&odiV*ljNa?zE4|1;^ZyNx1>r9*6 zrSoycBvt3+w_$7~M5eN6h*hpeZX{X{9I9egOLGRG-h;Dd-rra?!geay1@6$E}jT-X=-?p!gASjock zd>7mhJfK4u<#2q%;SGYnzzVUThT$+9&Z^-tTR3#@h8==*$?u>!5Fx=VCh7{BCPVr{ zHQnQLDG~%T3DQg_2_n=(ft9fpyteOya|S-Rj;`k-fPf=r*zYi3Fgu}a6zebRB>Oa) z;$SEDq;;zW;?-x zT)>*<!T^bm<@$6Nby(Dt1$Pl*jO3zq)~1lSa!$eh2pSN3JbTxor4hx zldyjU)aJ3+Ai>}s;ky7wvD(Lk?N0~b2o(Q-+(KQ-Y0eB2W>XBSTBWGwfHR$q1^`gG zP)2O8rgATUj{sD=!m4Ndziu)UML*rc*aAF6P=r*p9Q4#8 zup+~Rj79tIQ#PXYgv~Pw^Zs_~-U|tTDRu7!f9c)}p=c=-%&n(V zp1?{dujAj{PVjGumN-ri#D5%X2{7#@8JP87H&vY|;h z7!%IqlFT#P&j=4-l=k>;3o|&rF6f6@F831$-*E8MfQYSd0b)FmH3b2bO;!>!RH(s@A&uRLhv5u3sfj7~~@ZZrk89?k6bEcox5(F+Rqwe-giE zjSKWEy&2tkzKgi8n+4;~(G7E>JP;XwLD1e?-$OU%MacMX)Zz_yA!Pgy0(%EK4sy9h zF4xHA8r|7kBYETl1x!}vPHZCt(xua&rG0N4gz>7_`J|I5V+47|J@`pBjC+w_TQvR= zy=UCV?hF+PTta!^63Ps%=Nx7n;t-Uy@c_GO8V@3Edyk4Fa14h!alH3v*y&z_S>Lwx zA#dITkgHg;p^7!{0rON$1M;7Nh^ui2CPmZ_W+CF%S<@hIaZfw3Ya;{SyS`U8neP?C z0;uRaMNFheFr9&cL=O|vMG!d2bn-FxK=OtBeeWv=jKBi$&^5$#?qP(`Be;wSScYfj zUdtIb_n6u;_rtSm4p9eBG7qlZ9vRJn6X%|Wr}9R&kVA;jM@^5P)naaei+e~^Kw`F6 zC36Rygdd$bIW>9I-^vf!*djYAtTP1{BroD@ADmK#^ZR^0GqDCAwfL0eH+;O_tX2^z zay*}To`t&~_QGpOrczxjlw0OMW)ZXSNCxTS0SnR1G4=VT-qe|^bRK39nOgn|E1D9| zU_`IiGYFKI&LH4oQeX7tSS)I4^o!hk2MX~)qqz6*{c#MghPCqCA_#HV;Oph|9AY;f zgA?1wPMht1EhYK3Q$N=*}8m4O&D!sW=AYI8S2 ztx4Bv>Ba81v^qG#qll+?zxsk+s~`YmMsnlfXtq+)!MDROBUZSYPa~E~rCeMD(_Tf` zn1M2}wM;m?rI?i$GKeCWLm@H4ZSq0FgfCw#%xU~8RjOp0MctQs=tc6U@-jt-X1?WV z6kl9;`62Z5r>NP+#qBsD-fO~19>57Fc_t1IOgoQ_aKC2y3OtNv{$Mf#XVZG)X+&T6 zC`Hd?*KCAGk8&@)=~^aF-WVs(KYnU9HTzU*^8AS>E}Z0Ekq-gu4_(ygor}~axM6(K zYwlpN8D}`yU92S>u5^NS*^8BGA8#Usab&Lw|2Fqb{kr$Ig%^1rt(kivn}J1R2zJ6z z2$pOjr}mQsvCn(udN;JXO$^fNW;_Grn;P~SKY?^pu29BLQVZi=EH~o}(`aezv*cj* zr63R+QxP*_UW5~6E~)(}{SwDh_zK4~5O`k|>m~ew^V#XEYBB(BiqCjfs;U9>n0mO1 z$PfrBD_(|w-}4O?1|jMb0t207t?3N)|H=A)_v2;76GJ zS9WC1iA=r{o#39pv*(##ZY&SgB7&Cq0Gl?3%Y}@7j3;(i+)JB*p}0I~{1tKYM#Pe~ zaW2eV=HubRm#$=aG%MF@`TOJ0ES7nQNU;lOcD<4IlOUqShouPLI7MqWxjCd$vIv9F zeM}aP%Z*{0i33>8TF37Kr}h)4&O8RO<*I)_0j^T_-+;1~lHH{?*}V+igSa{r5Qjys zVNLvlHz;BZ#eAX2EiA?g%jZE-6a~)_*@Oa=AoFv zkmItyp?{!|Y}c=gTZd%V#4E@hlIwxi&=(qtR;!S&Mn$VPYK`=q0vrWc)04kAI+GOsysWxh(c0|;J5RSpuV-kMT8=JuLC zU+0yX`*Dq0&Ye2{*omi}K0ABryua0)GyW4AWi?|rUO0VrW_Bj^%;cG;TgKbeVrqi% zm$*uLjQ`A&|3V4Qjg)wezvg#;g9Mct-xukBMe0y%V1-&|)?2H0wOWHcweFvpM!c?* zGpW-jpG{4km_7Z>sdaUDV7014Jym~DR(*W(Y%+D?sk4urIelVwU9}IcR(rUo_Pvjv zKR%Uu{OQx3y&j}&#}Dn{AKJq|w8!8ddJC%=Ar#qn{s6Fx!Ceu*h8{l?Jdt_=LDD#C z4bpvQovOi*%dyTOUMC@0yHbLnVNbm%01S?}FQdn-~Fk;<*VBZqfH}1n$28^7RJ(*1OutW28)Ppvq14 z>U#Sx%gOXscdEOd?)zI^zYE_T0e1x45pYMq9RYU)+!5HUBhWHo39DyPj4|o4GbUG1 z1Av+bsSA;=Q-_ihR|5!p%^{}3gTS;c^RFfWtgNrZwE@zkSz`3l z20-h5P#z-Xqoc?AYaWpvfp{&4;MQFGNQAWRe8%HYG3osysCcbTLKXk2RX$%KB_M6gnJHp*kM7Gnh6(`&MPj1ZOK{oktI6 z4f+~1aH33Ai7P6gHB|AKDuWM?saMj4#xeR)p~rOc)bXbuS06p525r6iV{UE}&OIgtmok$WbVCt1(55=t zeiDVN3d@R^OFxE%++=2YcGgjFVtkK;2?}QdE5@E3igNsQFoX7atNEef1KT}I6Ku)f zzI>mQR+I<0u82ff$uyVc?1+Mp$}>e@IwZ~hW{wBGX5C1v~ab>#y9>+vT< z6Zt)F-#h2m*1qZI9olVm2_2X|uWa`o|6LPcpV;m_@w?fVcKokvcd@(=E&lz1X?}^P zOV83{s$D#$OO{715qWfJ<-NOod6$yLxQ;CER#ZIs-uC7FNOnz(EFVx_Swg8D=%x=@ z;u36Hub1Z%(|ZUd_py@GcjLtgWGUN5mM0a;1F#QLhBCbi&$cN`xExs?VxMtzc{@u- zmUq${iszjpEPY=2*dVb2ul+uXe4J%@-a5`w8tw=^^5M~Sg=CT|{7oNcZ?W(5T7bPM zzZbi?_R1>5?s+@bTdG~*^v`*(4G@Uw0q2YMtBzM5ftViu!=jq23W;Y=FE5d(bjdUE zi@WH-Y6CaY^#wF48ANdRnAUiZm(GGa{W5=~up(pPPMZutD+o`FhZIeoS2V(2J|?|JN=6(e7g~ zuld<@qJ81rmbtY6e=G1H^s~k=4lT*`(|oTFwN<~vUgIu6+sxEngQ&g7R-*Rm&o>&W z(&awX-j!;jHVR4$bdinbaVfcGcc^8Tk}DQx!Gf4#s`);0U&1e?y^Cwnf>@j25YQmi zg`LmAj5~DR2DEcfH@g z`Z~XuUQ(`&U>oJM-BtA>dweaYOwTIbVdd3f<>Nnn^>vXus$6}WU!vI6=YE>Ki_2-g z5qD98SKsC@Tzd%brsTU<-)>(%_ngd~KCJ9kcD^=hJ|I!oQZ+c(-rgz8imRE%-zU_+x6CYNToyuqN@9v-f(wDw;^tD6ExzD|JXy{z(L#Yo< z?uj z_WH=O%kNp{tfn>bdl(koVxrd5GS1+rchdyzo7-K2Ht7w4KI*S2JZ5Kxo>a@ zS|-zG>?0yjIW6x`7T^+KFU!3Nf;I#vaDFGzWH5wiya^fhc@=`zD+pT1p@^XMK?-^8 zO~wRd97EfFj>GzUEQPExj%0r+wHniQhN--1E9Cg>(tF z8=4vnVULxMkGv{Zwke~}A&XWif)g_^fL=&0_KDMfaa)1+bZ`%pF54i-LOVH8gNqc9P zAd&5<&OfWFmAV8S(;Idra6h6FIG3PvHLl;E8rN(?c&`b%X|l)Q#t$C#CDtT)|L(iE z-5`SRZI|G43BHO;@R8?rSAy3plJEW7LhvEnHNum$<5IMhRbv!!R{GdoQwRwM>uT?v zqL{#f5d5CGje=D$^X~_lY_!YKI+<3NE(;@RvUuhS^{nu>&E4sGs%>xGGGlf_X#v>R ze!`;Qq^@i~g*~7=n){Gie7SLQ42XwT{YgPocL(M2*H`RoBRYr5M1U#>>5Mn?|4(P|PS?WM+z=Rhn%inij$bAX45bZIV|Rx^!-v3j9=>8MIE zZ!kV(z2-ya1^-161p_N3p2Jj%MXPui2E)k#V+@IL1V0q2W&*;aDK|iDLr>Kk=?0Ix zU?edp$!X&Z67wFh@KX>GC-(}sZtHo%V+_!4`KV|C+YXO6bMS34rY>kTorO2TKGdvK zHSFBTAJz17y;;*!Vwcym*6wtxHb{3*KbUm)^gDP@|HWM|b4I=0<$Embhno^D9PHWL z@g(l|OK0#99Q_`w);dhv@R05b*IB3U#9Y#Z9|#=i^^4` zphO=(2ivYs{u+V_dW>+4Td_40``>WObzS*gp2u2-m(Kj&3` z6ULWihhO7WLDEbUVEH#OL{A_QQGRhnwo>iHL1v#!@u*+o=d$k34> zbLn?5U{(cNzr}Ct6?{Xxg6M_0S_MN!y;pxrYj>tvt7Py+Dm988s4KrJzFq@EsW<*5 z?&dS)BS`2!KI@Sq^SOmU&>xVWS&r38YG{O?`v-t6VxB&Wd-@~v-Z!vSnGo}Qf-><( z{(1H(JFNL7@tacl{~la>G0_HxR5~ImOW*(MH-7uF{wMbjetlZvvz-2TOb%zPKM7Ca zBQ6pQypl@g3)NJnS;(exxj?`liD%QnKuC|xaZfoG2*(RKUEdVzhT}^tb8lxzlm0K> zo+lzC(U+9Zy)>wJ{$y$SmpxxrzWCCzveo~JqWJ%r^658T*}CJ&f4Kg$Zw&4n@?PEQ zjW2HWFf?vRqB|udliV|&5DPo^SEVR-i!Ums zr}D0NE{*|;CZ7M`hu14~`E~JJB+6UzLcAxFGobnO+EEI8tDZS?QVk?DSm~?2@nWSs zuh&x5bR$ozYpPyJHS$<{Yt>SnVy31EGHVjp)cA~}eaB%3$HyYLY(eZKI9`iGx0!8VzWZFUaVE{>9SdTI7OSQ*A*Hr+R72eX|Y6MoV0pg1ay*t zoo;NAPM!SN*~zKX+;Nud|9VpzH#+0UeWc&HNW$+%cPdMBNtT!MOQz3>f;7LVF1dfm zc}MQe{?GO66MD=jlKy1*?)SUz&*NRn^HU+)pDdr~ zDj;uVdI4p+hls#@6NDfAE$~FygOuKr{g?QcL7wcpc$FnSu>mygnYfnq%Ip#@oXY=G zQNH_3)m^@2{2R0s5z&?aS~dPHZaRHX$$)4?&NAwF4-aKb!QbN%tv8Y}dl1v&oNNYh z{3n3fK^$|xFCODh@xb_x^uzphN|H`?Z9VD#{(&6W^^{hWxm5Q6tq;zfT1p-}?$q|X z?RH1tP9A~(@Q&OlH+L`OmLt<2wN+F0w_m#2*OTQ|d(VCDj(|G??g+Re;EsSh0`3UB z-$$UuZOpxERG4$n%^hwl%-sQAu-%wPq=4qeJaW~_5ZvnU;Y(LYi$N+=yJ}^wS{Zcf zxvv0@6b~e45jNmHyCObxUY&F@H%-VZi(ejzL_->P9%uJL5DOy|R%~^*a-NRhRwVSY z{G=~Uuya9=pY)~QXMBcLgH!Z5RY-jk&pRY#A{%!%c*?RHS7e=xbH^DVvQg!sxbE+`pPpaE{~dd~=f|V+h-<6X-yJ*ccTWH7KNEq_{^AFP zmsg0LM)S{#lh!wO+7}>u>x!MWcY?)EL-8Apod!9b6>F+HcA5%!{bHvf9*+#0Hhltf zox9eb{&8k0?C9FUQ{@q+tk3ce!EXnPny#$tvoIL+GyPYrZWe&b%Cucv`}vJ=`_?yj z+S{MrMb%O8G!<#Tk)z;g_D{IB;A!nI?0UIa(aVJ$p>J1ksx>*&_Pq8Op$COma}J*N z&fO~pPrFv%82n|8aQKn?x5FcrwbtjJH&=99r~;TfH741R-Y)zU!P7SF)ZqB_*5fSg zJ3oZ#%&Pluzu;+hKl3-iGTM1Zv5efnX|)ur7r-z0kqd!+tL*n&D?^Gco-E|l+4GYp zPEDPfom3xD0~A?3MK>oWXD2n-YWNVly{Ya5;5tlvcF48!3+{X^PbynV+QpLn&&cu zMwU6-zzs5u^ExM8Mwa;zZDjcgc2pZ#<}90$$C9}+Y5eZ9zST8h_Y2DyCF zXxc)-r<-L+i)b0@y92^t>%)MVpU_PvhgR+t|`hGE5m+J|)uIeK`PW`&!Y496w7J9;K`qzMYlGZ&_Q{Mc&=H1|@mh?ux0ta**d{c5n=z2uH9U*Wm-Lq&Y z6w!J^nu6UJLTsD_XGE^bZWJL+mjt_U#gL}z2I-!?>FS=jA@kM|GH*5AGyht3&)!@` z_bltc&O+c+>sQ*IKjgS$D+T4b(kj5p(T|OHdahzhG_6r1wDg?AraxE@k3}*C?{q z>MCfAUU2&HJojOi{Xv}Uhq)7#zG`a4b$r3;?=$kCzi!%RH%7O|(=Vj1tD7d+pA+mm zk1V@97yCT8X?A3FWANABCj0eTbasuyGU`I0i?TFB|es<)N%Z_v-blnjVx?Fao%Z|Lu z)LeFCv2n+m!iuO#;5tV92`%Jg3mc8Z!&;m_S(P6tB64@|Bl|_W|A4yVYhoi*EQd|t+_XiEq0q98BN5su%mut zEE3fsJj%`bozi;m6s7-*h{lurJtGM3g64yq*IuN(qEV@7h^1LwtTY?G(q+UH*NZu} z9WiFK!ODHxh)}thee1Hy=%q?+@p3`G;+tD+=;>N5z35{=%5+~J9pzl@{Zp>jPj_i~ z|12$$kUtvGV)5P>(veUg6w`tU_5`Ai$1(Lf!(=6qlYi9Qn}`Qvp?D-5hy}GkfEO^H z!o!zaK678DnW`e7W4%zyrr5MmQ&H zy!B$Gfv~}uTDp>2`d2b_B!ojEx*dDP!BoQ5JE z;~~b_P#BLwDIFjVBCuY^DCUREEr?cG$+V1n@GPkq_adQ@+mdZ8R`r(Q8^EIFt-pYQ zY8nsXA;x?-5|4!wfnYol3&*u!fNu@X@QrCy&4GI)aASt@2(mGYt6s>F1in8aR0cU_ wI72oEb2S|;WDS?FyP6pF$3j8B77WKEA~_6!K^{Un`|kiI5k`AtPj&7815r!DBLDyZ diff --git a/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_sync_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx1200/profile_sync_run_1.pkl index e1b0a91c43a2aa4729463032b1c5014206289ef5..a16fc7b95c5e580a856a21e2075796c699e7e853 100644 GIT binary patch delta 1024 zcmZ4Rn|0YDCWQu;snQb}z#w*tMh|~UW?p4_QDTZ-N@`hVa_W@H3+Kp9wB?dxcJy_b z(!-LMn43DKhd;TrxFkO}J}0F(zPK_kdCKOqb8J{Ahb%H@%3z;7VUY+^2H)hBizYB- zh)=d#EK#2!-NO_dI3+{Yn=wNUh~>RmG8DX-G8BQdk}m^ehBAmy=@BYQO%D!?4{!}~ zbM_DQ3vmrHoKl*UA+FgtrFM!&21gGsP}b8gI3zee*wH6+3ITJOpyp2ITOtl}iq;ZY z2Lf7{aXZBr?3Bq%mnabO4-0NXOi=y9?;hmj8}AL7=A;G(hJ?75rRJ4PnLGs= zm24U88N%KSK<ecy7ME`vQkdhPO8(FtU9yR4@WnY&iLhQ$8cG1E(S;prVC0o^b#b zb>dXS3{*7n85^e(LJ>2PbsRlxuJOi3@g`F;xVAoKoX4k`!PCXY(7?dRppd}}cAWr{ za$#Vwie!jpcsel%On$iBX?uVG<3WCTpzTZ;{s5(Upii9`xF_G+t-L)>o$&z|)IZo2 q13kUn!;&$L7q2Ryuea~;Wo!ef0(l#|SwL@Z_ecV&;)bhA)dK(*cr+3K literal 97696 zcmeHw3ve9gec#>zB*FK~q68_G*OV+BC~$G_2P4t|ACkx;D1v}wDJ*k5+%0goyqE4l z0(#isSxRDgCsFK+>rN-OrZdiC(!_E)&Lr(LmJ?5$cA6P;(#$xW+M~SFcAB~~Nk?hM zR)7ERaksa(2Y?_*f$|=Q_;$bD@Bcma|2_WirNLj=sdyCnU26HvJ@tHPalV?)YB{}7 z(W|wVx%YgvJeM!%7t*D9{nS;xRBwH*HFCZ63SQrw)vx9=ddnPqX8P0xB&BpwZ{DCVy!h|4pjAe>zwJW&$fI%b95|M zF6v`1q+d)I#xmusJ|>%HxzJ0sa#gD=wkF@&We%mTRnrwTx@88hxEybFUN{x)F2dU);M~<}W=udo^O7&WOEM3fw)p3!^G-~y7F?B_+mh=KOAo-HH zlRYV9YpL2|Dbq5?)9O)mhBuLQ)fWiHN7Zb7v7#Ske`{L4R1XD5v`o2F&(Al?jhb&n ztLo`O{zct4!kYB(Sff@QE97U#DvR}8xfId@!LdduU#Hr}GU=Mm`akJ4cc(JxLSY8k zlRDnaw#@tTb83-Y`$p7!O)Zt{>bY`BXPwR6Kl!2;gWf2$Otq#L<}_P*AJxD?v8JHF zZ+bDx?6p*?Wj=@x)~J?vVZiYnN>}G=qw1kUSFUAiy!jtpw)wS2zRud8d@=W+xq~Y7 zRrOk}g{*H^5hf+~(Yt+)!?txZ7!N>!5Zkw|gYF=Ui^D&zIYW37gx$ z)_ICj% zjLiLbH*!T2S}e$sD|fv6Q9P(c6D#7~7P%L=NM~P@*8yd2d9~e$NEi0UwXnb}8`6aX zSPEk+Bc0f~-IcEAAmAb$PII*P+E%)7D56EoFsvm)u@#PdxypKOxh@JWM_#SNs=DD&5O@~#YoRE|Gdl`K zLs}@(i-P9h%;|H_w{k<~-e>drH9>Z(Pl6$kYjrKT34bCH(t^P-L*s7n0e>K-#RL42 zb;&h0fkgt~yGxDYtX@^?IbCg(D>WPFfIpPz13DN8V_roPJ(EX*{xLmWEi9__YPwXL zD_4sk28p?$f*I4(g(m}=J0V#0c@hiF?{jYT7q6R4==+Q5*)KJAjBAO zf|2=uG-z&3b@)++t`X0R_(5&HnD2^a<%g-@)#PjWQmvjYWpp3Co&tYmRISykBdT7g z>FV6Um*z^ZsK5^OrS8J792`}zXv9b3tb%=$UXQu2mdWYF2uKk(EOj+a+`J33`S}#8 zxiYHG>KSm@YN=wnLc9lwc&U1sad>>sJYCV~P2Zu53mQ7dDqc|YC6#d|Mn@hi)asW; zWR3rV2>K*?F>_(!#HqoWW+t?Gy>8<~se*?z*Tp@e!m!qBM{W143iBgQj2~iK=nHl5 z-57Uo%G{JE22jhqUp5%W9tVaOI2&K3ff&YX7}b8=I*@z?XRGb$Zf2i*AliVtQ5@Sv zG#GK%E<&M%UyDWht}>W`joDf~UvJdes))tKzA%O&p`aEEb0(W@4U5IG7W;$!7F^33 zwgpQr2X5i8B_A~JNr4|;Sfm+FvzqTHSOgwhjy+B*5AnrmLlt~+AoYO9IEfX}3_61J zS=N$Kl^`5FDB5&T&f~jveK3;fcXG-&OWk*8KjZ|&o9<=xe<07IAsnS}7UfgiK0gx? zEg0@QKh2>crD!{3#RKe+G zZ7q(4gIe5yQjzEV2WyMP*>WMDd8~>R39v#wO|qVFF|e_3lD= z6u~YO6kCW>4|W#v#eAKPm9AbJ!?Xn`ny{iU5 z-CXJw^#_7MblQT0mM1{Z_%c(IjYg75gsp_91wahcp@nF(JV^w#M1R_F4<@FNHK&~D z!FV7V)j&6Q&L~#UQP7>MI7IBPmn&sZoOP8Qs~M4C;U=1qRgge>X+B@l$K<)39i8`? zyE9l+=nO9Ck(L==?f`Pop&r5#$M5xm)H;R-aQl8Jwz8 z7mk4N8c`1)Q<1*R#PyO$YLrN-#pS}4B54XRNWKM7i6p@9uI^Ktn6zy!kAAf4JPrbl zy;~`gMgV~)63tiAucd0$OsaH*Gioi(qDL4e`8rb`;2ru`o6MaIA3N}81bHMXOrc5I zwRGW%e8Wc+C1A~|l*{Q_>$?zGTN}(@?+p5151JpMF_3y4`~zVZWG6EsxE8w3CT&8J zBhDM9oT5YkPZ#mG6g;{G8izsy6Ec=&=@0$((=$N?`hrQxCqJ|Dl@=UVz<3AXMl4YagD}~?qD0Uh%wCqOJ z3&*xk89oBC=3U8njCQ*LV-$&T7=J9Ak19z-s#)hP#)z}F-|l_I z_;EI#VjW(77cq{}yY4Qrk=eQ~+vbj`plz64Lg+4niVH{M*d;8qmgNydLt%e^Vyi7S z>73!Zh)+0*swI}_xASRxLomW2ZC08fxq%ik;g zl$7m$Xp)u-S9Njp?3M6gWQgG$3OJ2Skffin{;Uwo4*H>KLCQhEBPd=jHT%z47~4`a zP9*%1IO5)U8JK%wu5&4xyB7Y)u7zy1A||ENY8CWjK?%kCFDQH3C)}~BUdlrL*~Nwn z#KREY`-Mu7l>r(heFt!JL;ZzgI~czV(sz*8-7In{3@wJ31KS;sA|L=_@xE{^tJH^y z=>r20s(=ZvZHj3)^n*9WEjgsJ8WiBAoe9ajHB$002f&Ge8H*1V3er3V&yfE#o_?;C z+{RSoceNifEsMdJ*eQB_wY+9=HS(GKHEB%^qi|hsovN>peR1$0RAR-5eVZ! zmSM>rV%fvUHiypFt8IYEO-a=pfb~er^yFKBXFNbfDckrcu8c?UM^?j%Z=J|ytJ4)I zXi$qi1UKnPE>Sy1lFGTB1`V>BA7)U!$su!Rqg2b!m-MWf$q}o~bKFubUNAybZP-%n zA-eYRiY65=tH~U|K(~5p8(_7uKCrQxR0qs0*Yxxi>Y~$lJb;PJA1@@q>Z5vYM?${OE>D}dA+2;ut}?C zOX)Byl7R4o)03wr+kG|%App77dVQ;+Dt`|{v3C`7OSzJo&7&`51-7X+3p1ovavWDs zwn)>}mRfmkuBO*pNws0?&>VdL))ix(hIw%M>GS8?l^F%(y}n6NzV&s*DB}8+mbtl> zFBSvQR&oyn>Rp4=1MAtR&j-6p2KMlN7)(>Km7{cd~YHq}Ib zml?R8mb~{~>n2+AN_R)jyy_i1BZtJePW@1gCQm-al4S;k;lmKV;fd7Q(=+y-y{CNP z|=Q+Q?4pK8I{Elbq1ZHt=V@;T`%!8)R3%J0SVW zZh>YWyumHoyv{(36+3E&)l_!0@qSGoV#~8no_+fG#97;S*s3gT{yRlk@@|$b#k}XJ zj}_pmyc(o=b<=a;tv2S#nU3|JPx+sg7e6J{^4?bh<7N)kW_b0&Ga-Oyva%)JB|5uMxgDi=C@mr#8P(yl_f zIGatwL@?!h=6t&D^Wi9?O`krO998M+{Dr5Vyf85dJs_^kLsFZAxZ}l0t~E+oM_Bc| zc|K%9v0vjALn1|jo{>=|yXnivYz~?`3h5UYA^m0&w-!^JBcoCXO{$9*1DBixY5zF5 zI_@Xy?dNbRAYoz4JjH}`T>4d=C!q-j%y)qeF0~i?HQs}jrIsO zgZz6YQx0hJkj)>{^MESvwMx0-^N(l@s<6-8H+j3&8E*zHU1h~vvN_&xe6Ii)Q_uww zS|>JoECMy1SgcJqdD@2k=gT-`cN_NSHXE0*Qb6NZbN#}-)X8{F=#YVl-z+v$cj&k{uxNN}EAKDjfo8o9N z6b;6qGt;ZA&}~zk`}}$un7DelPP%*>4Yi@;6b5Q1983se(I707v`CC|IIN0F-FRbe z2J01?XBJr>7QkDyL;nQSxWOBl!@4h&(s?Dmz^EDsCLG;8(3a?`dT%ro1G^~-vva80 zb%e`;^&X2xVWt8`S9|sB=k8#8Dg~k!dYMJIXjso>>XX2;oxrn;fne+sknM25#Uh$U z8xM=3P#1<8D@PUU9vq7WBH_5^5BKUQ*cYlg6hpzCSWg660s?8`KwHzR^|Pqp)C&!m z>!vkcoe$@GU|9Q(>j}B;F*cZc3_>F)zrt3rcb$p_qp&9r#`<#fjm^+Dv-0p`u}~1F zF27ZIRIYm9-bbOt9Pg{BYwYfY7KVQgzbL}6U#RZj=oySa^w`&c(6~mei zqQZ#MdK?jAS|ij0Ef#@len6ajZP21{OW}|A9*PAti@d|Zlre}k#Q2B74J4HH3N=R8 z5Yb|o$eA-YXJ~Z0?5Upem|Id@H=$)76KYqCQ*xYH5R9q|9jiqqBjYcRXfRlARP_|} zTp{)AHX<90RmIgdHWXizgL+Q~LJ(R#R2OT)TVFP5u|S78*6d)5v14inL7B)8D! ziju)o%cUz^I~Rn(?P~NjkMSi;NT%}J_MS$)K>OrI%+7YJwIp?S_BHy9+i+*cSTGJ9 z)_A(Rw6|+5ddg!AK!~~Fmw97|TInwGWaN{sx#SycjaTa!bmJ}Zu^3MD z!C0>?Csyc$KM4OTeV1Y5-`Gh95H%5KocbNMjChcgQI}DbGQQRejSP^Xe$`7_#-m`g z;!%J4?RR^ife7+z(Xc~L?(Hs|e!|Y&!Sg4e1>=C`z&u+9T8PvL zouP$c=Nol!>DU8p2R98y10sUoz&?Y-BV@;5yxV_Tv^&`;5e>qU*nzE)04D-?EVQ#% zIsQ1-0gWA@#Ub)b2rYOgXm9nPTRez~6cZYuR&^>D)u4U>|fMTE?;3tAwM z0B*n)SHC&d3k{apV5~;Ny)!lro;Ym2!FIU`XrT~3Ldanj!2eeaD}$7xu4dfgfe4&t zLCLmvrNbi#Hnl{cKdb8Vwy%I9aT@AC@0^06cMN{SLczXkQ_m8DB8d>J!yQ&SIHG{1 zM0>4r>gl&_t1uP84iOGGeT4+fDzFvxXFmR^4O%RYwFnBt!altlL*Q}(Ou1$Dn>N0Q zi3QpzgmJTKI$NK>_P+0KWA3&&Vdz>i;{Xeb55H)Ob^1f#PN>)4?9ljgTc6{BL=v2vi>W+<(RVpLS^B5Vm>Hg$}q94hJ9?^*i`VmTTPK_dp8+UEyHf!djL?2xvg@ zWtQI`NMM~3SjaJtdUn`29Q=`&$z^<}?BL!RROYIo`(uQo;ShX3IQUGBhvWVT35olk zpEg7B&$_{)DFHx>I?XlsUWL<*M1Rj3#_k?yIF6yq!fARJK7%igX&4IjUkC29L5l$x zW9I#Z@qOF3a)+Yr0WIY=G_*$ z9He$M6X_nMpR>h%lh1iv4w5S-SXVKD-IqJIWJuCMyY>p0=@G>8p;hhH6^Gs0bw&Y& z@SAmhEVSRYLBnYwj0Qlb-GOluj}W7?zf)-A8xGK*)$GKe zM>P>RKaKQvH``$%m`)x)m~z?3mJOs_=6d{Co5v4#!*Mqpcf(o74afJsMNcFp0wOjA z6WqpwCSTDBvFka=d(558Da&!FMXD3iGo10hiwi)pT~H_li)w(7jM8=`oMAbE+{Xjk z6w6ojB8+-f@~G0qf`?d@M94u4WTCVjkND#dh(!ADcjnGQISrR2!gsbEa2yQ$sFQ3Y z0Tvfwgw(#U`haFDf z7-uzwXrnbG?9j~VfABJg^o2^g<8Y~sUJxR|)X_m81R*oYjD_UMX&&x{V+KCBj-Kb@ zfDpmi(ki=>)xwh==_M^Q##LxpU!{OW^h9h>5n425N#Sh9?CT= ze~1UsJuVP+Qm`8*0xFgWkRCv!zwCY|hwMBcP&0u>wF8zwpNCX7mKl&;EF@Nj{AZNv z2$S8lX~9b)00%-2>uC%dEJ=F%j|Gpx2C)Tq1m6W1S~FNl!@^FgYe@pOW8aDOS0yp` z5LQ#@s%p8Q&H-b(8w~&y%*aXU1uAsdX(eVlAfWM2cFGb(utGw7-a!Y6htM{G>Y-St zoPZ6IcL9p%AkiNsfc};vfCUsqG+Bk9+dN_Mg~I&3oqFJ6!e2~1aM53U;9@9R3joe6_)GZ_jxPg-fb3)@1Yy!difkO{*0i#zuty!&To+MKd8m) z?lj2wUj+6pbQR<>iCiX;%OtwHnMCqb1zrw1CHz*tpme9|kF zF^oLpe*8(+jR%lmS2X?^y=Oeg?hF+Pd_Z~N1Ii4o^+BxQl|p zVmwUh#=V3=w>J%DZQId*ymgm9t|HC4D$=~4tkPfMz!)9Q z2ogj5OzFhFF>CH_d*9|RQd{OexN@CCoWYaKWvkrG8A0 z$lL^<_7I|g!tAU{<~Dc*KQeuCa^i@;l^e21?zdAQX9_1sZo}C=IH~N(?e+Q0#44Q9 zR=K0_v09^2L2SseT;>H9;C_!6UP3bE%3{9MGJlRm$-*NUmXC)hL^ns(7aDp)XTH&S zU_oSRxofOwN<4$fyk5&7G+sJ`5Q|BD(KpAUPgA2`;@&z?hz}Y?x`(fiqwp=Pm1Y+~ zhr{AtFQsP@t??+l);@9i>?!7Rc2)5zW$!SYDRO~cS!_-I%|I?}ewtM)JeicMn&_$w zy_gX$Mn+YOdl70)x>`*y+TYS@U<8jKg5pE!i+Z(;kdGP3f`@0>a#07%4#SM-;A$?7 z=q=?^VG*2q6`^A48o2^7M_2Ks`1mll!K!&*Dn-n&OuAGq<>7=L4ala*EnUlmhg*tL zc`<{yfpZA`m?00p2vvz&#O8(jclZ^qYbcGb&7w^{h?Vf=3;9`%U!{uWY@?w2<{o{C z{HMG^@t~PAc^bu+7G8N2J^km@Y~#{aoDd%{;S>+xgi|~dhX6De>bdsX~30$OmZ6 z+zmN#5JuObFw8C?S!0VCJ9O=8KOu)>y^2>iwYZHB(&A=33*4LB<262xbVDvs#!pZS z<6f*bV+_$~W$d%!VDhCP1{+loBVt~}4`nW>{V4r1M^pF)$1D(NUlGeC{D1Sw>8ofm z;%$mgcvh;S0raSPsDelkh$1UqhCkoh(TXfecPwHn)gWWW>IJ9>AcM_w)|5rgFT4oH zz`1f36AU4uE@t32nA}x%WzLFBz7U<@p1-pfm>+Je4wV9el=uLf_JynYjDD0SR@d80 z8-bnJ95nulczMHOMO#0Y-(4i~sbK`0qVY#lHH?%8fx&FrjVUo#OWqo4(hbc9b z7~0xC#wl96$*Un9l1119`zcvCDmV7fOdP;!);fMK__Uuqb@mAeELZ$}984wqZ$Md1 zNqeacb}vKqAdU_Nz+sVUSQG!?>l7h|V!TkK78X;5<@2B@ih}2eWI};RD1b;i)(3kB z(K(bax&PfU*uXd-=AoFvkm9o6&<`jC+k5YcTZd%F_^ZerlIww1*B9!FRxOijMn$XD ztM&A(0v`wL|7Hn*GUzALcUM?Bpe)Z`ZNDa~6X`g?TIRPK&#=(-{KiH3dwZUAXHa|w ziq-Y{X5}&(f*5l3^gNUftgHmK2*zwK*MD(hvl8 z%iJwj^T`vAE4evHo*&R6P<<3*6fq-w#vju@u=V`piRm;{94UZZTN1x(dMb7Nnd#?K zC!RlX_S7`kC9_ZYQq|9p$T$``Z?Eqqx zQI&&4sJA8+kGZv~&)0Zm=004bmh-1BJaOXbXU@%>y5Mg$W{v-bMzx!<6EB=TH$5|* zdUoRMGcDsCYB4pz_;Xw(J;q<~*>9yAC zU8&Y!Pp$i=rx2d&&lNCuUARdumM`9$2aBP*2tO$f}P|oJ*!oJbmtov!_qY ztf}_Fm1+<7)V}-43&$r@Pd;r_e*m#uUwt90`8^z<5Q-P}FKtgE(K z38{4~N`*2GZpxnS)oLwwznGOv@>)vY{blaE`@7)w;1)(e9(Zm6gj@7|9Dx_kfqK1( z-&$8Yd5Vs4l^Fe$0nmCMREJ3Q=*ZFjdPk&2AYPk8ENiZNBqCUMKjTrSnDG8_RNU-R z%z^yfcj)4RM%6Jrp#@S8s-~~uzIJI;od?GzT}To5qxA;3vOY@=MG8j@sIJGI45m!b zxmB4i!MSot=fT2RgMJzMH&Ldj#Pt-=8mf3ym9d6L)obZ|{V4rWp~ZCa)bVGYRF551 zgO*_L$U;g^MDm&*EU5t{x*0)d%cz1(4PHU zJxk+k#oyX|P|B*veR%GTz4 z%7*~avlM4=3_-_O3h?f=}km*stU z@gMh3@k=~idX64b?cyu$fL`4-n(0yJCroWb-1}xQSszETbuik>=++z?pI!2 zLaA-&rVm--5?orZm**1Cdk7`>vXWEx;l*)eDO-k{6AI-4*gce?Ozpt4Ey@xuhnqv} zGmbR3vUIq)o!(G9s>ae6l;;PD6FBt;DDnxG=Q$3)|5|9)FL%&`l?HCu_63KVyBNITCW%~Wz`MWGu=W~&d)`ca*_|@YarPP5I*hA3g|{1h%CJF^e>~%SUFbi5MwvWg#q+_9i$J&s z8Q2?sHr-;ldADV5&chE2oC^Jnu?L5mCk6fGZOKA+_n!6zECKw1b zNHt*ubRhYuT+H0vfHyxHFs+!*m)N7Wt6`z;3~~+;(4c#&h{BMr`@cKJ{e)fu!B3YJEX?IcJ9YdgJ1Z+%AjBE5+J+rGVh+qn0Cf912^-u^nJ*Dv!H+}N*Nf2Y%! z2b6nWRi0T=8iUF+pq4LlYFSdW9w!Qy&9Lr2F44=m8ZOa$mLbM)(Zw>{cwC})xasnVnw&+oDn3ycD&1zH*V8i2;wX5_1n)arU4l3161*GYPRn5Ff>UprpQ zdx_uj-k#p_^Kb8UrI;>3cQaF_3k6ps==`gdV*VsMpb&726w`D046ClMz6~e%K7Fea z&Uai1r{WTPDVN}L3BD@xQw~L8BJM6Q*J`(s;6pGR1XJsA=m$ks_zV#qLsk@U4sd!; z5duy|?|s;%_gs39_DW`QJ&!zXIs=lk} z{iC7i-L+A?-}K%TnS*fK3y=CTYnr@&_g&ns6FsQ70zj7@EW7le&!q?NBs~c8#RwuZ z@xV-70>FSj7)8|H-k>@+XWQ+)Tl5-O0EBOvTPRQlGd_QWiB@f1-pR1LcvYBalf5<9 z*Jp*lX6{VaQXOmWmKn2}X$!!<&I>+U7zz<2gF8=$s{3;uV+a-*X3`2qgtPFCXkC@z zo&I zbrkMK@VeV0cu5s{gp-Iv*Pz#zcp~UW%xKLo!`Q7cQchyHv(&i3B#47ZkzU+j67X-4 zF3x7tYNoz0TFaNN98oFW4#uaXSAEF5=)WXFaA2jxb68@rP#KRwH8nY4j3O}(;}3<& z8HaAjq#K*IuBU4Cbe#taGm;pTbF$QS2JSJMew!`C% zIe1nXRTs3X&O$3;A8M2<8g_2vk7#av{64H-{BZ_xM_9vuI|E)fsJ z34u9?G$?l`^i+e&Zcs#h-`pQyeHC%xP{9R{< zi2Mro+M&o)78Bi5WIL())zl~JmDOJOt9q7=i_Gwe8bm7Jm+h4GPNAF}64|EJq*NFE zkBjlLr+Y-cOyxJ*f-?UPm8(eMW$elGFdO^SZy=OGu9ns)Q~{h<{rvZoeP2e1x%t}9 zzxo?mI-3QtT}#y}uv-5m?iA~nHAEOdggNEQ%pcd#uW47)1%ySsk}qX{UHHx7yT|Wn z^VM>rlB((RMZ}Xz&E*#mxsSYTDZj<)LpAM(*nH+oLtpF3r`LymRV$=R^9{rc`<16F zdg;Vj^~6`(S*fdt?3XW>KI>I}8@8(pskuVBp1M}9UJ=y;g8F5RB8C0ZsZ=PK3jcep zQq5mY(o~8VmFDBaHkV;2HW$AB!=_|i` zRsWNR27hHr;HXRucT7Bd?l4>jTPfAqI{c59w}+ z2POB6r^Ld}-DN4ted3FX>8ZRco{M9EqKW7K@Wc1Ybonddxk!|6%M0`4`&fc=_Z^^FzErFp%Ys-){VT3u7Ma;l!g+FPv@YZR|FB_i$Ojm(wwJObk4 zxt_}EvyJ&wzBE@(VbvtsB2})*aJ#iJgu@*x6s{HzW34j~zm?sn*-$h(myU<$LUY;7 zY&JX_@W;b3eKs4(&PHOvv>phiL*cpE5w^D>1S~cTRjhWR#q)5l=*!jXmD-W9G2G-D zvs$KH9J`Rt>g5vKSP;LlQP;bwSFz$Aj%r~os0Jg!VEATQB1h0UW#7Y#RY;r@IDzSPii=~_+^e<{CS?Zk!RQKH0Vg>1c0E#uQ=v-og|He0JH zG+eZmBZkyskzz<`wVa41C8J8++#;Pi`TV(w$P_x1&3i zrMV=_%lReK%c3C7FRDxKA9CK2ceVff-g{+PUd}I>%FisH%kl!OGK>p?EuYKzC{ufR z*?yUR7-iunu3cWvf0@epk5|~Qy|0S=L^_6)%FFdho*&v7 z60j_1YXIp)QCaE(`^gTAfs*HFxxUHqliT+(6rwy?z9Py?I^YfZz?Pq_dc>`=mB(&L zKUv=XzWx3J-laT06|($f`Gl>2yp`!il(7#Hfq4srAN>}1qU=IS@5%m4{L3Iu_FcTn z5}()r8rDo)%X(#Y5f@J7e^HdTpQ-HSTgHDxOA(H337}QuKjEg^#g&Z4M&vAu#@42R zqp?Ut3v%x<7AYI(W;}^$5gCrn_!9u_qK>&=8ISR&cwqcj`osK%N}5jgZ7u1e2WdvG z{rBik+DB;Zu5tl~Df`UC%+GudppSgKo`UY9T+ri({heOw3@g^3|~C{Q!MQpoMrsuOm?;+etP^93=kQq z@=#z8b=^e{ic?6KQa!0Q*g4RL?kVcSk%u5-uzQ$Nm3g-OnP z!yd8CQ{fS=+Mnh71iurkYO1`Z&%$8PpQ-P$x>*3McBbWG-P|7b3*M$I3Ab)r6czRM z8#xNzX8nY#v+3`AVb=}DvVQEX2z{r5V6Dn2w`cqbhK+sKx9^WF8@vtPR@Wu?pk+#Q zUCRBtE7Nkb+UK4(kFrvI_J0)*-KoKMs5^z9B1qeYof-^ZR-7Mi{}b(xD+hn=7o5%N zr~VdLM%z0^mTq9SYDu^$cO!6bxCq?0%Ye@{Go;Yw$^4u;b7A7dsmW6_6YAq?fC9Ux z=;q|a%!CGG4Ie_s*HmcesuV8%h$`(uyBH8w4wmaW0PNk|!^syNSuv|;=G;1j4Z{wS z`jZiVJQ3Hz4*HW3Kl!r^w)H3b#N`wGAQ6}E@=gmss9#(@mmf6T@{QMhlGWipkAX?nwiny3- z=Qm#7>GB)hFe!I~TU?jl2)kw1RVjBGTdV2S`dN58J@rB(UGSybzE4NN2c2u*aXqd3 zCnvqR2h97xQZ4HBTp3R2x>%>=jL`LpeCNC(L;VcCb^=-`*ol(U=0pYpvAE`6CBIRG zMO_m7#%05zF0GUP+1sxEnHw+trXo_(v07P?-}uh6tJFVxV}b!}D>}U)2@0-{HkEbY zXCe41XLa>uY^_@V%pP@j9re#x)LpCc(niofyYY~+LmB_!4GO=tvKkuuG){j!msOU1 z4^Uwbc{=H+rg`kDYK1j?!Kpu6K|k$h)E>?w2gT_*s!d6HbGXLAD36FpXL~L*X2hx9}{w_4H+AHVlg}#w6G&TG6a5Pj6dlgwp-{w zim=^X{K$S`yIp?d2V6A>$D+3FOVHe$%pjDT;x{dr>vy^=<*wfo8%dN2>MWa6%|**O^Vdwnu-rMmp4NAVCX-({6-)< zoW(aEX6bPAqauAoq+AbRxakx3BO*N{QkUP@q=*Q#Zk!V-chNC*2F^WBL#n#mZzX=? z)GKX$NYD673=&DiZTgX)zEu}~E6@u-|`wm@P&?s28iXc4df|@U> z)$}#o*Dj5!B90q{A3IuaR0_J!k~4ycvE@vz>c%TyOOj-5bvyI}lFzwP?giKQa=IX;I0KTvaG{io16Bm^yR% z-1Dv6?vdP{NyU4&X#HP8M4sd~jUYS>nvZZ^dx7?fdbz40pk`&U+^GACR}olTFU+y+ zh%uuLR_<#?gv!ONTUWb`UMyD^ujch@zS+gPo~~BYi#`UVMEA9k5zf`#H|cu*beEP7 z(9#l(!SiRx$<;$N9`b7;zUV;I@g$~Rcc`o+a&pJa-HCWG7K%rpsU6h9F|Mm^JdKC1 zw0!2?Oe0l6RL5Grm`$;1)k+ST+t@7&irY#qncLYdy_RXTl3sH=dyz-TrPn`1ad>Nm zaviaQGu3oGlVdNc=018r@of?CQB=}O?lBK^YU2;|62e951bSB?3m`a%B4T9f%4{n+ zZf+?xim8Q6rj_)V`w-wZ#lzpqK)|gt-}o;crC}Qaa~sytBhx1*Cyw}AM(e~MY;funM diff --git a/tinygrad_repo/extra/sqtt/examples/gfx950/profile_empty_run_0.pkl b/tinygrad_repo/extra/sqtt/examples/gfx950/profile_empty_run_0.pkl index a5225b04f84a4c3c677842443e24863bde87e298..8a414167fee8885e32f13a33f9008d4a272c2d0a 100644 GIT binary patch delta 1902 zcmaiyZ%7ky7{~8-ckWN>{y58r$cU5_ z435n1boizS1Yh=I)Sx#)Z}Ls#o1z!JsCN=U6a;%y^lX|gX!jdzckDjT=l6Y{=e8FO zTkCXSmR;$5L=`{7noqG#k5AniOD9wQlzel1RMr&7VEWqS@vAai;Qz_xW1NlYj`U~> zJC&S-=crf9Hi-*0>W@pOIBZnd2{}Eb36xTCeH62p8RMZ);yhoB!NC3BdK)aE zw7v`KBNPjQr4-j$A}|#M5nhrCjp!Q8dxk76ci50cJw4;D!ba>6C^(D4e0Fk%V_jGf z5bQLvw^PGDfq+=JXJNfKx>y#X0a{bU0(8GD!~pblrD!S-u>k#g>}PaM6#%tw5Fi6U z@-qTt1ZXWsfJ^|r*|g|qG6VGaH32#V(9gF7s1hLa2Le8-_r z`Jj)XDEq9{qHgBrv)1oVY2EQ#LHi<4(Ao~PwoJf`=5w_ zpvGCKSyi_;Ugnl0^c?A~vmO$4#7MatfZ{KZC>Kz}Bc!Rjfm*%-lx{zsgEz>NLATL- j#xsFx)o(kiYT!Y0&OMFZ={M$uWAZa_Oy&OjFZ=%h=&m{G literal 99408 zcmeHw3zQsJd1lQ+vYw;SNXSn_Ss1%VjB5J*5FTap7z@!FNtQ+|M(C7!x~r$V)zjVX z$7n{ziR@q}a0<*qv*B>Ei{JnbCIM^$F(FxeAiD?7h6O@$4tv7cC2YtpSps6SnY1EYFv>Nf z-e{YfkJYRBV%a#JsmvG?b4I1ves6pDvG$|LzB*^j6|+X$>_2&8;y8jzX4YsIddziG z^-Q){nMoT9cJ2+iVxv~hEa)tkxyqQWH5X9mtg+B&51V~;quD-U_BN;6fq*$OTByz% zqYr1!XUe15YR(uHU2|L*l}5F$)fU>5&kmSt^mFx04V`YA;d8}iLCw}$dX0BJVD~4G zuV^~!g02sbs5_|VJBD|+n|Pl4FVm>eXpT;2vS*D-ZnTLTJ=Vbl}YnW@h-M%0}<&z{RQc=um(+3q)5#U|^2 z`g~!#xq%uC)Qv`~+~h6Zlb$Nvy_|Lm_grfY=DnBgDITw4u<4V&?J} zbbF7v26fR}!uN6?8+ZQ1c>nbL$9Y3EoGg*z_+SLnMH>Ag-aU=}CSKO6E5P$?n|u`- zxPP{qYn6>X$RuWpMm=Den(fxK+1qHg&0BItwm6$9Qe|Po_dz zBpz!&(Z=W6YgF2WJ$5Rx1hsIGr?#dGef%>otS;PASW{SM-EA~Cc1-hgjHhwclT1Tl#F_R`d0a@1~=rKxRbE&sWCEnc3-FM%8z#Cy&;&MzL}>fLS`0 zW?$l=O1>Hx9v(gwJWU@PX82@p;m&1zH<$v;|7LE~6u$P4ps>q-b{B$^xAzo6S3F+} z(eBf(Xwi_Cj3n4JEbQw{O2)NRD8iC1GY4-Gi@ltEd5u;U??QbY=FT-Q z!7Fn1VTsdX$&lad3kMTgDB_)cX8+WoBM-F;Ys}4WEE?ymZCyPG#8GIwCU+zi*Fx3| zc1`YRFs>ycDevT_nm1Kiv(rXhZ59l*RjoDL^+rQ+Eg6dXbvqS|YOxsqo;p4ns`oA< zQ!g*5&3dNN$XDyL*pmo1sla*O&Zk0AEfVz`n`9^w*5War-GLg}m@if`<>GlmCB)+D zI(i+{LaDgj^-W?&N`~=NiLn2OY^WOmj9EjSZspy>kqn1pS~SUD6&q%|&S)sCCHO#j zMq@2|fBStp8I4A@V9?RY#ceYgjbUSq@FaJQj*W2(uyWRDG}Mr~ua(c^T~?ObdID;^mLx{WEa!bicV)w~=kh&RF8tIR_s@lk0oTp+>e~5c<#w zSL<^bq6>~s7iVo1BMYqF&}TC>LOf!g>CGy`m4U@wdgrNm4G+f}o>z+%mEob0 z;n$WM&C|o8#rG40NaHD{j_=(+F*z}{SG^x2iq7hEb71e(UTwx`y6~@RA=iAzQb_1k z)O%Ntc}v}3leBHV5o@L{o-44^g6Sf{e$c>9JcU&XQzI&(VO~j%*m+ZjZMuT?TYopV6$o zTHsNlpB0FHB8iw54)HawbA!M_6V`&Mr2ld_)M!mNn#E?T$yP+HFV>=HuY9pkL`(T? z5y^NsrKMv23$0@by9G-vuh@a=)2}pNp@X(AFVHtm-!XKTFQbruFpLliD=dH7EF_(UYm8HsZ=7A(qg{UQz{utYO$afzXGHJx)Km9 zg86{G@?0Hk8u#*%N+u&()MsIe2gAvrh9~h~nAT%EMxA_X?LfmQh^G&S`HI#7)8fHM zFpPE3zslRNUp88^fKY@g02P2LW{hHfp>r^(l8Ba!J7AzdZAc0QwMfc;$ukGode*&K zpv#2CDCu}<@nAHZ)RKuVYxD-f72Wz8PA2idd=TB0MlinJ24j0;VRpJ&E@od_2M)^` zdbV0FXKD@OwK)UJBOem63B-f3L|BXXK=^nt83Fqz*##MIEEi{sO~Tst&SDQH$aRdf z&(?xk0i2^&6ok9rW_p2dXL{U|!D79eYqev=Ll6V{?Be~Qk53AQw=wJIparpoX!L;Vh}pBY;Q zO=DDMiWOs20BnYVhs;e`tQrKSEdrx$MlXjacKCE;$1rE!m;Ml9jC9M~TCScm>a33h zRLW>I1C$-*7eLUiCIjyA3*-UBo5gB{6C_~0f^dQ?V^2HXXAXeUsX-c&0}yIv3vF}l z{j(+G-tbonFP* zbNM2=zu4eD5Lzr_ZI08MGs$}Y^^dkvKc5c zw#_Mvwy|2%(HZSrrhHc92oM2@it;MOa^lpP7p!h)Z+UiWpkL@Wx6-2v84rIW*wYxq zB2sDIpA_UX<(=jZdY5Gb)6X>}Z0fIVS`&y7vZ%A;`2scxj5j9cu)wY2o)#!R`9NDk ze1^$MERx39RJ~=0Y4G9x!Z*x2EfUW~2s&xlO@y8IimFFc(Bh!S%MF^>OFqqFe5h~Se@^Q7x^-H;<9^UXC1U3E^e|!0I?QytA&8rekBBMT!oNHa+8LNHBBn91K zrmPA%jH|QgYeI{A-x=x!_nmm`{`d=;ltr*0A=Tn?D0hER7%~jsp-qG#Fp4F*)VncX z&On>olA5`D8$;1}%rnENx3q->gZ^9;a}ZJhaNr>Zu*6ZkVlHx_((y2`vtw&=q>$jL zOJ~;@^R-MR2VEG4=#Z*QLL`!MZgqBjNhp+Ax?k>{&fJ{I<>F5 zY$Tq9Pb}C)BD=8;2s=mWR3wG-bQfZ`lvC1a*P2QtK(BO>M6Rd)&eU4D;;efDB%!yh z#X}xy1U*fLAovV*dsSO@?o|0OWO5BPKn4`UuKuEeq?WLh20OpoB;;ILJlw6nu=+5v z5V?Vg=B_i6f^0O&htE}Klz4ZcE^}ZTliH}@i$m6%Gv?J?bD<{W(~IMnj6zgB*Z7RXG(0LnQPVDIeG26V{trl+`n#+S5ZsG?5TSID)(T$R~uL&AhULlNHQG9 zQ2Kpr5Vi-kV64lC3@|;qEX+-?>_VZx^VI1)<6u+(=G$8KPICA2jm&*@OFh(c)qJ1~|VCMbFlOu?3sW`hA?L$g-DHdRkIJWJy=TxU^FAFu#krMqey*k zJ3YXRHU<_G*(cIAR~?wbr$Pna@FIR`#q6J}7IW<*>7FA^n0XABeya}*%yqVT1oB$8 z9Ba&eGDc~e>oW7jhR(maVRKUs?^16y44o~bjrQBk4WuqXf*zg$kHz#j@2}NocB|H$ zI#DySBrde)K_JtB4(|5@A3b<{?<6$qxP}}FQFq|ZvWtO`b%dgJv*vmC0*Hh~jK&*=Xqe=# z!y`m!bP+VXrQh68&YWL>t_hQPx0wbiL2|4|^@UTR(;iv&xB8bQo8Zs+UUL;&^V{YG z6OwZwbA5&+nHZ|TaW<^fQ%pig^kJzbMB6MY8%}lwc3XqX>UqsyGMN&N;4ihI0AEC{ z;$Ev&Yk}af#%i^gk3*Ap^+;B=WL0al8ghmtO!KbU)Cn7B954!W1*fh$5D_*SsDmYJ zk!8=Kt?jXDvC{0cwJoeR`}R&AAVRO-m^ZT67NDiyPAAMhh_abA?i$#IW}us%!ScLD zCRsE!9@D*1~drx^1p5R;gk2!Ey>KiM0x0%@gB^B( zg>TzT<7>ABlrMx+n?6O3>m(z=(O1jczU10z*Jl|=dJy|G_dL3J0$?{=R86=BlGqntC2#5vf z3+2XEtp+X0(L(lNX1ley*SrPrskTsrJ^oWna)u*NRk9usWwR<~{jid>77quzSyX(ICu!b8OZx*p(qvl5eJEQwu@|pBZW+f^8CvGqk{0_){SB;0g%)7R30cXt_U)ZgoUXTakPh+s_#|IX>^O zwJ@9d0O}@CL4aY52Tmr+1-a^95zk4k`o7Mng$FlBrF8PSY!mn9^ z|JHPAj8<#%Q$aM9#jJJ{Ri*C%!4%e%o+KdChxMzy@63>*{KmxL2YP#Y#!&_^qHBx;y`+jt;C@gO2}mZKu$>v;b0othU ztDn6{SWbJQT`?}lzu(NU@myvIN3L^&}eqEE{SsV(|TQipv)GQgvs2*o?Q1#uUV zrXmD=eel>xMfuH7f|M*YGN3)keO}{V{%2+A6Iy1b2IDBA^OTQjWTEr!&ucUFYOAI< zj2Tit(euT55NSj!DxYIzk!vv#B-~7;|FfQWmi^-zG+Sp{*xi5cXw9hXKdkP5b}>qy z0}%)5?R$Ea=iyduUMKwt{am$v)@mLN_I#ANUir|(7qnWvILFKuy}kN`)>wRq)z9*q z1}p~4)^k0raryH{CbYA;=QT2LgG(OP6wEi-qf%R+(I9k)N%F=w@tgKVScVJBu_8rhQ{oR4T#}ws) z(!XY$?*IO?y~>k+^qX&u{O<9eQ~mTu^;4SPk9BxLp+Qx2y-=*dJ!3Jar!u*4a5|U| z8L?nCAM`7E4RM>7D*Gus(97xf*cb7NXOl(o z*7G6C-@8T;xSkcqf4_l0KV0(|;{9;VlN9fRYZMJmnGHn)cmvi`w3$jvD`Xl4#rmJ( zn{`F2S4mi}K<8k-$#N6aL$pz@SMkDgIlOK~n{G4|jl*V*&>zrgake#!NK9G6#0nwG zW@Z{;0rm1L9s&I5W_N|yw>`6VP71g`l#!=H26ERxtIL&S}2o!nj^ z4i7}=l+}qcgio#5x{+-$Fs&?&$Mgda9oaj1Xg@~s+J7`JiPjL{)lH`ex=LLNrBF05W zk%#?U9f%>4%F;8?Pkf((4j1&V*eAsJS!A@b2W2Qul>aneU=%@*9}@+sa{OG)E0c)p zC@<)DYa?>!ci$iPsgv3f?bEgM=Po~kM2d5hDWZJJRY2T|kj7{6-8mY%WNoxbD3{JS}gvTHd$!NLt^2^vDB;5AC12s>b`5YCP&` zeAB_>`zG~+Cl7T#bU#J=zUCg^*Sw0goE|T+x4ac|i`9GC*>%g*{{8yv4~a+Z!)<4a zgM4E7wm8mPoK*UGlQ4Flq53Y{;u^{^dFaFmYjm%$$+gz-c5jjuS zR|p@0{K*lJBOpiMS78J``8mGeC0Fxs)gB&BzL|#?C$01%4&Fj5RA84i~VDeJmB}&NPQ9{QYS7@eWHE=qv~55GiWTK1HgYIY$<*r$^Ko zaBMPVCX^a$w!n=IIAX{`p1i_1lkEtmfNkG$U*zNsog2wB4eY0(&KhTyL0q>Ry`hF} zH{2ueV_s~I(SPJb{lLV&lLys3V`|v(RDa1;g+oS>nE(hw-Q}Gv1-n{lsSfRT=~VSP zTaCnUgdlmUayY87>`5qve6$D$VO-!bXo*TrJUBJwD=R7e4M0X(;k;j^bSs2!`_>?8 z^97PWDaxf$k1o!!!( z_yyNk=uw{d18<&B_wsb!AJHp29Pd-kbW5XNUmAVf-1lXZE|tg4{a@bx{1fk?XW8*x z-WH{9yL73jvK-#$r*tG;pt6+3-PU_}I;)$*HM?(=pVBFw%2VhrKR6)9SEO}@mqwo& z;Pv(>7k}y+_+ylpFi`5 zy=lCDd;WYr`C%_D#hXYnfu;e_LllyRyV2POuO0abO!BQ#;cY_YUj5b&zhj!L+ zTTsM!5Q~weh_RX}uXNy|o(^2ZHd3)}|El4l?JEQqsS+10;%V}1}vvODR$tzADanhwh2W+sciAvuO_6D;x}8ocT>b~jLme-kVuh`NJ65jE+LVGL^sB} zZ48hoi9_)mdP>3rVk#64iDR#p07U&95Cs5*ppOJjldX?*ojf{VKZR6wb;`%~DTgV0mLRX0TVaL)Pzjx~j`Q@_mbkzrP>kGN-5P-$6 zed>pk2lV8;_Zh#@!}W!pV){bfT2xDah^KhdQNKr@=wpxac9x@06kIR#iQG@d@5TFz z|Jd7i>-iS-iQe*aZyqP0IlJNKo;e`u|KV-kG+w{Z$Ldjz2fcYj`5(+4@1k#Gm#_Zp zgeQ+E|IICiCylqy=jUI={nOjm2Qa98QddXvP$XcGj*(w#$H=zyDIAO8L}kV*>F|vV zza^ur917@WW^Q}D9712ZtM1!B#-S$qaBX0YjD!Dr}3gM0Z5c3H8T1}0hDa)| zMSYbuqTz@Z;s^3AX%loy2WDVugEt#9KsMpxGuS)!O27l&f^v=e3VQ{RO{t{(c|%qBo7#Z_6IGSW30dzKZ_+ z&UknKBf=);3*Y=Tk4}Op|AQYN^Q7_idH?)W{W+A*Wy)~VDxI_LJbRALRp;)}wx!F3Tg~+u965=j)6W}H zAAyg{t`(?{5D!MQRM<}+As$X?L5>!dlr605zy$1ELqTV)986&ED*_X6Jm5tIUQA$@ z6}fMuK7zyq-tm{1Ap5X5nmHi0e~AgC!}dlMe(--eGyl&$lYsz)srbN7S9z?{n5}KQ#H=Kw*n{JFU-Atc6IlKc zHV_Y9V1nPcxU`Q$?P+fwQU13Fe{@-Wf-|T6(^Zc?0riSg`JH?M5o?ETK7k)U!ObVI zOY_`r{|N3ngbt$JtIisn{t5>b-`!vHrco)UzQKjR@#b;z0e1bwFP+Qt_x_VNjn{AM8{D?-$DTPP z`uDxe16}k5?DEg7eZ41-DF3!~4|&pf`@DZ2Nc*QtTSsZ@C=Y&kTJV($NJB?y==c&DIws?BE#~VSA(@J3;bkf#40K?Ift7$66c1*w$$Wts zI(KP%_x5KpbmsOl8NBIq3S8%*t#+Q^6&y|`nG9Y=f*|rGlR-L1kcN)Z&`}yX-Y`SQ zR0M~ErEDgHEwV_(l3J3pAC^>L+|+>?*h%tYO5h2WA*r$(Vs?cw4c-N!e4c`Mq1Q!7rDU z$ECPn^9kJ66Ye@BX0U}6lFuN86jDf`?io|1kU|P6I)xM~Pc{(_LFF;#XK@)ul9pJe zV#AgW%&>*_UctM)S$qb_CR|<^?EQQtU(Ga-PRQ`1y?^cz#q)HV_59rH zy?F%W_%|2-qDy%$F248w`KP~}4t@BuJHF)^gJWyBlZFotk(CDhp1KQcTKqUPf@s6z zgkgLKuV*>x3MhZ?3sfDPF#Peo-uA`&8G5rfjY>Jy6+Tt<=5eYkJpSw6^fLqWer=dX zbvo~zIU}wduVRNlSh>QaDL2_Mm-Vfvw!5Dex1}6M7>%BCis)T^0qJD z&(^iy=<#oy?LDP|82Xx=~sdY4!y}+Ui7cGeqWb8-Npo`UNh{;Bg%i|6}vrY z{QYoDTFv{XOH3eD7^MoMRAH?D@~JRJl5pefiwUA391+E>DVM|q+d448Hs_jov-kvF zOknT3LbSlCP%kZNsTy3c4}!AGirj8L3nE^GDpPuJdRm}S5vha9m0$wNC$PtjMv=zR zi76zXKpq687AtB!b50)mAP;@$E!=Hxw~ngdriA_nF5Hto?`wv5PkQQx)v$6z2aDbk z!qFJ~C^(!oI#Ln|Ex1g7jkk7Sf?H|T7IfCl!UQj;Lmwn2keEOo`d~B{Pt}o!J`|)s zM(L07rSZoYj)me{(pR4#7KL)+GW7{=>%auJIWfUo|NPD_yjPod`|)pi^9W4w@R-*$ z-C6(B@A6N7Iq4Jpr{DKJ;|rEP!Bdt#!Qi4k0gVoGui()q`08hPJy$9_IQP7_eer&x zKl7&X_hZWrj7_ih%mF8z|B>z9^ed4a{PTpjyy#!-d*?lE^ZwcMZ|nQt>d7O@Km5Dz z^rZ3fTz2qp|K2}c$_}LLK*|oJ?4bV3B0I2E3}Z=12z=ZvhU3AM7UnjNONtI|?|=li zJD1Fx1qr;O1Dn(oq66xu9ZPro-YYur=Cbl!Crd_eKb+nVBqZ>TzY|hONFYTAQgl!@ z0z0RSiWD74(Lr9r@zhiB%-9HUr;OVsU)57cn44OaVv{^JYO|SQg{3vyu!wqHVUM|Z zno~F)^>u9!PeiroGS6(dqXQ7!K^v=}u~rTsus2-+2>3SR-E6#oz%DCt-$-2o2?)ff z+o1#m*@wYCujY%`2p9t~XUvVL54Vh#p*I(5#+ZoI3g>h(0@SUv5jAHtvSU;AOtx5= zNgE4+{EkN!GozMb)_7FiwMU&gbmXBOBPyh_V<^$oD$@&COyG>jsAQ(gMsDob@uLqM zIy^DbZnguZMg7<#yJ6+7wY*B9c zFTHt0`9F*N*DmF`D0lxm{nOQrkH2f*mreA93OiR z{ix3~_Hx$*T$P}+EY+!6ss%j7n~wTDss`_RhPSgERfF%Y7p@EZpFq67FaOA=?`(e9 zss@Mq{?_x3oqK+Di(fi_|9^X{H;vCfdw%`J9&aAezayV{T^CgWyL|6kyzU1?`R9s9 zx|HYr`>jv@bX_1_7f9CyH_~-MG8)q&zNa%K6G1J(eIqWZYw*er%E+wsqWTBuuJpY?t>QGbx6n{*$mQY zveZ^+st}wUnAmsnpmdrnohGkfr^&WxB9(wegReKFR4Szfmub@Qst(NXs+E8lAe!*7 z8N|N65-@`lO-RfjF@wYm5;KtgYhhcuT)0(YhF=EE5D7*STFmbd=m;GT9a~g!{59w_ z4?otX!)RBRzSpNHrTvJ3^~B(=P%;%w#iLQUV#KixT$oWhh*V@YH)mUV&8X{*;%rW5 z2f4P>Yt41+Hd}2~n(fK)UUNOWr+nE~JB@pm6PnVLA+4HQXEXD9qg-t^^hUOxX=V#7 zqiPOO!nw?x(O7JyonB*Zvzy}yMrEd0G1}=KbD*3+w>%60 diff --git a/tinygrad_repo/extra/sqtt/examples/gfx950/profile_empty_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx950/profile_empty_run_1.pkl index da08cfe5bc706e52a875f1e6a83c08c755a8f902..e2adabb44795927e6dd43d63378438cfd3012718 100644 GIT binary patch delta 1908 zcmai!Z%7ki9LMjuJNHL(+c|Ud?=%JfK+FD6AVki&>N;<_-IOqFxq14ma>6nxK~h?z zaE5YMg9s!d>{Wtrl`cPueH;vSKwVo_OBk-u*FY_W~;h~1jP#)rpcO+h{_eboP; zO#OB3j1wCb_J*8@YXYSd-HKv)L<1*mQG>%v9_(nH(`Mk+q?Ag5Fz=T_Qh31A)34!b zg_Z&ut`$tUPC(ctu(%%X4R9V2DTEudmnPK{anRVI4n>^J1PPs&*!dIM?MOj!4vP15 zOJTnIa@gYy@dG|m0A}Cqcr+J4jJikx*nNp7VmF-11yDaktL{N=_Cl~rEL=E(-JM zJJU|K1`7g$v55Z>2#BTo7PbWk<`0Bu%y(U6DTZhfc-pmVEk{n`eo-NNM38q~E+a((SNty|g%uB#bBX%j$S-dgnY znF0F#jsO*7etna`lz@?aivS%0sCnD1AE^|e!EXe}0??x$guYtyeYL9X+sSnAPeNmD znBVw~2)10_mP<`==vijVj284KR)+b^C7P$hqVH?sQ= Dq8~V0 literal 99294 zcmeHwdz2hkeP+!=(&#lBJ&-M1m_qo`BadqO{ZKs0NTZPvqmkv&h=Vaoxu?5&x?4Tn z?S9Nd#v9qhghzo{Xm~b?er>-|u-Z^uBka(xuSvY%5?6Gz#UV*;+cQ<&9F+ zsMTBM_EWV=u23?j(&brWeBLNGTJLEMKGu2^+1F={`9j8MnZ0LDk53^er{|1TzRTP+ zT}x*Q<=F{i$Q&zPfZd-! zuB_>-3%Wiyr0%1h?;AYWYT$Y9xkRHzy)itK&YUyK+2ICm^h~qfsLbi-j9S?!Q70yf z<|dY6J=o5)I-*YVKC-AlXpl8?`*5>f8!i=QhO0}Be5D-GLgC?NxzM1}!-^R;Ybp?2y-IK3C z1NY2Tvdxll2${r8(WnP3Q=`?KF}v%HmboEoWD0ZX60VL~SMO<=1BqZT5sHOk(Qr7W zr9!FJ6D@qM-A1{UKjcV_Mj~1y8Dgo;nS2lb%=7E>8}fbmP1fC3b8FkwZ{@cQ<_AvZ zw+90G9rzIQJFoQn%kQ2zm%qt;wd1>KYblT(Qge&tky3hYCYx6EgX)?4s#?8JJ{Q0& z9hqQX;_-5>5*QpDd@y*HJ~qto$?p8EtN3m(1y=vfT(2p7^&dh0fdA~y2PgM*Y^mweG1mOobCZ^~S=Xpq5B1Y)J&0RKPq>=VRe;P>Y6r8;J&^TFP&8pw4g270T&S;k=;|U~zRl7R7U? zVt$`{ESijIp*Y`=+O{03bxX|vV9XfmOf%;mk61Jn(ZUhmO5@3(7K$XjYY0`kk-fn& zpKc~Fj4^8%+b2dW5sGWrJ$-9*?2J1BmNQ1Zu7=bj&0G#|vmCxR^N zqc*4-rMjW!_C1m-KdNFnsE@Q4estfEdQKxq&e6w?$<1Bnj(R3<5d6>ySnKm?A`GTx z3bQ(GzC-GakpWn*>vQQUK^`&F^hSjN%fRwBz5l^Q4G+f}UQ`QZl>wrm!MjWK#@Rv9 z;`<0gOyDV|rw)&ePmWI?R`0=xqO&^P96daJSerE(E&!}r&^6b#77~0F_3rI5H`EL^ zOG5_?lz^Q!r!}ju7J8KMXAQ!i zcp{}m!ZE+4JP}G_mFG#G^>9bMIa6;G8qEe<60yE`R>cG@eR0PEzP#WP?W7(v|Fz%?o8)j1cQil?xCghPG^76>z?C6oR@#oSI%umJ7@_R57C_%zn? zvAjkkLeZEO@#|?K8jk=Pcc^$Xwqw-Ex7Kz%oQNh8S}^Q~(-H*!q81YOJpIksFzd}Z zU?_qWzzTpBvqmAek3r?0owm*!?Fr9$TJ8h}{F&@+`%DP64_cV`W(kM1E!qM2yS zZ-OK-+ps}&*kHDn3Uh@9!EK+tB^io_FwlP67LpQLEZM16hPbEvUtcm53u}oWCtljW z4kRVDP$!aO2k#}(9E}RFZKVd*a~qo>8A^aeuy_fss-yJb#r$_eb1V4N7zWPTb#JlB zXfTdh>o+lyQF@0#{(3#{kduy_OXW*CCZl0c85S|*?r$WlMXi0^Q}X~mKZ_Bq`;m26 zFrq*eexFE^cz#K1adKDefCW1nm1+fKV?$*SnE`(X=g*9-f~+yhvxTxTEHE~M!8^=t z87v%xr!5+zWkxRrDfanvWZxiX;1~Y@qKt{AxvNxJFlwxigjGstH4UI0VZSV^?W1a`3sV#Stf26XD(Og20;R_ngWZU`DnLwv~_kJW6|Xb=>BqpM?h<_jMX`k zDNx-+Z3W84f?ls>^zy+GD_N9bNf;fGor@p7I@w%jZe=r2WNew!7I|Z}rlT|3Lb`NL zdmL;NSUm*VHC-qq|Z#V=6K7k5V~eu>(+mYYcN%dFQg>dsO83L8Ufy}k54 zQT!9iEBApbV!BGMO4e8JLduYd4aC$Y9I6hhl_4i%AeO*f@L{4RW03@ir4I72WnqR5 z%3O3~EER@IjxWO|8H40H9P7YQ*j`Q7(sOmHxC03tPolV`f8pA^5^)H=Qypl>olq#L zl;#Z!MfyqElCd~QT3q6u5J&Ns!U_&Rv{lAB_C z9kd}X@3KSzu@-czg@D+9B?hivfuKoplvh$#y9PQCYY_v3i>E~rz8E+ej%vY_zwF!b zsqZ>%v0ETmwOCS2R9$>$x1uadtHI0o2vMciU+OH0T*l&%BBK?wQD2%Lcq|1ma5CQE z1CMLrLv+cnbP8k4#T^x zUH}skk}WQa@^qL4_d!cY=m3pWZwkV-NZg;}0i~=`8am~cSgS9NS5g*Q&Y0{7rjRP? z<_!7MQD#4M@47n`(Afc+X{6^? ztCZ9n-lV7ER0`j}hi0dWAp(Z@yTg|}u*_*4uF+7qmxu%u7a)K3+kR3+SBE-%$XnQc z39U^Cb=uS{&?T&`aVSIh?LMglG#7#$#IX*PeG3T|VS_}qGo{DzTOUjX=Wcy4h{m>= zUE{FL5Oz(Z7^%_DYW6_D2eYXFtY?Hd7Sa%ZA5!1bn&@ZN8~w|P>=S93>yA$2Q=x*d z`6Yf6irG6~DP&tGC%R5HT2rmT;_vo=i@C|RlR#d}sH4y9C99N{xhcI^sO$Wj8#K3N z@iO&h-O$-GT5rA8+(KFtBt66yKxSK8DI_T?)24s{*GAL>6BJS|IoX^!$(nw%P=CEq z1jVXwv>9E!5Zh9AVKVGRa}&L1FteCG=RMWh>_OFUv&hGd5JJP4%g(8WE4ef;FakV;plrtZ6U>hL5q z?6{^JNmBRW&N7gJq;-g*4zlKX_X3E7X^h4jhJ2WWuY*HGY;=$|yrI|JQc9m+g31XK zd$*V-D#3EBNVTO0LuWk#?XUN)N=?iI{rLvnGJCV7e(ri?l zSkBM3%=Lw`b=ib1E>Mgx5RAh%Buaa!pLTMafnfeyHsko}O#-D_yPr|O8sthwxkSbX zgeBuqVE&XJo!UNes@HDfNW+IUX7~_~1Y^l4$hN4Lu%M(ExH8L>yt&Orx0a8A9Y9p% zh=3y}@1i(%>I^V&BU{Kp9rlpf6=VdJxeo9fuo+O)wr4gsVFf*S`sn20gF!5Y4#2*d zEEPz{jxDid0WqlzkZH4-S_L))!~*n%hGVl@g{tImKJzfM;o9D9ZUB6$E)`&y|0I*1 zAxS8$alJhNvLW^1rqMJQ`sNCEh}7~6tf+3K!34>uXP`JH94rKK%)Ej+{Ro3WR0w}? z%s|Fm&~F%l4xd(@S;C$MaA=g%Gf-X}0eC!i{KPneeydX76iAesBY+zKrvvO!g+5|N=fcn-OZYbr3(5fXCS9wgm)yBv&wbv2{kHll^}JE5Ko2q_ zxHrInFf#*ziD8E6i<(bEAX+KIxUqzM=~_c07`rOh5qx|YdRNVxb)E2eCS9(S3*Z)^ z16du~@pCN`aFg^lAIw0#j;!S}1hk>NiCbu-!^RHqzCp%lkZZztvmO8^4{x4~y!e&Q z?e+t?yB?VVSN2g7dD9#rN>6`e@zJ~Rq;Fv8`%EuD|E=a`)){J@JOaRVf-U0e)XhMF ztCDimTPuEq$h>>us$vOU5Aa7FQi-Mu9Bn5%EJ$C&v;d4Ivp+l=NIux94XMYVQ^=JQ zErO)@F{=26k|M=VP#4G)?ivyx#ZOVC*PQw&ewwP4G>6MSiS)BTD+d z*s6w?cSvxU{kAKt+&WnBLn>HUv(D8oh|+IlrLk`lbHZ9mtJYo%9u8mG1655}4=^`F z*tJLpzp9~XIMS(V=_d5@t;}RfJyI1#>5gf_Qwu^7pBZc;f^8CvGql84_yYE-vV7iQYhlLq0n|;Pf&jw^51dSt3v$)JES{5G^?kik z$yNV4t@`j(ksv2e4ud*t)u$8)b|XR(UmT$Yn_N~9S!MC7-E>v)2LLhozT%q+%k*Fg zYaMxLhob!U_=#iPU0tK7f&F@v|82RE=9k!9n#%6?kItK5%rVn{@v zRufWN^p~U#`-EhOM?VpYceVj>Pa#c32on3=Q)d+AO`iZYnXjioevm7@`oI1QWyi<0 z^lTMIQbg)0AJWKD=Ut!CW^0vZRj(Vfq=lmA3X7oAh+I@Y&B`Lzaw4d>*>dl5o_Lo1 zKWb2Loo!-+|LA>Hqday(9eZXuN}mTE2LbN8yOrnRVr@|;Eed_1Qafigj|RIw#JsS4 zaQw4cwN{vC=8N85eO#+AKg9ZH_)Q&_gC*;^KB93E^nV!F&Sjt1$j}Y`d0100<7AIY zZGB3E0IsfQmq2?KGURt8bM6Zt*XEb&R-WNE&bpt5jorMFtC9SUo_X~5twC;Jl#Iq_ za5tMNA3#8V82tkWW^;?FSWvtSF$!JQ8^#Os`O$0M(Sx^YeNW;&W3^DW_h4p;@6jqt z^L4wjNfFf|IoB#8h-WYlR)X%AY`lI?O&F{xL zJg(57D!QI8RN<SFMdrg{M3Gn}wf;K)%ta)(;L3<0jvn(ZCfQo=RtpN|~VpP|a|-G+Zjp&mDj; zp$Lwz3O8>D5?V+Lso_{O8daf8nGB~xP;d+yk%$pVWTLr9CKFF*bE$MZ985$axn#}= z4Js#~aADLAl7K-yapI`e4K*-as=&QTO|PaKd15l_bvRkd19{Y{b9LfP>y}gjIWp(c z@T4${bVDcqAG7cvo2%$R4?5TydZjKz3-w{J{@Z%aZZ4y{%LC|t!S~>w{L-Dei$!?j zU(q3+!)kAS4x7F4;~naGphG;HEQ+_D4^sZ_K1JYqRviERHvar@&0~o7!!>WFcpqG& zXmHSMC>p>Uu%4pLmYZ5WUC%4l{}kV-DO#;U@_Geo2a64so1h+|jZ&?G7oN@Hbt~FT zy{>2+Hfx0bfKCf@%{fG3$`U442vIiDGYIpjmtXM^c*p@&Icy%+;B!(ZAc<7EB+$(- zAlhoX=#C#4Z^ku4iiG{sM7$AVR0CPLv^hYQ@%!Op}3WWo103AAR8D;mPA; z7|E;u(Yz#DLx5K|og(NG(UN=Fpt7_kiSlBd65&HuL0YG%EkTEg^=ljA{_&T;tcdbr zof2WdDlgJSc`I7kgA1x#q>FV;gzoa9|01OCjFwhWUaX@c6zeFmupd2x^LH0+C{C0Y zI9`OK&dj)W-wz{>%8PYg&<7%7RCE-1*w5907$T`GJp=v3_bKRbLH~+>S0Fz*c_uHfJQRMY|&Uaa{P7 zf7UAhF_F;<-Q`JF3F6Y$+tw=&;Xzchr;(m*O(;$&zLPb@)S?BS3$Ace(3W?|JxJ9W z5|ISV5@0#UHB{g-{pMvQ1B)Hcq_Uug;bmR0#U{smSz>f-L9L7)nd_K7AylUV#qVNP zn;TC}j-5`ER~e^zZU5=}^&@9aKcJ62Fm_`6G&B`vAb&UqC+w&1hYHtXCexaLMi+T) zZB2B+nY+=N=tYpGyPk#FDhn&nZ6Li~qUc+bN|(8@X3W-kW9ANAqm@(RQ^&^cJ9Bb+ zd@2aP@WmgZQ_EVuTan@T$bvvH*#L?6G@uRQP506bBf9?1cJ#WuS zEk`^p@3LAxa`@zgK6c;9V<(P}OhL_mZ<~!}Qpg{_5l6QG0OP-r@kCSiUWe@)jqRUfv{3-e;-4OSafY zIVO*vK5dQeWj49d8s5%L(quNU7?mpEd<*WbbMacjM<9Q41mpFC{={+^p);KIT5auU#80ukRx#8M?g?~ z;=2{0h!>%|r|x`$!{EQVcqvm+U#I#-ya+{p5sG*b3MyBGBA+`H`Q7~&<=pY^^5S09 zBktXysK*^I?nQYKy4x4|RtiOVkx$&aLwCBk?-YviE2WF)6z^T!i*lkK5sG+s=uThh zUbG|fxyMh$yW0`>B3*={ya+`;cPR3^$V5%=y;jE_5B z-pdhqDU3kNgr5>4qceVFmy;iP3EgQD_Q5`fA0p2CO}EOt_%M(+c#)*)-3gCpF}pP#7@RE#w%8x9$;%zZ2+ z>Gm{-G5kGf`0+Md7-%gA=r}2A&OT47mR=wm*RwA{oCB^RpWK1ZW_p20dg6!=`AELHjAn}u;TpU&&BB1`% zM?e1lao`6b#1rAg;U|7U0$Iv)?rCoxQT{c_XFHVV^_#g)=}-KED=c&=kGwPo=Z`l+}neDQ47l7JkP|DPu`-k_%7xB-{SQ=`Fe`@!0n3iG)oa>oFU5J`2tJr zQj*{EY2TVZz5nD*qhctp+`%NAUCKYM|CDErIOo^*_V}gq`SXL@y=lCDd;W|Z^yazx z`D52Vcg$P=h0Xl^x!)@rGQ4f^ecUy+>ECtwzQp{0)1PhjS zp6RZ%;i9fKT*Nj~v2Opm;i7$O1Q)3i7cJv%iHmLkE-IeEk|8mX(@vk5OcE18*N}0c zBm*gsmyW*HjH551W}6JcrLpF4^W}g<$*2|)&g)lzMBN+`y_&O%m|*GG6%xI2%|W7e z7L#Dtywq7m4=H1hjam#e@yyQf;_$J@h_&~w&0aQ*U=;~gri4^C3R1oGYS~PqlFg({ zNH&vXGu>u#?^f(M<7tSICncMSv2(9In<*KlLm522+#sz>f&sOBz`+Wbs;7<3#H2L8 zu9)hMHG-*Z@!PK_rjp_}TfBEe#BYqvbk&$hk(fwgqKYmtk;FvT$G7QQR`uID!%8*bPvax|s7ovXHv2@4p-ReSqxvV@LRf60qL+&~RX7OvE`r+gQ1^>PG z8Nb%WRfe8qDns5{R7-`3r+Cv*zelC$o5tH0Dp_61)a$)@#Qgfy`|jzWa$}eO$-YCLJfeK!Z%%sB zoX`JY|9!sct!slB)E=p>Bl##2Gf3~q*ULLHFayq@bso~Yj2S{fQr8iu-mbt$S=WXc zAf~tXjNbq5#RKj&{fR&E<`HZ11F!e)m(Kcs`d0t+hqltLPHM($6Eg^Pn$CN!t!BL8 zgWhL+F)+g&AHMdO;pyGp{hGf&8#8=+pEr*aGyHADFP&qC-y8L&x#!p5>%4iy{CeBJ zSs62YS?!D&`hRq3%n;rpF#|iSfgL*`ReWpuJWdjl_E75n{pS|QlR4K&-aR2tHeA~$ z8#<6dB!y$PePk~o90IK+SBV+cw_yh6xInDQH;T{T#SAv3FZc|dXmBr|!7eNIc(-Yi zh!>#>tkQ0rL_3*C9Z)U@Ge|x|=RIHY8J16~l4e1*vL&>aHq_EANa}06IQkkPKm#qw z&su6X3kroIQ7ybm9oP+Rn1QJd-e}AK*@TPFVDH##0W(P1gw1D=vWYA9&?03Mu*8-+ zo>HGd>N8v~eTGmd0kaujJ_Gq(NeajHE667LIA-YM`V7qRg}@9wFZ^)_dA2PtKJwqZ zc|`gDd*JIG%5&cIH|PD+PxZfa^cnV;-e-J4xk`P8ey-2J9i$NdgKIG$WUf(y8&{{V zu<#>q-?(hR#suqr;mzZeJ)GM14bM9k@Bd3Xy=lCDd;WYi>dhm{o6Wm@^$DEq|5on- zPaaYJeSb9KN#pJF`FCooS8hS+*GOMMk@^Y}6G$D$ZmF+u-Sibg;b>S`%)0mp;Y36W zMTIZ6OS>1@*oFx?)W{WA3^FP>?bXRA*qjG-sW^Ej5FZYwm!lZ>lC|WC=!bJDI-K82`$ClwytRO zu&E6YuwxAck+pUJfxW2+K)}I(7ZP{@fn8SQzMjem5)gRDUjl;6!{TJ-fY|#bAdn8* z>lOIH&*Y74vt;Px{vR5XwR&KGqoUW0bP0!j!pCd2o$oN>y5~xT-5tg8yQ{n@5!I`sSZ|ehtNN?d(~cp(>KE2QSURVy;rGA;Nu(N z_Ir6dtMQG1{@wWkbsl~!rzX7p6YuZu-{noCQYgQC+{*nQ^5$`}0si?9{L;BBf8f7( z(|G;%{Ce-Vy?I3c{(4)agRX#G{?DF^dh&?!Q<-s38gHNXZ_|hT)31@Pfz&^g`iH(W zzO-48HVZG3%|a}cY|}Nc84R&#R9huv*xCjewmO&08wD8#eAozLQ(g;@L8uzgSaCSf z9!khyZ`l$uJlYE6x0;*1hssLGAR&W<4CkeKV(I#+C&rQyE$DYtY$zT~i9=*pWHa=) zVTS&-fEg4IX0Qo;ff?F&X?yqfXEU_t_Ocng>2wTS`>CyVp5SGiPA1t5o!JDE%^+PP zNHa%i<|xe^ubY`;JetsAzM_dZOd9Otvrw%8P*nNkbDLynoyLh zKI%d886=-U@)=&Le1`Y_*EbO*t0;l|fi-iqsO^g}5ejJ`?g_!A>zIfnwQzWK_K*W@ z;DPPZ0zB**{J5{)fm3e%_F-=xQU0%*&vYoy<;HLMihuf39=(RO1s;T6gWW5dB>av- zr;y^N?|YwdwC&7?_Of&rw9b5R)=BZ=9`$?9e0c7Myq(qPH9WOxbv_ghXwPQlS zAM@sM&X0dN?M)xpsyx(b?C2?-C6(Pee4jHTyA9gt*L+XxCcaUb=oM&ywn@2#7!xui+ zp*)v-?EI#GdhjKr&G7p_^*-aNKBt!kVq!eGw}%Fv{g^g{M_b{pf8zD5Mq2@rkvm^t zb#*Bp_=UH9@qYd|`G+0ej|~YbC%t)`+6KqYdea~3r}t~aJk}M*y|et=f7e@H^l$VX zyL`0?oaKLdWVa`eD1Y+XcY4yOFCzUdcirWievPyh!~0+Fea0(+1iNQ=J(mj!-kI~ZFW%2* zpYo>h_iICf-}Um&2tHsB@5wDFsh~ak)ha)j9 z7V|k9G8~SFv?NbiQJvuCHcW7{6BC@jb4v%_t1UbD`2lYp!Mi>4MlWC2S^wi-_fJ37 zL7iZ2VgfHG!fELEsh@bC@r5f;C-~6Mcs-XZJGlQ}yzPtkb1=EH!~3;m2k)Kq=5f;b z-+F^L{c>anKYx?Ayy#!J?_7ue+4Jw+Uo3g@i1Htvd(@NWoPY26q<^}U9Z1=MlpWyM zXij@etAt+}tAubkh2x(5&WjF5!f`FQN~?rDZJ1z>bIrU_m|%2`^9kDZ2_z;M?`|Ih zi3zq1<_9LFilJ07lq!Z&#Snsx>*d@a9EnG?gr5iIa3l#!#Z}4zDc3F|ztz#(SMW`}OM(4Q=G%6x>K#`chrqd;#prgYhi3ubokeEPX zg6oM1qTys4pI}QO7)%9YiDWny;;M%$3Jz{<0|d9O1wa77flHad-j3G-AZS-6u=^$; zgA^RB-VRX;4y1yiR4^;qT-zB6*0lyHcYVBi3$FB%OCj~3Oa?!f4S3} zM+kGD$$hOud2R^y=b!UWSGGR>&Ldwm(G@Z$yzTiX-Ys_9wc!HsaxlT?zv1(Y-A>;G z_OsF06T(m}Uj;nHn_k_uTnc&AU zZyx9S-*e2HemR1J!K$~sm_LWU9q6D8U^6NH{)zpbJfi$`>^D4VG+-kAy>IkSm&ycE znLsKNNM(XrS!H^Fze0wHah$%C^u+}6NJ5KquLUb&f|s{pf|onj%o~LXE^cx23G97Q z@Cm4&b}ZfTdrHUUvU38zTvncIWQgc(!O16(n85p%oR~sl0?8+kd;-ZQAYX~MOg=05 z1Xsr=h$muN-0yUya3UDeV*ISe6*0kWZJ6LT+FJ!3wRSLpz3&Q4z;_z&e&fXic3F}8 zdMXo0Odv+x4kaeYJPdohO0IytfKeh@V}3||xM?&Ey|Gj^Mnt5RU(hp^>XL4y4XIh9 zo*9|0r89-{?1Zrt$nAS%IWuZ0WQ<4E1BcY<<0l{3H>B2##t2F@wDQan_>MT+$S9|0 zN=A0%)YN^)j-MDGB9}D9sb1JhHfTLQwp^J!aZx%fl12yYbNChF6VO@A3BOaC!-+Ut z8LiUj;1z9{;1z2D6F_$GlF=uSn83yYG`9$$NlYLy!J?+aL6Mrn0Yr)f(jUXtVPPv>x)0=QIz5sVxXEZa3GXSMN{!8 z>=vWJkX531FH(`&+@5LbRimcY3v*eWout~D*l2EIx0y<_+-OaXcAJ~oJ>|!nJguGcfQbR&~z8C7!!B`l=pjrwvct%*K!x7{31Fv_!qveBC8 zGW$zeR8lx^=!LR=Zl*Qyl)0hYoYNOG87jGBHn*rZ(zRKmq0bw&dZAKoz2$WILbH>pAKb-CQ)&Kwi diff --git a/tinygrad_repo/extra/sqtt/examples/gfx950/profile_gemm_run_0.pkl b/tinygrad_repo/extra/sqtt/examples/gfx950/profile_gemm_run_0.pkl index a34275a6a29de7249805002365ef13f7cc2f2815..4ebbc8f81a378c12f378c2efd47cbfa0ed9b36b0 100644 GIT binary patch delta 1943 zcmai#Ur19?9LMk3?%bXKHr;Ce-%2qB(Nw4&N}O}* zS0-Swp+LCDKNuO10^v68*7dAXO5kdLw5d5qBbMb$f5PD=7Cy_^h>A{k!a8pzL8-aIMgZ>r@vsW`wg^JNs{IR>|*|`*=xj z$PDj~oC(1J6V8@)f;7mmV0D63Pvw?!`sE-mivL-UOKmx%nSdu_h)yYD zNRk5cN^DL`YezdI4_mQQV4zF}^Ie>iVM>AhSE^EUm(VfZ$gQ9>WplGdP*D zHQ>;4PRIn#KAYh*8P3}ecn%Wxawg1>z=sL5k?7rsavoBGg+f}M$HMY(g8vV^;ykZkaO{!z zFaYY|#4#u~h?Bv_N`z{=?Rw%qrb!=Z9-U0n&P)TE`C+^HO|zN0t7#{hZg#di+v#qz zo=MwHI@4tT-#PcgeP7U)70M#;lX&hq=lj0%oyR@*d!2i013&lI13ra*k2eF2p=x3A z!dy8!6U^u4OSy8T+1NK(UYae;=N`!}&gG7s%`H}&mz#%Hnok4VKa)FKn9emD1CLG} zdj!d1b|Kfy`x=8M%h~C|;@m{;f)2fFrcf!(XD?&~RAXCip;Wy9q$hG0D$PTU{&KF` zJkjW@o&n|I!}+C!+~Fs)Pi5y1PcO~n4l`$li`-&msT?d_XdeIkp2m*MxpKAyo|}!x zxk5FsPM4N5CCPh0XD2Yb7|aL`GMPie>Oo@s;Gz4PRW!Qe9hGVsT)(q9;X`I&X=Zso z_s(Wx=Q|b(m5QjSHtWshGmXAVwb|G{lbbFqWan{pzjk%G*%(TMLWxK?5s4)dBf*i; zk>-b*7@EG^Vl)31Lv3<28H`6FqIUUAzF!V$-k;x|-;p2Ga=Y_;@_P^Ehfd`81@gC| zoB92(ws-jh6TbW#^S8?$=%^!~u~0M^4cqx_44i!M#Cx0h9gTgb3b}LYNpWAQ58!spH!TY-9*+bQ z&eIkjO#~yOar?BL0p-1m%L`|6Wwn~msmn{Hij8g(chiUy+wm}(Ohz1*1JS)Jmo3j< zP^;zaVr6!zyr5Pk^VkEK`OcDyhjBwE9NS1lqQTLq!_7=>?4B(wX6Fk}<<#ly{Ji8` zCz^=HgAqrfqZp95)4Cv{yUIE2rPDd}%<`-@8amC87W*4V( z0Rqq9p);&v+drh{<|{dM_Tc+x7oS$K6x8>dk)J*|tUeK>XVoEkSYS84#n-sCGM&%S zQzAo;oXpuQJux3SQ<%%3nJ0$TGr4Iz#wyqvOJJ1PL8SXz=(p`3}x;gJ2@7d z%T)u1&`ou5sj8k>TFl+2qL0hvMRj&DaCf#mR~c6CzWa%Dbj5V<@-?=Xb7GA)8>h<4 ziy7`$;BI+02f0A+Xy9)2Lc_w7bY>~DT$;&Na~TY8wQ|Ig7fm&Om5@jv}LY2s=GRu&zgOUpg;r%=_}0O0`g3u8JEF>)E;^B_fGXFrsY%b_3~P5-CG;Z)HHHrIy19PqG5>4Va~oa3C z5yyIUk^qw!SG5~cCca4lpLi8$mum!aE5PY1X87)eI31mz&vh$e%{ znDh8G_LlJwoz7*Zm*(fQrHXZ4;v-SoiyZ`-XgnUJZPtbETPhb8&MeIrrth&daL-H* zdzoeWN0PWD6HZTqXgnE;1w%=fq24`TSSVCU64M_9EFb|j91ez^)>I-i5)X!AvdX#? z8oPxsCbVwb%8w?n>VpZb+vbr@ghr#mSk&#oC7C2)w7LYjXQ^x*>_iwEKqTZeiK8LR z@2H#fxa)~rc`-MyLdP#uEkipRiQ_>VbGTEZ$q+Q7kS1H&M?p|iC27kuHaZ%Pj09tj z6P64o5<&3mc)x0T%7okRbn)lNVrfQ!giRE+-3|q1mDikZ(^}v zLgULkamfgj2a2OBL@#jTT1a6Y>5-gjoapz&gII2N0L?M(P|cwT})m;YO^#M zu#R6~7Q$b(u(YTN!3QS#8+#yBmtc~bA(O*2bW9jE0{Q2U|I9#R*L?P=3)n|QeN%{r zB>KW?Rle|8`0)T_>|u4fdR|H}&mVtjfF1_oi3oWUKtfsuUxKm?d}v!^u(SZFSHn0t zlbcob0x7hs=kJ4De@MOeohs4~m}KzfMY1{wxL|4FP*9^3I1eQR0)NoJ+T+_AdzTjr zvjz0C)8J7E{{pbIKr%kjChQG{t1Q%kg5VgvnrKJg~r@0=im;8RIq1qAllXTa=3v`W}`I z7B6XPl_Mw1%Q;>(zdVrtCyhHbwZo#E81<^XBwmj(>0uT6A#}v~3a$IDz3K4VleP`Y zs=GFaO&fwsBq5)CVsGs{gp=5KgCkwH)?4Mw)6X&WroN$TuueZ^;qBI&|iHoW3LCcusCb;XYn9=EE)SpUb!8(#4l{G#p zl{Jez2_-KWin!=;`%2kzcA-LqO?nGUs?lT&ZV6bygg9!UJOayf)JL4^%m*a|zok$~K#Uyyg*l;SaMckmh>M}rlBoTCSWSTt{w@qo<4-D&? zhA3PW#W(gTO2=CP0s2!U1pe^74n3JtXcRvz3vLYBe$3f$5yvJrqPa?KTemdd9L>U~ zG&Xh8xNAp7L$EA_%&SyxyP^u#)`aHCv{AiPx+@o!&gK@NoTwI0DDWd;`2I!{Hq#?j z#gd_5Ebc4?7w>_TVlami{AdI>OiaQX~#RRf{nLuxnGh}y+ZgOFLr^(Oo@{--zA#Zl1A7I2?jsOmj7v*&P`n--@1vdnS1)=U zkAk+tZ4ntwLQYObocHp^9z>?WzO+=q_6Ap)WkizUQFtWml6_<}8baI}2FvAc+am%P zrgIpL#VX?7Ot%Ie3Ai;9M#q8lh7noXEnTKgM#r#Q(n=E!O}MxmMj#rF#jp(Bo~FhXpC3A@V|qcEDJ z9cRSlnXyawdk`iAqov913(RB)N+Pr-2X2!g7`;MK=Y?QUm=?&ej&Mk#E}~4zcRrR3 zEoAO|EVOXvW6{_`G$Jp%6ehz9;kXSlyr5%KC9GzMP|8Fj^e-I86VMP^3LAWo1eZoX z%;N|T4K#Lfa3{sUp!{LfUT#k85s{O7I+fxEY&N#te-bw$A^y}4@S9K?180{CGtCnd zz7ti%S{C2XuPA}y?f5$pClW;!m&VdJIF{}grFR0hv177Y?zA$odqQn&!y?Ke=+ifg z;LsDrH&YW-Q9MMn@5Uc)T@ctgHZxP6Dj`6NI^rYB6TaihfR(=;qRj}<4^1d1Z0xI| zS~Mp5dR*A8G8OXaVvxL71Xo?A?@-$4*2MAS`<)=pfjc0!L0PpJBeV*uBlt zhBu@HHewrhh-!f>!XJ9RF-UtR;vGaV+B-_+x%*UYf1WDkrpdpg@2P=o1%ZV3i+K3Q zWLW&;@b1wQB3BkRqvFRgZxb~Hmd|Gw&dg*HSe^+yI+?8o0@&|@QxBe)7*^?O@{xxh zcx3E2qP1ky6?xqbA}7(vdpS%+_lf2udjS+a7J57=8-_2D9Ce3=g{BOT>2-A#$bMaJ zY!mWQvvEv#QKc8MJV&lB3{Bt>F^t3+79Cu|xpK$%DL}>Wi7S{RE7I93DZhv(i6J7f ziJ092sIomSSXwFtLWhEaYJh!Zclln|mSYx(VJU$9!5A*UHBo3a$zZsn(D^zCH71t| zi&gVMjl93nKX&|n3eg_OozG3<5sN_9=EPW|AMQ$Vlw-$2u9`*oW)?U94)#-`rZGVC z(Zu192|VE@`f{N0`1O??8rzqatEFY^Lyu#>&NlpoMJ*Q)94zCE2xMA5Oc5E4;rIe# zR-8g7%!es{tz#y>o`)%7P=cTq*n~BN7scZv2=QeFD=t(+ zmaR_kOk=MufobP`ZpA)Ghl1pOXJ2gNlaE49tIiZ=5oGn2hA$)rqp=MFC`3CvK=ku2 zjX|u;`=;(cK6YOS3)CR8-a@ge6h$GHF2%qeSB7v9CeTQ3U`NNEfd>|1T$W2E1j`)G zPd_Qb#`pC#wj)5KbfK`=Y&<9Y1E}J#1R0ruZFE?Dayhr06VFT;Qh~x?{+vL~XmyBs z&Q+!n8o-C00<$8(mY9CO*vtqC|8Yo);>Y1HJc1~V;NqDJc#vXC&n;%pAdck-w*2&i z4;>R5_J#s@xOW)NOm<;L^z7UHI0Tg;n-fvLop6&edf@U>5O#y3~PRl zvpBgCAhxpQa`u813PI{myb%Vi=Cg49EG^!J>%o$J4n=f$(tQ;Lu1V0IM0600o%KtBgF_6`1{r_n(1<_UWKDoTQdy z-hck-pFmH4hS)Ct13k0vY}_K4q1I_=1DbDHjIbM1niQ>-@;hRa4C_Ya;dd6o*Z*OHz1WDG2H~e^Ux(ml5)(0n@ReBPB+qoCva`7C6RF98N-k*p!4KaY61V zk3nwBFU??|gwNqI+Fl+9Tc*M@TA|!E$7O8f%p*b|$5tkekp)1To@Vr5fUn0}lN$t>Z41-<-$ELXHNB`U#>mV!y~ zBRnFNx<%$w_n6a?%qlHr%7XVf4exVrmw=pVX!@9GczvkYNPFBakIN(fXLUx5=LY%I zhF5tv@OY@0r;Vi47X2S3h8@yNS~;gYJlg?#0d*=;C=m}#KB_3IzYTdgU&%tQqT>pc zkN>%H>u(0Lb0q}&k_@Z-(;$WLUjEtN4#J0kSf)y*LU!C=o10%clb!!maA|h7k|QvZ ziFf>F5COf=6P|!o_A?ShldQWC*m11{hPo2?y`c2uR&sL`Es&WloQDieY^VOY=sr4{ z;Yvu@bBjB^Xe%f1+kZWXIIFp3JP$wpa4EMq{*XHU`A$*hEM#o>nLg2{d?~n)J)a?0 zZ|2-m`3bFgPyn+er2m%DC-5(VrE=k{IB;NNQ2EVZr86Z%pO={mg8t{VS^88^I?X<7 zq!@f+=1W0}LdQvN!2r%+kSm+|;QO5*ES{Cj%mpw~n5I*z(@(sJ>FYGJ@AEQaZ055F zfIpj?EtA1i4BTh5)lRYH>8C3ZG=KG6M{(pynO)06ncZjkpAARn(zwk4` zU;V${cjk+HKM_yZY4v?U;28cG2Ujs>Fw#mUlP{Dq)60dK%xol?h~=W$nefO=1b+-9 z=rX0{Z`ab4w*P2ZQ$oBQRX(fyEIPvbjp7?@37|fIi%|PNq3DW#Hkpk_LWokE9i7dE4=E495tb|8M8wd=TM)^(7E9`*jQ(mG7vH{Fd<}?Dr`Ki9ENe%#JR}4 z0(LtC=zKxXc18KmKd{!mRV&|TsgTcMJN$$O&-2V^vs$@qhf5XZ16sLWBCmeYrTkyI zln)W%z8wndHIj;={99Z&$v=OAawqxcZ>Zcs{!xN>=}ArrLdJu1s08O0mxK9iC9i0I zsk~ZNg5@RhbSXhHsS0S42vMAyFE3%jXJ##r3vLsL| zB!gD7XOQMWS6<1~EG!x1Tp>l8Yw>EM3`wE(QqoFnrLbs|6mP2%((T#lX)}T-6#{u> zx=g2K&D9tkl86OYAWAZ^9iW_>UKSF*smhe;CJ$!rfA5L0;}4GWj9*(y%ZPM4h-Bht z)eBh<)T$NUuQ*S*vU&*NP#^ zN&5iT^FGRHz=F^Gb4vF+$?c5K`!=V%Z);?1^8HVsLfncP0P7c=rWjF6t@*OS=Kq5G#Y58nTt%-HzJ2TvV)RsHSn)^gO=@@-nnN5@V~WX2ypk$&jG@sqEr@quoQ z$83%7ec+L!$1@K+`k=Yf161rd%|6Fz-X>a}NEO8_--fj%=z6ZM?I*{_Gw*tkyV{Se zxy2#5upC=V$rcdG!k|qD|wfygD>91(%uUp_A z(DJ9X^a(9ZXz6Z^k7X_WNiB_P>2?k7Z|K>8-#FuGX-nJJw`%$=F)E6(RnwFvS8UZ( zoBl1E{QfQ}bpNxr;pvax=bx47*>#x)J}uL)eOIQ9Ka=T|pO@)RX(Pw!B`wb>m%F9E zq`^OC!P_Xll`i9PYOS~CueJSa#kcaqd~<58x8^syeJelAH>cKmYksr&zt;Az72i4@ zJZ_v?>#h0CZl6s8WMZKotDc;4Ij0QIsiuK<@ODcw9g{V~aq32g%eh@nxtvp`!>P4> zO`}#gWjJff^carY<2=)4I4eHqt@TXDnlijK<#rg}+P)RuEzf)~ot@e|a5<-pZ_RT# z<8$f;$MDQQx92whOovm(x285ahaas-+Uy!s(Goe>5}pV1)c5gSe3MI8B%pTRI3 zRFSdKBYilRK_^adtf|?!^x7Xeu^lyGX0#(GIHXebK64^Nq_f1Rc-H;)PFNr|S$rUYTGYRg zP|lvCqiK&1t6JC{Sxt{rmtmO=7)t2iB)xV+xVW@xy+8z7>sk3uQ+lhaIEo{WSbPef zCR)Z}p}078BICHh5jC#A(sLN^L@iX0&>wms>;1=$KKg+A&Le8X(78X6!NK8NfnvP_ ze9CTP-%h}ZwVU8jBgF+L@oG7|DaLR>u4BMY;%yiBOpY+^!?rC9?n9cXe(K#PPdeJ_ zi;rQ+;9$2fYAeN$BX@KMqK5a?q>bz4!|*b|-!k>|cRzCs=8Yk3W8(DX!yo#-d<^=O z`+n9A!}!l9f8c^I>9^Ob`T+eDi_<+C5Q!+fq!)=PevD?Ls1E1I=WFOyx6`Iir0 zY#d#0h@y*)@%7d3O#c@3_m6*1wnep>$5-!GC6cZGRFA3};Sz|Xs<-tCS+7vNN&prL zR8Qqp@9Wg+_(OYme65%=|MKDIG;66(slQ|&A7$rrB5>fjTZua1Ja<>WxRN>G-+os3 z!=AfC73DtV-+fckdG4plEb+-7qliNJ)A`~&@Za{870UUP>i=$UAN4g==b$9g5s8dul%@>pL|O27Uj3CYp_JVG-9)Z!I(0gswo5J5;Np#im=@v z)?1A22C?7g)w-fgtuO`WR#wM6+h1dB*Y>;gZ``wgt6lr|lH%RJ{ayMu?%BW9uKjz- z*QI^*&uwhR(Yr}y)w2gL)!gxkZoeo0<6Zf`+=c&_R@~d~7yOTV@_)Gt|LZBQ_0`{% z|EVtgU-sai#;>NVk9l$MLjP`IG$&YjHK^UooHNv59~iu(=P6fGj%!b)dj%btTfp_8 z*cGUh$_D#}6fPZv-bLkH^EA~%#~btyHcd5y!11B{6N7v7njz!~m+ohxi25t~HRq_D zN?#t_Ew)=D*T+#Z7~qn7MZMq=WeXy=n>ZSlyc{PEsD{$Jd6e!&j>_LmkaA2!iR>46 z1f;|_UK2-pjmV)d9UQG+#0zmp9LQ_Ii=Z#_3$ov+9}H?Gk{|Q;_QVL87(W#)E%dfhAVK>_1UZ|tGp zkTwe_#mpIVM1_8O%J@_^7%|pZA@N~SL7#AsbE{2})7pJp^xITFthXfjV@v~AG1=8_?VbDkr$zNKIvzWI0w9pydop z8giYPU;~4@#A>wC2!IA^XnYo^e*%gzW$ze|*ASo}3*Jjyg=lmZ=<4plMLU)TMRy$0 zzT3>dEE&W)xEHl_fAMJ2SQ33M0rfa>sON#}0K-^v6o@at5q(?{eV#Akg!>@JonW#1 ziZw~ta-L~iWhw0()Ccx>i6*d5xvL$!LFvZsSTrtN*;P1Z4%&f#EyS^8_lr}VaSQGVN650=$FJXw$H6~ zsJ&`@ZjL)lyr@*<&V-yaSx_b$I+j`r||y+ z{uDv@YZB(P;$JhuAKXm)a&yU$* zuHAm3)4%n&oqp{W$-m^@-UsIYtKa=cHr<~3VdVedv+uRRY?S_OZ!FqyC4J>I*(64j z&h<^aK?`qg#j7ZvEnbF3F>wuM?fv>IKSuhBzW{1CvO&ArC(X%eFH?5q$!ayTaMs0? zeXfLW1H*<4Yp=nQdqbM3i@!|M(X&$XI{;tZ$ANWQbQ0f;GQQ4OtBHj|^!EGCw?K>k z9K)_fVEaYXy2&8zlLqN$;lPDK+Go6Bx0*rvYd45NTJ0I6Jq^+|W!%#wx9Vw-dmR{@ z`@9XZQ|O=lF;90cULVpTgY3HEMPbto$`{@Ff4v)&XP zioXw$?V6RL+RmOb1k=z&PZ_FKj?64&mP>f60Nx}~%~mU{gWMb;JEZa?I^WR}FZt7z zp)g)D+WB5t7iFkVDnsv*%1|F|r))jhYRb?b-Ehi~)1!*!XKP32oZjr{oG#G`*F8F= z#-gTC%%MA1=EqZB!Z@i*a$PyEE=B1YorBm5H(ZKcL3E7IOBwTuP9eJep8S)#uXGbX&Td)5BovIr~F|@86s}{Ddy-3MD=zHMVeQrO_%sd8tqg~As>go#wC>2 zQv^{LWoi8Nx2QCoQt(T+7_waxuxltoxuhqR3}i(Ksby(WH2@<^?CI4NbB$T8sgF&< zv!wTz)F=s`7I=zsNqA>Sp_1j+w5zh_)2Ul=Dc@6ObnnsTDKj6rR?kkVr_6}gs`r$c zEuhTc>yJU>XEhDV%xLFi0?r?Jdv^MzGLx6e3^d`btIYhH8$_AWj~X~Xw6?l3bE)Uq zaaATrRq1(lyo7C1x8!~0e0RaK;`0{vyDDQn&rUbb4xTbTbI3eD zw+Q1Z z{-<$8KOI8P`!!`l_XZi^(qmUz&Y1o_DsQJw;ja^2#;QYAlBy7?Q95K>^jlG`j_GCf zF~X{^5L~-P*NSp_=f*<8dZBDITKZU0j`D2mebu3C!gF)Y%FgMYveQ#`G*8V<@41OY zBEh8m3X{ci6B!u^M&x@`yDB>anzDn4ZG^E6(Cy4O^ZomexWuJ$+^7e{HL#u zyWlIQ>0$T2ZTCCYmv&$Lm^}B)r=Il709(baZOgbI(>9 ztc>Uwdi!)`R7Os`+1|&m`;6zF?Nn6l9JG}$ZUyvh=c4cWQQ6K$&P9iA;d9Z}p2DB>8~pU;m%=y zm;RCG$sK#OYyZ}LUE1&T&)&*7dY979McZrTNa~&!XNZ1J{z=*K9>3Kt{I9R@_}JU; z7yOTV&VcTDvEvhG>z(m&n#KOE{Ev6xf7OG3YG0mJTKOXr+I zx*)>izbs3qCKX*t(c$#=>e|}XO9ZmwPf>}#rYimxQi%NWCrW*EEv}SQ=iH7A*J`g& z9jQq8y9Qo>nQkGY$P+r!^OEk+n-oE76e@;R1a7-V*Yu+l{A*WSVovKdy;t=XW9nyV z;Mp0|A-G;oRpC1_)1(~Z@c#S1t=q?*pp5fV`Gy^a@$dfg|IzXJYs9~r|0gyW#(&$+ z&)Vu_`||nzreWv$=4&ME@92eT_4mTGL|E3%FYG6y!I-14AH}B;qA`B6lDB7Ln|5YJ z3j5oPH*0msF34LqlkgkuFuWOj<|S`o|6_gjjfm)<-c90o16Mbuu+L{o^zE6r=B?N) z?0@hMdmpcdu>X)M+qquCe(4^)A3VQ*e_1cH^uHtQzwHG(Ov&Jxa2mr^Phn>r;*_y9 zJB3Xb=KF|4J72^#m5+D&XNOru?^4>C6MLbi|)eixZrkpDxc_MJ4G=7=T}!D#0-tKQ0hUn&ECDU|`g+$^N#zFC%kyE33} zjBNRr+o^q1wOjsmT!yy3@s!IsRiT6&T~{#_{LWJv6|NmpUWqc$v;5m1Hj`a!?|RI* zKISw7JO`_u!p!=bmmotlU@q|%*1HIEtDeH#29%Y}7UsM3Z``wgq^{_5!E<6v*A?&a z)h+Y(`6atNEdM?)3VN849iKSs_vD|{6>t7myYRnWV`19fK8f-v?aYV0mfA%aRhRkq z7S{W_@;}~%|8B|(EdMF)p>bX%_uVc3YM;O5gvatD0(H#g;;PeT9U1e~lom5Hha{b;DRsr$FS7d<~?C zaNa3Zg@3|WqpKU_GniEBEm>=XA^Ot0aB2;GKs%&xQE9zqhGW&oT&nW~&xR+5EX;3l z&%?KR%0^Gw*ec3KjK199sBFaIvEaza#yk((rLyrZscdYg&51XY%~CcHW@Pa^=o{gU zpltNQjFi(9CiZH>jErx)a(dvcdWt$55avBaRapMJ2z%G8%=A1Dp5kuL^WY`QzrH*V zPGLsh>SH^*)7?#$b2o8bz9$OG0EFH23Sp<$=!s;cQ%{j+LW}@NeRzH8dN|>v{G-;by^{s?Xd&!PSDTV z0x41S)0GicU#L`v=I8Gg7v@tvVZJBKZy8}enG7cEA83Q;U?d8k!Ul!;9a5P8v=rv? zR~P0_twvp90`)TkU;c<4hBu>M{BF_(UkZc&A$6Z){p6mT=gh#rI%V%;OEI1qFmL3P z3w6Gorf=|6&KthdnSpbU$#ympMl`hZm7f_nmec#e^LzYDdYR50tsni&z-p0{8d1kyzI zx1Oe;tXRv8`H-f_5WZe6m4`I=0KTBD*AXPawo%K^W?9=FGt!W7ynnytKN{yxhckv5 z6}P|L_Kary)m@iub<~0p|2O`JrVWPi|N8C!)K(|)rF`(*S?BuKD9q@Ep0dzW7PgAA zFcMF=d?#ZhIT{RaP+8b1m4&=i7Vu7`%~uvud#sj!eOqM9Kk=!TQcmWn9+x!=5E6wB zP#TN~B2qOxBeiY^j9*EZQBPU0Kaiy5-#FstnY!#LEZz9ZKtC-1a&0=TP$(m!f6jHJ z3dl1D?i4m%*uUs5Y`)%weOi_BI~VTj(_0mg-??zd#u0SS^?m)7f%P$(KX(r0nFHtb zNmx)8UL7d7Vqjk+EoFZ#&xJ!v5)=u-_B*w~DYIN0?9CG0Z1E zk_?V+P}m=o!v3>T*cacuX3c%GJOg%NU*8Z}*q0lmb(TM)u)RO-in>rB@So`ZA?0QebUpbV@5(?_Lk{Dt`3Vu(8O5SGWFJfyhH znv&gLdIzjEq%fdv1thf=)B071-lTZdC_~iO_=?ldA(RJHP)B4-^^6#l^-RQ#pwOjs4^z|uUFewA|7T>{>BBGvzT?o29D%Gs^5O+xwKUKru7payp z{C#{y>4BqeVlYJJHAR6k0H}0}@Cjb}9q3BFZ73w(#zSfYpzs%p!x&x5A6^gvL0-@K z1%$Dl6a;~r5*G$c%GXy?LVPEP`kE8O)tXMdEdc=AZZRzQOX(0?s;BDBZu$SYuYFJU zw)c+F;7smpFgriLG@Y$3m4lh-rP76^Hn(3@%$cr`;WNHuYNY*lMmZr zjAsMPBS$<9DbfSrR73Un5-lvq);W-F#-zn=L{#62Io5J-?(T0=xl&Hu4v!q-G0B;KRJeGT2>E$ zxt6#a5?^wbSM=8o(eKGWoei+#7) z8pQP#fZ}K8fIcqfM=cBf{xu^M!iBVOpLWeyPPAlR59g@^6SP`XPD+AQQdFTdv_%V4 zW~CxufhrXluB*2?R~SF@F=Q(RC@m={4>iWHrX`^?5f;~^$M}j;-haO}XCHgQG0xA* zhwU(of39%BW&hTn@5hGTZ-Zg{|GD{(ZFRDJ`Fy|SPo3*uquBmlxKB^kuio6UzWzPr z(QqP|j5`VYqp?^pA#1u?_;;JaeRdmf*3A<3ha4>YyxH9d!am3L(^%ObMYP%Wg#8~E zN6LMQldxaYEd0)+_xcy+_1J#r9*%nmyCljRWBZ*bY__nzUs(8^ID?=m_+M4L#Qm3Q z?!q$NQ=2KQi}sH)_fB^=`smp^j_r3IAELjW@}htJh57X{ul7mUm){d`?!8%lUxQy0G**a$wA9yeoff53MmBs2$*B~kIY%j} z7!d0FWEs6Xh|~)D!7BiKdBrLD38W})_Cqj5{e__BcDeql!GF*mBdTV_L8=MPxI`l) z&K9IwOwXELS!;_v*KM4OwrVVlZ*gUy7w*$j2DXkekVpn2j>l1`4((yfh`o+r zd&+>f<=+_l?=*rv%fFYfM(TposA1vZN9%9Bi?F)c%0#~=3|sFj%D>)~iGIvZs*AdC zvs?bvfp!a`H)d0xVJVKw5_-+1{Fzf}&8N5SP_4Pnn1`@G=3MIsCdB)iH~wiQ-IDmE z5|}BG7vVrl;+xe3Qb8o_6~LIWD4#~X#6vjGm{kGbI!fVLsIjL(S8CdVY8~}gQUs;9 zK#B?dWJ$fHnv^9fm+A#WfWA-1Cd{;t>nMe?;WB;_il6^mT$yO~&JOgVh_;GnAsGt= z6OLyGl8Gq4PpGSBVXsssv{#JorLBiIlg&~l?4AX=HPYW3L7C`z7Nnc_)rJxA*#YAN zj@BLLS2bai^X7CN;+nZc$CTxp9VXQCEc84J)-`F&uY0cV*S}}sTLV(CHVd@+JsUE7 zw--WnipEB-B~JwJ_vw_V(!^+nxw^tv#z&e+ zP)3|;(uAl?x++r=$9Q%@AQ*V0JU`#JxU%p3I-4A8#g!_x#0Yj`i6$En`gM();W%jQ`XB`mbE@rLg(YAOF7{>tC>) zInX!v8$ns%GY2|Xv<2%A4WsAbciwLAW6Ebdb701$+^hK>!M-us&PL80d~T4>92mK# zmSx`Q%)y5z^nURCKK`s;#<-LMU*RyM-+$f?Q_{XA?A$tD!C7O=yEDWpge~V9YMqn` zx0#`Od=J1!`0f;@5RMG}C(nbqM`mu-->Vhf23l>`+dnRDK{OQ-6(8Drwimg@)5^*=d^UH8>KzC1g6qSCmQxLE~_Y zN_1hL=)m*g(_3h-QCVHUk+K2xV@0o}ypcBLPMh9pM78Y{mDg*ctu^}Ig2;oi6D3vq zPBFpA`nG6YAfcorzJfq?^%j>I(^KG5;*knvZ($N)Q}$Aw0`LDn_s{HePf*77@#SyW zVHm&uE8{L_7TB}zQ-CrhdHrlLT4-94Rx zWxnOqF2r~t9EXHxZ>3JY1&MW_FYV)NrNOpc z@T90Om5w%OpK4L58OE5eugN+cLMkZ?a~fewdL7Z?8kvSP<-`utt`V$J6{5TuTlY+C zab=r#j9nv~D+J5Si-EJ*a^`GyzJhsL$}E>=vRg-4NG5_Kj$urZ zNNmFgsP~(ch5fXx@Mf}E$^yceEZ9k&&9En!GRv*%g3HqXK(h{|qrL*>$PUT@(NKRcxVMOkLAk%IRn@m?gtlpX;q zjRPyzk~+j^of?N3aljG_*^-;IkA-z3j$l=z0n7fHgq7+6DH~7<(5mz@(3uP===%j6 z(~+JF1)dwJ_DGJ>DXLGmfQ%oNrL}e#kJ5IHYGsKkJPUQu*88N_5`F0#VR&%WGQ#rn zy{9bnlm$hZc!yL%E<$rzE-$J=7lE!c7r6Ve^T7;0$)L_c7C)~R7S(e09P*XNht;|H zr8C+2Or`Kt?nrgH1g*+Y0;PvO0AZ@u3l6D;Swp>=o3G^56HALZsTqmTs81DtnG~aV zuBsl991%m*_OPl;{G=!Jl!fwKMW{#l0~5ad8}qkUp@NG-CnUsawXoKO+r1aMR;&>*(=!RZ-n;nMne`EE#F8ET|{7>I-t`FTr zXC6NIJ$oM~cgQmjpS;ya!;Rl_cR~NfPj3DQW4isU(nsZYs8p=@l=t*~PmJY8&NKl3 zZC}9yi}1egmw3j<3hHa9G<`>*{01G4=kLXz)XRt((*S3(u#Zg{eu5-PyQ(k$*pZFUr$lJ8$|yakFFcUco+UJd+<**`AmZwedGTA)^@-b)l1VqqI zohlWAS6#otoE9*r1XYHn)W62O_+j%uwMN1|mk18VjL<142C`OuB&`m3oxdUFs)lQ% zV1x*XW72N57@wM?M*bA24rWeP6y;LqZAs2Fpo{>@xsJ+te}D1```l8C#`*utkK17w z|2MPmc05yH#IL+{$_B&u`#&;mtCQ_3r|H@Czy7In{cCh)p%;eK3q#s6o`q;692|A@ zEJR~aBQ|(u;SEw*&<;qy!FaR!KKj=#!oGfh>DG_iVR$om`?tU2f-g6Nr_z7pSdWjg zIeHdo`_nxO&YL0C1FS|NM${K4sB3*S>njOE+B;+K^UD%hD(ravePp7Cviu&mqT~j+MJP)!xDa>;@r=gAjZhu87eZzr001VG5 z3Gvjz)iK#ldxdK#mC8%IhF804_2gYJ_gv=n(-ls&6lNv4H;$Q+QmXUnRf1@@jCE8p zDZUWxhCM{}+I%^|)O}aRWVurqA@4U}eXXI~oJmXmZ8YvsS1JXTZW9ujzqe(v5U^(*;3*$Qw(r3~2fb;>WY~4DYrOn6zQd!C7&XTs^O zp`BTlZ? zxiC_YiU*ZInb2BDw~P=>q0~A2EN%a~U!pGHfxjl(sw1`f2BarM3tCzu`gNgHw0+zb z!idJ81gvo0tcpRI7?U+zZ%tqJRK|Lqg=T*5q5RPC)4jJL_1=bbbH5E~Pa+gbh9j|1 zI5rXrIyx3&iTH*;0R2Xjtp7&Z+gNemELk5>NREyL+JA2ZS^t$BS=frQK7WS6`V8ZI z&#};REYLo~>%p8}-zXw#wdYvSk0dzFg3+8>zx$51k@bt0&o9UqrEjsdAfSSj)Id5cF8^<@2? zVSmdQ_D90uU`)QU$YR(ZiH-)t8#L_SZW8uyr|pP0lY`H^$I%OCd>H&u-44U}AO6qY zUcSQ*IeYnTrXvUcZ_(bzmUiUeIqk^7Q0K^jd>9}~$9Ci(T9tHOX%x{n&fD8}iX!^6 z59@F|e|LUfFO!_nUs@mfk%Ls*4r4rWV1LvOiO(ny9qU7k^Q(x$kk?#dOxIiPF)os3 zehae9@5ukM`HPwZFgSWw@6wD{`k#hJAP52v1iPoZ(~?l5-vPkB%I8 zwBNOV>psSJZlhnDKiP&E#u|lWt|jh<6lvIZs-b$EANJ&*lm&NO;%T)D|1Yh$&tE@g zCuN8;a%$;vm-%1U#>WX4z3a-qZrFFGaM^=@8b5jDz1`Rn0g*jIsy)uj{IF&YlEU}w+#rL1F3Gn zu>cNaj1s9kdOfwC_VNDwg>O2{4Iqs3_Y=O;wr4owE6Fd}>Zk=H{zw1r_iQkXzgYdM ztxn=Ar|HPR!M1b#YZS%P6ZU(;{?-xp6X9UO@a9t8M852Lz;5Kl;xW(%lE#o zysCIf`!}+zPOFj(`*LMEug`JM-tmb6_cglixxTNzEKer}y!uxYCkEX4H_CcWvi{_D zyUE@-l4>W2#<`7>kcCNK(6K0ro_1{BLkS2pe79XQB(gQU36}qaoP`1WpW@#Lt3g};?)J!=ITCzkneLuNf<8n?ZXh>=e zTw+YyD=#SR7RYPV+9|&3q`o@1qg>rbX{Wr?J{`E0t_y1&M7oD>&_fQkOs% zI=X@rl%`LB*7g6$!B*52i~;{n<3Ck%x4Y8%TlJK+ZhT#0f0zD^d-iX&)lFCG`31aW z;htaM`uhbqpx){3CcC|#ZevYywWqqQ-)<3{5rX7)$nq)3@inS3)^Yw+%DJ5DQ{v@B zQl56gZ>LVR7&g5^SN;~m`PW4IJ`_6eFJ<$Pcv+CH1Jpa%b%8ah2l;@GS)-u*l@!1n z%Bhqj{L?~*@#6|Q#Tu2;mGl{qMpEmXjuz#BlMtNp^LvZy0=1ql(9;FBj4qIj1VeEl z@>_z05@CGaV1ut3x>G6xe<+oKJ847V&1AEbft%?>z*bZS_(Xu)18>z+7{2j^{hs5X z=QuEnUo3{nblr1(zy2Ku>UM8spx&|zz&t|jmQWT}+#%pN5aO+bsE^SuLP9qEaFtRN z80NK{NKk4_{eARaQFEC&4`syXTrV9BMz;V7-w?@7Euj51l(o3kbc(NJdZi|XdMGsD zgnr;#T={);4g8eezR_hOVd+&y(p}MB5$M1%rcky3VeXSLpYk}ZL6lBw4hJ(;ogP2` zx41IV>M0XFWn!x+6A7FyPB?z#Ad%>NE3NCRgzhpa6L-<3#GA=q{ZG4HA`12Qiv8EG z*zgohiHQ6? z^4h2T;CCJR*e5@7AWs}HmY%BG2?SZ9m($!(YfdK;{*OP9?QG;k!nbbWk0982r`DWK zBz*R(dOvu6S9ZL?Wq$P&3IESsc9@dVuXyk&;wlb(* zQF@ z^cr1}^_`zUPL${+K2`xASS_L>@KaZ7;?mHLF&`2iRIZD*jOEM)DI-u9@E_`e1Taxj z;dzjJN#%h+Nc;y^HM+hc6cH!RsTMv2w9IWer6KJ@&5}Y}P$oYFV}U=EIbjMAWyqKI zROWbp|MQsh{2HUwKJgwq4C9YHv*7Z4;MwyjzjAoV2E+KbeDN7u9rb|gcm9KO{cCih zp{GprPBd(p6Ahy{z8p71De3V`qtQe#xxph1Z#IP~z1euPZk96f@-9*tZd_%;{1F8E zrrlE}yp;)gKpDVC63ji;M`9dvGbz4^<~}}j@0KwO2wWpP0lXlE% zRY3VyO&+UCsyx2h=#5<&{_gh4pz7Sw>0oo2g+Y9Y!~m{8gQg6d_fq^xlJ)|3SF|Lg!>#65uJm2i)xMatc)1Tj4|D446=M5ftc&ka5cq{EJ zyq9d2E`cZ|i(^6Gb#DY+q8Ftkqs(4yl+uuIyyh-V4qc;LK~bb*L0>b*=OnG~sTHr* z5eMXHcqP|@q-dqf`Td@52ej`YOOxASQoQ+;IAx0FbblBA$36L9?aKd4K5xVM{+?rD zz*chRr>jsczTXKA)%**Sz4lG*9SF~D!;W}dqS>GT{GZeWSpeI~Z zl&g@e>nS4!!^kuQ+1q@_=q?0u3|xgwo)-9iP@*5p;!<<*ub`b5jr9F}3`Z&W8>7_R zhMqrQ&?E>>uaOvVEEpi{y2#a*glpR{B_%IiidwtkbK^{w<48nG!LbM{U!3RXe@~a# z7YG#p79q9ANBWAtfP21i>0F_jSM$?PW=iFy*}{A-P|3~D24`|-ht(&SbIZ9*^+G9k zgbRcDbD8O-(uIsxH>}R&D$_?!mb24^#kq;xg~05=_jiCnr7)d)TD|w3>d6OBy!YU+ zTFzCE0HYdQJaeI%s~j3ubBozC^SPNLlaD-{e(<4V!_8_lQ2ZXTSnVr5Pb{W8^s@8w zOViovQaLD?SX!uE#%Gxh;+7G&Rz(dD^<&f(8!tP*+YVG z@5sZ=edNKQG2r~PL6ztPDMdozWN=h|Y7Ksg|A>`#Xr=jdGm!Tezlconuka`Tmd4(D z!=uSqG9Hb`f+OLijCU%28mzP0upPN04CJNZ424+nacc9wUVh!m$TLByZ}@iw^GHq?ActU(@Jw@jve4avkLzNhd;=jkqa*+6fbr|zI z8r#I}+ARJq>L!%pm$aeVHbXl~vp9@=qp$K*^SCmwT>J`h81*515HAslj7F15M42Tc za@d_4D87ugC3PQDFa8%QBhtn1;L3?~EEI}`gHhcz<-ogvbl(O@)8Zno@f|4S^j-`Kvu)`;x;+}hA`Qvc2D|ir+fMF1v*3)uDf}J^-ix~ZJUUSDRSB4_j21ok`8V15whc1RS^?G~N^a+#0*x^5^#n?#yrLYz& z7*(2^^k-wM(xj#4+BaLoF?%b`Tdk-qAt!AfHhak7z4KVQgPne*J0vI_URmh# z=@YT~Z<(%8rmGxWBN{!pR<-n%jc1tQWUp5_CCGY$5x)Nmu!8K&01pKC85U&QK(>A; z!}g)Q-SPTmzaU9jLW_4-=*injO>OkGT9H{@7_Kb99JYxJ)Mv1eS{-h)RbWv>uuuQa zK67^o1jKYSp6A4YsjQFzpvUno=vG$92+*^K5j`n8!~&FPM8K!8$Vp5;3-Bm~4y-a=m-NF&a&4=jQ zf0E?-1z1Qg@`y5TAyCUDh%$>AsMjt;sb~S}S3NDu3e-v)^hFd?1Y+j5#S|r0+t6fs z#D}WM6Bk3WH9)Ix18Vae%}WVTrXlo!TyY;zXUAw>_5&4-A~X3$w*zI4Lza6+?b`Z> zXGnKWDd5K*&`Os9g=awtQFi%%+2x7Lv*?l9`(-MTyou`5$rK84#KR6a{BwI#4*l?C q+VTeVlf6|yTkG#>b}NB8zCzn)6;O5GX?C4Jy;z5>=kH1lH~a+y3qP&^ literal 130773 zcmeIb3z!?%btYKoZh~Zsk0vS6dfH{hu|c|3#QVXB8Z=vkBDYA>BE^X#D;qn^Ob6=x#MVMX|_0@e>Ar^mp^nSzgTNM+uF0zdJ^dVnf#gJbiUObdF*hC6gZFXD=aPK_kA$;q1^nw>7|+cK9ns(xz!Ax zE!GO^ba^>jma+#7aRRf8T2@Gq&F&dj?j8RsXm1JvCy*3v;>ABmXg=THq(;9h?WRN zox^YnoVPD7FPzF()LJ30E-#g>_^XO}7q zYE24{MbII9G#rm;(S+n=6+W6sCA3(=<#s0Fx6T$9bMwUy<<;rj{Jc%{Xfl}6;we|I zsaR5rM0<$7rIN>TI-OTfEzjD=BbtgQwNS+E_K1ap2`w4!!F02@f|E`$W8p+Zi|be4 z+WA;G8PP&+YXmxPtYytzc$7@%t5r3m9$cQC#noI4VX-Lu!>3zvi3>FETf@4K%fAmZ zKrH;98)I+CFt9hcbyqM*F1@_82T4R9G(ZF8x zLZ`yxbY>~LT%O6*@>vXTt$M(AE}Ck-ecy7mvTwe4YG3($t+2F+xf8}SezBH6R|Ax; z>(u+28;XmI^f14Z1z%pAHj6&f75qyyAa?)_quZ-{-G^Tdk8k^u6vfgJcUzJ$vEy1e zE*C@didUe!dS`X{RJB&DE!V^?i1}>43uBQ;QcK2N@4{$2p#>AN%Ffz!&uqlaEjhCu zfLS+nPxH1cl%4tWG^uDJ$yE#U?%n-#ADZl>^w4sxD?OOPTYQCIYKC3;1x#~yTqW`Y z_v!-O%k$>i&Bwy8UCVh``1Psx75>q+pI@;^2v3cu+Y>bwg{BgT^mycLf;LqvX3Gn< z2Sz-eN@}U7q_VEW@dTdUv6zc0K@)ZdJ?o13Y7L9Q*-EjNx6jvj5|4&>%uNqW#PNU) z#+(m-aVy+{1qV#!;$gkL67fh%izeM}F9-|WQh!L<;agZl4wXT@TnAgQiB?%AzE9}~v6W9ox66w;DWH(6t$ zNEoxY2iKiaO43McOVE6lD)zySg`hR3+@@PR6v505_gD$GoX%Gk^Ybb!{bJ2Fw73(X z;m6$WtOPMl>PoQv&T4KI+*HZhwrIs*LpGiqv@Oi=U|0E>+G15mkxS}P{T-nKj&_r$eHwW&3oyh&gU^bZBxcDNm@;P zf(h#iJT3hT7smrVscgMV5vcBXLs!@eT+DgH9TO&EB!BoxR%AP z^a;Yewu@Zplhno=xvrI3Lh5U}n3aBBjG?{+^k2G4zd*R&()7AKQ#XYt083Le4qun6 z{u_zKVI{jSO@@cjcrIC0>u4+rb28}W#6qj7SWlz1xucw`*)`h(QP^&Lw@gJCV1aPbYqV1Ng;(BQRt+d_U}sd8S0CtUao zyD^f0T&JIt_ua9+>$H^ztfq^4XC~j@9#DCx^r38$fWb>qepoSCrl|q$u5p1$5}okskQNk34v4r7$XfSYF&H zIm%pJEr|qHv0$o)r%XC<4(1S7nmBgcyllsUAvgiyVbC3vrfEu52)ZK>+ry`|Q-&=U zmd@lCV4tYAh-fSr4kC!FrE~|Qg>^U-ibu6j(mhyNdN=eG;l`Br5?F=cqQW{a)Mgu} z1cZdf?W$2Xx6Uu+W?VH@8_6yEWm{@k~YLM#gBUP7O7_I0Ycy;_7J440tv?Aiwc@orpwy|*^L@EU3< z-CyS5duO#!oUN&W;P@Uj6o>7+v|NVePkvdO=ok&sD;(Mkh9n*iN3~SgeS#qxKA)>0 zU?nBDbA>w(SpcSmyE_fzh;CrAxzDvRc*Ed{f;-Ip0>%^Y>a#u}24tIXbjN~`cq)vU z;m&qb{#-e?ID$+HidL=ZU}DcwzH8Q(}E0jEFoF?WVLax*jP{1UtgZu%B+PGCBP z^h^T5nFk<}48qah>%wiVKutMAv25qH0wsK;p}5O^5lO-|4b8E~^u2||ohvWT6c=n$ zEQ0kVgd57?B1RulYyNLSTz%m-{5%CXyJY^B9fA8WP|v_eL}ohFQS5FqRn{O-1k|=^5$} zSuz}(v@?z3CXJaFG`=k$Z4%z}9uLYfnrW0=P+6K+XK^L1d}BdGOSNP~cGPm&V!=cN zH%+f6@L1Qmv#&w}VOt?Ujbx#q-d8JQLbAkOyP#tNO5Tn!$Xcd(Li$xXL(W{4B+oY?X#_^Z6bwo zTeni&fUV|+2an@MB*ORn0KX}vIdW#HIMX^Z9#VpbxLh+z$D5c`_ngzEYTySd#DMjDD5HMd+~=`7YsH}%*<4d zm64`J9q|$8Dc@mb#4g_^(PkLrd#02l4)Ik{EgBPjJ**B>h749C*WSKep#K<2WEz90 zD>T)G?wL|$%TGhb(k@UpH%*)?R`pSy8X?R^OP6jE^>kvE_3u%OqS*&=G&NNEaUF@e z+ePjkn1wqg#26>e05T6GE08$zmO|_W{&x$$3yBl18gU{gI7yaY|Ffjp%=dufWqSdBsMh zt@`ld>=I_2o=3c?If85t%-Yc$u4h(0zxFh@&nzw0vdh(cR%j&E*2kM$C|!VpP0*z% zD|;)I6toj*1jx_c3_i$ursR<$%coc_J%8_dS;K2G0-Mpb$3*p977-FX*Bqra6B!R8 zA?^NhWp2N!ug}NI`DqF<8Ea}FS4A@6gCa5hahV_g1Oj~YgveKffCT(_w@GR)_5 z3#Vps$S}_a9y^+=1p-*_v||q+nHpE==;)*Gedy7N!^qf@nO78eyBBAYi~^YB1lli} zm*NEgd_4GsCL2aLkz#dw#)YMfpy@RY7RY`*+uR`ZrB?Hh2&BqDWMz(`T^O3cqhc6I zGE6!|h4Yne_*0;Y;S)#jM^xdAFDlsh~^jIA%tQOwN!#~wU9u|J3jYEoKn zp^Q~Zr4Un>GGPxZW7rcDXr|V%reoE>BMXTx%jGgMX7&}PKPZyNcMLT*B1@!vzPQ+G zJ|iLosN&oNnVx}#bX@)5a(+24o|-bR0zj>BR*+`(I^;g*tJBC2;C)YlS&?Q-LVs8+ zXGDbm*e^v{SE>hsu&q3)^v;AD0X3JU!%L8f$u=IsarjDseM=T z79k9^PD2~e%1y0R}rm^lq^nmQ@xJpV{;6bb2prTsY8+0~g%8#oVLMvlqp_Q>>Dd<5$+e z9lLPKT5cJ6e>yU49y(BFQaTfvcGLnV`iU({0Ek6Nm=fpZit;%0w!+d3)=9)19;fBy z35aC@p3;GG)tr1lj?vh%vyNeTV-V4r+hH>F;~{xDicFK zI&wu4rYM8H%1})*iA^xNJGu02y8UQ7AZd=eDQ#CBOTPo8!lu$r!X3uV(mMFkE=76j z&?A{4pD#TkHmxhlu<|Y1c+vf(^`n=z%Y*lBP?WTiR5s)9n6gdr15C7OS?wtQtImwe zO*$TdxlJ8pEr>DwxEzzGLMVN3Ui^FgJMOS7OJ283uWi?T5hh4bYD_pm0#5; zk@wl3{tXRb1mrVSvsH5C{@UF9(y841=d`8S*=nB9NGaa*yoR)17z(H1nEl5RMOUo{ zFxYgp48{gB_$QhS9H}jiFgFlm>t&q=D4BS7~XHN}dm#_U3 zoXt%y-iLzzV_MyJ-`woER4mA|jK{{OPf^T7=a24>ox_-|`uY&iZ?-=$gZC)>Q z(zg;^8CCfH=KG6E{(q;yO)KfWn*{9g&pz`TU;2RdD}VV@r+$a;Cz1&(t+6f$8q**1 z;40=5jI@%?7K-KU^m1_~I~z_VqWMT}CKQ_q!r6Ac3D!-}x zG&;iTjp7?^3#1``n@ESBRz3l|Zz{t+tY6>zKb5VUm8aUxult8aKDu$p|IyEG8ol?k zTejUhq-^@;$cD`$BPjhF-mWP1qklf4Jms&;a-&i|{^vu=Q=2X*L+_|7ttZRdn_nMF zfAQr_TZbTO{pm;-H-x~vO4tR6GJO^0k z`e(Gb;-{-^3a`hkhMv<8MkSiEUoXG={U0pRVL-3f*(gVuP{->QP`2QYgUX=l&p(x1 z5n6B#n`HxqTCH5&zi%H-3d^T7_`CK!nw!ZlEeh2F_9KjGJV6jP&^V3<|C1OBrzSGjZ9C+b2GE49K0ThNMtrSn-A?#9ziTD zU)fJSWc86p9@NE91N-Keu(7<7E$3T?F?~9RJ(F{& z&u3}(@En}1vrAbh8Cm!cvP)I=9#{9lyK9kLoV*5cy94NcK+kqX`OiPF*S<}M@32+K z=dcrhO2_Bx%w)4VT(-kdMfr#hHz4xpANGKMtp|LJ7!Pey*shUW6y;BG;HLk48*n%M z=Wht^qW>rw-hGl+H0XHH4i#-~aak+mss%;=OYmAn(JD(6>QXfFsS0e;2m#K|SC(+W zXJ&A@741~Ds%X+|&`3#yYFk`bUH}kNmK17*bkJJv6v_hl$|G4?l_is&E3`;!E#7jJ zB`wqmC9A{%g-e^Xct@4cZqH0lTL}bIDCE`Y3hkM-R%3KXBNkGDDe1&cgi3yTS!no{ zD$B_reK7mr`;SZ)^bc@7ucKTBZ1gNYmvq0A-p=&AZga`& zw$8?(-@gkL;#SlV*uLO0&4hYsKi>y9(erbK?G^^4?au{5HvMQ_B3kML{dhe{>kp=~ z1Vc@hKOR4+=PJqr027SqUwF5~WBUGkNwA<93CB#IvJvnpmoz?n?aeG+-6GGDIwH+eTH|^Ac>bOPjK@w=$iD$HIPv>g zS#GXHl2yCYcr4=CX*7{k$auOX;o5Qu-9D@Dkz_8gB*xj0dxJEn_ zjbNLY{zeWFx!xip9#j+K6zt~h-Ke!Wx(6e@)!fcb$itJa4oG@EWn~faG&hJu6H0$5 zoxq%GZa#W=@>q^`qnq=rwjFz4_TXd3-k+U(|KuZwjv@4Y3ZaG5cmd?G_aWEf-1Ky7 z3aJIag0`u9(pu0 z`QFEl96$7E5HGbVWg%2I-D@il9zJsH__6GXiANr5m8MBz5}}mG(bT5WEET@e92Fy_ zf;=nAqJ#pCmQL&CJW7{VLw#C3_Ig_#8q{jo(drKL^^pgUWgmXm`N6WYBEgzgXGL@Zt?~%+S4^JL{U5$?nYCP&_eEUO>9z2|V=&^^bogN{u>t*)2 zUgiy=<*9T@T;*Faw*=q5vuoq=$;s@GKg?Yn#;LW%F*&hZTTIIqkpnB6#6HnE;@7vu zO@wjy;bX`2(S40gZq|plcat=k^H_@D*lisoAN&dZ0rRk4KB1TA^zwqPU_Y;yAJ@yL z^>V+C_mE!xcUpk)dxw7hpY#&StN49SKYvv(KdG1h$aelE{rtUpSeNkVp<+tQzQPyjj)|HC&TI!d7-4^+6QsaL95T5?{ z{qcuo`PJuS`74)Ynf)zU{>AUh@@p%ye2+eITwc=8xny{+@|Se{$8B_LrML5CIxg+? z_VcT4|7z*&^03@o+UxD-*SmeYJS;bt_ImsI^_Kr?+rL_R`*`rUacQr&pI`6x*&jgu z75cH=$tA&s~L~WUVIqN?Q+R*E}0LP_V#siTH%uM>?QMKJZ_KA znJ?qn>G|AV&wT79vjE^ z50^}DFPTrT_U-gMUX0J@ESDYM-X7!G>6wompU->oWj@@GUU&?*x6k9(t9_>HB?q@} zFB#un>eIw4yb*X)jzFu4cR%E(v%(wcGkYWZWJXhH5gR__=P*p$RAgp!YzVtDXuk}R zN038|x191W6Mg@Po&{~KGi{H&zw{Z{5zPyh{T0Ye79UBV7LBhYRB~r&W7-qrs-AO4 zUeg1$Wq4)-CJ1eur1x%!_?BL607RO#aVlSHN-tIw8*$_Yi_hU3Ma$SK6c>9=WEM9h zqQ;H4dXD3@sKwd=`a|zzeelr1#~xB2IG~12o%<7+8XU?ODbqW^d+gTs{RHe+dy{W! z)U$%|rk79;?xX_%ua`8LUU!eYO z`jI~r@?K~jypnz5;$Br?IQviasHzh!q1fB{l&n{%UL^$E2vR-4sovMEHRK=L#^Y@q6Z--&mnqpYm((akh{8Ix3+7J##@(aRj{ZPZ)SSKhDe< zFv8-NK(G9$(4TzDM{7=;G6m8|%q|5}%JNuU8L>jFm>1JT-3#MNo2h$Y?DF+%T~Us$ zFbDTmE>84qf0e0wwm+bMlYRSlabW*mR{HmE*MR;__U+%rf&F{gH=up=uh-a2qIXA? zi+y|Wa=kY_@g43f|KvdVpB*6o%PYOx9~SaY_LcwH0rFo-_nTkC1LZ$9K>la@$WP-} zSFTL-li0F)@V3du}z5S-fDE9*wb@532nCm8COg3j)t{@KQaFu1>Wf`{ZhCiw|Y z28538M4z^HWf(0}AE=)M8!aEkUv6`BoFF{jz#xgniHAP26&?%1B>1RC^rHk9E%TT% z{+%uA8&eU!^^B;+HLOt^R7((vKMEMGjpdvmz!5yTSN1XE)<(}r8Fg+LDTZ`g*h>(j z1z^w!kQ64VrOYHB<4Qc@^6EZ${e8;wEBy4@3W@1AdPz~}^+fa=)2{%a0qYr!J^hT5 z=?OL(G}qWM6-8C#d?V`nxZO{QGtu$^0UH&O%x*o$HAyO)e2haJM(KVGv*e=%YMsxx zA5VauVI=Ee(Vm3RTo?@=7A^F|8Fx_O3K%Vs3|(3rGk^3Jd`CBpMr99qlt!_<7-KP7 zGWTESs`pqC_vSs2SN3@{Fd7C7=iDB_M=$GeQ$Owa_h7C_E;LgNE(XlZ5q%)ZNA-Dt zW6AF@wFc=IOcF@E(JhjGG*0aZyZ~zEnJYtdEHQotccNiT8Z#4zsD$5#7ERE-qdFXt zp=Zq9fSO@6zYTgwg}Jp=@FQBrfDk9l6iH3jhPa7)TJ+w0FVJ0Xl9=#RrWE|*vKPc3+%f)E$p~hQDPOzM zaRZZDVCh__w@H4CG1~2slxuVwaX*aKMf>`i3y2t7l2Fo{`y?s3Euy8dg`@^G^^<0r z=>44rT#OINBJO_7WuhJ(B|HcHy%I(k#K9^h&P=?-*aJ_tC^>N-WNU;W$BA2Jsfj<~ zP#c1e=((gKh^TW=6P4sZbov=!vTb87NIoX{tT;Fm{JHHe*KQwl43^qxiRQ9=mKZR` zl3WQx%0_LBA_s=9zI7zfZ@{P?weO@p(UJVgvh2&$<&gp3D&!nJ*J)E4>6p?H1evsf zVQ{F+yW>N^bgh4aWc!GJb+@Ax$s?`!aYY&{X5Nies*{k8&Nb6EV;J=f;L+DQy1m4v zu|}HC{enVOI(13uPOmEd32w`+%n&CbU1o*2+PMUk#za5UzZb@DQ69|SepApgy+0^(DC@p&iC zApc5wpnt{SKl)`S|N1Rv`JKIB`M2cn{_S4=Czk&Q-?_(uvsV74U(y`3G%VcyUte*r zZ{ht~c(W^BRr#^fv#=+oUccL0-DSKGBsNs~1#r8O9p1Ge>2A*Sb9GlgSgU0h&h&6~ zpDpA2!SFG|8*K9Q-jHtW(l68H@Vwakj=e9mCiB%%3zjYOpKKc-?X7cQNdG zhPGd%v|Ak3KIyRj7y`U-So_R3?0R!pkK7;*Yt?gD_jOp;mC3#?yNi7tc5ecQ^booG z3+W#A&|CgDyTh5T!|=g=?%FYBvhUSVlpEb4-QOKO zrXZm7B41&im*(=-ePk~Aly6^jDzh~2g9;lGWcbhz225HU2t(oWs)QL3>-p{r3QU6D z_s`|Jf!}DG@yO7@9*jB+x)nO2x=fnN4%)=AsF%hLL5*JdU!-#h@edhrf_2i2+40d4 z3?h`7HbWcWYEWcWRfSA8`LmXtI$@p=RkFr^kRbk5Gj>oRlgG@n)24HVykIjXjT(vZ zE!`kue0#|Xy=M^FM^2ve_`Gg|sPq-A|5t4c)i-;_5L{$ao-tIb9++9mE|>Aj0la3S zmaA3S26>aG^w7tY#D-k)-96`fMv?22z_-{V-He|8w$PBp78;_vneXNGW(y79aJG>3 zNyyJ6B)6>VC zD9CnVYbn!aX)-#W#wtUsCE7OyZ92fBpKOz?B-la|283!d;ykk|u}=7`wRFbxWW&9H z&=hPf{CDpMY_YaMVa6g`O;OkulGIKftRHx`kozd|_*|VW^zo|=40*PY+)LouLN~1~ zWQ2djCm?N~vXxd+0v`tIe%g0HU}#txLj@Taf+4+rjiHSwK#?U1% zFx1l+AydZ-4E0mi$z1ED=F#1Sz>KfIu`tk>^8!Ny0z(aqvQPPMJ&d7C3Yi>cC7sTU z^W6u{sy>?tFqDM)-$*y=_UoGwN(3OSMbJv zh9U5>3L5Z~5NQ$-7C}0`-9z$7005Du3}eHXY8&;P34%Ak57~NxV<)XDY)%<8+%}cS z)LSt)m~lFFg1_o8rU+`_)kg;Hlv7U9_3?}y^QakD;07^vPI`eK@qsGO*jWe0PBIeI zlBREn8fH*39@WA!CWQBt_K*CGNMq;o*wlb5xDmR&`DXrKZ~uOeZP&b5`jpdebK)@l z-G4RGH!b!L?%v>9U*5{ws(DkjvC#0AYWHP`wnAICR2xgYl?T!DwrW{p)Z4dJ+j+1t zVo(q_1Irwl`DVn~$CrHOZPiXLs=m$IE*G}~{*Gknj%e(?28|Aql0O6CXpZWi0guQ+kMZP%=B8)L>}N9AO?pR)c2L~d3Fv_H|e z{ei}Ux9z%rp4-($ZYVY-XN!5xnD4)9z3tlk+K%Qwci!JG_&V)gvJt!+xiPI727acV zh6#db%qsQC?GD+UhN)3E*GPke&5y<^3=Clobnw^U+|fCnXoIZ@PG{zGiWEu1gWwrZ z!5pDO{D>PFQ~ov%@RQsdX*!ao20kPv+BVu1h^TK&$fL_`DxLKmpR5mdjU7KIoOq}g zm`*cF;$E^umLyLz7hb=2{kikn5w`jI{>*orI81-)|M-CG^V>|n<-|`paG3s`TbCSl zvVCc6{r~&i>sumA?5ZOp&u#Lu#Ngcavc!-Tc9Uj_874t07}8P^@i|!W*#=Wh1jd4_ z8E7nQ(9^`Ev9Q5>v;O>TPD8-Hm=y1J;_zZ{=YQyLEPU+_=R!pMPi`fZzloz8(^%jw zE#?K7PEX*flJJ-rs_{mA5AAEfuc;0~7`rkDce(K9k9B<1Dt9!~&u-;&Q03j=yoRtb;k-K} z)d^ptPIa_h1ZKkpI&Q4eaYiWDWM)V+fy@Ue_xKN0USLoRDlEE>< zGuSyv)hS0KJS=E~d^U z)fqW3Xc^o7LRN-zsfzf6ilN|FveKw>H)?&gWl^-g2TO6n1xreeYf6kIj0y0H9pn*v+L6oFm(~q&C ztn21L#{^|1Us2L9H6(5(O;mt+we{v@Z|L?L#g@9+rzs7z4h3 z^{i~K^T>c_40wAcc)$3~ZVV)Y(Xf^ZxkLt%!2~`Qkyx8Cuu&QVKQ1E!;-e0{m|QPo z0BJAy6xuK6L4X_vyW}%jo%aS&A%txSxcJF z*k=9|oACfKKW^Oi1ccsW^nsVeA^vzsbJ+Gd%0P_+L@__m3hA zy=Vz8BMZGSjL1SSj4K1icbDj&``Aymm65dg6kG2r^m4tw{KFGe-%C1*9k{p66c-1` z|ME)j_J@W1lM1)t*3z>BcC;#w3`6ma+|7;)mX$<&NY&;(J5^uImLk+IuLyN0? zcl+x!;{G<_zR*on0SG@R<83OLkJ`<7mkKu3kdn<~1!cd@$JWv>c+j2z1KWy0@Q!kM zf-o9sJC-yD{1Y;3pK9%1eW>fJCoO3>8At|RZGm0Kp+aw)a<)n}5?;5YF=d1-nN@=( zDB=p86$4`B_G$5<)K@^k95LG9!dSfizWTrO=$kREX*^&4k4_w>zr*+Ud*tLBXoLM;m&p9c_r~yA&Ln=4a_z8m2XDa+ShmXBY zP``kZPaSlC(q}3fsKL=ef;H$6Kgp>vae0N0x%98{5tXl?-h57H=JTU42CNV|GcfA< z4g;f4RNF~2Mpdh}rTdNJg9fQSOI%)bx9GbqEgvxHAj~erl=OvXHP`;#O+HjP2 zwd)SfppQ=|_;v7qklP-bHgtLUb1Pv0ERYUAEa^QGuMjtR|rBnKPUW? zHDU1Y)R}_Q`C(%SnnqgCx4AwIi^rg2s8MpdznQyHt0<~&n_#pc0mfv4Ff;gzQLE$m ztXm4Q4QET9C9r)@VeGJ%u5nmqeDDAxBnowKdJOS zrPRY1C_`m6u27?m3xl5lk=ZNqoLu%sja1R0%&!;}85|oJq!Op`Q33-(?tTNCa5HUr zPELQQ_ZJ8zUH_R?ah!%y?K5+2fGhnZ*H;jBkUv>ULxLfVm5gsU_|VUa$Hs#S$GuUT zN!#%{Z7ZFl-p1FS4FqFjwI7G)Y%dd%K?Z|@pYLzlu)(LF`8yh4Me$w$s?FrjXt{ZE z%GZ`ET6TJ=e12)U7N~x(R?9A&*?(Xmzpzv}f2No}8#r~omd{lxx$}XueKR>MthHFQ4{os7g&n(HfnA}Qnwt?~VzTPzWTEp#jLqH@>Y{%hC7ee)|H{^ox; zahU!qzy5D~&?_hDp>yc3-RpxrHU`kLN9Kgg6*3kr*~=w^KgU*HMX0BF4u8KDpPxes%1A6D$&?rv!HJZE# zg10e%?=<)F+HVZtTo<2m^yi%S_t6dhx|EFm`rG)^T^k6}X&dLAw1CtB?&3fx_8Dgx z!Hc}~(Fe$fIQ5ib2YP`Ra<(D4zhu|iO;#Vg%Jwh_ZR zf+yF~4&V-|{NMpe1ZATwj?+Xz-!wb1YUV~z>p6@H&=E}onASQvKOv-Y!dZq(FoEPT zL#!pQ&%gTf&TB*b&Cl=CUv}a!{qb);&?E8Rxc;v$ryV#PTxU-_Sm(wD>JTRv^+MA2AM3#~DNu+_40V5|9NyWCm1OW%!P z3|Kb~SQlbi!l4XrlTdtF%`*lZ#*nvhptmtVs=l{z;QFNl(f#X15C%mM3}w*G7*IDf z>b+IjF=&FMv^Uz$m6M)6ilPc*pi{TkOS8cLVn$GMEo~EEwOqnTVb8d#*|HwWcw3qT zkOH(g7V5V&5d1^0O!yBHreE+dK3Uw5E&1wQWu9mWQ<^G*2JwoZ?L|U_WgZWXG+3eN zLs~*!Fr`|V>8KX6rdiou=gPjfOW0HP*N?KFifCciUBa<&aLu0oZI*PQZM2;5VsfJ? z`?T;G%D!_Mq{Y_m`FF$8ao(U^!qh7LjM7u~`+NSqy#u|KeVRv;p66fT{Sn4pW!z<+ z=ihBqj5X-`d;X0qWdI7|3Rhp}PB$oM36FCP4SU+2ccbKWb3yjKdXmsbkKVlgc! z-x6wz48)QVEh={%4BTkE-4YqtPRkB2CfCbYK$?(kZ-KGw-3Z2l^^8dc&R~;D~wBndk3jfPJ zP8_EH7hib32fa)Q{NFZw!L|Omn`leGzPp@#O#9yWEdf9BcG=EawgfykDz^lC7}J36Gv_dFz}g<%VYI^ZejTqUdDBnSCB_zmd1%O zjCZN`ZXJ$!yh~kDIcbUWMd+gLHN^GG>cI8~^lviVk3U%h{rh)C>EFNM0sWio+rNtg z`*+1Rp#5(DoTE62-pOwT^u9tA80dF>$X0N2rZFR)WGl!c0qD+JgW}=<`LC=ne}jg3 z4GZ}vm3|}e>;U<%q?y0l7z_`Tp8^AJlq}=LKJpWe{8oUoZ%l7q{~vtDd4H?o)j_@K z(~7?>YyFnd&$Ow9F$x@{`^n#_L#wVc|BlWJny}!O>An;wlO~)D^lsn_&6|$KlwkLz zKU6Psf0$bacxA^&yQE&vfq++5ozgW{nFFiz?ON|`37GY^2hUf>)w8)`?Ew8z@&3~X4;_5$A@zX+YS^^(`AP@4e_TTKfJdGdu{oO}a zUXw|s|Mk6CXL@%oSGw2V?HXTuUH{4i%jecU*#hMEr;%`%->gfz*#ajA*aBm|OuH9v zg1@4-%dMN=AOx*`2)4vQlozPJf&Vxb+IMG_jvHy|_lFgbW)6nNO*(s{EnDxTfo|!e z)ThZFpdabn@<*lA-7}y+F@Y8*2qsgM%vgkf*(V`_s!k9qMyCxKz{3vb^PsR5`qej9 z7}u>N%f;HF^_+F}b#5zoNkU%S;3l^fjP1jTXh=)C?kGsa6IyDG?-JT+NfO#gOB63A zH=40POP*mYSQkKETJ6Sy8AH% z$-ey~bAm>xw>khT85q6my}&{L@$&)){mqF%#=`EpG5h>&=RB;77jzmy1F3lPHSwp7 zz(c0ZP%0a`60Ry8ovV{K;lc4g0qhm3%@9D`3x;BER;FdJfpMhy5QM=2bzV>!W(K#F zNxxwEQW^yT*p!gwMPt>}e_dt+GNxQ^o6kfuLc$EBrW!U@$S^V0`ed!~d|v0uzLzBA zDf{b2*-r*TT0H3%8AyhcT1?gqj126uWC`t}1?74x`^XZyX?6s7%Dz+Cm+uOa4>I%P zj2~tf`zpI{0%hKd4D>ZOZsaTodSJ z6k{17@K3Z69gt9p5E%8$2lP;zrDjqY)=`3K@?e z)4___0i+>J0P^6&5U`m8X<_VeYvxiK8_((aPM~a?Yq=d5Bf1&jR%<-J*SRrJ_qGRn z+k@B3_TXeZq6OWJfn-WctZ~c0tpYr+h&i3851iaeW2{39ohwE-EIGhS*FY? z;WfH2v=4|GL-46H1oG=V3F7xrE!R;=X+YrTGsDa{N4wjW-7(7DZOd-;jXLSoH*W|g5JginF`*X0v;b@PVwBj*878L9($34wMP!V z@K?_Jd(7Wy%PW_GTI#%pP-b1A#ubJ#uCFBoBh}u@qqDkum0?nmmp*)kHR^in1V9wR z;7mpWIJPB(-YF!1)>0M9Ji{_wRXZ~wVK4>`j_GeBVk|02MaSRtD>?<&Wkh@a392PC z_-rIyUf{;%Bq zWydp|>6Pzo?V;=&`;+ru+U~$%`s0(TqfWLj^@H>!_xd-;Sn&27cxgyCc^cA|L@<~N zMdP7lDiw6eK}rQfcvI0DErs3EQrNUvgdKL9@6_uRcIag(&|3Fq+iD(pUY2>!!-6Omik;e`=@yPtgKutJ8qV(ZK9Ri~coH{nsx!AWIs zNq21lwm4*6r!A@a+bQ@su&#$20RgGy^N2$e*Cp`R8D=g|q+u_()=>M6iOaASTxC)9CUsLLHQ2Bzr^}ztL~zR_=yM>;VOdE5i0E88q6s0EmTQ6P2Wz$L!kPUC7V-;AmGftc`Lh9HovT!G z=L2WUwaOkea%y?j^fsaUCEp7WX1TJc3ijy!Tww3x=d>(v)mglV=$u+yR4ci&IIli2 zuFlOboyyH;tHlrH57d^+7$_42O^{@kY6GxGEzX+i)%<)luO3-i%*)P-P|D{@zf66N z=WFT#DG>=oZI7$Q>yErPCG9@IBrEq%*6~B4IrmBi@FG-o}U*wm{gR@2h&Q?Q2bG zee=Q=YY$tv86)Kl?CN({R?8wRk*U*IyJ1icPNr?HA@f1vQka5G0W${IlX?!fe45pG zH-mxean=zoKi}86vEYR*ys*Vh9=0%yg-|G|MO=-A z5WcYxlr;l$oNl)m3%Aqq!i&l0pK*^dnq#2t%T64o|9=j?t8WbR7eC`(e=}{E=zP`L z$F{y<;u(Fz#8`L3gk0Q_L*&>n;r~O)=e6cIed&*!?YreT9sDx`kFVc5h66pWpYaWa zAGyPcBR5PK4>bED>sY=2t9&lPvu~#{&abpt6}LYm`Fz;N7|sVd@i~U~k}|Gac@fI2 z(ViXS$|`fW*YNqS0sWio+dncV+~sEeR}{u@$D{YIGOk-W*|&ce2lnrZZ$SH^f9|qP zqIdEe3bNh}iy{|pa?>>-I2jxANC0eV8yOpJTn!3(6;kik73JlX-t7+y`6rcr9G)E@ z|CMw<`G*I}Pmzm$@;}>0ei}dd4FzL7>|r{la+rmF;29dwUVEN?_M$LJ3CSxL_w{NCm4Ds>jRf7ZLbOcRfLMrGo zdrO)v8NdM5dD4c5Oj=Skd`wMc$0yt9h+gA-#at`v=r3}pbh?Jl$B?G6O5b?M;IDyi z)JkX@kqHqliPxW{{7&%TKnO z_PNi&W1@fVbz}<24H3OLTz6&vLT_dH&93azETqVSc|O{&RDV-IQEqf&;L1e*{?)}t z7VNz?&dtiAo3ZfHMpEewd>Tu6-e}Vi?{0>=+;+ngG%FAu2wP~XD*9oK3W zjiHKTU-F8X8*#9nQyPyIV#dgu^T8-(`c0!QA^ssZoG}IMw@{ozyrP>*RBxCOojP$NvxVatc6vU4CEaP$F_}zm z?Mq|Dy^f$TwpQHesE**QAFOj@<6B;2!i!9-m&imU9@9e7q_9OMBB{6*S>xxO@0P}f z{*i~fX*uD=vfju^9Qi{h4%45S{g*xH;e1hi%JW}!uUEES_{700O*BQZ zfs5an{sJ$fH%4S)Y@6)AAr*~^u{>i>z&2UuQ!c;k(#IkBkq5bF!O2C{_b|vBXT95= zhL8M9+0I(_G<@q8{s@F2v$++wJq`cqFO7ci_5Ba;_`iExUt>?hgDEG@pgj#YyRqU$ zE_z3%-t1{k!-y-%k3jUgdvE+Sronp}{8;UL%KLxZd~f^HyqbLa`y0Efp|P%X+5n1# zqE}G*>x}Iz)7>r3=DLnhoim`@-39|Fz0$XD#>u3qCM`A?lfn#Ty-xL+HV*NV(zKDl zB+nYFRLVF*!$-%2!yAYh-(Er)ZyWt$Dx*}z2JGIcM1I|v@S}A%J}`DBq$HUYgR6wl zUqpUD=}t@Vvdf1h%=n<6knNJO<&=Qe_qRW4zP<*g*%D2k39*7{&0UG z>cC<8OLrb})KNdU{_z?2`Zs7#L)$YZJY!Z{!IYna$Ewn~s;vUPMhI`C6 z>-91wUfn{8NH?xAVNG)~7Fk}FJ!7K3F)?V*!zJZn-$bA*0}_FT^$ntYJ?!N%Y!J1> zjPt?UpVLbJT@7+igL7WUHg1A6CU4J!K7Wi^L+d7A@tc3oL!(W`#N%CKg4A*P$?=fJ z1fMxeD6?tzkt1CH!IMg-ZPaJ##JMB5XWA&_j)yuO*?Idf5F}{w_rq*?kzmpk@vqP^ zDe(?UA44)LCsu(oK{y1H(Tz@891GZG)EV+gqlF-yx~PVyj0}m+iWy7RNUD?K^Vfo3 zw-s~!OCmmDhBUy>Xpxoab*}7tdm6knrFE006io%SsC)scEpiYGCA6eWU>caF^j4|t z>)&a3D=jg+nA|wZzOmHto(8$}84DmUjm|v{1}$ULo(00U?rAXaRMgmaH5j!Bb9<44 zlk~9v*RL~jFlf(%7dhycvwss93&ND+yU|)s3eWpRh0QT*t-K3guJ?{S5Qg_Qq<*7^ z$i-E^4ar}p`~L?IIlacTuBu{BgL%aysBsQuw=J)0r)@xuE4|yK*n$i{qmItf3VWV) zE;bBfCxn%b`5eWL3gY8lE8n)nwG@l6;&mooalouNuhhX@r*c#IRf{pgm_`X>2(bmL z0o5?INqZ(3+b_(3Mx7uGL*!z`IBNhKh6wOrCJAh5$XI(tIE@K`(+-MVBg~D2W8@|`Mz--6$OG3e3N~(NpDL9 zOyxeVZEs73^S-Bbi&qJ6OGSTM?do5)8uwQ6~PP0bAd3(7EttWoOPg0BOs8zTyL~{Q`43T{{(CAh6XMJ z7{meff)m0MLv)_Oe<_Q!Tsn1vc0(5Oc!FRO@1%>ZVASGeR6c9ZI-oT}>OD6@NG|;R zUgx$z-OEDqvXItG7E(ME)nf89(YC-uJQ~uH-EX%2+hIl7v(kF96)5;izk`#~zsDcE zooVboe1bZfibtXmq%%ds@-3~Uzd|jLn>(hLv*mmRAMsn5$%-!owx%{WN5$#%((+=h zbvQlLyhWT7-t=;73g-e7UmPTaDYbd)Lhf9)I=@t_W~JRPgzEE#ykkk(TT$k?mzb6;fejh77xbu?qG!QqN=!QEK2_h zh7wqO$v%#SP0bDBBa5xlmr*yRlzvYix(ze*9mrN`9OuoU>W5l~m67GrH*iM74r?wE zjs;_>P%x+kQ%Tzd_nzP-g%w!>DB;W;s>pP49)6@{X9YI0n&OxH~G#@=Q>mz2|%!RuqkG|S*JYQie- zoVnvSqjS4M#9{D9wUTa0jA5T1CtA^hEaEn&=jIQQVDAO%EZjPcRwbv3S*4(;qshS$ zi-ZlElCwxea+0XTl98J5L@Gr9hHz;b3Q0Bm{wFVeLzj73gT!6YmkHNNP`X&e>&8S*jA)NzS^O=Vgzll zsINu`T@=4R-!IqChOM8CopeimFq$?P4Tbf?iT}GUMj6IRTMX1i6U#>{SzssAf=cAEsJB&4)3y4%Ck=m}&(z z9LJO!)Y2YIc|dtHnDT;}8p4zh)XG6j`9Wc3SZwrq+X+ zx{9eFsKwiu3empqVrl~@VF6Pep!BDh3WIw50#lu!zP-ez8UdAkgQ+O3_#RU+Q0_0F KT58`BP52G|y)O&^ literal 130351 zcmeHw3zQsJd1m*_NFKc(ejr)4DJx+0$m5!RKNOFnr;TWhBuirlM~HGyPxW-Sdb(R( z)zXY?2O%B_Y+(`H!Gt70$O#Uc5Ns!5A&{Z5(3GRY+%pkZgz7P z@AuzZx2w9UdS>*p#`bL;&8@ocd+S!+@BYv8Lq9zf3Ww--vJ^x>V;XoQx03J zCtHPLr9P9_=bhYZrz=LYR+!gVE^AoR$_1mWX()F}pEo)q)?iCFJBO_SbE*@GSfgX* z#;iVex^S*g8!I-Z^)b;q*M(j;8ZEUs-sPt7CQ|<<0r$sArXW#iX9SxlpTVCTqkvaaJ#!YUw2v|09N8 zD~+5~#+Ad3x=tmnyBFxLZkS^x{%AA;>TUiF)+!d!thF)8BaOC+x+YmekrOd>G(MU* zIilR%jY_JcsnPVw5!Te3#@a?}tX7#CYtEbHMm?d%;$v-enW`Ks77U%u$i#rPK`UYe zlZc+sk#o9ZZK;%$S#}*6Q7VQ~ZC;bM0nEH{Ku`~duJQd?V5gH$J!v)hf3|T zJZ7goWepf+$GTx!FIHv?HC*ktubx7YOf;H_WfF;KE*VoZ+0J`Ac*zI!dZ)bG+UQD5 zs;P96rM9QagZ$%AURl1OJX~IF->oaJFK-wrZ#-Py6v11Dx3zrZmAs@F_0r$X^49zt z%C}zPTZW}QF;Ko^2`>zu{nB5SS8DBE`RXr6gO_zVF|jpVPF~H0TF&)bpXG7)T8l-~ zNi~ygw` zYJ#Wq|HfH22>c-z*LB^vm*b}^mWrvVRLogi-5oR*OC{7)X3-t=XXL7qtID$>#?lEj z6VC;%s&q6C}9O%GS}v-S}~c>qVJa)(v#eD)IfA_mu<$feYT zy%W1@FaISf-cWDPPU$VhEbB_U(KL9q&Vo-yQ)()f3Fvh^noOy=WWbk}>b*@bv}*H; z*(%hHQlm8sbWGY2LoC^Xp2LdZAW1rz>=*@pL_& z#)N0HK?o$COD5H%4U>Jfu5IZ!{1$a(s$KHVM?9Cpqp~4%Z&wpB%yTTkAF!{|RqO%o z^LH~5PiED4oI`d`jq!9;O(%l~@8EUoCLH{Wx?v~@WpBGw0x>|T$AFru+dwaPZ^wr6 z);$Xxdu?LE7?!{oRslQgJTqji(wuX8Rac1|K!@Kh$slq{b(0DdoLDPHrEZvodQp#1 zb`7V@QN=J@BZ^)#bfvWYu~PjB1xrGCthewJ+eejCD$y7Csc7@Wny|ImD3*0P7imN# z(9RTy&~a?4GNYlHQ=`h1UIgXA&}IuwI)RDRrkM?P&W${Dq^VNQ$j%dUDh9_Io>MAy zg`LkvNA9i}=E)J!;z#MMlgB6~kL}raVB)~!9_3yorl7ML-R$2pxksJRP0x8;u}{*a zo-;I^rbWHCg{>P}I$Nq8>tUP@H8HNpPEJZtMT9fZ$WDyHE(PMpbVF-5rwgX8VR}tt z+;h*GvhEBz1{WK3Q=c15(lxwDjUpyv+CD>HyRH_)))1z>(6RD% zKRLODcL=>JVEO5@jn@1be4}yChQabooAXUwogykvX_j6CqSF}VLQ@wD<66z{g|M~K zY}6`5GT7$sHjM;*`Z>F=kayd6^iDt2m-KCHUj_?OZA!N&AgLDlrS4O_%>SLv-hw@vKbHx;(@b}%f`SO zv1t~*$zFR(Z`Jjh0^WSZdb=t#MXwu|5()uc6ffj`>M`fg`4;$%uFUA|((c%u1uq z2{T}hgMb7fmHRsRL2ErIK23+lQ>RwZgf1 ze2-Y>4m;hFxC6vuTJtAjCnF##j4DNQjzM`|>W_wqOv82{5J!-27m;?*Vb)SmiH$3) z)y-K@Y3xFi(|Sp9A~=yrncD?A)QED=eG01q5J+5+X><1=H2|a{HjQot|+`#~SvmD~FWUvFugIsDh}oXNVDpo`^>$UIcOE z6ZiBv=PMY5N~|KRaH2XJx);OSgl+7oHT#M}b)#H>` z?&?>@m6Sy0*0SO>N+Pr5Fp5GAwVO9oP6G9(jZlbF4ocA=d0oY${FM2HDdINV8&a6ZX*rsE4D zoZdPJ-E<)n4t_jUvs;J=Fy$gt$TMckxD4?a$@1*#5G^jVR1C}ANbW|LzZ*rCAImct z%SOS>viX{9KU{AsMeG9&*{aEH3(_-PX5i-AecH#&3+jA+65*oJYo$CRt{3EAsls#4ixwE z1No4S8g_0wkx0xI8_juz|6=h`c7Ya@d+$+>J~H{1BZrl3C!(`Z^@V(GvkjHwlbw91 zV6Cjw#a+12h7vwle83ffb{^akfRuI5O zK=l;Dk!p?IRqF_-yMy`grFOS^nw3RQs|LS9FRN`__VhBHh0;eq85k~_u@{ol%AVts z?xSZm8I6-PP%i3wpe!kkUA#MXmx?nT5I1wBl-b4^eHI)%#iQ?n@`-dBa?3V9-gRK6 zq+A@w(r7SbtIwp|B-p zdMkw}1D3UY%=MmSt&LV)Fk&GI z#buYg9frb997>zMb3N(>Ey2;%@zuN8C|63R5{ZtE0Cthwe~+#Yl}|y=CzbZeFHurD znpLr6g1fw~R4c$1gDu46nTS*l%12qhA|Tb8gRDq27K|6IjgZxVoCJh5f@e4lnH)%c z_#{nGahhOgI@V`~hxNH;p*{_z2^aSUr7}?vbCM|^F@}=j5ao;akub3~LBpEQXb>@m z6^C$DEF09-OdQh56Ojj`;4iaQRUxiE>0yyb-pGm!O=T3!BLt)xbB4od0#M|F=0%Rk<1P@J625NAr zTz3m`j4IFzGmBNP;#2JJ zn3{=qIX?kW>H9Y1(ipAK+4W3B z8t+Cb+ecuGYZn4DbJ?EWH!g?A2PI*jY+E)-mJcAx*l(lQNK1`0vRaLrQo^I`oduM} z4wMM`HXsQfP|>FExozySO~+a^6`pSdO6MTs84Y;nVwrRf%RAj?4XcD1iwNYLqp{a z*5Cmx(Q%nTBKZ^>g3lmyD00siR{hkSS$8VLQ#tf zzV&7N@*!*JOrtX0Ih+q4HepGndSWOPid5f$zxgy(Q7z%p9eZ73-GoUQ~OkeQ$>kXEmqi z9ITfbm|+_*1L~6rpN_S_Z%|lO!M^K zW@~1bVt-SPH}xXPAg2sA-QmHo_S*$HmC>?A}Z>8qxhXhz^P>t~9X)pBfn$3np!~%(xJX!WJMMTf5&#{gs9mgi-n|GS zPef0uykQ7pl6>RHC=*$HMMX#07@x9MunoUs9bgivT$ZynLlT0dFnf$mDfJSI5puV% zb7gN?T&k2!B`ZW{P%cqhWi@}9>jVh?QU^-=71S#3)n=m^iH@kO)~>3k)J%g4f)pCK z61fv83MdThT?KmEm5Kr`J2njt5%DZ2Sjl7%xXxsfX_t=EBE+p9S(v!B0@S!cB5&QX z21G<;01twA|4Tv73*V1ERkBP!Mk^@(2WlyT+Uzz#IIT)n>b$Dw3D7wV9ydOUqu^K(!S#Ivagt>RRFENgl%oeyFfq)i*=PAtqsm`*ERyY z8Niz?1#^Jy=W^^j`VbDq=5(b54bI(GILb&e)(V``agN66#i6TPtHH3?b$tKCo?TIJ zMqKCmHDqgnbeu&?+6RXkrFcyL7je# zA!aIs|FE1))`=jPJPw5_wLUeEQyk6^dc80OMYM68G44Nj=m0w*EJ+nlj1JSO6=tW| z$o_5+mQvr(nzfDFz+AFDRG}YMv~A{$+A?Cy;tchs&}tRty}4ld=$t+RHLo91&grcN z^wNrg%8TOy>?VN@g=r>xQRM=}<{EWa?5ZJOp=GLc+F6oeXFfmdkf&H%G>wjN#X`MN zuYgXE4oquM=mrxDClgYoJ5hvk6PbT1(s>LTc({cUHH=2#6lfAL2;y(BNM=O9Ou&;T zt3~`u;}&uwr5%rvNu4KDdlrfGqdhkF#14$~?d*_NT7k3SHfs&*47E=7j$xYEv3K0B zn~{pG56BKk@2>t1PMOvRq1pxm*kBa2B86IU*Z6*ZFzq=eo>aR}ZB*HzaZ^-!YsZKJ zf<1_}Mb%!)APRt=W8f5Fr_?k9GZBG-Dtjs!(>+j^*m5L`GRSwn-fSSNw^BOp?Q*vOu`T0SY zf^za}3FK5Y>_HQo2C9!DT$Vtk%67@GSStmZdhlJgH{*c!E{sVg- zdO*2vT#36X`8waEVtR!zM??trEX^h(4)>B~Dru%yXhBr&T(7^KD*|sAA78}bg4ooO zxm*mKc$Wl}D}{t7{LKB=@TOM%9PPT7yrET}r($wDTgb$)+|KqgD`b)dC|Q8Oo7}@z zptCLMKZwxsBqqtQr3aS2ok4sKW*xl-AL)#S6bO#nVr-X9iJsr~DmkPT=@^+E0BLxP z9biBrbO?>TQx%ri znA1HRrpRJ>Q$^Kq4A)c(ZP+EVGm|a;@unzBcgzMDwIHmJF^m6*a87LV)h`mc?*#ND z;NuQ-VyM%dC_;^b9Q5@>PT7&1YP`?3`F4y6yMUC-JTyd5HxcoPKtIj{w|vWaPQHDu z$^}V+Y_GS%YwE5*S_P9LQqjFT$?$!K$_s-t1$!|k1u*z}cDfg^h z?%7>DqX_L{&Yaoh29U6n_C&j4UXFd&oM!X6#D3b-8+s7s#8insEhVJ3=zo?v91uQk z$BR%rvm20i25AaHNHRQd^r2Ab-MCp%e7sfzTJ=_I6Gqy*%w~qjkz|9yJX95_3THi%|af zZME5|oMC2*{r&nC)#yHn_0RDe0~Uj8_QQEbeMgE7MvoJ~f?5PnA;XL?IoWj(4+P>ctkjz;65d)35yQ(C0$$L8AEH zgu<&|puFcpD_@|Z7eb*oz4Ha@OQG{ezl|?gn1=%c%?;L~y=D&~8!C&esjd z5JZu-OvT<26u4`_zZo#9BW2TU8oS2Ea8qtif%iT;cC0Y1H|lI_1+fIKxyEX>Gqd+V zX0r;xIt4Bw@5!h!HKxQ<$z)QA!@Wzq0H>tssGdmZiA*tBN)(If!gMKDNJITUktk(L zdTb|b6KFxqPt}?dmR&&-M5kSqBcteij zsREo2%phIY$TP@{R)LX-cA>p}@*n9FulI?6qEGziDSlu$Byc^;hCdhaKL`&zh4>&m z@G*)Hzyl!_HsbV<3e*Fb6jEpEZ56_g<&gcC;>}h_Z8gZwNk}Enk}Nl2Bt+{q$aypY zd-1qK>Xc!GR1TX}f=z&SmD%(2x>02eri|>~RAGI^mXNcMo z>sfr)HX!bQz5H@Rlo#JG5sL4dNEhYpYC~IbL3N9C@x2qFx4h`T2;YdZ^iGQM;`=H> zQ65>?kH6pIE)?2kHqSzIGb;XjoX z-xcxRiu1WR7u((H&cA@S_n|D!sVM)feV!NR^rNC6^3cya{t3kOlo#iEdxP@k_ud~1 zsFT_e?bEgU=Pf^jM2h3zODCr&Pd@70KXEHUnji7(-1%OzH{h(Dk=7{G^is%ct2rB=)T^^vY=jSa?wx1xa-ub}F(0L3*u?Ee;40Pz- ztSdLkikX=Z^sh{U4DQ%QdYP7oD+T5pvck}YwWX`%VY6qLR6I_Y9czQ2mrd*oq?K7i zM8-gutQAc4m&{aEe*$aPT6J_{-|+%DS8;2^uRs2vw)dgqk7)ZI*>~u`acGK6L4@yA z#~L{PAWSXH6^orb^ybM8G)xp&0~maM2tk4F2G7nkSqKscH916Q>4v%{Z97|?2LfmV-SZmR=}wHo)ex(Q=FwEwtv zaR1x1J^LmPK78P^#yhxJ%L!l0H`y)k-E%mv?K^V#{zC`%OxLyI+@^fkWWfn$3o zvVw6`1-Lmk9zZ;KoG!U}A0 zH*awwG{l>PVd@#GZ^0IaDaXXYOgulw6IK{TS* z@h@nbc^yxO9J-u=*Ypf1f6NaV#w|Sjv&VS&Pp5e}Gt0x5pXA|ZKhDF)7~^X3sS)23 z5iVY$H-{)E!roMQCucy;z;c*@e(xv$T5z^=P(Q`7QXCn@NnXT@dPOMmiBQz(z4yj1 zHGQ%Edh7M}OOzM=@7J!k9&zstMg87*aWCo@p|^dJZ?RC67x~1!H}s~9`+lJ)zgW5$ zr}z1ba-tp)ig<76O<(L@v@7y?=TF3Y+Y$F7U4){%2t__`DDr#b#l1)up@_jk;eCL)s=Oz!~+?pjZf$~yCdM8QiLBV0?sjiag z=82D9BO4UbxW_e`Sg9i2UAH z+Gw-Qjz2g#8OS%QTEt5VaXvt(`eoehjo^d2={)fwL!p<(LXQ$Z@iTv8e(C_Qq~Pd^ z@TIZ${(!jil;>^#=+7g{f8+dPeaiFtFMPIN`g?zHlU@H~|H+@{*#Vx;`y+bggu4er zkM>KWYF`?=VD0^i#ag&v?fc5N-tp*@G?p)f9%ucf)H{{`cwAvQ{Ntx|B%PH`LoXij=W(yUm#2c#LyuCM ze>&?=qjspgSYM;>_UF0s_2twbzUZ(2{2Kmz`FO>solwlrrjPwmpZCY9|4H>5zC25< z|IfbsU0+!~KYaem-wjUhz>yP7!NH*aPgY-~Ps!y&aAn9*XH5-Mzlz$HGx#Hu$IOeNWB=!pAuM0U|rXxk|z7ELg}nsy8peG@F3uEdi{Dfp>dc5%yr{a1VxNpiAuGuQv?+8_NVkD-uIX42?6u zvc=HVwfLk+{7fb^g}HDdUQ7H88D>V^mbN<;qWAAw-1oP{&k{dN{A}&G4){3+_r(Z00V`mSAM?(&*RmSVqloDmBC3>$$5KP`2Z#m?})?R~LrggFWPH zCMocGh2Q^XnZR#H1VqAbDFWggE5vGWi?Cc55s<3`pQZQ*q-P|2mhc%oW9vR!w!T;} z%*b8|pCx>Tc0`TbP)J`L*9tz9*F_Sd3>xX3e4 z@u#DHp9u7mALs2XMFcv!MuyZ}aDI zlaJpx8kEl0$0xJ?G(Nx1`uoO%{yd_8Pydwii;JlMeEs(AzC5D*&weTHOXKbH{(bl_ zgVQ@#2Vd?Ul;Xt_UuqIxN_?5Gl$6P1d-fe5imY<45+kR%k~}N%W$TiBz+4)6Hjz#T z`GNy^Hj&G!i6x2`ujs*-%vgljU*GygKX&x)?>E2T&m+n|dBnjaR8Alzk*+-+oc`!K zLV%<~w>0%b(xk z&*R3I|9T`So#VH=)BZGGzk@FyJ>t(J`nUb!;`mbA)*oNq^}E6NQPh9?cY@O;zVy3e zl;XE>mr|Y-zm?**T+!2NUCQ|UEy-YD%tNV^awoQhTI%nh;6 z{a_}OOoGXuWJ+ot`eibmQ)ApQ;iB}*l|A?pD%jVbe(A@T4mK6^OOENh$d1!DF@*Dk zOld6i^XBrJXz->hj7mzHe8NaWZ!MRCFD3ml2>&_wP|`1-fExHZYmMJRnWSGz{`ho5 z(l5s({Zi5|$ymni6Xyn`59?RVJVGi86v?^Z9(eK0A_Ww+?e((6lRP~;>%am{8lQH zQKLb2Z8Pa;Ld_+XW`1k92VXM7p#opt^`2kvBVXsxpT7M8e;!f(#CN~gr##0eUyJ@$ zVEWPZuO0czw>;n<<9`mh4%+h zi06Cvr<^$VDA2#I`K)(;-k->?G0eAU({?1=s^zU~c+T2He(pkU% zO!>Rm+Mdl*`> z8uR1ZhNFm`lX~5NVK5P)kw*@I-CX*RT$UZ}yV1Wf6A$fM()d(P1xK?C^*t2?Qk zTWvL)QoCM^NCyhcc?7wU5OxcS;cAlRDc|Gj0~U+nDI_wY!sWcnugHkXw3Q^#lt9yQ zL9+ZDC}h&fj2h=~(nH8hXOe1aiG<8mJwS66?P6jhyq^iUvABBDc@=(7-%jShTbv{RSfgM>2{$Ofe^N8Hfp$!;Ba#yrA9{;(8xw5 zzFda*(y_LbiKWy)I%Wn2Qf&?MMddJ8_rS~5%M)HY^sME;C%+UqObIU~yetn2FCLfV zDOSp1N;ymq+!QcL!b=G+r5?GoQv^5K*UrpmCY$WTC(p)UW`Bu#K5KgL<(lPyFTp2& z&B$R&d@1pz#Fx!^sYfp5Fr^&knTmcEq*loum*h+EDt7K%hU81f*=aTv6)4if9?xdd zYK~Jj7o}gW?ZKC8-T2b@#K!{VB;9gwXFl!EBiQQ?-uw%F%5yo)R3SKB@yTIQuedB? zFa1))Iw7Sab61a?m+F;c^y-m2rFlND9ZYW>LeIzPRc{S$J4OHdo>@Lswr8bL4%z zou$ZOqJMY2NS%kPqaXc%zkf(SkHj83nC+*XeB$|T_%BWz)h1@yEr)sdd;UD`_4D%g zgVOo>xba{7Y1EFp{Olc{@~tt^zawY=k1x$z{#W1lFTOmY{J;9uH4DmN{_@`jr(Zic zOzT4=QKyuPry+&dC{-YH$ECRRGo#Asw%*n?GJG;FBGvL)t=MSJYjzs=uewnjpKKM1 zmHJFxpO2KbKi17Wt)HnB^(T~j?o%cY9)4u|D0y)lM+sA{PtC&(2AErVy)aeNr^k;T zJ97WQLkC8sp3h7KTv_ZE5Ob7zK3CmbqjVE0-GoXvq1VMtXbx_Q5`p5Exnxq!a0~2< z;>&eC_;MZJfBF7?z3}D60Q#jsFUtX63jR3FmB3W)P~uBR&qw0R_LQVwO8TYLBj2M+ z`lX~_O7Tm$tqf3*TZZaAxokqs1mVkU6c&s!OH}V!--9pLF9&=X^5IK|f++B1FtQ`d zl|l5&-rRorr9U0YfL$w?ZtO_&6J0OVQ+mr$8s(MvQsT={=!)H}OZiJFe<|fJ!&PYq z(~@>Dr5#L|&S<@oc8ao5@W<0ZcGk1eOjeC9(NE-t9(=ihFp}U8E+6{k^1_#re#zx$0dJU^2c96{&-h?97;S{HOS0*HU{>1HoG)4m>YZGCBtz7 zUVinTW__Y*9DWM&T{!g3|Le~q$`_8Ceadrq@*}?yoc>9l_~r6~mqPr~=@rcizoXDi zz6^iWKgMW}8}i<=bay#sFxPyIx3d)S%V$>$@k@8FspT&D^*k;x8&wnWrO_`p_u$LT%Nbu1{mK!)6x6KV&?|n~n@XeWx+?dG zS30yZull&R4v8;0{aMm4CH+#e$0d7Qvd5)$b!lDwPHQbJY?u))BgqZb?#NG;-(=n4 z_UT0CDqj`qJ=sJYMlQJ$kB5GlfM1{367l7YJ^1p*<$y22ANQEQbgHLP9-1zdTbMt-V&m8$R<6ZtdLOyN#U%sPKn11{?j)wf^mXCx6DE?N3qJd@pV2anX6!}V&@0z!~f5zWF($AyP9{I|@ zX*>DE^ZT`*b>gTO0_CH8*Uh;0oIj6SK63PhpmhHHGcWqns2z8C>u>#eMF0Nnjz0-p zKko8>@~v0O-E zWTp2<=C)gUe>^a`XaAnb%d_irAzaOoL-cKK;@Nb;EWo_GaMei62duW|3BqngK4n}f z^-$Lw59^XX`I@6oCR1rO6R5wF%th7g63wUI+=Cx)cJH4rf3TnYoP!_#Ysa5Q?9Wer z|FH%4=g5PP2d2Y(dLVt0@QXvA6lkV5q;zCEf5F;EVHfkbb6u(Vp-bU^f4Jcv=|CqrQ*lu&-vRA!jErzzmrcqzgs`&#JT6vjUT`IqCbxtzkT$Ng3>uY`^7)@r%^lZ z^8Xxq!na05|91ZH*L`W;^85e(8@@cEe6;*MUm7pZ`*+(HgVQB`l=?YRKS%24aA)Sg zV+BK-)n^;6c_}|B|%4O$yMdT+X{jop&P|_bI{qbJN7hQ0ZNZ?m!CbILyoT_1aR!WVQGN)APN~>^| z*vKbGm6=*&s!-F6$~k@9Y&YS7$Q1(zLA6G)H`$38Q7R=@x}w(%T{+yS>)eab7lvRj z^O^FUR+^Dwm|G&rzveV5$-gFIUcP6gO8j`>bh}WCK=-ao(v3Y+&*z`0H-&F9o#gW0&9~{XqnK>a$`C|x-I;YtG3)e@%+FN zbz64i&!O5Y(T=(D8kV2?oNPLsP-C1+?ZuDToSI!Ce%#W7AGf&i=K6@GRk6-(1UeBfS zuOIt+fBQlF>nDETD-LY@?d47)kjfWoFLUx^ulS=M0!jGjz&8mWpTNOd zk{{jU)}k;W zOSk`Scj=8Gi615T@hX@fm+WiFzLxB3$-b8CYke0}1efgVSJXRqHl0!v31(#6Lx0TX zU^sG#?i085;Kyx*lmz?kdeI-jzV?VeI@oVH;KyF^N2hNBhf4aRq(AP8cFM!$)g$lf zNcv-;)hbB(W1(h@DjvKz?zxxr$6~=SBYP$NQPLl!_+!1Uw^TA$&M1vno&K0h#DtKe z2S4Ug88y8`^N+W2{P-5g>%sivZEpPdS0B6BNBqsf2QU7DKaaqlX80q0%5(E5H~d&| zdT8B+C-;8ELRZKL&$r(3=#v7OEf4kUOTmvvn*n1S;N~AMGV_nl*l6wvjViV|NS@+P z2R`+ghg?0!+gXZu*_}kea_cgg_ zJgG*PXx``c9(;K_-+%f3e!cMJOKZLKNe2lD`Xu$!iKRP!PwBXH@wffC>^uy;5*M52 zaPnQ6dqh#*3ElKbi7)*a(Txu!zU)iCl=xEeza{-r^1mhjTk^kmaAC+-lK-8Hrju$S z$ozON3jTMLrz|Qzc}EYvyn}F>px-PXeCc2}fiJz7korxz{P@x-D-h%>DL*OkrNo!z zL7NS}(pn8GK)a6bpV+em{`_D6F_uJC__VsYTor~q*fVFjgb){eqRh%lO!}Gy(Q>ys9Z1%y=5M%7^bvV_Nud1Cd^Iba;bP$ zO$8ihk&?~`MhcFzNAW%idTq5d7z$M<5krDEp4H}xMJl;@rZlISfI_;dozYuHrBUy^^*x1fsO_199-6h7i}l^PNT|si|A@1x zsJ^%%6haL}Oq`fSYvs{LCf{=8Ff_uf73{m#sg{wR4^=A&X!cf2oAXV*Q*FV6k2PSN z>*NR9)ix3_?;A7GXeO3PB%-M#%o5usLp*I%{}}C-hpT^rfL9(Cl~@0i;;828w{R6y zb1a(3si~OQa(a5t)<93q152$rn@A;+v8;;f?ageFng=hX=5#it#*$qx!|oZkXTusQ zwRFrXn`|*3z0;aS|B|t|ioZ+i;)*4A5jCe$S@^N*av}q|SpCQD0~4#k)4>#^<)S$? z?fAp%sfONAJRPSR(LrA|i5L`yv5*!Tr&Eo16rvh3dzit9qKS9{ddEwvM6481317;r Qk|b;=+QzrrQ@Z;90a7nIG5`Po diff --git a/tinygrad_repo/extra/sqtt/examples/gfx950/profile_plus_run_1.pkl b/tinygrad_repo/extra/sqtt/examples/gfx950/profile_plus_run_1.pkl index 4ea36d21248e1839159bbd78461329fcbad814ab..e4580da4b499aae77c82e8ce3b617a5d2ae9d93b 100644 GIT binary patch delta 1874 zcma*mZ%7ky7zgmX-EB8hx4Bugwl(JSX%^SExh-6j;sVeZ;*m6+1zXhQ&D1@k`O6oxmozxQ`RU~&ZEh|k51Bfj zo0x{{Ttq!)mVz^n+(!H2$u=VB{FIi@nG&O0uS^mxZ9*<_7`es!A>!@7W!{93cD-4N zt71$^D(YD1Y|11)-J--z(kaJFL}@ZGe!B26w6ME z?#^o&=sy}soC%MlV#!G2sCn%I@fzQjI(m^&xZP1?-D`o8tnQyfO8DA+kC*#aw&IK? z24yvx64g*RDULZsas+HQ(-SI``AKpW|rM7t^fML)}6V5 zh_5znEHd;SP<<_!Y6P|5!&DQf?ZcRA1{Dor$^mNiG^U)Ogb1cwpfV$vY5}!z4pVMW zy=hE&KoxSBY6Z1@gALUG?d_!5JFO;7c7w|ctU zeMmD({2|*35VjH}aVK#I7!yd0moK&xAS7AX7@RLzU>EoTyMd5px!HwpgE1dBYwz#> zUcIjBs_L21WliMQw&r!cyI#Gn_j|wp@y-l>tgZAZ^f}%RS(}>0%ED~jn9&Mmxn|ZI zZEN$PdNp4xn}?0cthw)`S!uSPY>zzFeiYeP&zL8RIkRmI9zC+}FoKFPXSNG{*4nAM zkt3YFv6m%WsPM8ag_J}o5H=FH)R)2H49ST{aV}e)Mta#?S(MjFRXnZ}8_nb+_y)G*8W zk>l#PdaznCsibxLBHc9`&9Nr_=yV6v+w5DcH7uf5Zefr|s;wsKnqm!wjzzT5=xFTt zhK^MZJ}AHR$^KtI@Us$smigO(JE~;7wsu-uVS*vmwhFutJdPrY0fBC*mK&}XAPk)dP?%S$XP)L_-qp4?ius{Y zyDU$$-I})g8_l+L)r^@d&KYHV+iQP&vTbcjg~O>xCK1h~(-|!q&$Qp)#;?5Jth5We z+^Ly}mWsq!YHPYMz~2sq)rG4HLxr{W)rP`wVdF?))4{^#5PoL(xfZUzkY6cod-*@k z!nW*N3fG_G&kS>WvcGWiGJY|5_{;ybyit?)!l!>B96YZJvB_x^B$O;9Yg`rlXu)=kt1AiL)!3(jqDTCiBj& zbX1Fl6HCr6fk5Q!x~xn0ihOrvQd&BkW>d&JyD}Lq8H+42y9%F_vr5h?&x#lcXS8^f zznDC;Dng58I^~;H*5K6sgAcU}L)PX8i{>eNlcC;=%~PSxs^4fszgRdHO=~e;g{Ssd zJf=lc0qgCbpyG{{*4(sNSDOV>ZB=UxUadfss5TMP67j%#GkDX66MTv9f|FG5jiymA zFR0DBQEB9>^*P{U+5xKAw2Hnvjbt)$EfWd)!9@~4w2>4~@-?y{U#uAA;%QT*ZH=ev zkpzCLkxbAc5lN@gS~#8b4S0dNzP@f^`=~mu59+7l9ttI1pFuTV&8i~cUa3&sH z=^FL`C;Z)vM$$EqMGG~sphx8CtrC~#gtIr(_Q${?W5`mjTi>LFWNG>uFu-i$Q4Z5 z8R^6m=qC-LIu1`4XLU4lVpN?rbKo8t`kYatJ(!qndb7%Qx}l@@*EGr*+Hq`N!_BdV z=hb3GWjnOdk=x6S=J65H;#0KQ$>J`i4)31WH@Rf*jaJ2;u3i3sOLLpyL6b}7&wGgZA+n=zWE zj^S-K#y!`pDeIP?jc~47X`1s*@DPmBc)(g!qz!S~x?Xg=RmlYq0NDLqssTBI0nzdM z!enh*ciG(lA<_)-89b`agT7${78;=kr}4}q8eSu>3$kp?%oyG?h3KJv8A*nNTK{Wj#okEquy-j>~pt$B>ygVAW1_2vOozweQj zQ4WC?>r^?a9!qH9Q8h(>GFq5(VRDnc0=H3E-nCfZ3K@+eDPk|lzIAs?qcz=V7Mram zTO?`4@hra4XezG7Q+zS>tdsF@TuTR_9a?-{tJ+ok2ZowmbX4{x>slSuae0AW`ShB% zN#C}0`(jKoEk#5j6Q1f&$dl{)O1EH%w4#AkhFpuws7f`AZ@2rj{ZjvaZ|p1GNgU2B8nBrb3~)q+eM;* zdzpARtwmzNI}dAU4nm%yUYqm0^kcDjN=v3)Fa6H;G?t1-vBeAC8*Qe&ViB?u0O?b8 zNR_;+Ni3C2XsLv&;?CYNo=AnYWZ1uVW?Q*+0PCof18=rr9u^zscmn&bz#7w7x`KA& ztj4VX(~Z^~cFeTpz&-+7j#;yqU+|1YJe^8ui6p<{?gcPTd&pF*$DVeWt!_Q5&BQRo z@t|rG(L`8FMENcGmhAPk>FCwh1h&Jm!2LHW#T-fW=o>qjv@9P*7l|g`3GpystLAziiq{tsFh6^Bb`xGQI`66;L~r4>2x2IRo1=Q;S}YkL zt?0(WS2wG*D(JJO%4h;cezK&zkJ!q{j8>biWLSmWQ zY;#Nc4$zCKFC2>;4}nt{RddaGhU9sv-x(y9hOI#0Mv!kOF+1omYbmJ2rd8J3+8jt4 zyU^5(nOB_%&M~R;J3&K@sCV3@B783?qOgjoJFN=Ip}4j{ZEggd2dgPGZ<7Vx9X!5@ zNni6t+H4%g^UzLon5yWa_B@nI`KmgJ3etuuX8?w z!Bb)uVTKdw+!&v#w>X`{NAS~wrJDuprrvv~H8(4`(cXUU7Hy2GAUHvCmK!L!^lf!I zE_b6Bi>N))CwHTlgaLoNPwqx-alnP|B`>rFQYh;JeerON;oSf^ACjU(1qSa0U3}dZ z{&pnfE8b@=h{E62W2F=@f)!g7V)*OvAdJ#Zs^q-!L}{FG%0THZ1f{$2#|0ZoyQyko z!j}dMrM;AQnK+?zFJ->u_@MMoYUUzCg3=`G`=!PKr9*5yIpkVm)h`{PymD2)FrlR6 znCmM_&?q@(DPlZ-y{XQs?$8!7u&y>Y+C^evQBQ)C2nUlwj@E^<>GAMQ(Ktx3R3w0P zgLE$pLN3*Vk=c(7yDh}lH;^9O**`c02olYVtX46St`pLQ)s`%CNNz>k?xm1E2 zekTqctN>0g9wv?~9uMYDT^MJW3;I>)2Qdu_M>NUt%`<#mI8^vYO$RVx+s9LN zyM>T|QZAZ8?l9|(^U#fv^3IM4(b5V_xjwlT$+hVG*P_tMlXu3$DPs3ie9iN$EF_*x zB!i_9uJ`}7M{JrLs+L@iAXF_!23)F^oC$o^W&+=M#YsslG8IX_W=b^|VL2*a`%=qs zQH$S;peVEQv>c=HC`9?G08>P)2(;Hoc1#%hI#%?Ds>MpP+ls!h+8WqBxtFXl2F-ah z*R}?rtlrMP!y154huPT=!C>29y22yYkPzh{#Tvw3r_qMxezrfWn5bdL4aZ`!xm>lj zpz=>7I?BGF1@+E5)I$$V-FN>%^@d~NIoSF_y|&hZ&GGSeRxzyA#frGQ-)DRb46<{Tt=YPY!ym5*Q}2(H_!TELQGu}Dv~Rck}J zYRo`n$JJjv9gf7}VJ#f-&Olx(ltI$T9{M%wxf%ph+i`0_@RCVD7TcP;dlVvxD3q;i z-6P-FZX%xvMFV;yw(^d*x5+4UD14^7)Fy0I!kyvqRlKoLgdi5GP+WP*+hG)j(M{B~ z{boD`tC#&mgLmH`BMQZQQw@bjM}WFW>)&nZ!-{9{GtSuBL)}9WgGLB=jvkgi8}elX zz8GvJF3&i`Vo5SR@`^;LL2wXJEgtE?o@|1u2Gk^=tI;*Uox6pW94x_H^>$L?Q5bRc zdTvAJe9frLz+l1=-B4%Zar_*jc4xZpKAxoK)yqU{Gt8^`cm_~0tmb4U38f;h1r9rh3>$@Yh|!o2-WVSdFjE;Spb%*Sd?N8?(um-@~+>hEl=HB+4P&aHSF zbX3H)aqAufSehq#8c|tS@7SU8VK8PIYKXj-jClH+z~k1S6UB5FJ#|Oo(B$jcUwB`eo&XBv_)E`+*_Ti2JDBrevcGs=^aajZ-NfVljO zbSt%yB^_z~Q0>evZ{H2aGT}L$Ko8(iaxTWdA&%jo!bWRgpB{t16X9Qo8HUj+ZVf=Q z3LcC?@Ixa!BSFCn@%JP3$#!;_c|;lRB(k4P+gi1E3Km*a@SU&YlU1z2lhxu(`(U>3 zU=u!6O2-BjB~{&Dp-}fVX=?tjT$l9?2>PeZ7yfWY!kD z)k$@L%mDEU3B1`1%l;UvZGhMIet^%?Lx{14cFz|Z_P}KaDd!Kic^(RVO{=V~I!f9D zSEjG7>(g}2d){Xk{~)5gl`T@NKd4xzeCJE4tJ?Rs`R%ObOwRsFzKRjHp)x=|dGBdk zYYp72PKJEs`D)`#wbImE4O3@8q|yF>wVo8VNh1$SK4sZJAt^pGMQ*ssT7!Dvw1?E% z54NUxPk&HYb!&dS|UZyHmB_DMxB%jk?oh=%Yw7zXSOeOw(!l^NR03K7i`wAS5=(^{**!wnFzb@LLK(?@27fW7)nFP{Y@ z4U6Vr(A(u)k14@AS_2pAUr*ZPUzq2~mB&`YE(#20z-EovErK+#?KQjDx2`F6F`Z9I z&x$L`vtks423!TvZr)F7P!g=vBDR1VE<-mu+g;oL?-l@WauUn|wqL$6ap)*E#m$*w z9>$xytiCYgN32!YrDGqB-HWrCx7I>bvGd5@$=y4{kcqg2f#@D#_z8rP3Hxt-&C`Np(sn|NI7Jhf5vHl6#RGV8aEs9@{C)aEpMDuWb&B&UI0gq>2> z8yJZYJX6_2$;3UO?K`)RU>EARszP({G0pGrGHgRjU!$?O=zGx8H>g^<^K@GTO7pWz zeA!wp(9}B?!yW`YfI3dP$k)TxAQf#}pk*6PjaL&QfYj{@kJiOgIw25- zX1JyFhze9PXDpBmlX>t{4HdA{#wP$~HL$&~<_9QYSzHycC7!L%{~pfSuVKBW#zWK% z0GoQw)yZe!>^;eV2g?WAIYcz5V`4oDd-Y;-oc@r#{N8!zh6_d-^V!nmta<+##Ad@0MDFOuEYy$&|&>0vOXI`m>k zRcIN_+Pih2K{7KHyX=IS32&W>%N^kt&^j=@3#$J>D?h9TXt91JMvY`(6@ zX*-g$#uK);x5G@?1yn9OnjwU`iNq&@ew+tx{gyMFzP(cQf+Qi^eNf>gJ*PnW1C##1 zoHIA&34$cbBh6U;LT0oWTPrAzF=vIIXI**;S?Hx_lGwZQCEdMu#eqnWW@{;7OHevU z`?ITU>9wsr>J?uLju@C~icA=p_44IjU;cT7JE2M|W7`m&!}5a$I=Iy=a$vaVO6xLc5qVW_G!LBrK;r z(XJSm!++kKVdJ^Xe%iyUJb-dys6?NZ6H;6BFHIc|2;XbRi%>kXtB`jVX(~c!GTeLU zsG>aaJcRazhJj;>$Q5(rBY&uD`B}}Ft-%>6Nz|3kXe7>i@_o;1u=apMnugvW&Hc~L zmaEf7`B|-+&o@lUMiTm=pVi>&6UquFposrDp2e2)FHynJg;ua>$qK%#&DN`}n%*#H z$yid)7v~|-r*11>WVeqS?Ic3pKU*341z$X6-};;eZ#lCqK+~VOALRbT0d?Z@ohbbz zB>pf6`e?uMC7gIWuailheyUnOVKmfBYw{Rxh4p4vYQ$`dO{fc@o2) z=NAq53NG6Z=UI&#HGLKpX(wjBq>*b{oK41>gX?B?yJ)ZP=QL>eH}shWXetzQbly_# z#INFEb!s2@Jil<){sO$>o;34yG7zBq-t{?qt~4=HX7kIqn$1=2KtTU6iSD>#Ha`!) z-C|COk?6B0B<5Yvhkg%R1~5DA_raZZCi+=#9$4M^eebg4*HgT*RuR8v@q40B{JS3C z2kCXUM(L-x@BG;htxNy@XJ2@3`;N5u{fJ({kk@$^;CttTQC0ZJDmXH{SkrT@;*6e& zPS2#0(|LH;Fp}Y!Xeax*e%Wdl*k!+edX?X$yr8@fiQ;{u^sRk~@}5yvzeGhJSCqH^ zz)RLwl{1IFgEv?o5BvLT8?CSQt?KVr&eXo6^wC{E_EnUU|Ku3RIq)W)0|Ijpqe7o` zJW#~@NWiIr{l$hgO32Rl_piNPrLSAjdNiB1cQ6YOe??RD$u}ECkh| zLbF+G>>L}zMWHnf+56nsVPnRuR@l-CvIJ*xjg`wM=k9>cW(kUQD$a(y1J=M=M8(+~ z@wggICDTzOl8hz8W-MmLQn`3OmdhoLnS91bLd7%|%ct{ZWJEat^Cz>ulk_;%0|)ln z-B3egk9p$#N4$LbO?io<|L^7mr)frW*}Ku7{r9xn=X-p$5pkDrshYUrya2`jU1TAn_txhP6<)90K-r@qfql8dJ41;hd zb2*mGE*P|IH+D?|tat?#cZV7~YHW zq4`RC5P`eA`xmia5siuWinyk-^sW-+#rsQy58D~(HAHQR`7GXR8xi*pufJXu<;D9; zgyQ`s(nWc@T4fu)P~9S3yzfNlEid{n!ndL<{U$|u@qQJdD32`c!++o6N>OgZSBewm zU$Wn~BHZOqkw|g; zed**B<>_!b_eWfckj6(mJ9oZU>;*V&XQX*bw?y|!A2EOD@m-7^OR)chHt;V$;*P|3 z(XNPo9AARUf59#fP}ueHmM5=I&{l8%$ZF*bZbY>Pn#L?_=-oyXZpw;znXt7wBB?N( zt7vX#5Uv#Xd+GR|v9@-MJZ$j{(~8Fqvu$k@eA(ngAXjD$5{-c_S*w`oFFB|x{SfA? zwdT;|#1VszRdHLy4z^U093qgTUlY_KJZci$G)|$FG3q^Qp%-VwQXywqp!}m^-9I-}^3o_r%ox2lt)V{SGYE za?IEAHFnE;b|1{@6Zapy=fM7nsq<=laH+=QzQ#A+dw9>Je(%x!-8Vf*(SeV-FYqz1 zVl8KPmDp3h26KzmyXfq?YHDIazi+>|)d5^~x44NMQqNL-i?%pK zIVSfXIbx6QIX1b*9^T$f(quOAHUbCP-XmyV_q%I^IHEW4Cuo^@6Ay>HbvXi8^ax}I zICg&iEqt@lf1HQ^;iq``WAEkRqo3vByY2Y1XZiI~gCpK2B3vrJH(ityp*LM#%Mp+x zaQR1|*Y8ujB}C}$sn}MEEu+}Ui+EA52t_^-iu%0Q-uUIFi++kw)Z-0BecpbFa-#nt zpSQfYUMgMGFY?+v}_ zOI?e0MLzHNiFj{2;##DOP?Q&;$mb13es8?E7U?1s@gfxQ-cXERul7ZLZ$0AL8;bGq z#>;Ct0$0Kav@JN3Hgh^7B>UWiiXm5) zofAZwoTrUg__D?HarOPyUua&x`#$o%cud$_aN3C=d5aqiSCnd)(Udvc+0> z+?sg#55E8557E7RU;kd#UrN12{q-X%%i+I&N=MQeDoa_sZGDudE0oTmyVEzyPw5md z(!J$34U6#=b$5rajGY}Gpo-AvKf0dx+1=4N6Gyp3OX#2S4kixoQ??)S=W)-!n{q+v%EQ#=pU(Nys2wUV=GVj%{yY~x zznuEN_91`$XV&rO>wWIjH$J|+|K?+#>hb$=>i=(>f83X6x%E%J`Z-@&>VbGZe|63C zf$41=;RIK3aOnTTrLWPe;YCRhAyqgD@EdGa-k{Q zg%f!#@iTOo8M`fgcdEqq?_B!SZ;78JewO&zy6rOH=P1rZb{|~q;O7`lLW=NXR$&0K zIDYQu__@G2Xr{gJX2s7(R~&v8tZaAqX6K*}DKG!ZOFrhG*i+79zcTjqUm*^<^lm@@ zOL-)ReVI9IhQB{>q5Sp3lD}56QZb}egH5^3XLBcF(s#aeHjGml^hxP#SUMYqWivnG zY#B}l&yBy1B@dMdKcGG%@w3Fw5E@%|*~;}fqtOiQk@#8SXP8Hn=^P4qSjQ#8&+$0CbO+<- z6r2ur{C6+TUk`Hpd>@cAbg02%J35rFJ@JEqI+Skw@)JMe&m-`6|7%u{@?1x>|Bep^ zrq?z+{vod8EHs=Q94Yjhsh>_PUGZzLesfSRJ5Ns?X0ML3w+?}CgW9Klxbf{fC;a#L z+kM?S&ffaNiqLruZUOcCbfBC2c{|I|fex=bYwJLJTjtlO{|s)UO9%R?Dd#@K^G$!n ziKF`#y+!$s{>GuN`}4TxPwrbm>3n|t*`NB;s2z9tSO3kQNA&OYoBlj-e!0tk^Vz@f z-(x7_U94h z3;((}zAWtUV*{%H;SGcU$%bxa;!C0bEAW&zbmPkxclqz}QsB#P-^1IvRQU3>2b}v5 z&-dwHbmHi_iE(h_%P+p{&*R2-zxKO9=^Vek^N;;$)Q-FSyZ*(WNA%CUVQGAs{!DLt z`8Vxg{3z=G?k@(XOMK}+k5TI1MqO5UQvX)!-*QXO=0eTX$QQPjpKID)QW^!L#*Zq! zgr_%3#}LvnL=&fq6)Wrv6OPscID;`nlP7rulI_T}d<{rJ+s zE`q=0SVkZkC-n1|PFa!5>!U%$i%#;czArhJS=)GOTeW(yrsJ zk&sSksTlJ}wxoLLRbB9sc@7iw%w-ghd+19C+^qz7DaGTQUX?Q;DdS;VDbv%b_lO$sC6YeEGx^ zr+cWMIrP)teZZebl>fOuJkz5*$0uLe@mOH`q2Vh>{qn@!{(Jmw<>ISf9^K2^S&n#I zS;N&Y9YU7qSqH-hsbBuWLFYci^S$;*oj6bVj{0Tmd4C?aczfn0e|mVMa6#-rO|zg^jH69aQd4}JpQrA{*6XOQ6$jB z$ql3GNkan7i?1#y{ZvUmRnkw@8|bHMeTT8(SC>ZQ zl?o0yymU^O(h!oTxT!dkk~`w_G~A+&Vx<=QTIxprm3ONY!_2}1I7(ZQ7PQo_r^ zfH=bAoQ{c=I+#)iQ^L!>lGMSpqz?$fXjdRKh%2G*5*_G{(V7B}}P=d7@*-9?rf3 z^QHCV2tLGNBo|f0Cb#t!;%}gA+#xA5nS{nIdAe`rHFMxN=N21-t$8P(c)sI5=ft_k0_8h8m}kD>&*RpCy!~52>3n|N{T+WAwc{@T>3{X-5&e65 z;-$d(tFHZ5#^^|{-!UDx6k|c&Vk>*FwgV)K-eJ^dp~$jhhXW2TyoMD6FFOYjCBn&w77Nn8 zOoSmFk1P{kZs@|78{F&X<-(Vn0_aPD;8p^@6yk9jD+hJb6~C7FQsPTRx!@x(rFdM5 z$EA2&ipL9AOYyk0AvfP>@pvK_kv!k5E@kpzEpx$vdp!c3;x!hMtLQ^l=xEO%W}DzGn&=9rst|s zJT83{?bf7tT#Cm>)nf@QJgTPXkMvR0;ZRo~e;LUnwRn(=^+Yrp(^AVk!e?U_zT60S zblLHxpTAsL_)_whTs|!2<5E5@ePGUprlovb%E#m62tMnUrRta$d9x06OQWIBnRC_p zg2a~+Uj|FqS0KKOCgWN-5#XID5yjy?k!9k`OX*6o*FHMt&m+oT zebU52=aga3*OFh^exu%HQ7fVP76me%=3h#+ODt5$R8j|72jg)W4KRL`om) z7w4pGsedW;FQx0tH_*Jt(Z7tvV8z3Y#69*su|!(4{bDZZo@jFyzTE6yKQ9-){O}5w zj|=`%;!8ia%L_lE0Mv zrIe3L`FN%Tph_U94!^dqf_9_2Yc34b#<9q!dXw31JU?8cW~cC*`l{NKLozsEyE z-Tp5(DQ9oQ@Y^4zzq$3}N2r7g z9R13V?)!0H9#Q_T_0Rj#c>BD6e|ao8{UYgCssn}1p-@R*r6`k${?c!(hJnC`PZgU5 zwUB#6uhpyhV%ZFFFU;o2QT36Q*)sL!Ld_f(ky_ytv(DD-G#oEzHge-rbt6}-%x2An zP=5P+I+eT*&4{aY+VU9445>2f#y?~b^q*rFem@b<${f{%uPpqS3`ezipnuCGkmYjmkZ$)aIac{`^0Ctv`*=KiB;Fcm6!0 ze=kgZ!q>L9{7(*k(w9e+pZtyId};3a{}0#y$G~)nAEkYcw9k?DInw@H%D0#6VSNSi zkI5AL3kC9zsc>9Na?h4a+UH!?g&(hTub-ETfAr%=2crr3HpglLu{oiw8+k^YzeD%q zq|%lD{2gz)%Ct1y9xSPplj~gYqvRiZ^M{gul>FnJIN{>)F0U-a`0Y40uaSEeH4g)i zd9_$k>&7W^<8ypeoh?_Vjk4Y-o;Jsutr{p=R}2_ba+}$e>_m*H#k?zBHOmcCJy@-n zTulF^K{#Z87DsU;&89l8GTv75k92Ue)Q^<%Yso+Q4`2MAmS3mhaV;9?KPZ)oXz691 zuk)5J{P-3^N`jxcT=)_4Yma`UgWgsGew6ZShkumv>(=yn9{(-n*Yf-vd47&OKS!RQ zWA2pa=e+Uf=cFSsEyhuz$3IXymeJCjnOu^8+}ed7x4QA;y!LdU{LU?(ygTX7Bk<>| z^FQ6AJeOalemyun+(ZA77(js^9YpEyg4A~}|Ja?%OI_&sIor9r<FRpZ??I-|%|QRet@?f9G#MNPhjnfzNcFm3V%C8F%8Sm*P%Q zzN7#6+J1i?_xu=tC@7sj|ND;n)2JPH`G5HVe;(1lt5^TOPM^E`=g7YY{_|_TJfi$B zKDqI^eNZzM_a#D+{x{zWPM7j)DZiHTYbn2$@@px-{vMWJr_*795TFS3mwHc#n z>W!w+Y>Y4-X3OM_eWhFJyp;#*t%}~r70j6ysK;Hk<49rCq?BJv`E?MogSMf>k2sicRGrssOpG%MTh*Gz z3BM5yFuGMYb%VRWH)lcucRzKB4vUfY*V6v_7VFBwkI8T(u0^;P$Zq~I8IFgwDEAz> zB!0Z13qRhl67VDB*B<>x2LrAI{3zwuQht3A&aaX9QQ}95AEo}I)PI!vkGC;ny}H?G zmC1CE90eV2RVrqk`739M6vy{Tv23w$owbn{X;r3>(6F~q+pu?bNn2bPv9bKmX#~V3*yl;T{f4tF+AOFYi9rfvs@pMPO z?3@3~pGWAA{j>V+9_6`wTsa+_u57q+{6GHHM*{A#zuW(#U_eC}_UJ>3co7EaN51+A z-p+FLBmZa}_Ydiya(b$wcILnG3@*FQ-~QlFJNF@;?-Sp4;ymR$`jLP2U4I_8e&f}z z2c`4*@yi3pd(0na{(XJKpGWj>wE6Q(mcQlZFZ%L`^433n-Iqr9DbkJvZs}o{Nx25Jo$KEDMJH$gWLH5G${;lz2dMLTbtqw z#OiKCJ|LTX#jW39)Mss-3&~Gjar|Ti^6f;>`8CN1qk1cOyD)(zm9#4#Fs(%(ZPojUp@-|DS`5Di7zF8DfvsuUrPQ`@|Tx`zl=tr zS|Z4PI~k3owHRkMmz00s+=VZ1UYYpP!Ei$U?bUgtep4<=#}}9Wq*GSlKd=3^h!^3- z^dBklrNo!#=^seqONlQfzLfY<;!BAyFAKg*rL_q6LhjM8jKM#7e3||w-`a&Q-%2P= z$p0=Ee+l`xN59g+h${hKcI#IsF@M zYTE|F+Xg$J7%8DSoMFt;=wuD=wZ3q&Xr5|YDYs6FU2(|C5Te71 zje6Z!;HM+ohZm0sD@186D7A5}Fm0SnDjbDPV1TtCQW6<0(vg-N!tZh9u{KHlSC{h8 z>Ml(oh7`x|h@>;|Ofr^Ez-?}Zzn@C)LL%~7n{zF_X4Y}6&D@O6ep~JA8fz`P%vD>J zW_xm1zqO8CQ@&iQoy9fFiC;Kn$g0*gbH=>hC|8>ec-^TR&0K+HRIM$PaLPDoHae}e zvqRQ(PIElLtjrcGW;@$w4VPz73H-8ZkAbxomj64C?48`bGu#$qG29u8sFq3bmpB`W($_XB3Tnt< z;KV3es}DUib>IC5+l3)(6??C>O9iB7l~NG_jozvmI+nd%s^bJ7tG{u&ogHYET1dpW zZ^D9+iewVeOgx>@V$PFtw^8~Q+AR!~eh&e!yiZhK`ZmQ;&80uUx4@b+>4+9}JdSkL zOmm>C=Kf{Y91SND;Y1uGl<3$JEMD`#xzwDF;E)gd{5a2u+oNF(=IbU#l?}ESkFH^j zhLe$43a1sv9CL@oySQrET|~|4H2k>4gKyq`V1L(nU}83SJDAC2VoBU-a5eVRajFp= w^i+dMluV_xRF47kRU;mSzZz75WBWR%^92o;(;G2Y0;)+wwNyrsXr}i60TXH%1ONa4 diff --git a/tinygrad_repo/extra/sqtt/roc.py b/tinygrad_repo/extra/sqtt/roc.py index 8ed0e22a8..1f1df5f8a 100755 --- a/tinygrad_repo/extra/sqtt/roc.py +++ b/tinygrad_repo/extra/sqtt/roc.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 import ctypes, pathlib, argparse, pickle, dataclasses, threading, itertools -from decimal import Decimal -from typing import Generator +from typing import Any, Generator from tinygrad.helpers import temp, unwrap, DEBUG from tinygrad.runtime.ops_amd import ProfileSQTTEvent from tinygrad.runtime.autogen import rocprof from tinygrad.renderer.amd.dsl import Inst -from tinygrad.helpers import ProfileEvent, ProfileRangeEvent, ProfilePointEvent -from tinygrad.device import ProfileProgramEvent +from tinygrad.device import ProfileDeviceEvent, ProfileProgramEvent from test.amd.disasm import disasm @dataclasses.dataclass(frozen=True) @@ -39,17 +37,17 @@ class WaveExec(WaveSlot): insts_array = (struct*(len(self.insts)//sz)).from_buffer(self.insts) for inst in insts_array: inst_typ = rocprof.enum_rocprofiler_thread_trace_decoder_inst_category_t.get(inst.category) - yield InstExec(inst_typ, inst.pc.address, inst.stall, inst.duration, inst.time) + yield InstExec(inst_typ or "UNKNOWN", inst.pc.address, inst.stall, inst.duration, inst.time) @dataclasses.dataclass(frozen=True) class OccEvent(WaveSlot): time:int start:int -RunKey = tuple[str, int] +RunKey = tuple[int, int] class _ROCParseCtx: - def __init__(self, sqtt_evs:list[ProfileSQTTEvent], disasms:dict[str, dict[int, Inst]]): + def __init__(self, sqtt_evs:list[ProfileSQTTEvent], disasms:dict[int, dict[int, Inst]]): self.sqtt_evs, self.disasms = iter(sqtt_evs), {k:{k2:(disasm(v2), v2.size()) for k2,v2 in v.items()} for k,v in disasms.items()} self.inst_execs:dict[RunKey, list[WaveExec]] = {} self.occ_events:dict[RunKey, list[OccEvent]] = {} @@ -76,7 +74,7 @@ class _ROCParseCtx: self.inst_execs.setdefault(unwrap(self.active_run), []).append(WaveExec(ev.wave_id, ev.cu, ev.simd, unwrap(self.active_se), ev.begin_time, ev.end_time, insts_blob)) -def decode(sqtt_evs:list[ProfileSQTTEvent], disasms:dict[str, dict[int, Inst]]) -> _ROCParseCtx: +def decode(sqtt_evs:list[ProfileSQTTEvent], disasms:dict[int, dict[int, Inst]]) -> _ROCParseCtx: ROCParseCtx = _ROCParseCtx(sqtt_evs, disasms) @rocprof.rocprof_trace_decoder_se_data_callback_t @@ -129,44 +127,7 @@ def decode(sqtt_evs:list[ProfileSQTTEvent], disasms:dict[str, dict[int, Inst]]) raise exc return ROCParseCtx -def unpack_occ(viz_data, i:int, j:int, key:tuple[str, int], data:list, p:ProfileProgramEvent, target:str) -> dict: - from tinygrad.viz.serve import amd_decode, create_step, row_tuple - steps = viz_data.ctxs[i]["steps"] - if len(steps[j+1:]) > 0: return {"steps":[{k:v for k,v in s.items() if k != "data"} for s in steps[j+1:]]} - base = unwrap(p.base) - disasm:dict[int, Inst] = {addr+base:inst for addr,inst in amd_decode(unwrap(p.lib), target).items()} - rctx = decode(data, {p.tag:disasm}) - cu_events:dict[str, list[ProfileEvent]] = {} - # ** inst traces - wave_insts:dict[str, dict[str, dict]] = {} - inst_units:dict[str, itertools.count] = {} - for w in rctx.inst_execs.get(key, []): - if (u:=w.wave_loc) not in inst_units: inst_units[u] = itertools.count(0) - n = next(inst_units[u]) - if (events:=cu_events.get(w.cu_loc)) is None: cu_events[w.cu_loc] = events = [] - events.append(ProfileRangeEvent(f"SIMD:{w.simd}", loc:=f"INST WAVE:{w.wave_id} N:{n}", Decimal(w.begin_time), Decimal(w.end_time))) - wave_insts.setdefault(w.cu_loc, {})[f"{u} N:{n}"] = {"wave":w, "disasm":disasm, "prg":p, "run_number":n, "loc":loc} - # ** occ traces (only WAVESTART/WAVEEND) - units:dict[str, itertools.count] = {} - wave_start:dict[str, int] = {} - for occ in rctx.occ_events.get(key, []): - if (u:=occ.wave_loc) not in units: units[u] = itertools.count(0) - if u in inst_units: continue - if occ.start: wave_start[u] = occ.time - else: - if (events:=cu_events.get(occ.cu_loc)) is None: cu_events[occ.cu_loc] = events = [] - events.append(ProfileRangeEvent(f"SIMD:{occ.simd}", f"OCC WAVE:{occ.wave_id} N:{next(units[u])}", Decimal(wave_start.pop(u)),Decimal(occ.time))) - # ** split graph by CU - for cu in sorted(cu_events, key=row_tuple): - steps.append(create_step(f"{cu} {len(cu_events[cu])}", ("/cu-sqtt", i, len(steps)), depth=1, - data=[ProfilePointEvent(unit, "start", unit, ts=Decimal(0)) for unit in units]+cu_events[cu])) - for k in sorted(wave_insts.get(cu, []), key=row_tuple): - wd = wave_insts[cu][k] - steps.append(create_step(k.replace(cu, ""), ("/amd-sqtt-insts", i, len(steps)), loc=wd["loc"], depth=2, - data={"fxn":unpack_insts, "args":(wd,)})) - return {"steps":[{k:v for k,v in s.items() if k != "data"} for s in steps[j+1:]]} - -def unpack_insts(viz_data, i:int, j:int, data:dict) -> dict: +def unpack_insts(w:WaveExec, pc_to_inst:dict[int, Inst]) -> dict: columns = ["PC", "Instruction", "Hits", "Cycles", "Stall", "Type"] inst_columns = ["N", "Clk", "Idle", "Dur", "Stall"] # Idle: The total time gap between the completion of previous instruction and the beginning of the current instruction. @@ -176,36 +137,24 @@ def unpack_insts(viz_data, i:int, j:int, data:dict) -> dict: # * Instruction cache miss # Stall: The total number of cycles the hardware pipe couldn't issue an instruction. # Duration: Total latency in cycles, defined as "Stall time + Issue time" for gfx9 or "Stall time + Execute time" for gfx10+. - prev_instr = (w:=data["wave"]).begin_time - pc_to_inst = data["disasm"] + prev_instr = w.begin_time start_pc = None - rows:dict[int, dict] = {} + rows:dict[int, dict[str, Any]] = {} for pc, inst in pc_to_inst.items(): if start_pc is None: start_pc = pc rows[pc] = {"pc":pc-start_pc, "inst":str(inst), "hit_count":0, "dur":0, "stall":0, "type":"", "hits":{"cols":inst_columns, "rows":[]}} for e in w.unpack_insts(): - if not (inst:=rows[e.pc]).get("type"): inst["type"] = str(e.typ).split("_")[-1] - inst["hit_count"] += 1 - inst["dur"] += e.dur - inst["stall"] += e.stall - inst["hits"]["rows"].append((inst["hit_count"]-1, e.time, max(0, e.time-prev_instr), e.dur, e.stall)) + if not (row:=rows[e.pc]).get("type"): row["type"] = str(e.typ).split("_")[-1] + row["hit_count"] += 1 + row["dur"] += e.dur + row["stall"] += e.stall + row["hits"]["rows"].append((row["hit_count"]-1, e.time, max(0, e.time-prev_instr), e.dur, e.stall)) prev_instr = max(prev_instr, e.time + e.dur) - summary = [{"label":"Total Cycles", "value":w.end_time-w.begin_time}, {"label":"SE", "value":w.se}, {"label":"CU", "value":w.cu}, - {"label":"SIMD", "value":w.simd}, {"label":"Wave ID", "value":w.wave_id}, {"label":"Run number", "value":data["run_number"]}] - return {"rows":[tuple(v.values()) for v in rows.values()], "cols":columns, "metadata":[summary], "ref":viz_data.ref_map.get(data["prg"].name)} - -def print_data(data:dict) -> None: - from tabulate import tabulate - # plaintext - if "src" in data: print(data["src"]) - # table format - elif "cols" in data: - print(tabulate([r[:len(data["cols"])] for r in data["rows"]], headers=data["cols"], tablefmt="github")) + return {"rows":[tuple(v.values()) for v in rows.values()], "cols":columns} def main() -> None: - import tinygrad.viz.serve as viz - from tinygrad.uop.ops import RewriteTrace - data = viz.VizData() + from tabulate import tabulate + from tinygrad.viz.serve import amd_decode parser = argparse.ArgumentParser() parser.add_argument('--profile', type=pathlib.Path, metavar="PATH", help='Path to profile (optional file, default: latest profile)', @@ -216,26 +165,28 @@ def main() -> None: with args.profile.open("rb") as f: profile = pickle.load(f) - viz.get_profile(profile, data=data) - # List all kernels if args.kernel is None: - for c in data.ctxs: - print(c["name"]) - for s in c["steps"]: print(" "+s["name"]) + for p in profile: + if isinstance(p, ProfileProgramEvent) and p.device.startswith("AMD"): print(p.name) return None - # Find kernel trace - trace = next((c for c in data.ctxs if c["name"] == f"SQTT {args.kernel}"), None) - if not trace: raise RuntimeError(f"no matching trace for {args.kernel}") - n = 0 - for s in trace["steps"]: - if "PKTS" in s["name"]: continue - print(s["name"]) - ret = viz.get_render(data, s["query"]) - print_data(ret) - n += 1 - if n > args.n: break + prg = next((p for p in profile if isinstance(p, ProfileProgramEvent) and p.name == args.kernel), None) + dev = next((p for p in profile if isinstance(p, ProfileDeviceEvent) and p.device == prg.device), None) + assert prg is not None and dev is not None, "must have program binary and device props" + target = f"gfx{dev.props['gfx_target_version']//1000}" + sqtt = [p for p in profile if isinstance(p, ProfileSQTTEvent) and p.kern == prg.tag] + + pc_to_inst = {addr+prg.base:inst for addr,inst in amd_decode(prg.lib, target).items()} + rctx = decode(sqtt, {prg.tag:pc_to_inst}) + waves = sorted(itertools.chain.from_iterable(rctx.inst_execs.values()), key=lambda w:(w.se, w.cu, w.simd, w.wave_id, w.begin_time)) + if not waves: raise RuntimeError(f"no instruction traces for {args.kernel}") + run_numbers:dict[str, itertools.count] = {} + for w in itertools.islice(waves, args.n): + if w.wave_loc not in run_numbers: run_numbers[w.wave_loc] = itertools.count() + print(f"{w.wave_loc} N:{next(run_numbers[w.wave_loc])} Total Cycles:{w.end_time-w.begin_time}") + table = unpack_insts(w, pc_to_inst) + print(tabulate([r[:len(table["cols"])] for r in table["rows"]], headers=table["cols"], tablefmt="github")) if __name__ == "__main__": main() diff --git a/tinygrad_repo/extra/thunder/amd/fa.py b/tinygrad_repo/extra/thunder/amd/fa.py index bbb4d4685..4498b30e7 100644 --- a/tinygrad_repo/extra/thunder/amd/fa.py +++ b/tinygrad_repo/extra/thunder/amd/fa.py @@ -19,16 +19,33 @@ def _sharded_empty(shape:Tensor, ref:Tensor, axis:int|None, dtype:DTypeLike|None @functools.cache def custom_fused_qkv_rope_forward(q:UOp, k:UOp, v:UOp, xqkv:UOp, freqs_cis:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int): - code = (pathlib.Path(__file__).parent / "fused_qkv_rope.cpp").read_text() - threads = 256 - thread_idx = UOp.special(threads, "lidx0") - block_idx_x, block_idx_y = UOp.special(B, "gidx0"), UOp.special(N, "gidx1") - sink = UOp.sink(q.base, k.base, v.base, xqkv.base, freqs_cis.base, thread_idx, block_idx_x, block_idx_y, - arg=KernelInfo(name="fused_qkv_rope_forward")) - compile_args = ["-std=c++20", "-ffast-math", f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", - f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DTHREADS_PER_BLOCK={threads}"] - lib = HIPCCCompiler(arch, compile_args).compile_cached(code) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=code), UOp(Ops.BINARY, arg=lib))) + group_size = H // H_KV + q, k, v = q.reshape(B, N, H, D), k.reshape(B, N, H_KV, D), v.reshape(B, N, H_KV, D) + xqkv = xqkv.reshape(B, N, H_KV, group_size + 2, D) + b, n = UOp.range(B, 0), UOp.range(N, 1) + pair = UOp.range(D // 2, 2) + even = pair * 2 + c = freqs_cis[0, n, 0, pair, 0].cast(dtypes.float) + s = freqs_cis[0, n, 0, pair, 1].cast(dtypes.float) + ordered:UOp|None = None + for kvh in range(H_KV): + q_out, k_out, v_out = (x.after(ordered) if ordered is not None else x for x in (q, k, v)) + x_in = xqkv.after(ordered) if ordered is not None else xqkv + stores:list[UOp] = [] + for rep in range(group_size): + a = x_in[b, n, kvh, rep, even].cast(dtypes.float) + bb = x_in[b, n, kvh, rep, even + 1].cast(dtypes.float) + h = kvh * group_size + rep + stores += [q_out[b, n, h, even].store((a * c - bb * s).cast(q.dtype)), q_out[b, n, h, even + 1].store((a * s + bb * c).cast(q.dtype))] + a = x_in[b, n, kvh, group_size, even].cast(dtypes.float) + bb = x_in[b, n, kvh, group_size, even + 1].cast(dtypes.float) + stores += [k_out[b, n, kvh, even].store((a * c - bb * s).cast(k.dtype)), + k_out[b, n, kvh, even + 1].store((a * s + bb * c).cast(k.dtype)), + v_out[b, n, kvh, even].store(x_in[b, n, kvh, group_size + 1, even]), + v_out[b, n, kvh, even + 1].store(x_in[b, n, kvh, group_size + 1, even + 1])] + ordered = UOp.group(*stores) + assert ordered is not None + return ordered.end(pair, n, b).sink(arg=KernelInfo(name="fused_qkv_rope_forward")) @functools.cache def custom_fused_qkv_rope_backward(dxqkv:UOp, dq:UOp, dk:UOp, dv:UOp, freqs_cis:UOp, @@ -109,8 +126,7 @@ def fused_qkv_rope(xqkv:Tensor, freqs_cis:Tensor, n_heads:int, n_kv_heads:int, h def _sharded_empty_like(ref:Tensor, axis:int|None=None) -> Tensor: return _sharded_empty(ref.shape, ref, axis) -@functools.cache -def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink): +def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink, window=0): def grad(dou:UOp, ker:UOp) -> tuple: do = Tensor(dou, device=dou.device) attn = Tensor(ker.src[1].after(ker), device=ker.src[1].device) @@ -129,7 +145,7 @@ def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, sha delta_vec = _sharded_empty((B, H, 1, N), xq, dtype=dtypes.float32, axis=shard_axis_t) delta_vec, dq = Tensor.custom_kernel(delta_vec, dq, attn, do, fxn=functools.partial(custom_fa_backward_pre, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D))[:2] - dq, dk_partial, dv_partial = Tensor.custom_kernel(dq, dk_partial, dv_partial, do, xq, xk, xv, l_vec, delta_vec, fxn=functools.partial(custom_fa_backward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D))[:3] + dq, dk_partial, dv_partial = Tensor.custom_kernel(dq, dk_partial, dv_partial, do, xq, xk, xv, l_vec, delta_vec, fxn=functools.partial(custom_fa_backward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D, window=window))[:3] if D == 64: dq = dq.reshape(B, H, N//16, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2).permute(0, 1, 2, 8, 9, 10, 11, 3, 4, 6, 7, 5, 12).reshape(B, H, N, D).transpose(1, 2) @@ -149,7 +165,7 @@ def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, sha return grad # TODO: remove write_flat once scheduler can remove reshapes between custom_kernel. TestCustomKernel.test_simple_reshape -def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False, write_flat:bool=False, sinks:Tensor|None=None): +def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False, write_flat:bool=False, sinks:Tensor|None=None, window:int=0): assert attn_mask is None, "attn_mask not supported" assert is_causal, "only causal attention supported" @@ -176,18 +192,18 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False attn = _sharded_empty((B, N, H * D), xq, axis=shard_axis) if write_flat else _sharded_empty_like(xq, axis=shard_axis) l_vec = _sharded_empty((B, H, 1, N), xq, dtype=dtypes.float32, axis=shard_axis_t) - grad = _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink) + grad = _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink, window=window) fwd_inputs = (attn, l_vec, xq, xk, xv) + ((sinks,) if has_sink else ()) - attn, l_vec = Tensor.custom_kernel(*fwd_inputs, fxn=functools.partial(custom_fa_forward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D, has_sink=has_sink), grad_fxn=grad)[:2] + attn, l_vec = Tensor.custom_kernel(*fwd_inputs, fxn=functools.partial(custom_fa_forward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D, has_sink=has_sink, window=window), grad_fxn=grad)[:2] return attn, attn, l_vec @functools.cache -def custom_fa_forward(o:UOp, l_vec:UOp, q:UOp, k:UOp, v:UOp, sinks:UOp|None=None, *, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int, has_sink:bool=True): +def custom_fa_forward(o:UOp, l_vec:UOp, q:UOp, k:UOp, v:UOp, sinks:UOp|None=None, *, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int, has_sink:bool=True, window:int=0): code = (pathlib.Path(__file__).parent / "fa_fwd_causal.cpp").read_text() compile_args = [f"-I{(pathlib.Path(__file__).parent / 'include').as_posix()}", "-std=c++20", "-DKITTENS_CDNA4", "-DHIP_ENABLE_WARP_SYNC_BUILTINS", "-ffast-math", - f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DATTN_SINK={int(has_sink)}"] + f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DATTN_SINK={int(has_sink)}", f"-DWINDOW={window}"] Q_BLOCK_SIZE = 32 NUM_WARPS = 8 @@ -247,10 +263,10 @@ def custom_fa_backward_pre(delta_vec:UOp, dq:UOp, o:UOp, do:UOp, device:str, arc src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=code), UOp(Ops.BINARY, arg=lib))) @functools.cache -def custom_fa_backward(dq:UOp, dk:UOp, dv:UOp, do:UOp, q:UOp, k:UOp, v:UOp, l_vec:UOp, delta_vec:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int): +def custom_fa_backward(dq:UOp, dk:UOp, dv:UOp, do:UOp, q:UOp, k:UOp, v:UOp, l_vec:UOp, delta_vec:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int, window:int=0): code = (pathlib.Path(__file__).parent / "fa_bwd_causal.cpp").read_text() compile_args = [f"-I{(pathlib.Path(__file__).parent / 'include').as_posix()}", "-std=c++20", "-DKITTENS_CDNA4", "-DHIP_ENABLE_WARP_SYNC_BUILTINS", "-ffast-math", - f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}"] + f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DWINDOW={window}"] BLOCK_SIZE_KV = 256 GROUP_SIZE = H // H_KV diff --git a/tinygrad_repo/extra/thunder/amd/fa_fwd_causal.cpp b/tinygrad_repo/extra/thunder/amd/fa_fwd_causal.cpp index 3deb93bf4..2dfb10cf9 100644 --- a/tinygrad_repo/extra/thunder/amd/fa_fwd_causal.cpp +++ b/tinygrad_repo/extra/thunder/amd/fa_fwd_causal.cpp @@ -269,7 +269,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p qo_tile q_reg_fl; load<1, qo_tile, _gl_QKVO>(q_reg_fl, g.Qg, {batch_idx, tile_idx, head_idx, 0}); + #if !WINDOW mul(q_reg_fl, q_reg_fl, TEMPERATURE_SCALE); // Use sqrtf for clarity + #endif copy(q_reg, q_reg_fl); transpose(q_reg_transposed, q_reg); @@ -288,6 +290,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #endif __builtin_amdgcn_sched_barrier(0); if constexpr (causal) { const int kv_end_pos = (min_tile + 1) * KV_BLOCK_SIZE; @@ -337,6 +342,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #endif #if WINDOW // window masks interior tiles that causal skips mask_kv_tile(att_block[1], tile_idx, j - 2, neg_inf_v, lane); @@ -401,6 +409,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #endif // Finish softmax for QK1 exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); @@ -469,6 +480,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #endif // Finish softmax for QK2 exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); @@ -535,6 +549,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #endif // Finish softmax for QK3 exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); @@ -597,6 +614,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #endif // Finish softmax for QK4 exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); diff --git a/tinygrad_repo/extra/thunder/amd/fused_qkv_rope.cpp b/tinygrad_repo/extra/thunder/amd/fused_qkv_rope.cpp deleted file mode 100644 index fcb7ee6bf..000000000 --- a/tinygrad_repo/extra/thunder/amd/fused_qkv_rope.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include - -#ifndef ATTN_B -#define ATTN_B 2 -#endif -#ifndef ATTN_N -#define ATTN_N 8192 -#endif -#ifndef ATTN_H -#define ATTN_H 32 -#endif -#ifndef ATTN_H_KV -#define ATTN_H_KV 8 -#endif -#ifndef ATTN_D -#define ATTN_D 128 -#endif -#ifndef THREADS_PER_BLOCK -#define THREADS_PER_BLOCK 256 -#endif - -constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; -constexpr int HALF_D = ATTN_D / 2; -constexpr int PACKED_D = (GROUP_SIZE + 2) * ATTN_D; - -extern "C" __global__ __launch_bounds__(THREADS_PER_BLOCK) void -fused_qkv_rope_forward( - __hip_bfloat16* __restrict__ q, - __hip_bfloat16* __restrict__ k, - __hip_bfloat16* __restrict__ v, - const __hip_bfloat16* __restrict__ xqkv, - const __hip_bfloat16* __restrict__ freqs_cis) { - const int b = blockIdx.x; - const int n = blockIdx.y; - const int bn = b * ATTN_N + n; - const int packed_bn = bn * ATTN_H_KV * PACKED_D; - const int q_bn = bn * ATTN_H * ATTN_D; - const int kv_bn = bn * ATTN_H_KV * ATTN_D; - - if (threadIdx.x < HALF_D) { - const int pair = threadIdx.x; - const int even = pair << 1; - const float c = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 0]); - const float s = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 1]); - - for (int kvh = 0; kvh < ATTN_H_KV; kvh++) { - const int base = packed_bn + kvh * PACKED_D; - - for (int rep = 0; rep < GROUP_SIZE; rep++) { - const int qbase = base + rep * ATTN_D; - const int h = kvh * GROUP_SIZE + rep; - const float a = static_cast(xqkv[qbase + even]); - const float bb = static_cast(xqkv[qbase + even + 1]); - const int out = q_bn + h * ATTN_D + even; - q[out] = static_cast<__hip_bfloat16>(a * c - bb * s); - q[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); - } - - const float a = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even]); - const float bb = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even + 1]); - const int out = kv_bn + kvh * ATTN_D + even; - k[out] = static_cast<__hip_bfloat16>(a * c - bb * s); - k[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); - v[out] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even]; - v[out + 1] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even + 1]; - } - } -} diff --git a/tinygrad_repo/extra/thunder/tiny/tk/tiles.py b/tinygrad_repo/extra/thunder/tiny/tk/tiles.py index 98cff727c..f27ee77fc 100644 --- a/tinygrad_repo/extra/thunder/tiny/tk/tiles.py +++ b/tinygrad_repo/extra/thunder/tiny/tk/tiles.py @@ -209,7 +209,7 @@ class ST: return cls(uop, rows, cols, layout, base_shape, ker) def swizzle(self, row, col): - swizzled_offset = self.base_shape.swizzle(row, col, self._uop.dtype.scalar()) + swizzled_offset = self.base_shape.swizzle(row, col, self._uop.dtype) row = swizzled_offset // self.base_shape.cols col = swizzled_offset % self.base_shape.cols diff --git a/tinygrad_repo/extra/tinyfs/fetch_file.py b/tinygrad_repo/extra/tinyfs/fetch_file.py deleted file mode 100644 index 7354e2430..000000000 --- a/tinygrad_repo/extra/tinyfs/fetch_file.py +++ /dev/null @@ -1,38 +0,0 @@ -from tinygrad.tensor import Tensor -from tinygrad.helpers import CHUNK_SIZE -from tinygrad.nn.state import fs_load -import argparse, math, hashlib - -def _python_hash_1mb(data:bytes|bytearray): - chunks = [data[i:i+4096] for i in range(0, len(data), 4096)] - chunk_hashes = [hashlib.shake_128(chunk).digest(16) for chunk in chunks] - return hashlib.shake_128(b''.join(chunk_hashes)).digest(16) - -def hash_file(data: bytes|bytearray): - if len(data) % CHUNK_SIZE != 0: data += bytes(CHUNK_SIZE - len(data) % CHUNK_SIZE) - base_chunks = math.ceil(len(data) / CHUNK_SIZE) - tree_depth = math.ceil(math.log(base_chunks, CHUNK_SIZE // 16)) - - for _ in range(tree_depth + 1): - data_chunks = [data[i:i+CHUNK_SIZE] for i in range(0, len(data), CHUNK_SIZE)] - data_chunk_hashes = [_python_hash_1mb(chunk) for chunk in data_chunks] - data = b''.join(data_chunk_hashes) - if len(data) % CHUNK_SIZE != 0: data += bytes(CHUNK_SIZE - len(data) % CHUNK_SIZE) - - return data[:16] - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--hash", type=str, required=True, help="file hash to fetch") - parser.add_argument("--len", type=int, required=True, help="file length to fetch") - parser.add_argument("--dest", type=str, required=True, help="destination path to save the file") - parser.add_argument("--check", action="store_true", help="verify the file hash after fetching") - args = parser.parse_args() - - fs_load(Tensor(bytes.fromhex(args.hash), device="CPU"), args.len).to(f"disk:{args.dest}").realize() - - if args.check: - with open(args.dest, "rb") as f: - data = f.read() - assert hash_file(data) == bytes.fromhex(args.hash), "Hash mismatch after fetching file" - print("File hash verified successfully!") diff --git a/tinygrad_repo/extra/tinyfs/fetch_raid.py b/tinygrad_repo/extra/tinyfs/fetch_raid.py deleted file mode 100644 index 4cf00fd19..000000000 --- a/tinygrad_repo/extra/tinyfs/fetch_raid.py +++ /dev/null @@ -1,42 +0,0 @@ -import json, multiprocessing, functools -from pathlib import Path - -from tinygrad.tensor import Tensor -from tinygrad.helpers import tqdm, getenv -from tinygrad.nn.state import fs_load - -raid_root = Path(getenv("RAID_ROOT", "/raid")) - -def fetch_file(item): - path, info = item - h, size = info["hash"], info["size"] - - path = raid_root / Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - - try: - pt = fs_load(Tensor(bytes.fromhex(h), device="CPU"), size).to(f"disk:{path.as_posix()}").realize() - except Exception as e: - print(f"error fetching {path}, {h}, {size}: {e}") - raise - - pt.uop.buffer.deallocate() - -def fetch_mapping(h, l): - mapping_tensor = fs_load(Tensor(bytes.fromhex(h)), l).realize() - mapping = mapping_tensor.data().tobytes().decode() - mapping = json.loads(mapping) - mapped_files = mapping.items() - return list(mapped_files) - -if __name__ == "__main__": - h, l = getenv("HASH", "d734f5e3be9f1e9d863bfaa4fc6c1ef2"), getenv("LENGTH", 175866113) - - with multiprocessing.Pool(processes=1) as pool: - mapped_files = pool.apply(functools.partial(fetch_mapping, h, l)) - - print(f"fetched mapping for {len(mapped_files)} files") - - with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as pool: - for _ in tqdm(pool.imap_unordered(fetch_file, mapped_files), total=len(mapped_files)): - pass diff --git a/tinygrad_repo/extra/tinyfs/upload_raid.py b/tinygrad_repo/extra/tinyfs/upload_raid.py deleted file mode 100644 index 5dfed6579..000000000 --- a/tinygrad_repo/extra/tinyfs/upload_raid.py +++ /dev/null @@ -1,32 +0,0 @@ -from pathlib import Path -import multiprocessing, json - -from tinygrad.tensor import Tensor -from tinygrad.helpers import tqdm -from tinygrad.nn.state import fs_store - -raid_root = Path("/raid") - -def upload_file(path: Path): - pt = Tensor(path).realize() - h = fs_store(pt).realize() - pt.uop.realized.deallocate() - return h.data().hex(), path, pt.nbytes() - -if __name__ == "__main__": - raid_files = sorted([p for p in raid_root.rglob("*") if p.is_file()]) - print(f"found {len(raid_files)} files in /raid") - - mapping = {} - with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as pool: - for h, p, s in tqdm(pool.imap_unordered(upload_file, raid_files), total=len(raid_files)): - mapping[p.relative_to(raid_root).as_posix()] = {"hash": h, "size": s} - - # sort the mapping by key - mapping = dict(sorted(mapping.items())) - - mapping = json.dumps(mapping).encode() - mapping_tensor = Tensor(mapping, device="CPU") - h = fs_store(mapping_tensor).realize() - - print(f"final hash: {h.data().hex()}, size: {len(mapping)}") diff --git a/tinygrad_repo/extra/torch_backend/backend.py b/tinygrad_repo/extra/torch_backend/backend.py index 4b600a4c0..85466aa82 100644 --- a/tinygrad_repo/extra/torch_backend/backend.py +++ b/tinygrad_repo/extra/torch_backend/backend.py @@ -4,7 +4,7 @@ # A006 Lambda argument `input` is shadowing a Python builtin from tinygrad import Tensor, dtypes, Device from tinygrad.uop.ops import Ops, GroupOp -from tinygrad.helpers import getenv, prod, strides_for_shape, argfix +from tinygrad.helpers import getenv, prod, strides_for_shape import torch.lib TORCH_DEBUG = getenv("TORCH_DEBUG") import torch, pathlib, operator, functools, weakref @@ -73,6 +73,12 @@ def wrap_view_op(fn): return wrap(ret) return _wrap +# NOTE: list assignment raises IndexError on an out of range dim, and the index must be a tuple: a list of all ints is one advanced index +def _index_dim(self, dim, idx): + idxs = [slice(None)] * self.ndim + idxs[dim] = idx + return self[tuple(idxs)] + view_ops = { "aten.view": Tensor.reshape, "aten._unsafe_view": Tensor.reshape, # when are views unsafe, and do we care? @@ -82,15 +88,13 @@ view_ops = { "aten.transpose.int": Tensor.transpose, "aten.squeeze.dim": Tensor.squeeze, "aten.unsqueeze": Tensor.unsqueeze, - "aten.select.int": lambda self, dim, idx: self[(slice(None),) * (dim%self.ndim) + (idx,)], + "aten.select.int": _index_dim, "aten.permute": Tensor.permute, "aten.alias": lambda self: self, "aten.diagonal": Tensor.diagonal, + "aten.slice.Tensor": lambda self, dim=0, start=None, end=None, step=1: _index_dim(self, dim, slice(start, end, step)), } -# torch 2.10 handles this natively -if tuple(map(int, torch.__version__.split('.')[:2])) < (2, 10): view_ops.update({"aten.detach": Tensor.detach}) - for k,v in view_ops.items(): torch.library.impl(k.replace("aten.", "aten::"), "privateuseone")(wrap_view_op(v)) def _get_view_ops(view): return getattr(view, "_view_ops", []) @@ -99,46 +103,21 @@ def _apply_view_ops(target, ops): for fn, args, kwargs in ops: target = fn(target, *args, **kwargs) return target -# similar to https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/InferSize.h -def _reshape_target_shape(shape:tuple[int, ...], args) -> tuple[int, ...]|None: - if not (req := argfix(*args)): return None - new_shape, infer_idx = [], -1 - for i, s in enumerate(req): - if s is None: s = shape[i] if i < len(shape) else None - if not isinstance(s, int): return None - if s == -1: - if infer_idx != -1: return None - infer_idx = len(new_shape) - new_shape.append(s) - total = prod(shape) - if infer_idx != -1: - known = prod(x for x in new_shape if x != -1) - if known == 0: - if total != 0: return None - new_shape[infer_idx] = 0 - else: new_shape[infer_idx] = total // known - return tuple(new_shape) if prod(new_shape) == total else None - -# TODO: can we get rid of this? only for test_flatten_reshape_add +# a chain of reshapes is undone by reshaping the value back to the base def _try_simple_reshape_view_write(base: Tensor, view: Tensor, val: Tensor) -> bool: if not (ops := _get_view_ops(view)): return False - shapes = [base.shape] - for fn, args, _ in ops: - if fn is Tensor.reshape: - if not (next_shape := _reshape_target_shape(shapes[-1], args)): return False - shapes.append(next_shape) - if shapes[-1] != view.shape: return False - for s in reversed(shapes[:-1]): val = val.reshape(s) - base.assign(val) + if any(fn is not Tensor.reshape for fn, _, _ in ops): return False + base.assign(val.reshape(base.shape)) return True def _view_write(base: Tensor, view: Tensor, value: Tensor) -> None: val = value if value.dtype == base.dtype else value.cast(base.dtype) - if view.shape == base.shape: return base.assign(val) if _try_simple_reshape_view_write(base, view, val): return idx_base = Tensor.arange(base.numel(), dtype=dtypes.int32).reshape(base.shape) idx_view = _apply_view_ops(idx_base, _get_view_ops(view)).reshape(-1) - flat_base = base.reshape(base.numel()).contiguous() + # clone, not contiguous: contiguous() on a base that already owns its buffer returns the base itself, and scattering + # into that is an in-place write to a buffer other tensors still hold, which setitem refuses + flat_base = base.reshape(base.numel()).clone() flat_base[idx_view] = val.reshape(-1) base.assign(flat_base.reshape(base.shape)) @@ -166,11 +145,6 @@ def _index_put_impl_(self, indices, values, accumulate=False, unsafe=False): def index_put(self, indices, values, accumulate=False): return aten.index_put(self.cpu(), [z.cpu() if isinstance(z, torch.Tensor) else None for z in indices], values.clone().cpu(), accumulate).tiny() -@torch.library.impl("aten::isin.Tensor_Tensor_out", "privateuseone") -def isin_tensor_tensor_out(x, y, *, assume_unique=False, invert=False, out=None): - result = (unwrap(x).unsqueeze(-1) == unwrap(y).flatten()).any(-1) - return out.copy_(wrap(~result if invert else result)) - @torch.library.impl("aten::randperm.generator_out", "privateuseone") def randperm_generator(n, generator=None, out=None): if generator is not None: raise NotImplementedError("tinygrad torch backend does not support torch.Generator for randperm") @@ -231,49 +205,6 @@ def as_strided(tensor:torch.Tensor, size, stride, storage_offset=None): def _reshape_alias(tensor:torch.Tensor, size, stride): return _as_strided(tensor, size, stride) -@torch.library.impl("aten::empty_strided", "privateuseone") -def empty_strided(size, stride, dtype=None, layout=None, device=None, pin_memory=False): - if TORCH_DEBUG: print(f"empty_strided {size=} {stride=} {dtype=} {layout=} {device=} {pin_memory=}") - ret = Tensor.empty(*size, dtype=_from_torch_dtype(dtype or torch.get_default_dtype()), device=_from_torch_device(device)) - # TODO: should return with requested strides - return wrap(ret) - -@torch.library.impl("aten::empty.memory_format", "privateuseone") -def empty_memory_format(size, dtype=None, layout=None, device=None, pin_memory=False, memory_format=None): - if TORCH_DEBUG: print(f"empty.memory_format {size=} {dtype=} {layout=} {device=} {pin_memory=} {memory_format=}") - ret = Tensor.empty(*size, dtype=_from_torch_dtype(dtype or torch.get_default_dtype()), device=_from_torch_device(device)) - return wrap(ret) - -@torch.library.impl("aten::max_pool2d_with_indices", "privateuseone") -def max_pool2d_with_indices(self:torch.Tensor, kernel_size:tuple[int, ...], stride=None, padding=0, dilation=1, ceil_mode=False): - # TODO: supprt stride [] in tinygrad? - if stride is not None and len(stride) == 0: stride = None - ret, idx = unwrap(self).max_pool2d(kernel_size, stride, dilation, padding, ceil_mode, return_indices=True) - return (wrap(ret), wrap(idx.cast(dtypes.int64))) - -@torch.library.impl("aten::max_pool2d_with_indices_backward", "privateuseone") -def max_pool2d_with_indices_backward(grad_out:torch.Tensor, self:torch.Tensor, kernel_size:tuple[int, ...], stride=None, padding=0, dilation=1, ceil_mode=False, indices=None): - return wrap(Tensor.max_unpool2d(unwrap(grad_out), unwrap(indices), output_size=unwrap(self).shape)) - -@torch.library.impl("aten::max_unpool2d", "privateuseone") -def max_unpool2d(self:torch.Tensor, indices:torch.Tensor, output_size): - return wrap(unwrap(self).max_unpool2d(unwrap(indices), output_size=output_size)) - -@torch.library.impl("aten::arange", "privateuseone") -def arange(end, dtype=None, device=None, pin_memory=None): - has_float = isinstance(end, float) - return wrap(Tensor.arange(0, end, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if has_float else torch.int64)))) - -@torch.library.impl("aten::arange.start", "privateuseone") -def arange_start(start, end, dtype=None, device=None, pin_memory=None): - has_float = any(isinstance(x, float) for x in (start, end)) - return wrap(Tensor.arange(start, end, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if has_float else torch.int64)))) - -@torch.library.impl("aten::arange.start_step", "privateuseone") -def arange_start_step(start, end, step, dtype=None, device=None, pin_memory=None): - has_float = any(isinstance(x, float) for x in (start, end, step)) - return wrap(Tensor.arange(start, end, step, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if has_float else torch.int64)))) - @torch.library.impl("aten::convolution_overrideable", "privateuseone") def convolution_overrideable(input, weight, bias, stride, padding, dilation, transposed, output_padding, groups): if TORCH_DEBUG >= 1: @@ -294,12 +225,27 @@ def convolution_backward_overrideable(grad_out, input, weight, stride, padding, grads = out.gradient(*[t for t,m in zip([input, weight, bias], output_mask) if m], gradient=grad_out) return tuple([wrap(grads.pop(0)) if m else None for m in output_mask]) -@torch.library.impl("aten::slice.Tensor", "privateuseone") -@wrap_view_op -def slice_tensor(self, dim=0, start=None, end=None, step=1): - slices = [slice(None)] * self.ndim - slices[dim] = slice(start, end, step) - return self[slices] +# the functional scatters. without an impl aten falls back to a path that assumes a real storage: "self.has_storage() INTERNAL ASSERT FAILED" +def _scatter_into(self, src, dim, index): + out = unwrap(self).clone() + slices = [slice(None)] * out.ndim + slices[dim] = index + out[slices] = unwrap(src).cast(out.dtype) # torch casts src to self's dtype, tinygrad setitem demands they already match + return wrap(out) + +@torch.library.impl("aten::slice_scatter", "privateuseone") +def slice_scatter(self, src, dim=0, start=None, end=None, step=1): return _scatter_into(self, src, dim, slice(start, end, step)) + +@torch.library.impl("aten::select_scatter", "privateuseone") +def select_scatter(self, src, dim, index): return _scatter_into(self, src, dim, index) + +@torch.library.impl("aten::diagonal_scatter", "privateuseone") +def diagonal_scatter(self, src, offset=0, dim1=0, dim2=1): + # a diagonal is not one axis, so scatter through the flat indices it picks out + base, out = unwrap(self), unwrap(self).clone().reshape(-1) + idx = Tensor.arange(base.numel(), dtype=dtypes.int32).reshape(base.shape).diagonal(offset, dim1, dim2).reshape(-1) + out[idx] = unwrap(src).cast(base.dtype).reshape(-1) + return wrap(out.reshape(base.shape)) @torch.library.impl("aten::slice_backward", "privateuseone") def slice_backward(grad_out, input_sizes, dim, start, end, step): @@ -341,19 +287,14 @@ for dim in [1, 2, 3]: torch.library.impl(f"aten::{pad_type}_pad{dim}d", "privateuseone")(functools.partial(pad_forward, mode=mode)) torch.library.impl(f"aten::{pad_type}_pad{dim}d_backward", "privateuseone")(functools.partial(pad_backward, mode=mode)) -def upsample(self, size, align_corners=False, mode=None): return wrap(Tensor.interpolate(unwrap(self), size, mode=mode, align_corners=align_corners)) +# the schemas are all positional: (self, output_size, align_corners, *scales) for linear, (self, output_size, *scales) for nearest. +def upsample(self, size, *args, mode=None): + return wrap(Tensor.interpolate(unwrap(self), size, mode=mode, align_corners=args[0] if mode == "linear" else False)) for i,pre in enumerate(["", "bi", "tri"]): torch.library.impl(f"aten::upsample_{pre}linear{i+1}d", "privateuseone")(functools.partial(upsample, mode="linear")) torch.library.impl(f"aten::upsample_nearest{i+1}d", "privateuseone")(functools.partial(upsample, mode="nearest")) torch.library.impl(f"aten::_upsample_nearest_exact{i+1}d", "privateuseone")(functools.partial(upsample, mode="nearest-exact")) -@torch.library.impl("aten::scatter_add.out", "privateuseone") -def scatter_add(self, dim, index, src, out): - self, index, src, out_unwrapped = unwrap(self), unwrap(index), unwrap(src), unwrap(out) - if self.shape == (): _apply_inplace(out_unwrapped, src) - else: _apply_inplace(out_unwrapped, Tensor.scatter_reduce(self, dim, index, src, reduce='sum')) - return out - def _copy_between_devices(src, dest, cast_dtype, to_device, non_blocking=False): if src.is_tiny and dest.is_tiny: src_t, dest_t = unwrap(src), unwrap(dest) @@ -404,15 +345,11 @@ def sort_values(input, dim=-1, descending=False, stable=True, values=None, indic _apply_inplace(unwrap(indices), out_indices.cast(dtypes.int64)) return values, indices -@torch.library.impl("aten::_linalg_svd", "privateuseone") -def _linalg_svd(self, full_matrices=False): - U, S, Vh = unwrap(self).svd(full_matrices) - return wrap(U), wrap(S), wrap(Vh) - # register some decompositions from torch._decomp import get_decompositions decomps = [ aten.native_layer_norm_backward, + aten.native_group_norm_backward, aten.linalg_cross, aten.addmm, aten.addcmul, @@ -447,12 +384,20 @@ decomps = [ aten._softmax_backward_data, aten.embedding_dense_backward, aten.linalg_vector_norm, aten.binary_cross_entropy, aten.binary_cross_entropy_backward, + # the C++ mse/smooth_l1 kernels resize their out tensor, and a tiny tensor has no storage to resize + aten.mse_loss, aten.mse_loss_backward, + aten.smooth_l1_loss, aten.smooth_l1_loss_backward, aten.upsample_nearest2d.out, + # NOTE: only the "out" overload, the "vec" one is CompositeImplicitAutograd and overriding it loses the autograd kernel + aten.upsample_bicubic2d.out, + aten._adaptive_avg_pool2d, # activations aten.hardswish, aten.hardswish_backward, aten.hardtanh, aten.hardtanh_backward, aten.gelu, aten.gelu_backward, - aten.logical_and, + # NOTE: no aten.logical_or here, its decomposition reaches aten.bitwise_or through a path that checks aliasing by + # reading storage, which a tiny tensor has none of. it gets a direct impl below instead + aten.logical_and, aten.logical_xor, aten.randint, aten.eye, aten.hardsigmoid_backward, @@ -495,7 +440,7 @@ simple_tensor_methods = [ # reduce "all", "any", "argmax", "argmin", "cumsum", "cumprod", # complex - "avg_pool2d", "linspace"] + "linspace"] tiny_backend_out = {**{f"aten.{x}.out":getattr(Tensor,x) for x in simple_tensor_methods}, **{ "aten.add.out": lambda input,other,alpha=1: input+alpha*other, @@ -540,6 +485,8 @@ tiny_backend_out = {**{f"aten.{x}.out":getattr(Tensor,x) for x in simple_tensor_ "aten.where.self_out": Tensor.where, "aten.prod.int_out": Tensor.prod, "aten.scatter.src_out": Tensor.scatter, + "aten.scatter_add.out": lambda self,dim,index,src: src if self.shape == () else Tensor.scatter_reduce(self, dim, index, src, reduce="sum"), + "aten.isin.Tensor_Tensor_out": lambda x,y,assume_unique=False,invert=False: (x.unsqueeze(-1)==y.flatten()).any(-1) != invert, # NOTE: axis=[] in torch means all, change tinygrad? "aten.sum.IntList_out": lambda self,axis,keepdim=False,dtype=None: self.sum(axis if axis is None or len(axis) else None, keepdim, @@ -555,10 +502,9 @@ def wrap_out(f): assert out.shape == assigned.shape, f"shape mismatch: {assigned.shape} -> {out.shape}" assert out.device == assigned.device or out.device is None or assigned.device is None, f"device mismatch: {assigned.device} -> {out.device}" assert out.dtype == assigned.dtype, f"dtype mismatch: {assigned.dtype} -> {out.dtype}" - # an out= that is a view has to be written through its base, and _apply_inplace gives a deviceless base its buffer first - if canonical_base(out) is not out: return _apply_inplace(out, assigned) or out - if out.device is None and assigned.device is not None: out.replace(out.empty_like(device=assigned.device)) - return out.assign(assigned) + # writing out= is an in-place write like any other: through the base if it is a view, refreshing any derived views + _apply_inplace(out, assigned) + return out return _wrap_out def _inplace_op(t, new_value): @@ -566,7 +512,14 @@ def _inplace_op(t, new_value): else: _apply_inplace(t, new_value) return t -tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{ +# the three arange overloads are one function at different arity, and dtype/layout/device/pin_memory are keyword only in all of them +def _arange(*args, dtype=None, **_): + return Tensor.arange(*args, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if any(isinstance(x, float) for x in args) else torch.int64))) + +def _empty(size, dtype=None, device=None, **_): + return Tensor.empty(*size, dtype=_from_torch_dtype(dtype or torch.get_default_dtype()), device=_from_torch_device(device)) + +tiny_backend = {**tiny_backend_out, **{ "aten.remainder.Scalar_Tensor": lambda x,y: x%y, "aten.floor_divide": lambda x,y: x//y, "aten.floor_divide_.Tensor": lambda x,y: x//y, @@ -579,8 +532,8 @@ tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{ # inplace ops using replace for fusion "aten.zero_": lambda x: x.const_like(0), "aten.fill_.Scalar": lambda x, y: x.const_like(y), - "aten.add_.Tensor": lambda self, other, alpha=1.0: self + other * alpha, - "aten.add_.Scalar": lambda self, other, alpha=1.0: self + other * alpha, + "aten.add_.Tensor": lambda self, other, alpha=1: self + other * alpha, + "aten.add_.Scalar": lambda self, other, alpha=1: self + other * alpha, "aten.mul_.Tensor": lambda self, other: self * other, "aten.mul_.Scalar": lambda self, other: self * other, # relu doesn't have an out form? @@ -613,7 +566,9 @@ tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{ # these don't work in out form, they have size 0 "aten.abs": Tensor.abs, "aten.logical_not": Tensor.logical_not, - "aten.logical_or_": lambda x, y: x | y, + # compare against zero first: logical_* is bool-valued for any input dtype, while | is bitwise + "aten.logical_or": lambda x, y: (x != 0) | (y != 0), + "aten.logical_or_": lambda x, y: (x != 0) | (y != 0), "aten.multinomial": Tensor.multinomial, "aten.masked_fill_.Scalar": lambda self, mask, value: self.masked_fill(mask, value), "aten.masked_fill_.Tensor": lambda self, mask, value: self.masked_fill(mask, value), @@ -622,14 +577,7 @@ tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{ "aten.masked_select": Tensor.masked_select, "aten.all": Tensor.all, "aten.sgn": Tensor.sign, - "aten.acos": Tensor.acos, "aten.any": Tensor.any, - "aten.bitwise_not": Tensor.bitwise_not, - "aten.argmax": Tensor.argmax, - "aten.argmin": Tensor.argmin, - "aten.asinh": Tensor.asinh, - "aten.mul": Tensor.mul, - "aten.atanh": Tensor.atanh, "aten.fill_.Tensor": lambda self, value: self.const_like(value.reshape(()).item()), "aten.flip": Tensor.flip, "aten.scatter_reduce.two": Tensor.scatter_reduce, @@ -640,10 +588,22 @@ tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{ "aten.add.Tensor": lambda input,other,alpha=1: input+alpha*other, "aten.linspace": lambda start, stop, steps, dtype=None, **kwargs: Tensor.linspace(start, stop, steps, **({"dtype": _from_torch_dtype(dtype)} if dtype is not None else {})), + # the functional copy_. without an impl the fallback segfaults on a tensor with no storage + "aten.copy": lambda self,src,non_blocking=False: src.cast(self.dtype).to(self.device).expand(self.shape), + "aten.arange": lambda end, **kwargs: _arange(0, end, **kwargs), + "aten.arange.start": _arange, + "aten.arange.start_step": _arange, + # empty_strided takes the strides and drops them: we always allocate contiguous + "aten.empty_strided": lambda size, stride, **kwargs: _empty(size, **kwargs), + "aten.empty.memory_format": _empty, + # TODO: supprt stride [] in tinygrad? + "aten.max_pool2d_with_indices": lambda self,kernel_size,stride=None,padding=0,dilation=1,ceil_mode=False: ((r:=Tensor.max_pool2d(self, kernel_size, stride or None, dilation, padding, ceil_mode, return_indices=True))[0], r[1].cast(dtypes.int64)), + "aten.max_pool2d_with_indices_backward": lambda grad_out,self,kernel_size,stride=None,padding=0,dilation=1,ceil_mode=False,indices=None: Tensor.max_unpool2d(grad_out, indices, output_size=self.shape), + "aten.max_unpool2d": lambda self,indices,output_size: Tensor.max_unpool2d(self, indices, output_size=output_size), + "aten._linalg_svd": lambda self,full_matrices=False: Tensor.svd(self, full_matrices), "aten.topk": Tensor.topk, "aten.constant_pad_nd": lambda self, padding, value=0.0: self.pad(padding, mode="constant", value=value).contiguous(), - # TODO: input contiguous is needed to prevent CFGContext circular dependency assertion for shapes >512 (see test_cumsum_arange_large) - "aten.cumsum": lambda self, dim: self.contiguous().cumsum(dim), + "aten.cumsum": lambda self, dim: self.cumsum(dim), "aten.logsumexp": lambda self, axis, keepdim=False: self.logsumexp(axis[0], keepdim=keepdim), "aten.roll": Tensor.roll, "aten.logcumsumexp": Tensor.logcumsumexp, @@ -652,6 +612,7 @@ tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{ self.ones_like(**{k: v for k, v in {"dtype": _from_torch_dtype(dtype) if dtype else None, "device": _from_torch_device(device) if device else None}.items() if v is not None}), "aten.max.dim": lambda self, dim, keepdim=False: (self.max(dim, keepdim), self.argmax(dim, keepdim).cast(dtype=dtypes.int64)), + "aten.min.dim": lambda self, dim, keepdim=False: (self.min(dim, keepdim), self.argmin(dim, keepdim).cast(dtype=dtypes.int64)), "aten.cummax": lambda self, dim: ((r := self.cummax(dim))[0], r[1].cast(dtypes.int64)), "aten.cummin": lambda self, dim: ((r := self.cummin(dim))[0], r[1].cast(dtypes.int64)), "aten.nonzero": Tensor.nonzero, @@ -713,15 +674,16 @@ def wrap_inplace_view_op(f): return nf # the aten schema says how an op is called: an inplace view retargets the view, a writable first arg is inplace, -# and a writable out arg must have come from tiny_backend_out so that wrap_out was applied +# and a writable out arg gets wrap_out's dtype cast, shape assert, and view write-through for k,v in tiny_backend.items(): name, _, overload = k.removeprefix("aten.").partition(".") op = getattr(getattr(aten, name), overload or "default") writes = [a.name for a in op._schema.arguments if a.alias_info is not None and a.alias_info.is_write] if torch.Tag.inplace_view in op.tags: fxn = wrap_inplace_view_op(v) elif writes == [op._schema.arguments[0].name] and op._schema.returns: fxn = wrap_inplace(v) - elif not writes or (writes == ["out"] and k in tiny_backend_out): fxn = wrap_fxn(k, v) - else: raise RuntimeError(f"{k} writes {writes}: expected an inplace first arg, or an out arg with {k} in tiny_backend_out") + elif not writes: fxn = wrap_fxn(k, v) + elif writes == ["out"]: fxn = wrap_fxn(k, wrap_out(v)) + else: raise RuntimeError(f"{k} writes {writes}: unhandled writable arg in schema") torch.library.impl(k.replace("aten.", "aten::"), "privateuseone")(fxn) @torch.library.impl("aten::equal", "privateuseone") diff --git a/tinygrad_repo/extra/torch_backend/test.py b/tinygrad_repo/extra/torch_backend/test.py index 0b45cbf73..b6f53f944 100644 --- a/tinygrad_repo/extra/torch_backend/test.py +++ b/tinygrad_repo/extra/torch_backend/test.py @@ -83,6 +83,12 @@ class TestTorchBackend(unittest.TestCase): torch.add(torch.ones(5, device=device), torch.ones(5, device=device), out=a) self.assertEqual(a.detach().storage_offset(), 3) + def test_out_refreshes_views_of_base(self): + a = torch.zeros(4, device=device) + v = a[2:] + torch.add(torch.ones(4, device=device), torch.ones(4, device=device), out=a) + np.testing.assert_equal(v.cpu().numpy(), [2., 2.]) + @unittest.expectedFailure # TODO: storage offset assumes a contiguous source, use UOp.contiguous_view_offset def test_storage_offset_non_contiguous_source(self): a = torch.arange(12., device=device).reshape(3,4) @@ -166,6 +172,15 @@ class TestTorchBackend(unittest.TestCase): expected = np.array([[1.5, 5.2, 9.0], [13.2, 17.1, 18.4]], dtype=np.float32) np.testing.assert_equal(y3.cpu().numpy(), expected) + def test_argmax_argmin(self): + a = torch.arange(12, dtype=torch.float32, device=device).reshape(3, 4) + c = a.cpu() + for got, want in [(a.argmax(), c.argmax()), (a.argmin(0), c.argmin(0)), (a.argmax(1, keepdim=True), c.argmax(1, keepdim=True)), + (torch.min(a, 1).indices, torch.min(c, 1).indices), (torch.max(a, 1).indices, torch.max(c, 1).indices), + (torch.min(a, 1).values, torch.min(c, 1).values), (torch.min(a, 1, keepdim=True).indices, torch.min(c, 1, keepdim=True).indices)]: + self.assertEqual(got.dtype, want.dtype) # torch's arg reduces are int64, tinygrad's are int32 + np.testing.assert_equal(got.cpu().numpy(), want.numpy()) + def test_isfinite(self): a = torch.ones(4, device=device) np.testing.assert_equal(torch.isfinite(a).cpu().numpy(), [True, True, True, True]) @@ -373,6 +388,22 @@ class TestTorchBackend(unittest.TestCase): for bwd_eps in [1e-5, 0.3]: for got, want in zip(run(device, bwd_eps), run("cpu", bwd_eps)): np.testing.assert_allclose(got, want, atol=1e-4, rtol=1e-3) + def test_groupnorm_backward(self): + def run(dev): + x = torch.arange(24., device=dev).reshape(2, 4, 3).requires_grad_() + w = torch.linspace(0.5, 2.0, 4).to(dev).requires_grad_() + torch.nn.functional.group_norm(x, 2, w, torch.zeros(4, device=dev)).square().sum().backward() + return x.grad.cpu().numpy(), w.grad.cpu().numpy() + for got, want in zip(run(device), run("cpu")): np.testing.assert_allclose(got, want, atol=1e-4, rtol=1e-3) + + def test_mse_smooth_l1_loss_backward(self): + def run(dev, loss): + x = torch.arange(4., device=dev).requires_grad_() + loss(x, torch.ones(4, device=dev)).backward() + return x.grad.cpu().numpy() + for loss in [torch.nn.functional.mse_loss, torch.nn.functional.smooth_l1_loss]: + np.testing.assert_allclose(run(device, loss), run("cpu", loss), atol=1e-6) + def test_batchnorm_unsqueeze(self): bn = torch.nn.BatchNorm2d(4).to(device) x = torch.randn(8, 4, 3, 3, device=device) @@ -516,6 +547,15 @@ class TestTorchBackend(unittest.TestCase): cpu_res = torch.arange(20, dtype=torch.float32)[::2][1:4].numpy() np.testing.assert_equal(torch_res, cpu_res) + def test_select_out_of_range_dim(self): + a = torch.arange(12, dtype=torch.int32, device=device).reshape(3, 4) + with self.assertRaises(IndexError): a.select(5, 0) + + def test_select_collapses_the_only_dim(self): + a = torch.arange(3, dtype=torch.int32, device=device) + self.assertEqual(a.select(0, 1).shape, ()) + np.testing.assert_equal(a.select(0, 1).cpu().numpy(), 1) + def test_slice_negative_dim(self): a = torch.arange(13, dtype=torch.int32, device=device).repeat(8, 1) torch_chunks = a.chunk(3, -1) @@ -796,6 +836,86 @@ class TestTorchBackend(unittest.TestCase): np.testing.assert_allclose(w_tiny.grad.cpu().numpy(), w_cpu.grad.numpy(), atol=1e-4, rtol=1e-3) np.testing.assert_allclose(b_tiny.grad.cpu().numpy(), b_cpu.grad.numpy(), atol=1e-4, rtol=1e-3) + def test_write_through_detach_of_unrealized(self): + a = torch.empty(4, device=device) + a.detach().fill_(3) + np.testing.assert_equal(a.cpu().numpy(), [3, 3, 3, 3]) + + def test_square_transpose_inplace(self): + # a same-shape transpose is not a reshape: writing the transposed values straight back would scramble the base + a = torch.tensor([[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], device=device) + a.transpose(0, 1).add_(100) + np.testing.assert_equal(a.cpu().numpy(), [[100., 101., 102.], [103., 104., 105.], [106., 107., 108.]]) + + def test_interpolate(self): + a = torch.arange(4, dtype=torch.float32, device=device).reshape(1, 1, 2, 2) + nearest = torch.nn.functional.interpolate(a, scale_factor=2.0) + np.testing.assert_equal(nearest.cpu().numpy()[0, 0], [[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]) + linear = torch.nn.functional.interpolate(a, size=(4, 4), mode="bilinear", align_corners=False) + ref = torch.nn.functional.interpolate(a.cpu(), size=(4, 4), mode="bilinear", align_corners=False) + np.testing.assert_allclose(linear.cpu().numpy(), ref.numpy(), rtol=1e-5) + + def test_interpolate_bicubic_area(self): + a = torch.arange(32, dtype=torch.float32, device=device).reshape(1, 2, 4, 4) + for mode, scale in [("bicubic", 2.0), ("area", 0.5)]: + ref = torch.nn.functional.interpolate(a.cpu(), scale_factor=scale, mode=mode) + np.testing.assert_allclose(torch.nn.functional.interpolate(a, scale_factor=scale, mode=mode).cpu().numpy(), ref.numpy(), atol=1e-4) + + @unittest.expectedFailure + def test_interpolate_bicubic_backward(self): + # the forward comes from a decomposition, but aten::upsample_bicubic2d_backward has none (nor does + # aten::_adaptive_avg_pool2d_backward, for area), so training through these modes needs a real kernel + x = torch.arange(32., dtype=torch.float32, device=device).reshape(1, 2, 4, 4).requires_grad_() + torch.nn.functional.interpolate(x, scale_factor=2.0, mode="bicubic").sum().backward() + + @unittest.expectedFailure + def test_interpolate_inexact_scale(self): + # torch forwards the raw scale_factor, Tensor.interpolate recomputes it from output_size, and they disagree here + a = torch.arange(6, dtype=torch.float32, device=device).reshape(1, 1, 2, 3) + tiny = torch.nn.functional.interpolate(a, scale_factor=2.5, mode="bilinear") + ref = torch.nn.functional.interpolate(a.cpu(), scale_factor=2.5, mode="bilinear") + np.testing.assert_allclose(tiny.cpu().numpy(), ref.numpy(), rtol=1e-5) + + def test_logical_or_xor(self): + a = torch.tensor([True, True, False, False], device=device) + b = torch.tensor([True, False, True, False], device=device) + np.testing.assert_equal(torch.logical_or(a, b).cpu().numpy(), [True, True, True, False]) + np.testing.assert_equal(torch.logical_xor(a, b).cpu().numpy(), [False, True, True, False]) + # bool-valued whatever the input dtype, so this is not | and ^ + i, j = torch.tensor([2, 0, 5, 0], device=device), torch.tensor([0, 0, 1, 1], device=device) + np.testing.assert_equal(torch.logical_or(i, j).cpu().numpy(), [True, False, True, True]) + np.testing.assert_equal(torch.logical_xor(i, j).cpu().numpy(), [True, False, False, True]) + + def test_slice_scatter(self): + # the scatters are functional: they return a new tensor and must leave the one they were given alone + a = torch.arange(12, dtype=torch.float32, device=device).reshape(3, 4) + out = torch.slice_scatter(a, torch.ones(1, 4, device=device), 0, 0, 1) + np.testing.assert_equal(out.cpu().numpy(), [[1, 1, 1, 1], [4, 5, 6, 7], [8, 9, 10, 11]]) + np.testing.assert_equal(a.cpu().numpy(), np.arange(12, dtype=np.float32).reshape(3, 4)) + + def test_slice_scatter_casts_src(self): + a = torch.zeros(3, 4, device=device) + out = torch.slice_scatter(a, torch.ones(1, 4, dtype=torch.int32, device=device), 0, 0, 1) + self.assertEqual(out.dtype, torch.float32) + np.testing.assert_equal(out.cpu().numpy()[0], np.ones(4, dtype=np.float32)) + + def test_select_scatter(self): + a = torch.arange(12, dtype=torch.float32, device=device).reshape(3, 4) + out = torch.select_scatter(a, torch.ones(4, device=device), 0, 1) + np.testing.assert_equal(out.cpu().numpy(), [[0, 1, 2, 3], [1, 1, 1, 1], [8, 9, 10, 11]]) + + def test_diagonal_scatter(self): + a = torch.zeros(3, 3, device=device) + out = torch.diagonal_scatter(a, torch.arange(3, dtype=torch.float32, device=device)) + np.testing.assert_equal(out.cpu().numpy(), np.diag([0., 1., 2.])) + np.testing.assert_equal(a.cpu().numpy(), np.zeros((3, 3), dtype=np.float32)) + + def test_copy_functional(self): + # without an impl this segfaults rather than fails: a regression here takes the whole run down + a = torch.arange(4, dtype=torch.float32, device=device) + out = torch.ops.aten.copy(a, torch.zeros(4, device=device)) + np.testing.assert_equal(out.cpu().numpy(), [0., 0., 0., 0.]) + np.testing.assert_equal(a.cpu().numpy(), [0., 1., 2., 3.]) from tinygrad import Tensor class TestBackendHelpers(unittest.TestCase): diff --git a/tinygrad_repo/pyproject.toml b/tinygrad_repo/pyproject.toml index 5a7533ea8..a039dcbff 100644 --- a/tinygrad_repo/pyproject.toml +++ b/tinygrad_repo/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tinygrad" -version = "0.13.0" +version = "0.14.0" description = "You like pytorch? You like micrograd? You love tinygrad! <3" authors = [{ name = "George Hotz" }] @@ -84,7 +84,7 @@ testing = [ "pillow", "onnx==1.19.0", "onnx2torch", - "onnxruntime", + "onnxruntime==1.24.1", "opencv-python", "transformers", "sentencepiece", @@ -111,6 +111,10 @@ docs = [ "numpy", ] mesa = ["tinymesa==25.2.7.2"] +autogen = [ + "pyyaml", + "mako", +] [tool.mutmut] diff --git a/tinygrad_repo/spec/tinyspec.pdf b/tinygrad_repo/spec/tinyspec.pdf index 4e380f5d9db2f68a7b1fa922f8b1c353bd40f2f9..1972ce8b51c709cafc1911f354425bfa6c3fd63e 100644 GIT binary patch delta 64462 zcmZ6SQ+t>V*KHe{O&Z&_)!4Re+qq*WjcwbuZQFKZ_j~r)d+k54j^`Y6tj{3G+z`k( zHh`6hlZ&4p*4f3$)X)~zW8+#!+8%clrRP-rDqf^6`{v>zRrmp!kSQ(`Z@U1uO-qpp zSG>i9@VB3DMMr7wB1>lyWoFo#hC){Mxwz}Hx;|fMST529;W^;y;NUP?MC#oqtfCav^ z`j&7;rg-uE_uq?JQYl{rDRASYCt4}=GaD1eZwGJ&AB7b;6U$Z;&l0Y}qKAqDI}7y@ zhKF>&9+J6rh~lgLvL@&=og6v(sF26R(miWhi)rO@H_3-%Onr=e`AN_%!TdQ3)jCIz z37&ieLVO8?;QNuYE^zD-+DnFe02T^04kGHauIPzozgg)GtJwn1D!{$JeTss|>KYB7lG}0pHl(&B zLBqtthR9-A1V6tXVbvAgkW;|}V0Nv<^=s;f;8B^7 zm)g^0yKrW%)tOmS{;WKE_3nm?`DP1Zc*n4Xi07*k>uo>>oY3Ho^BoHpTyDrTyQ-db zRa7ozyoA210VQ{f^jfr`+nheTB?%gBg~-I*6@P8$+DjM)WwC@#A)!5pRY z+)O6fT^`?oyF-vf#5TI4xWk0d7<=>fO`H}s^&%fBVh@LDY`l)JkLP^m3?{z3HN#?PQ~5!IrQ9NPj;=ZZ3%sT)8&uI~I5}-SN(2pAK*d zPsG|~hj$D7+0#db8M-9R$ii?E3_b}=lGUASR+}O(y7sz-a_fDb{0IqHhJIU!AKYx^ zU0M(GSvW>HxB`;A43B$<^;sBqDRiTR<-x#oVeD=dB3&7l!LT<^U*yX*sG5%%R9)!c zE+p-2w1WMQ6?WX|*)0^N$DOjLbdCg>53d8y+Syc4xJDC~r2$@>(0J#`Y>S$Nk;wYc zN)cbzqxcC@X2|Y-QeA4W+z|xyTmj7Ki?+!#+s8Wa-_ECCk5NHc810_E&Xa{UCKiCi zeF7;P@ngJh+7f>ilK7@~ZRQ$g6nw|t^-_iL&bAC%pzkRxtWdY=j1yyfD) zP!41wd0Zpn5C&NrRf9>iR_tU+*I#pMoTEFAoCh8EDPt`MDcbl^x3)gv-ZC`3yv9n$ z#0zW);Xd8lZKadrlv;hz^lD@*f&%T4qNYjS`E%2pQ17${=tBj_|6cioWKnYZ9zlAslGXl1~Mk z$qqHunnRWNpWMRG=d@X8L20SBraGBX>a){ey90`&TTp09DVB7bCwCxd)LGf-Bh-oL$F8>;<9rIY$Q{XO{sd7FOO-V90 zm*%ugE9*8gV`-?bnB7zBk}qrXQDw>7TVEGvAgzk#h2AqE$#YRwkj)6^f~bUgyd9#O zzN9T~2{SPMYMFB8r<%9*#^h>ITij*8*iZ8-B^_~vABj!Bwq-St>{3>^sXN}Z6{TxiQ*fR3W%dd;9i+1&nIE)ewvn957`v`*Z{g3{rsBC#G8E>-8Qid za&$T)eq4voZ-FT!N(oW88>F&<_h0_VyW|yu`G)q!$41n7)dW$!8A!gF%k#_K7_Ztl!}p>Xy|u^$36+Vm zfEHe=cI)3Bkt72{c(&>t1$SH=kv!>Z>FZu-{?vQU>DMU2$jr#uMVVv)w0PG^T^+jj z6E4j6IgCZZA0lxn15mWt+_m;4d;(e?Vb-lr!E}g3UjTN76-hi!7d&U}v!cNnoYf@^ z4pz0DFxdKWNB3Ss^xjE_gzJa2I`a09CShA?6!H`5+h!TYrY1y}$hJ5l^8Qy7)GK&H z7OsrtDO)_(@-(%E%nvH5m{%3LY6a%(+_8Q$c;5WNzC>KV&2><`;Tp<T8DbW~>Ny)1#n4G@$7ANp0~!33bo}I=jdq=KMn>~;HN?Oe+}@}W zRR*f0*#>kZANeYoQLVX}XjU*6K|;)1;HE>TCzM}VgE0C5GBZhi6K|feE#37wp&ijd zZyUdR=Zy7etLZT0I=!=)b2)vF$I+avy8Zad1IUcLL+h@ZGa!2D>n$UtF5@&}VTLnf~Ag8<%MR`ZbRj zYkwrx?#~&rD9gDf-5ygd4fZBM{wOCa;S3g<`XjMH7-0;d$KXp_xj(C%<{Qso3jgGm1%13MpJu*8Osr zC7)8dc26wl_D&>%H_s?yK_LLQsL%2Da+^VV}YbpJUMs1$L#1!+^mJ$cMQ3&R_-D+GBUc zs(0nX4nb2GwtXK0yttpNxe1RLf%&}NiM;on1}ZW#KcUyu&N@ji7CnhIxQ=}aUuPB^ zBo1D-+YOmN5lN~U1>v`?Sz%Dm(5Vk}dj7B*@RkcZLG&oK5T}&u7Vez;#Nawjvut%za=HrOlJ<>7u{)fPnt14#y`dDe<*%wG%!g zJO!Sz)<5hi|48cUwc+YlX?CcE%&rclWMhme_kP?=eT~=+D^h<~Tm@Z(UV9bnt~?7f ztGtEYC%xyH@=u!96_qN=f7{?P>whgpNJ3ow<(1Y5?EYxk9;Kb-0(-@NN}`7ed-vzZ zJ`J`?B6BAw?1c02;0OV0dgEKrH;Mj}&?wM;ppU>0MIm1y}Fi`eI9AhSnB~oS&%FW2Kndn!3#|CzrDY#WOj6L_Jk!y>Icj z0pmu!==El0j+Sioz|(S|4RUG6*hbtvRlB?e^-24FrFYD}Xi0VF$bq+OtKZjtI&)g5 zeT53~DmwZGGk_P;1%I=(`B5QbM))S#;HAdqGm{ ztnU|3Dn9h*mc4G`Ay<_t7baLcXfXxs3JCZhiR zIqv?zIfnv%ug#l_4Ryn{RAbkznEmYEe)}QajG}hZxoG?zGQ>i)kqedUR6ZV$9CJ0l zp8vBzPc9o7{xH|}g=*M|W}P%ea3P^$>vhj*!itC}jV26)H*>cp?J8yIimyTSU} zr^zoZdZo3lP>T&{>Miw}ypRtQ8GHps}B5cCB zQfg|h$j=$2NVV~6z>*}8Vx4yTr4c=7c)n~#r}9;7$}W8N3pzUiNnHO$nF|{H5_+GE zYt5S1QXm)*!&vmJuQM!a7chM6wEi4#`_Qg-g*)hqwBu}uLDn0lx*~o}V}+P))c8Ra z4QWq-FC)zW3(EToLdG0IOg4~+p8HFKOIMg{rEbAWgA`Mv^PE_Ualvbq=F-$altw{X z>=nf2!U7aePM}|GKw_*{S#AS9XIipV&1qteOo$aI?uII^r_xJ~W}TEVU{`h@hsGvd zjZaDt5-FkqnfzD+=gQFNf!#f~GOzhmN97mSi!CGc!?pM5Z293R<335LeQOi^5h^#bye!3X+Sb z%vhX*ZFJrqfDKpr6`C5~>K1K3GLzuX0MAN6mS8!cj-c!@RX3Zgf5lz1sk+ zZ|OV+J9?IDEK0+dMHSkCKKOdfLFtEg!K~1O`i)jJ-F}qiN(#A64iqGe(UeKE(UtK7Euz0Vm3;Z5y==nte{V1qk4bz{a+NHm;^m^YPEk7@6Od({gtZy!7%QHSseh ze)m=T=$+%&eL9=cOQ!|fU(Sscm_{*u2lwn6&LXe0!vYet%wpVsHXB}Lb6*@5)<(Nf zqv4*GE%e7H`fTvVnNCy(N3=*qDTWz-91wtHT!(28>Ol8F1{aJOWJVy=uQs6Xl=~2! zd3Y{$ov_YU*G3L=gCfSV+Kd3^UVA-;3Levjj z?MsfTtj<|~(P9C|R&{vzHVge75VWvHh{rf6-8a&7`T4;WgiX;0-KR78blY+f-l zP>Qw>#I}p0zuN0gc8Q7Ov@Osl()qfucA#YN%!<_)sdCU`{w7od!r)TQ24HA9c(VXLz%kcr#JBoODcZ##V$1Jl)FOF|3oRMb z7d_<~<_Xq}Ltwhxf+W0fYS!`&!Mp3U0Bb+6Upx1tPQOEh&!my#Y^xuU}QD)Dd>+TgDGgAiQL1bOZB zJg0mZ>>|#(7if8Mv6jcgGumFI3&N09Oh2s6DZidn(OfAckCmW<))$xHBw`E94gu#q@hQZnTMG~S z&<PGfgbgg9-(~adkLo3%7=x9>nL1>?{m z3rg^zpD77R-8`DsMBmXCR!;Ejb;2$*!rRG=4Hndy;sB+- zE?E3he#Vp6s%ja&H|0q)w$^+qN2xePN^HwD&n%NYoD1z7F=QN|YoJtfq8AUtT|fIs zC|e>4VY8Keg){ftGWVq4H{=}V{L>dM#ATepLl`rUE{AIrgY%-WxtKwuq3a=MArI@p39Vdft8D#>z9?&ooE1&Rt+`pz%;mXCTfNaS zOy^h?j}A-i#(zI`Pnl!H z(wQ`SZ8luWm$k7w6r|x^2%_bb@Od^e3B4rb1gu{ds1>I6e!|R_Z-?;L;`9XX&Er<> zwbNdMR6$y}1! zpaR(OyU&xo%opG`fzIoxBl353y0-1vfU?O7=E$-&Q_XwoFKBHD@Ho?weXDH5yr)7C zRbsjt&H;X{lscTj%@obG1-jOZG<42g5_!PC*TolkI5JT6>y2dW`;*i1@8~_Pm0q3b zo3U26-!!czO1VTWmW^#<(%HqF)DKlW$iOIn#aGYYu+iAwyGD@?qe7wM z*_apsIDqZD`Dm1%hmF}`5WEGhY4Zobs9EOvC@mo#GCnp2YR)ewO!GC*9)igPI?-odaF1Iyrs-NCe1yJl6 zQbm2v=9Q5pkXpJ1g*o3ij9+R-hB7Ppz}K7EA?c+}3uFoK!?-+w6^atOEf$O1u9I5F z(MvvS^Lo@3iQHW zWB|muxyxm*JQ=Ku@vBz%*%VX8*rbiFa4xZQ_ufT*j;`?0q)% z?dED}T9spJemSh~(zH$iK$Ze2_jJ`$7JprzTRGu@9@k;TM&7$z3>lI1LQFMt&ze;Bfy_>Hzl~Y#f_IQFp+>DwDTJ#aVxh|FGBl9-Sh^-Y^ye?^E=Bj!#E}yqTgeO z)oIVgYE+hHBo8=|AfVWt(Q0@*?w87TS9|@~;xp4K1nvd+=yZ`9pNFJIhhJDZ<7_7R$8vAS$ehY&E^1D$;)2EGR}-XZ8Hn{rQ}ur}OT6pT zA^s{2g#O!md~V+HoWj4GJiGb+HA$@FfI4u0=@ce}P2;F2r%~Rd?Wd;zSQDYA-5khI z?xx;_2?f}Z`mE=g zu}`t-lEuKGNLjiOISLi*VE6z6+>)Vds(+)i(NaHXLR$O5m4Z-#e$0;DD&j~ihZ;yx zHIUiAEUH0hV#0LR07YEIfa|sy)F3P@+lL%OyXIh0AS5M?xdH=T+-(FWmHZf-g31Lm{KdF#7m!bI|Rx zqu$K9CGP!5hK*5pjPl(JD+lv;Sg};~jfmdf<{e_#-7>_)`0c^Sd+|q`{xF0Dtp%ew z@V6F|Qgw5Jzj*};U7vO}1-4@*cDvLm^njH)f02I9{Hxt8U|AUBr9J>^<)avl8+Nbc z&umfCkcWl6c#q;*W&v-;jb_Ryrg^^V0DrjAk5#!F_GPJlHM?;)ytykrlEtw3{b+6o zNVMm3iF`_T% zeVBd0*CeD2cus+S4qk8JR;$g_vUSLDNqRiONsftu(@~91IE#n5al_5aNtgU=7w-Yl zIl#fcG5Upe`OBQdcTOTAEt^8BUtmj--76Qs%8@K-U8JO6z)0_fR@Q(P8-m%Awob}! z9+uSXyr-9B%Tmn}1ub+lu@4*^GGbjWs6&IiS_j<%Jn~7);#`c%_(gVdC!K}{?Xrgo zC@bS#4sX+6s2*pSq`6amYA$?wka`u+7ucb$|J2Yf`Lf2m*Hz^k?u$?Bcstj(*X(eL zt0@rK@dBOf)HpQpRF$pT77`@PJjdT!2m;Gji8GT5wkZ2tC&L@k9o;5x#%r`WIgpnZ zWN#e;nsDWyJ7<3*{QDXeI9-!h!20nQwf87krR-Csiu^`{U&FgM<~B7S270sl!7lK+ zF6P=<&-BM}XmcGN-!!Wu&ZwuXR@=4^0UozZ=h=e+CB6C&uRj4cd#q|i5|D;qQsT9g z6SuP154)`~C?mgn%2V}Iq0KF_qaAU<=#sl`fYXo>l%lh}&PqNeUUG_UWbP6Ei2bez zs(cVrT-@NL4X7PdGwVO|#V9Ic)I7nHgKljiw@RA^==9dy$V3b@e8kj4+@e%%5}L~Mp?lj;Uf)YP=O5M5 zz@3cl{pni+Z^4Ek99m1z!#nr8w(THmTcazKU>K54;6Fd8sP862ryK zy13Fm)%4CJ&cu9L6g%e1Uxo0xOlMnO^1XNP^+z^Hqch0l#O;T2Z@H4`ds2aV0njDf*38VurcIPhNIf}_ahhe&I$ldbV9k{4TXvp> zDNJGPW)A>j2mPj9wSo;w@!v%SDJI;r)ciX3LL3H~pqwD4E$d7tJ_7E|&9Y+vOe&}+ zvIw_5>Gkc4xT2^^PDN($%nAMe6oQ(zR0VoTi8I>jz;`dT099Ma3e64%029rUylQ|j zLeLKJtL)0u_&XQZh5o5JWv;M2ItYi>q&}w3;6B%i2U!k}ESKrELD`8zr?K67797)& zXJ8>0v@iKus4_0bZXIWWsB6m@TW6e;K+6EtF03aV8SF{t6Gc{@UynyCNYO}T%%s8T z1TU-1+sb3x#p$)-7CSWxNU^2T#PCo}3L5C24|>AQ*;aVv7H-|?*z=Yb-unn%m&G~h zRWwRoZ)omUi!JK!{G5uXmJ^0c&tifzkzB!Ii#n!SOC+=GD4`Hhr>$!O^y9N5VTghP zp5!i%`RcRL7@~*r)_d~l(h$myJR8-PwJ@1<&1%P<rBh`m zukhZ%5}=!?YCQoepXCO^qrv+Gw~PwwLrh|bN;_qx!hVz`|Hx*Wn!jYq)@5OJbE{`x)dIfzy{ADm z@e<{}e1CEI*mxg{R6ITrngmHp{YSLH_kxVu)w^>we)R;FQ&$gQkoZ$IB(} zpvOmT5bi+P(G)%wS(UwY;5UZ5Ng^71f7j1=9z>NW<@UUzrP83VnF1~%E9Cqr!Zd~2 z`$gSV#n{Wq>6rm@RpK1qk`)E5i){zR2{c485%YEUsSp{ zZ*O4Nz7FE$0WV!vVjp=PHv~1E^cEiCqfZRa3*`w;+x^Tsf+79KWzzuWDF1%mdGW-P zCB9M2&-f%u^QAC&J9OlLtC8Q-gNJn)0GB6yqNARLv5q^k4~>Qu+K#fFmlL zrw#Xmgc48@Y=Y7Dn)!C@u!C$Oi=h8Yfvyhv9{-1J1*`XvLwM!?O9dGxpP z+wdWoe*hD&u(D@KR-$dvD|?aFpRe74WDej9d!>nvjF^XtAohM-vhb&lDz%%+;*74T zo{7d{&AqFgJkCWmcebMa}V;pX8HuXyZ~mTlSl`HsYI)g@|GrL27@BbUJ~p zcqhYq^PILVtn&jR!dvz14Gw@>{rE1l3z*jZbVGe7l5{&Eq`Ws&*d3V~)j7{XM zp15my+4&jowo_h%_*FmW8bpC6)dv6!7GCA=hiXnkds|{MZ!5~jMO-aTgPkl8&$`W0 z&Rq)|@Mstj%+;KtF)g2#dqu3{r^4XYD?-`EbP31&Eb;o^;NH%#@D_U;RO}6@>s0P! zNqa<>zG1fV7+$^FJzO$9H+5ceuADSgi&A zb^Zgx^QDAJT&Cs(Sh$$j|F_0m>u4rw52O3d)++3syXwxreF-kj9zZERu}uo((H{AUE_1UA4T|IhYD@31-b z^_{s}F9iVB81MRucR#Hgm7{Scm&JZG>R144T{{+F@L3NvdY`8!_L;TpxdNL$_aVqQ zS%XrQ9J4nM4@SmVt5WULjf1PJSvau_z7Nxly*62f1BZN^uAg!{&URlhsIT}oZT1-a zlvwz62{7j-zI{@khyN`}7nnIP1r3IK0mkpSiE(dO9se{K~rRs5dkLh9O0&fGY z!nDwmt=hy*dC4TYNgc75rIoO?64$Sq93wO%MV=ux6u-8Z=eA1ITW|HgPKTFG0tOh3 zNmcYPaYmF=BulFWfPH5>vi|u0Tm~RiBy@<#N9@-j<2yH^pfHNOs_AzE(GU-r!)G?~ z6i(1?si)oI0;1xx!;HA$Xmm?=oqeu-uD`0ZL90;bUzV>YR&sibGVzG?W7bPC&IqQUjn4#Re#F4vhtWV8 zCuUny9AeCv$MV(0kc2XEMbyNrswYK6%OOVW^{tOTTRt`)%k@Gmwdgk#>465d3vHib z@0Vkc-SYqN58OF9W{48IFl7^1$uwrmuRB}a^&jj{M!>3}Fz3!^1mqCJ>MKk;&w)`h^PruLXrgSwURt_M^23tQVy`60}JBhm+kZ*&CqWn=RL;jy;9T z*jM0VF{Q_5GAMxCCGD*|Gt|H#vD(@;kA^+@3Xc?OIltYnU5te$8h5DgC4rdl>;lrL zC~j*nOGk=1NaDBfKToR8Xzf(Ja)Pv8dMTn5q=6@(zGyR~hB z6etndi8meDGe4qnZAStDP7<3nz)Li;^UJ2kXr}F%kRJ2y;T-rKn=p2qgPv8(1iBx% zX?hZa3c+Ols ziYPpi^j!5PjJdiLoBdvb;}-q}6!8~Rl8JQ&$Jy&l7|&ADrY{@lWQJwQiZLS=j8C+( zY_VBiiLHvbA8LIUZ$lbiCeq2}xGD~OCp&xNW1ZKu!~w1>N4wFo@jpSRyqQ)7n>PF; znU#E&&ZU1$X85YhSkb5k*^T$?R(lfAH!TuOefjt`v-o7Lyq2@s7@O}#B@?sVG-JN+wo!bjtt|?gXs}ZUx;Pda662z^DI(*DLevK) z-6ZPOY6J@;k(?x*>1K#`l}F3W%;ELMxb@TeC?0?}yQ1~>ul4z7x&JQuc<6Q{?ZD>!=<&M0dN}+d+8I-|y?)p; zBLM7Ud1eQT{OT-4Gxjp>cT8VciuoVTMy*VTs@fHnG_OC&U{=8wPa>2+WHcHr=pHK% zT@82As39CBsXYMR2t%x8c>-J01>L%)&cd}{si6ZUL-oZVjfBOzE|TV?SkpE(tb>FZ zk|0I@$w3pqxg>(p zTq>aj%qJk;_}po78<(GhibL`tY=s z@}K#5r%NRnd|U3?=!8JaA>RfIOdE2$8g!1|%)@0l1N!JmI%@th(A55gv50O^E zOrgyzo}!7yt&taM47s8bbv$53nC}lJ9^1 zMxANHwJeagm*Ig7G&wc=$Y=<9bXJ99BbJ}PdH+0AoL~vkYrnz8ey(xFmyT}r0jaY; z`G~6SJ^Y4TmY#zF!`*shHfL$=mmEnFD2`RaC@z$@qB?pALdt**T93DRW7QDvH0O1h zey+q9jy+Ewk{%N1muMf60LH5tmor)W(R0fzaeejOxLqDrTonCwIjG$cH97In#^^U( zc>-g`W-QiAP>!vhKzH595-dmNk;rtd+bz4ioDkBpr}1M}hkHpPSMV4z731mKmW>|09_Q(dytxTF`duzM75Z(j zNp_kovhJXZtW=NJ;2o7EH3SXQki3w1%ko)?cb3ldd@xxiGnXS>|4<s-G)f_o zi0rdV66g#q^#HkujL;upSr;=-I6z(gI8KoYL0fxzEjHFb#mV`-UW+&jLI#d~H7Jhu z0C=NTSrf4TqZJ7Xh_ZAsZvpvGX=~0i7g!6n>@5) z`j2eYf)l#i7jg!qH|HFL6<|7fnF`IN5PdEnI$Nhr@<5mJb2W#wn(O^mpX9Qo#4?8C zDt|t=?0I8nw+h(1d!NC@y@zTcY-h5tYjq7oU!V5`IAhfbE!Dg4Zc*(-Jfd$}7=lTD zn`MH6iOI&r9HJ=RxIp^~&W>+{WtMZ7Ex^^b!#6%*kh|5QcijLA|U3?d?AWPsC< zUgw{j767e)krDLO{GNd=B1G5uR4mB7H5FR|H6~)E8Q>OSij9IUcE;+qO5_kOJ>5FF zfmHXrNZaO1%I)MxYS@k|8Loz^Id#TLu|7Gi->akJ^}&p?QHzY-4^mBi>3j!aqIY8) z1^FgJc5n6Y=T;^Zya_o4bucynCx$&l%*Rd_{gnRF3_v;@PR#lZTfl<)#^JhTi)QR3n)Ex8bDvl) zn&f5|j=~}zZAqBE%ojTOa|>-H<%*cM6xZcvQ8NploexeY4A5d><+NgX6d=S3>DQtH>Bd|~e{0L>aoC?|T@j84C0(l7Svbdu*4B1!lyVm|x z@k~d*(~wJ>Ue3OzFg?3E_GfLz=cml%6e0?mS#&!jn$A`e>? zjnz)}eM3E*`} z0bsQwQ55TBjCao?>FoPN#JKs`C&!E?TnFBUo3y*Ee{_FA6orN^g^Z*p9hH*G-}CG7 zN(z(3tprZ4Fzl8lio1`C_j;gh(3^43{@TgCoVYX1oEpaMW6+guFi;*{FiLE{wH{4x zY%tG2Vz!g@-`Ef-iNTO(mZc$g0C;++)_xW;A3*E2-VmQeli=eAa&H+M#Z{47?TR?a z_5mBWP3kQAumxNR`<8FTV1yVd$EIpze*Yb~pTrCuQf;xVYpaYD^E+WqYQv1rEtlE; zqq6CSsx)!+bc5#lW#(9#PW3HblB7#!_`2Hl6|g5<#3V~exR3(FVrviKBxle2 z{=C@MV#!c%c@)WITiHxi=gGe)M-yOe>8st!vCW)=7VWLWPlfz1b>84gFg_F#Jm$nf9%n z1N2)9?4N$Tzmeyr0#;87kTQIOxY;Tq5`so=caFuYFqk)L z%0wS5^)t+h6GT-T!Aag;-Zo1g{cr%x5$|CtO&fSxw zM6%Emxx5q97kICUgdwY1>l+_`++P;Yk^6CE8~b5^rZA5n3wOb)(XXYqU?ELftc0%_?7#^HZxM%t&m#vYCpokz4h@5Axea<-MZge(^T=oH%{?wf}mSts-6~!Ig zjP~e-zrhW=wA_9{)hmOC9lW%_@~5W08pv@*9)Pfg_(|Je+4F|<#e90fQ8MMw{oe8F z8hsRTmg$@o37XXEAe)&~T~xXz0SOT(a@AHI_+U$y-VK> zpSt^VY-`lSm9DVdw4SO(G=dzBXsC=-#*f*>yoCtSPi)?{SW59BQ9U|7hMqsNV$$9i(t^w^~|>!8DUT2X?FF3G;@k^0qOB6(go z4Hbo>u5F~eWA9rh*m(|IOEI0hpy#edWIvo-2#@mkN{ZMTdeJEwt4vYhW9}R(#2YF@ z$V`f+C6EXr0i11Us&w<1*rKvf--DD)A4E zyiVozpIpbnV(~y>Nmv3AaBXEd3Zw!vIoaL9I+M!he2fyv{@H9KXKiqC`*T)GgdZAj zF`tHatE)IlLtcB=&EPN%@eey2ruV8BlDLxeyD|C(e!9#{zk(s6?5aRPgks+@-`#CmRmb7|==|6XBQqATFfyy$4$FhV-R^r{89vPvY z&-_d~qb)!C`1*IVy~5^telKyo8czgM`Z_Y!90qa`)Z^U`Ze{;9mR+@Kkw6u)-?~{% zD?vi!E0l}^oo#q+)@-X=p%>s7#I>#)tj$ggMYla+v_nl{0M510SPI4#ILc{zOdO$T zWi*ysTL$9Nyh!SjU7GHvH5?Nu-;rFQ`n&Y(!#6XCL+0^|nD?<`im%B~F=!zdo(Dt? zhXNswg$+;fw&(=xMoD*NQwB(|BCg`KP$+Tg|B>~M(V0NYwrIy@$F^-d9ox2T=ZkII z>e#kz+qRwCd%QcwIs4sz>vyfSs^%=rN~q5VyMakHhb&g=k&M)b=jlc?a7?RK>Ds53 zEn6mz2FQ^WZ(Dbodh+$VZe^xk`-q&XDZywhn?HJAt|mJ(UkCQTsw$nZ%C^$x&3rIS z#^QlG2GHPRD_<$!AGh`ltI;hzS=X7581k|JCfVT(a^&t zuIXyPb{^9{hXcwc0pgTY3h+EM%_5E0NqEt{0i2Y(MCGWL-Z^&5x#}Yo?ko{uSJgX> z^XVy3x6S>=7Uo#tEkSQK={O7h?lvXms8uv0u2F~DUXawdpYrZmf?gbniM|%kj1nVl-kPzbtAQ!hmP3K!O)yZ z0JQk~Sm`uJXxub)o#Te3^R|ow=HiTlew0=$7Zt zvv#siPT$oLqKKv&oGa*a@5urOHmE~$9d+AHEsjzjz3FuJttCIZW7e#xZ7aLjuzH3A zbI&x5iHx(jOx-h!5e7{)Eg0FZTB~d$cSF=|)3nA+Oq4#1ZWWqRaTH*`T7l2t(*g&(In!GP_1^m#06BJ{H)Lu!r_Gmnlk)>3zmBw&JK%Gy z!6s2!p%h2rUbX#rMV@|yAAVq20O$gr9O|w&5EIur25T;O=n#D+O7negVNLWbFF?lF z)I_iPe6}^s^_Bn^$Mu69zN>PFVg{gFy<{q;;04#WiI>F4pYOj}5O(m(s?w6hyKbVw zkYR${uBNqBE|QOgn6kn<(>5b%2Vb)d5+N zV%e+BL?j(9uL6)_M_p)_uZ+L7R93h5msV!#dj?AR4U|v}gcGH%q5gkC1s*K7Wkh4p z{ifxL^#K_v66MNvvzDmW-EDe(AB9Yl%Jo>JNVOfzs4!#si(}BJ<79*t&It?f(Y-~* zy!0+qF4@=D7mC|&Pq!7z0Pkl15iIIkA>xzUgHh#*e`vs(;T{$>X7elTfq7*Z>etEn z-gkBeR$Z9^1nV(ssjRwH_5o{umGkeOQQb2RW^|4yj)Of-R3@T`*vrRIXhOv@xZ)r> zrXIY&t6G}=zbG7)h&la?Pn+jiBaWE_$XG8|i>ff`V67#z^C7M0z>M;ihQzC)C@G4T>ak&MPdFoWn zM#&gn!V1r7aEtDtrIGG1&&j09lDvvSw%35W-&>8JPTZuI9JZ=Rr;KE9uY?UTLpCm| zfmAM?Pa6s@piLV$fYVy;lSdd+lin&BS=IU#QJgM&{bI;8_5s!#gw{?W9vtW{(>ZM^ zJ=MeuPnoSYDyFM*-m>OElyolLSs|+-m;yI=NBIo2CSdkX%(&;$vF8!ds|`iY>~opK z1=b@yNsELI6jUV|a~FbO==JuyupX~pC{Sfl6$Z`+xND750KW}GOk#xilEKKwGaY2V zGbxD1Sj}g(*!f?_m}hJj?jTX5Q0%BWRr<-td*ks6886}t@|#Vmj>+5*vL$o_56-FE zu@q|9&m*_BPy^mO8Q&Q4RFqVO*DEP9$4KjDz6c61>CcSr zDinf4Ve$(k0DTrmCb^@Qmagl2=N@pG##8#&K=awsw@7+HjHQxqzff3W9%W|=R&{^u zNxOSP@=NCyLT%Wf!XwXYWG;}8Qr_WGSy9$n+jU6&h~F{+GV_#F+DP=h1Hyx-CWxAqT_3caKo{X~^9?YEnYyaC z4VF9ael1?&=D=p&@bV5>QBk9$)#&977ln2b1} zSRfY#Ah^S_l4rYzu^>Ot(&9U%=npaMQtlUqtsb|eT&>~j-w1o$KN)?;H!?b@>Uh91 zU<36ekXnf|^?=AUai5p+16?Wp~u`=VXBS3w

B zH5dJBA?v3Qr-5ky2HwM*<{}WiLAf~LH%0)j|CnD(z>?KdgxeXT=}_D$U

@qiDbxG&`shp;G?P#dw4Q zSbb=fjIqO7KV)k*xZDGQpaRe^G?)~%B8A?V#Wk>|fVYMqSc^YN%Eo+3E28y(&@Gl( znXOY?^xU84x|UqxUXn-1?^H}X(Blo%W;p^#Y?j1;gH}bIh=(Dpk{j;ANU~uq>g&0Z z66XdZ`XG~*ACycmFFI_A|C|_2+C}XN=bLHq%RmqR5XsHX?M;b_M)0s}c2cV)rR=p{ zhNOup0;uirR3{6ms-edHNST=W>@;^Y!A+bp=ROb$JyGXNdA2VjvQZhFDCV}8a=l{!V(F1lXZ^o?>t!Ud^G`KoKj$8@2}w@rB{ zQ4n#lAI}$b6W1#iPi9f3Y@v|WTYASAPZwqm0Zuf|Jh)(Lth?rD?#(~jx&W0RHmK+c zj&d^4*+ieDu$A1lnr$N-L~`yIPvs+A#ZrC_74K+mVRg~+Ku}LDHvM$2gB_CMENRB{ zfnh#UTV{#4HhCKZthpBZOy8>^i(feqySr)9+bx;&#>_844@)*CT5c`Qalquo9I~&# z0IpM^IqoDO#pB(Zv#knTiF(tvP(5GAQO(S+>n_~NxB^SZhh5%eIG@Ysl-7HXBA;F4 z#E$0)_d<$sKJrpech7W8W@_1x2w=mN?Xr~^E3_i#O#k$=nyELLXg4XuoyxredT0zh z?7I-So0e#YHnrCj!-M0T=!s*(t2E9XfXec?)9DhugFSCGjazy;es+oV7$r2o%CHOu~K5zVjDlE$m%+X5H_`2-OGI?z+AMG!N3_;@;AtDAvbrZfZY#(^I z=j#}7b^>oLecWkF0Ny>phxBjGP9*Ud<)i0-{yO+#gvj(YjUok zt4XGY)&uE6??v-aTU}f;dfoh!CDy{eM{0(EiQQFf8GQPcKKBAs2o#uTqyq-dP9!? z-Qg^@_4kpw691#n>`1|nNznx(6>MGBH7!!s{3y==Ovtv<(IzUh0_)LA&fQwcJcZ(&! z$C~E-L~fpA&D+`Pmvo1?K7j*9hd3WzN)YYd07GJc?-fe_|~S{Wfh$Nl?+aGw)> z#fF6qOa~ja2c2~nzAqD%7crX}Tg1FxdkaJ45o5yVsB$u8i#~!mkz=VXM$+OPDh9R3*b%1E{3wb&d$q;d$DQHD$9EIyktx1xNUXk*a1c2-IDYd39nZ%<0KnV~bChA{+a5vt~e za@~x((vZ2A$T>B3m2eKq{W#-G`_tAjJBn2Isq3Y0FYn@{Sk#w)%CB?X=k32S=heb0 zkJ)VZPUv*3pXhL3fGud`tUQkX`>E{)DCQAA5&9EX;I6JEoDp;5DN`BOt1fEI9kl6GKp$1OYHPsFXmaSSmD=u|P^QKJ z4Pz`3)=*t&)c5uStTB|>R5c0lDw$l|wT)f&m-3i=vA8@d!0gjr?KzDEMzTt%45x#; zjaPX%@9{%*$iW@rxnGfvxQPQW_k^OB&C6bQ1&O^p; zcrrv$r1o+F(C;v$6X#yFZ+Y|T+~)GG?m?(a*8~K9)T9BYGr_`g4ktQghvzSf#|ja7 zl1}vtYze&UKq{=~u2>v{PV^iDXkb6(l;4yqpQ5~B`_YYv9@DeQHC1)wi1)c$xdnJO z`h_X=VbReWQC5q^&V-|9+RjBW;DQ8&F2|RjRD`e%;Lw|#L^TE`Cq#W{?lJ(%YrqKQ zTKRj+(yw)pQdi3$GDx%rB!C%ZO=jDbwGL{VlwigThPy>*q%Mjw9RTR(ZoGa3O80C2<6QjpLG%KMN!dkW?Mvd zr$T!UUR@RRX*MRXskjqKq5g>)fK!Q1g1CaDHKDUCc%C*P(}01p?vWMp zn}f{$g26DsWKwn|;C9%l>1?5I3A%zsRF}9lCOF5e^|1*rhL7pT4Y4VNK9**8+ruX?uud*}P9F03;*j!Xo|`RFBH(y|e)`jQX=vkKX64+jkrk6+!sOb3u-X{V$eyFZgD zjAtDL=c~ux+@(QS?^;_eXLSfkTxwpu@#h4Mi>!HG9|wA2KV>(N6qfY!83!?ixjm%a zfMZU`(W;EYCG2GS>~;v#edcwY1kv{vc8IN3-&yqWV+O3>Mpj97VPGqYA&`+NXYS65 zuX%e6LB1;4pW}U7c$q%IH`Pglwvu6fS)c2ok$i>>H8v8dWfc&)h zy`h%NOhR&&jjvbL9R1XQ=T^m0tB*vFjh2h3hw1@|VbFx*Mw7}vv{or>Q>gW6sS#kq z$s({p8Vw=Q8ncola7IjVAo|$XDv3Q2ug{#*&XF2lFVtA_XZitZm=bHg^$Tvcm9@3A z{92;TkF+Ut1*05kJWzLQS5h?y;Im`0j6;AP**a;4*pkJ%UssMIKZ@^_bYA8~MlL8Z zgyqv}mn>g1BWx7sKl?Ifztq=fLt529zGjyqp_IvK(;0gGcQf~U)i3391iYTcr3Jq5 zph-Eg&TfbaA@T4#N)rYqZ&^=nS*IsfG^a;g;!#^2W%#|^PRD4iUA*BPAkFT=vQV12 z=Hkgc@-z-H!M1=f0mE2m7Wsu32ptP%YNT2KN%U*DBlhzJ!*YqhfmvPbc-d{IW%$|W zqM5vBF^(vZ_GjlgyRk*9C6!K%VzOyrQ`%#%?+@p44|F`A%N8g7If@yJE@<)mj$Ih5z)4G@zh>m=o~~@c=Tg;Q=MX>ud$7 z+lb&&L@*k6yDjvWneL}#ai6ZHJFu}D>}w@jIxqcSY&P)<^yQkVA+}drC?wO)f13^< z3MwOW#=0ZPZgAMwPPWXp_oNZt0rKW+PdQ!ZbMKbhfLM$gxf#SbEh0GwNZQ8Fc-d-;;udN@wFsrB;*kZ(EL*J zFdRiX@XNwe!oZUCIm$o21tUYlC-Nz&nx;tBkPFWTE8-6js^7~_c4c8 zqyEl{%`0f-`c&3N@TI!O@}*K%vzO0r*IduvJHr?;!+AfMM0Ya^A#7wHhT!uNP*#R2VxAmets`Y)P{YQ_ zZ7_^@CH(V6$GH;9Yu@dB?3kM6tD)9(Zgz}6MS+#3u|jR3k9G6sS4V%9nl2*KTBJvM zTMLHf)!R|`Qts8SNdUjE56*MV(KpI%%Y+9yiL{UyfO;AMjz;^~Wr?Q*Gepycc)}@` z@h1pdw!x!z=}(NBd)>7@Nt*7~>bWA@G>cmmK>rsj^EGPl23=mlAcetv)vtweYZo;Z&(vDJ!is zZxhCKsR)o>Yb_qYpDd~ttM-jFb!!e;Hc2V~;j3(eB0|Q-jbalaiZdnNB8%V=`in3S z5Fmhw#J!)-IYw5gR;do{{oPh7&Y{xjJ34LUrdYy1+2J)Zu4JrV!E*9_$aL~?qJuZE zk^ z1Eia)UrN+hURF)UIm!D|2J(w2CD~hsU^X|=sv@JM&~9L2<60--(ogogz2EBJ=pr{a zQ#vY9p@u$$0t~fz0!fgB6yk!ueqZh5Vgu^sC?s6dJE(++!nNw`4GazU0|*rQ1JoT} zM;?wlNUjq_TCLH^AcsDJ^Xae+jY#P9K*NLRjMC7fsxAyg9TdamFZVIAsZd60)p2*o zj~)34Hmy^Tsl%5B@%m#S>)~3$n^8B&)(}A@qzTB7p9huu!*_rGh<*m^hp7yX1OXuG zVqTNf<*W(VkTb((pyc*p`Xg5*t%*CK@cUC&1usclf9ushf13*1Lu&1Vn?aTOIZc?h zMCW|KztmS%*fds+H*eb}ZmDcjZIe1uvW7A#(38>R++K)`4Q3|xlj}4TU=8x19dcu-pt*e+?&vu_uN*lw($HafSM$_60< z0Qg7d4#S#XFgKRdw;puukxu5b0c6F?d4{ipT(v!g^we2X}-hkjMy8SWX+|*wxX*IOSopsTQ8V9)IQUfxWj<l``JB3O>zwfTV25&^!LJ`rg z6|L#`<_o9U7FDL|R+}7X@e5F}Sa0Dk^4hte=D}KlSnuHj=@^j`<^hzlf7`K~Lovl7g)A{#*z7fL^7gs4r3l(32dXx;kXAl@HFS|cb$5~QI|(d> zuPxP~tnRlq_&MwLEe#PK2VB0!0E%#dVC;Kh(4hwv&J@Efv!x_byVAZP|H%}H{Nl>KT4g_l#7b~F~`VW$Vf zFkq^TN>iJs^s>|=<5r@D?C^+VR*M^9tqmXZ`dphBqAk>7THuOw?9;c0Qq$6f8HL^% zL*ku)SN7BRoj7bmy61tK1*DlTG}l2_D|EVhKDhPq65*>7V}KH9qK%1$%7YAG!XwX@ zugcf6-mWWG$))u{+OCC9knvbA*6EUowlgq za%wm>A?RnsGvK_TN%9%na4{Tb-2?bq)`nQaoFw8oKnHc|JOto;!>HS@S z_Z!TO3sMRU&CekB>B>zQ^ik>YfymIy# zM=ZD{aMr`;t!1xzDP#M4!X(}!8w_?`N(iDcq$Q~Nx_T94@gB~LJ-l2=+C^oF39^%x4 z*5zYLuyncGBi=&kDb=2@`@quTNX+3CWZ9p=9_9h!w_ydnmXRrGYz_F+M{+8lqdDjK z45--GJPkU2)y$_<@4yWpls-c}=;7DIID`2{)Xkg~D!$N&uQ1HFPR3L?7(gprsN*(2 z8F0SHPWLl3Z@esxVV)FhEwY40P#qPG6C6?_T~05jj`jVJxEOIi7n0tkg#F>nVdFtp zF#iKmPfyAD3z`fj$wSg(GGK;zz;9p66+px1p8`EBl;Wyh5*s|(UXF2Cs!`%CYDYAe zrO3_P#@A6M)_YUOBE#Fi|Mw3Y!a~}QFBT;m@zNe;Z7Y7U62*@8Cpoxkk(c9)@Kr{c^XO2w4cP~<%^nnhxG;HB-{6aXuI zX)0?4Af2$|Emk~`^A=-qf80QN2E(1u7Dbj>ZWRB`GH=Ddtd@lug-!n8FbW5TM~LVdv`&oAd`l zV!;sg5wTv(ib%zTK!@;23EHL2WgjKOp@R@Dw`vg_MSkZ#Z5!uNcDbo-#Q@rEZo$F` z5giv5-ae*!@A`%87*ZOM}G0H=ke#ErG z%mr%Vik3QD_FigH@%)`_Ism3Bmimn=?x?}wS!nzd$fuZj*AWa>hq|1%ll(MVvcaZU zV7s2u#^U4nNJ(daD$tKb_Yuby!Scu-se}Ck*t=t{!ft;|X0CCAgFSfrx%au5vlcsx zqt>oJPngkjtA*BY%xoibRz~JU;H7HHwfw#a^@6b$mE=GCNuT6pQh=VG=EdAAOfL{4 zUdr#Q6O!)=Rw~~aYO<00S2S1O69DjQ4+IPR{%|ez$1QioB>q zK#9N_gfJ+uhQ%-89RM+S+3b4D*EtY%j7_t^Jvh7)@R^2c?B%)ZV&kJmf`E8y*As;i zdzRKNXT&vMVeR?*gX*3@BW``q>nmEa1Kz0lf=?Gt5GY(xwRplJNnrWqdGQ6<<>_K* z6`s3NuUQolxc|J>RW)HreySjybNXYncNFmlTr>Q7ZZu>t0Kjh*v-3sER|bM0=w#l`CPr&ylU^dB0jkYiTHL||nRS%F~s#KuGHX5@`~8XkpB z`B~;<#TqJ=uUBx4`1uuFfZIrTQ>v^kb`=4sF+~%2>m>1lVp8tysypSEWens>rbJMh zWb7uI`ey3#)CED6occO=7cp?m(}o10f5#p$FgnIL>G^=EA&Ew=S{)% z!Odky$8LRuEFss&6)NDGK&a)Dalt*dk6oJE&XaVXN& zoT*avaeN)m74<9>ygnqm>$xzZRZ1 zU=75pE-73n=!q|jpI7-g#O0q?i;gQgr&?)5;mwvvb)7?Fm{EAt{IPVgQ&lOdZFdkT z224(Q<5y%cqGR4mCcVY+Z!H3I9iy%%IfTg81lwsCd5Nv?{prm7wJZHEG9^2RFDYx$3wU4;pH(?AFWG z-RXXp%@o=oj7h8tfd|wb*td`%AAdEsBM`fIIb9D~qCz6cZmb(PyWQ2=zW?T({^8RW zu4YBQv!|i@DQ(yADvp_?C?J^eX~ht${0ITBuyS12rcf7MNQ_kB^(M*}c6!h%AKk>pFn1q=`R zLJ#^%U*7IhM)QzrA%4&bUXFnBw*mFh(+befrmbch)pP&0Ot}`&GS7oGlaNB3CQ6r| zu?R0+Of(D#?B_Ker58RTHKZaNoRs$n|3vxiI4b`}X;H)2XV9k*CmiAxH!A3l^09}9 z@{xZZ@s-9wyh$6lAC)F=)%0jH_QlSA%A*4qP}#}KT;Gh%!_RA5(GmtIS$c8b+~3f% z|I;Z++zsUburU8urT*W)`p>-u|4XcVtSSbybdewiiTp;)Xl9}HVB?YqHcpQA811RK z#(b9-+;cKJnatXPdZ?$c;SReUaCyoBcd)cMI9=Y-k=YkNSq66?%4o(*`)uh}^+7!D zC>j z1j6h$m$$)R5alXLxaS`pZ?;$uHNDW}cHXvs)^xD3k1R@ilw*R}B9gOc@p(yB3Iu1A!9tdZe+4Ve1S%mCvP^)4#4?~XNG#+3 z;VB2SQ;Aj*c7n$|L8|_BG*I3t&@Pl~Dvq+;ZM8Of`AYxGvnEKK{0S#%*BL1-wF3N zS=ARTrT`_?oosCSu3S-m*@Y=P+OKGu5pKoB3;4An4rws#|ogVQ7qr@JylM4$#59=Bd z$}>W*jRWKxAf<&>uf)&D$`eLZ@YktJHal2P@3rt6{Ep$(1oh)OY;A`ckIL)iCVh-K zFoR->h4yo+xMpzTZ{#ipkc5Cb#K)IXeF zJ=Jo#;{E$@>$?{k)9s^@&kzeKwz>afjl{Mnl*Er%%0&Mx3pa@;$$yn-S3cD`RkOzpONuc*;0X-zO zc&_4~);(rg^i-MU@%=)N1}GP^{RAZJ92G+hL3m<>5(&WQleQYKYa*=S<0{*0jYCDp2%A6yY`8o0%5fcj_4-%e8N;sx7m{^o!#P2!i2x=|@@(zDIV=D^qSJ_!f zW)w%FjL>oTs$LIozbW<6Kck|vKP_5V*ZSh?PgGo1)2wFbpW(M)4I>#*uOIb|7>0J} zNw%|=CB0PKdqk#vgA=!6#Q=;^5_TV;00|^|)6_epYw16>wt{`+oMVx( z8|rdgJw{_W8LI*$^J9aqZq_3u(sc?$%3+a7tdjqf%SEzkmKgSD zj4g}Z(|N5loSG~(XDD3ufv=E?WrYz*^kmBvSwv_d(xooRNCrzq2o?TRiW73H79UB{2=G3a8grRd>L_ zmGMJ%aadRtXN{Fum?w{)T$g-RFB&;4dx2FSpJ~Ss+-ZGrOAlZ>9Y2n|%R%9-7Dp6z zn7eDWy{_9K$v{dEIF@!GuDegu*B<7e1e8iyJ46UpdWyHTlyqJ5P~mW|ueT zd%fM&eL>^qkRl@VYrE(XucBOr+W!Cx|Bmexlq)5N%swsHpZtyzwr*cT21h^zPn}s{ zy|;HYM3$G!i$?=m&@QFkO)J;1*!)o{061X6=nC~4h|+*GhAhvM2fVvmI~yk% zQh$F7m`~GhcaWb&bo6l}Ut$EEOyrABxrZwAPazO{UWA|We3)|Wu}Il;3@1%Jcz%hc1oq67Dg1R!ez5a z)B$r5$N!5Rl^r!5wIYqCBq~ot58$OxQdXUqqOkev&gse&?=MUHx%$>ciMXKEZYuYC-N94(m-^-3-HPX6P zV-7VWt9@_4{?!H=okhmm&Deu^%0Y;c;Scs`Y$aBdwh8trwiI1x@hWhP0{|=87(9wy zME3gYDi1E;U371|SwPQZjoU(8|h0?^~;>yER@01on`;#z?wJ+TZQ^D;FlMFd?Ai<0bFZ1FxT zaZNapT5Gi^;f=I{SVPa^{d3$VVKlfe@=FWiKFq9iI`!CfIi7VhJQ%bBZ9L! zj76ue9ueBstwKe0=sIA9C4JdXMra-H{h7>SEilx$JALvK^Sh`FK3SERBzHOtOUyhu z1s;N#(|=aKltrS2AQ@wYX1m}dg=8pv!)=?GWTvJxwdKpkVI5HVc##*=kaXNT&O}~z)YB4bQeZKJ;1*<(ep9pvG1C$3<{t}#50VPw#ry1z zzZRO;RUV`lfUm|qc$l@roPmZ5IYAYzrdL&GK$XM4qU$=Ec#W#D7KzAx?iryiC=P1fwt}Qym|v z#yU51`HGNa-xV~66WS!cT-%LhWl_^m2uQASd(k>k8hX6?l#6Syp}y*KIh3+iN*^iT zA*+@+V1Q4ri&ruOAjctgK7`lQ%$WAjf<}B~U@B-t8;^LExW9d{D&zDN_MgXvA-E)%kTr(L447*&aO}Fic%JWFWiQy=0;G@P-fo@DtXl{`U|7O3}ocioHPmMk#+Di}Tc?}51i;Ra_)7$5+{-2mTd z2%)exf8HbD9cSnFLQ&q_PJj^NisRUlT%Ovgx&Og@1Hu8f)WL82 zcWO!yi&0(azN1L9g_c+!hy{X80rp*2bef@)C(H*{%5S@v>!AhjR%*onzW|F(_%H^=(lFnp*P_=*Tr zuy2>}EHo>>kS1H_|HANp&0@l#zuih-@NRSnGn@{r1NMnMX&D9WvcoyvI4y(VAuhy4 z`_!|aDSJGX{ZaM{`cc6r&ym;X3KzSsY)5sTf4RifdgPRwBmDCy==Ikil7T(r_V+*B4Ttt=Ybm&(EBzlgdi^ra7Q`1Ur*J zAaI+cku^G9qDKx(VRd-P343MA2?>sbWVJn8L!r_3hqs0<-jtOVyFa<@&4%=O_N$av z=ZJv&!fN^{E{H5kX7$~yU&eYHgPgCZvNhw+uZVSSHM3UC{eP)5gIFS+dN}f-1SAxS z9ISgJG=m%dLsW>7Wi5a@%b^4Qd|lv1_{Q<~|Nfq2GtF7Z_R3R0{o1-38~U)Z^AvJK zNwo@m;pe<4pc=xjv9^)_pNNwvlEs@ymxuuYo0;oYB8NisKJluMn%IU6#0o-Ou&J|K1krociLrY5d zNco3(8Xhj0XS12QI7`>_3Fh+d&=ctILF-7mY6@w0>ay|^==b5QTo^7n@~ea$Uy10L zdj9L=NW`gvzEcW*`>VTPl28Fvj@fRU%tof0h*B{>R8C~HT){MNG(WH60I+C}N_tH9 zEgyXia#ks!qg4?7E*&BnGM_gpP{ACVW3-_1@s6T0*AXeAl25Hv3nuHtP0Ye8%-~il zmW&No7`k|(q<{aoVBY_9ws82aGnQ!w%#Rj016I1*{A+s9IN6<_B2;QbEAJP~&Q$8! zSyY^(bQ9x}u3s*)zCV780`L;cG#0JLWA96k6(0VkmK85DS#mZ-Ko|`%mYJmQar7r&VZh^rCW?T&rFUJd7=G*{1DNPO%Tnf zzHE+pF3A}#2*GX2TAB;MTB{f_LVK`VaOcylMU4azNb$IOaPyMG*-d|{6aUiN)UZ%u zt$*Nv-b8gmHzBuO1aQfI;g(H@dU1G&RsTk2Cfxd!E;}c-OPuC46!;cbysBrtotztc zFO8F<xeG(1YHXAlUmP52 z+>V)SW*fVaa%FXiQhL6d&KlESKbTmEP#L zJL1A?2b-h~Cw4!Co-x$?Hde{{q#vc^v5)9Q$1p&aMn=F3{D5ZI0Kjax^NdyYUUwAi@9eXUK-guD>}&Lcq>+A|dkQTgb?a=jB2gzf^5iZ~ zQ@V5LXDj(EPoN5tQtF4@9)XhMss_c}2M93Bep=>>$N-ds3p_f5Z6{aP@kaM47tr)k zeXl=G_SPOsv2n%TC^9sjN24vgWE%L$V}=e88PsG80sO08uQ1uERGDiBO4!Jmtc6?sZ9LrY6Z(N1zMSM4_z5yT~^_5C#8n;mG6|l zOb0F#K|VYH?yCMUMJWfzg)&?9)x&O=vd{6TUbYOsA+4_i-h5ix^~F-JYt>G&*}?VS z1dee3{DJBUs3{=rkSbips)xfl5N$Fqth&&c1c-LmQ~mw3k=c7QIvx|NRGw*+>FPK+ z(D}H}@P$2oJJE*-O~38)v)t8mWx-Z$vVhg-C$kYTN!r;?)l#aUVN+ePHBp??=69~X zI}PytvUYT{h1=2HQLNh`qNb*#o>IViHo@g-t*HwiU~E9iRM-`Y`CEckU{xxcZ>8rG z3;SkRuSHpbMY{CEkHT zAmaeZ&-s@9pu=OeIT*j{;QY;#^bJibj1IEYSo{XgHHg<99yP_Y7B2wuwBnSDVjbXFjY<{&@lyjgHQ<(L+*MoUP`|97~ ze%k;wjpC_umgDor(>#EP(u*c>rEB%HQkkfVNxG3rAca*^az$J68Q2+)ci+oS08lhhY4svp9RMDhM zoP(*vtU!~A{S=-pU2rxf%fXpF7T4XVLw_!~)AwDz6Sg+>&8&+O!_61bd-PhhS=SN_=We5S1z)N%FcS;5 zVY(&5AT^QQC6&uOd-@5-n^ZyE^1V4AevTsTAs>bG|9<%(_O|&52GG@O0z{Wl4eln8 zTMg{G-+sE<-T-q?g_}8j4qF55*uZ@IeWtH}fbP-)EjZvx560ZN05Jm&dHsBGLf^`} zAPzXsYo%%NFa03`bcD2uT|sbASDyqSJI)P<8=2!jG(BFQRBm5`(aTboR|rd5>ujI^}mVdl>5a=~uadP~~pQnO7YELU}`XUE%SR>F8;}yVi z{CdCO74nB|uhYJa0fC8MdEBBcx%`Ns1#FZ@9TL^&p1Yf2jr|sCkao~-(0Rw+{^(A^ zPJec(zu@vOc2Ti(-;X16-7lF9ST%#bI41`3w}bi_BwIkFFW}d zH^C!;g+K@)0dhc?W^$|r-pwTFM=3-`Mr?Dam&b$Dcao&rwMhK2TeU$GQ>^Kd$!%r(23Qb3i>0f2>N$xfRO5^e*pdmb zT&^;6r9Q#lcWnhT3`cQ58&|D&-S?rCZVnji{|9?Oguf(F4@iHR_$OVs-&M4qpA_u~ zjHZuu<)+&Cjs%nutPDolvk4q?HvYXkCOiA1?fyOz(BN49@b>)6rS>ezprBX}WG>{2I09fQDQqL1!F`2F7dV&M|JrT8}NJ{Gld zcygy?hP~{@KIK`*r3VkbdJ9i%;tb`;)W|lW@@5{w=|)twtEkMBU%;t1N}AF(9G%7D z=a$aHqTueg_U-21i#k-?um2�#h*dvn@#>y>K5%h-Qy0p>ekS|Qh9LgksP0hQ@_ z98PDUqPi-x*;>QZ)>`+rw#qhNypuyv7;t4^faIgE*a6B8nA~x>L)3E>c3u?=Gl5r} zNp)m4aN5HOmFsF`YieJwJ)^w*HEIHJ<*GCbmt(FbEa6Mq9K{vvv4Uf18s#ei$gn=N{(+7=P;4^qc;YBM;;04H zAK(r)O5#txj~dwXFy02w@^TZAj(ddKeP5a8G}l5AB4AOs^k7r0In*z03o_1I8)N zX`<-W+mn-CN1rxxa2R}@*cT!+v}wks88gac+er7kjXR}2cl0+tJ8|JNPJSoXDF{!- zjwz?agP#sgR1g){FF`t1w`5qwo9Q+SDr@1}OU;wH2G1>XXv(*GruThmXYzL-XP3!ii9h|R98H4 zMx`eRn!F>>@^`^eGBIV5y#4zk3NfyeqT$|0wh=JQkVUQ9!ePY|jG>T%$8)X05UJn#3sByJA5_fZ78hg*&=jp!-f*VDGs3GE+*YhNM!eGC(& z^%mZ3hW6z_&mqG9hHdUS%yNM4IUKXI!9-XOtPeH-dmnp@{VtM-dj4@EN3ReM`4TUA zaHL;k?BwwiCr+Ig5*i!s=i?hGn>=yil(Ck7_gONh@v^1vo&h1z5dnTO_h2vC(($qt z<7F#DqJ^V=vXLRdG7q`0d!V;0#9OvbzDcHtkcUUee8NK%p%J6T%hvh&MaZHB~t`@$-`xlzH-^3^=sG37KQ{z%9i_i%7Y{1vT@^NvIx0c<{KFq`r(9s z36Tn)@gdguq|7@cT;?T@boUF4 z82=Y&-yT>Q5+39phzVabHV7Mn4aG)dW3aKz*_7V08 zHV2!FIbu%OJZwI;09%MH!WLsov1QnQa%=^*8e4-oV{5T>*m`UOwh7yeZN*$LSIiA_ z$2>4k%nOra-k2}uhxuaxSRfXJ1!EytC>Dl=V-Z*+7KO!NJFs{xK{P-_i+&ReK@+}~ zVaqJxv&EuCgeb$t!A50sU;LifS$y1ff$d>jhSy4Dk~Cr^QEfM^+t6;ux=-(azN~v; z_ZK}pdYtR&-^;F-cds)0$@X`9Z|Qxt&)`1EeJ=H#-S?COaGI$+T7L7%^$#LQtnV~4W0WkE7H*lF;_ z!6mT4z9SiMa5Kkmixzdb_vd1NiB<7g4N4XTcHnqHnl>wrj?<{((^%x$`BUsB&zD;G^56XTsMKYr<7ZfAzr(2D%V`tzfTDGqcxg zbMaPs)Dk7bqj~l8(X(EYS#M&KW`SmTklkFr@Qtn%sr>M9 z_~o`k`wo{eq?2cQjr(D`;$q=9Ec)oUl+=%%=eK=}N8_>6t@q%pOvb)rlK?@R3GU?_ zAbSs`{)?o(rk&rhZH%{NcR-t>+`5R{qV^Cj- zi#Dr)h1GC1LE1iT9lw~h=T9n47hB;WRkoZtKaM%Q~M%)nx8mzc!RRWH2~(cFQ#qQt5PRDm6o$ zk$?L(B!pCd$^cx4h!a8P{?TVJ;hDH&BW#yKXGFtEXwJKn$3_A_h724DSF<1VN8*@O zK3ka3Xy-K}=n-8x!tWuhY4l8T>8fJ>ad!XeoN2QuYqlu4GaNiE(r>LB1()<(+t zs{S0YASLT4%l|@<#eYf_6i74$yH&5JQTHLEkQ9-BPeww?m4mzM&Fm6_oC~G4u|TWb z!H|2m25nck(c68h_e8K^Et<0}RI}ki-9d({*wb)|ZfxDTon!NM0Bx+0GP8y1xxn4$ z;0(dZ*UW=RNC;GGxV0#@-BurDQDNfT57olK(Idoa1}b=IU9(vr2R6fHB% zcM?dVH+$}Cj%0(5fVL(I)=NxAQ1NvAW7HSfO&BJ;Q@@Ue1hhX0#bxXgI8cg`Byd#b zx8~jSi(Eu|38|@3)OPeXi&~FW9D|x;VtyNc{8-wgJScBvVk+Wtm+iCrqWS2FHtNyj zvlBn~&~bdMVSB3!lb6^WZr)+HCV1M~k<{Sli+_dqPxpRf=ELVwXRocRd>Pc|4!&)d zdUpwR@zd7pH?C})$5eDmrH*qq&YCvw(!F!_SH2UV=sg>+jX4~&Qb2gHfbd}s9uoS0 z)1Y?G_S@~@W;oPLHuJynTcuW*4*eA-38^@NfHSD`9zNUf(~?`sbWGBp{gEZGeM8jz-8lB2goFQ!R<8_aWC zR)q_jVpIb_k$Xlafs{BF9VKExN@144n3>KL=QW@DhCX557Msl$dmFqqQ@&tTggfIE z;T^V(c5*&>eg{j=ovSk%b$N6{iMgt3huJ^gH+36JRt%VSb+j8 zJu#n?B8>!U_-?p;N40{3@f<(g+Lfobi#sAAhdNZH`-ZdB0Y7cTt7)g(7Zu?ah4{e7vx0y!7WtlAOa=1I+ax(?dI?!O$r)fj(# zQ!P`wVGUSGhr0oNDjSf09>vvD>7cTtoC(?6qA8<)yn6L!b6j(HCA+(+DWjAo-&2At zVAL^6b(%7f+EI{LoLiPv$)NcNlZNU7c7`*FDUl(b^i=SyvPNHDWM;`L%KYrg+I{r* z;DbmO+I04iuAjeo)QE-0PafLSe89+@m)wW8U>e&+k1la=cUNzJJ0ZaD)}MnS>^bFTkK5?uL z{wyIM|Cv3Mye(dT+z4$gq;9bi9Ax2g;+N}Zu3XMr@64e1Kb?cy!~1a6IZ+-g6~j8X zN_q~Jt|9E<@2>~npms65-=7O3KGwWO$e4GVskXTPvEaVf$5!{NiQC54?rW4EGhr6r zZVSHcy>0bPnA?`d6W9txFCD)7?ZY628JA>S_IDdu;j24`Mau{Ldru}a=yYZ z-jyYR{1V@!@8UB@8lRd-AkpX_25Hj44VlFRY=eW6MKAVxz-0o(Vbo)uk&B8M0r;5Zk}H z)KT^a@*R-^Nb9504+OC&4*m!iL&xmwlgGc&59A1cO;`X3rG1=%(T@!z_USSslc~6* zq?A+!wV@L@y#D`{sPhPcb=klrr~=PS&Phn5g1kVqh5t8N_`f}G?wTZFmSla|_Ep>7 zHu4^yEm-TV04L=vBH2j3` zRCIiQqQ&gN$R6h-39|9s`7AO*o45J=g1B-Tzd5*dZ7dbDLmgxJ95do|bvkXGP~^B! zB3eQAxO()7CH2><#GB9C^Q`kS_Y zBj3S(AMcpAopy3@TM@$kN|4{6!GCblivQSv$fNIWLH}EuvUKjQx~S!6)e3F9tG>Hw zP&cD+^B2G+lp1IeJ|dT?n$uSpFzc0}j7nx|;KUpKv1U{&Q!33(M925th3D=yI8&x?`rM3#f5wC{i24B*!hAU5 z^LyXJq4QCWW!5Xr{6s6q-gVEyX#AzH-v=*IFVuH55(#x7{bda727CSdL?A$aGqa^W zV7RvZdt!w-OqP1Ig+s_Fo9sOE6RKKj6ZgJx0IFbg$_lb8TY7y2ITqxXGp zoflROvZoqe<{TPdfcwT5aP13!^0y9KUL}1p@(1)jgI@ksc)h!kCBxYS>SGLplZNO}e5dR~r9}_u5Tt@5quir}e{;1c28IGRv@|IA4MiB1$atcNe zWWVEeN9&H$H!e;=xM0IT6dUzwXSC?)IWhberAm#!u%m=QeW61DTm#2Z&}k2Jg1up1 zrXU{}j8vA|s7ji&2$8FqO~@t@iDyu=^H8tT2xn053-_1i(wXVTtUp~C#tBz$g+}Qu zCG52h;({(&HSNvc~;Q%ath7y78q9_Y@AAAnU z^UQH*lMlt;QdT|QCB{$wRf`MhyCMp8#? zYCsJ{d<=+=E(3)u6n>J#gP8JiP*%vm23SuNmVt7?1@r6pjJ9rn7K*Qm`6{$Q`lk1u zRS@!tcx{5m<&S7&H)J&GkNQ3y0qw4}?5{JiXwjeIBrx_n(ty|yMFbO{5D@A@JFRRN zgkz6u_rAQ-I(36^t3dGbc@#YSaO=a*O%7)sRlMBt()p3^J#rkzPvJk4p7XqkMj>p) zqNNNuXs%yne5GZ7i*?XlaP1^_uBerUBCyDdCEb*(<6>iDnI-cS6IN2&1(l-8F912E zOd|mu9$#;)X2|5ZHRTmetrU`V7QPF@NP*vQ@T3Wz=qd&9qMtok))D%HG!x_{C_!?n zj+w9WjSZ)_q#XY-o=vzDdu2<*y0U?ft8QJqMg0>=zC|B@FzB;Cl?xgCwqTmBTSmck ztDbyQFqx1bid%tSBfG~31TLnQ0?!sXLmu~OIj!17-zvFV%OV9oMY=HJLo|{>Vwf+f z1o$>CJ+mTEM|M+(iz4SSvbn<+Cer`4us* zhAXAdmfW3D&Y*rAzM{zM)H&+fsf)MZM)@^BMnouWa{hyR~_X3(irE)h+j-O~#}^1EWvQ6LhcD;-9Ss zr)fcdVgjR4Cnc*jcC~U3u#-mi^2KX?i$iJ@Y<*a*>Wg?g@`g!$H*Oxa$=7rC44<># z#V`dLU3$`MO9xmj&Cu)1${2$I7)s0R+z%YoRnU+)^w}5e+wIxk?%j-n;;Perx&F;o zEyJh>OqM>+key{00%k9BWKvTsO~kTV%lqPg4z#n(B3%@J|7N|=ljc7X^RXQRq)076 z_S%9nH5)GRM;^FYVl2+NT##>k(9lyN2;#~3Bt-iob~u#;wjVkK4(2k)iPOf@ z*{7KF%EJ2xE*dSc4&P79Up`|AUpYO&`oISlmCgti@Y~~ zZH;EC2qEkHG##&MRr#5iot|KOVAW1j@vfY`7hI3bMEwwrq$muPL%aDuvgBzf{^=iZ z06R%9-3m7E@L^U3ZA*)!mj_)aQn9gz!JSg-R{gUF#fQKOjw$s9o3>c&c-?=`@b8KF zRUN_7yyP5ZygDgK!=Qis%Lz|Q0BL}KQsyM*6{hFr8X5SHS5A0_5NTjp;tHj-G74_s zL``Ox-43(O{RVY7@|&bFmR%8)#2|%$6`WRB)BwlOrPV;2%T}m?GLjO?-4DbMVLUCH ziRa)DedvXCGvP@L0foCQtg{>xmZmd?yc$qO!!bqvXj-9VTqw=(qiC4I!R66^4!>Lt zH$%!yzJlMuKGL({>j=jnx%Xo5u!Mme9Lk}g78Wx#T%wy5Bpao+QxGKg9jcR{-ajTb zU9Y`j!-vnU+nt+_Z^DJY{>(WX z`}N7M;34^nUk{xGiI5hQ>YB9EE+usdP|#?x!sNBb zfU%@8zok01+BZcDbQS1Qx5{ z_)Fa3uJyyL_7~h+Cfi+udc0+_=ox{AjQ(h>3+ir$&J5r8@6}u{0u6?CNVq5@9{dO; zC5*8gb&=By_2Cl? z*%N+8kS$%G-U6D*EE-MB1{2}fGY4uf)iIL@a<1SQVyZ=!cWL$S$zgVI zyb{+Lby>!Do_q4E^fd2$7P@qH;K@c~QUP6P$SM#_oW*BQJ)s=DEI=E5_a6H>Jn8f<^)3H;S*S!&%(n-=B*iHb!M|Q7rO- zU%>a_HdFoX`gY4I1c^1c=T>0!v@SwSibyI+t4&ohNP=b~DO`_hjcM8GRKBTXzutnR zuagDI{(@y?#Obos5Mll=M;w}o((vf`plu`QF2&w|eA-Hkh;8~EHKkOJF+D4ffkU7- zoQzU%gW9OpP>G5iZaV8)uPP3V7#H4(^u0!S=0zQS7A&oD56Yt z4Hv&)zWxBuRu;smsO03Gp*n`tbgfNl_Ug2|tPI=TRQfG2@vfXh*|+4uzm^fFE8Y=K zCF!(jiOdM36WE^#SLbTe)2ZraQ1*Yy=tvoV@D+-7bf+kMHNH*}vTzwq-b{-Gr7f`!i0nF9F%=Z(-D&I}a4~MHP&z`y` zN4X+98g-XIj_--*_*$T%E==2d~WtrN<(>jR<3rcME zhswJNz5mP85`shBp&ikFDF5n&vdYThUFX-eI*vy@HX#x7-+Y`*9Lh@YU{U42t+U>kq~;V>lYH99 z$A2>DUxbeeSL^f!HCq_q0hII*Bw9Q4zvAihZ=d1*{9GyWw>Z-0$FC24$&g-uIW-`; zmQ7a2C52J3**i;Af=HTdRiO`rSHIAAN>R9}|NnH%{?$&HZ4!Z3apG5TFk5Q+SDb-q6=lNfsN_oWvi4!XUey5)trnVF2bvU=kDG(%pMs!|sf| z<;D0h;PttGFfuqkJ_>Vma%Ev{3V7OmSP4`VSJtk!sVX$k zC`Dt^WvW_&ASz2-CKD4EP(+F10#TGk0YN~NMs{#RHoN;Z&AurrD7b(^lqFJX5|?O< zNyhCYIx(3T&51@$h>3HnePsUsH7-de=S-hNbyM}a?tS0=?)|=M1!ZSPDHIBSr@8YM z>DNXlzv|~5pkEVfI~634kY7d)f-F$7sF1aT?ub|O=+mR6ey^tLHuL&F>!NZw?o6Uv-HX7+PR4<dUm>Yl|EG)8L+ zR#6F5BDIEEOC?h&R4SE!PGwTrN&+u=c~YJ>pzSC_NaGnLYssTGpgIIfh6|!%ule_BCOgvVMTlP-qyCp?gQBk zh7zHCUz@p>|GYhadYYKX&NSePd^}3S<&A}*ITyc_y1JVLZu%M&`HOIj*;R}7h&U2h zHFt2V@21r7mF@2W1>@^aSOk^zEZOqW4rq`R;0eRwB)BQ9MG&ICH|aC{LX`m2&QH!= z(|kYiOFT*FpQQt;|2C#{ax4Rr|8?tuhy&n;dTPYHF#`5~g9Zp@`gXh-DelEzsy8F- zg53o?7mmVWW|xGb4v7b&QD~kULdE2`wmqE6YV)A=yCAQ)B;!(N49 z+#NX;2CM5gqAfWlx5j1UIZFplB6U{Xr} zcHAyX%o-@7e^b~X_?kdqb!NChDtS*)C(J#62$MS$ZQ!!!2XOd7Dd&^Q?8rvNxyIu4 zMM6a8qNHFRk2Jwc?}>1VX{tthTZ^tPJt20iTD)Txk0(xa$HTDZr!g=I#`ipG5XXT= zjSHBPBj)bLgZ7;*owfUU7+ivfCzwSQ{6aSMP=b^W1?fcUHaIEeIkHvV9e;F+7p9_r zF)esZ*L{v3&eE~qY@rWdK!y%cng|bpm9S$U3xmF>YA!27#tLCWUO{%YCaOKVssw0rA1!0z6Pd+{CdUCvN>3xN{da-mW9=nH435?kl)0Qsk6N*-UXKFT-F6&GZHW>mk=W4=FO$QwWUO#*3K*7he z4+-3YlzpM|(O7<6Br+F^D=P91@6c42ZY!@8IER$9-uH~nJh-2_eo0C_xM90^V0&Xp zjoIEml(qetU$~76&}bECqD1+OYA@@`4;uKrcbZ%Cp?Xzu@72L^l2S8j@B> z@=elhqdz$Cf!s>QRM+DUvU!RGq9nYI&B#Nkc_J?CZ^4Cf3*DTThcbAaY72Uf;ihMp zOnQUBM}y*of^O9d}R{&*gR9wY7rk@uH^)KPi0llRuXc-0{b6JSqRf z@Gf`1B}*gM33sx5EphzBMa$8e_WymIQWM= zYy{l@Cju^B6uwTdMZNaFqyCkD)?f3wtxV^DX!(P;(%K2%t6>d)9tCbV73W|PkH)L; zO3>n0APaIqfKd=FVDD|}z^&t>X9)qz12f*{7X~Oh@BTyhsOz)l zv;2jN`Z=BA;kQbs25HF5AnL>oj6bLV7o~gzlGUK`?ZdC(zx$2CFX5rLJmC#^^V>df z60l2+dLqIU<}inUfco|*davP-@XpTe#^Zc<_xgk;u`v;aMG(Q*OTek@g5yur2oBqJ z5hrg}!=TaM;K@S&q-RVe#^nJqE!5ekxae4w-jqAc5sZ=Iu0_ z>+6NC&iBik_|Hyl&k^c)fV%na z2Lnn#5Nd_NZ2h|^u~lqKK-mGBIcV9XL?Pqd^IN<4f7gAtUBr^?r4B`Z$FB?6o-7~2 zz@(BXA;gO$KjZ8ppb)6mgNhq1hsbv758mFtByi<_(z)pZ&@6WqPyP#If0v>e?4@Q< z{Uo65++`8}v?w+L?yxT*q^X-l54dcohgcOsFh_PeVoNA`WJ=QT~pql9Gx_ zp|_V__UWA>QrOqe1Kme3SXR`Q)FJb2&WA6<=F42RJOQ-oo~-tSeM0R1r1B|s_Nyv_ zYb4G2J80+Uhm*s~gbikM@zXrN97uDk5oO1J3U%ue%2)Dhu@Az-SMEyLl`D2894@-H z(Y~YT^Gyph(|u9K$aPnHHVgH|#c1Qx+!O=3Rg|D@jRI$AX`qvO-$%RnR`5aKA=)%N zC-d=!EX4(Rky5U(u2k<@U$Zf@c;m(*q2K90K6LSBRERWL+w4lBq1+0Wc-w>1KB*??a!X~qMpGDxbmOUkCA8)gNv>!m< zDaEtlt+dVVr(Tv~@ms(1Ka_nvd~v{3L-R=i?*8e@MpkT%fi0Nv!m>rZj%6EmqZWQ=Y+4WHN|uxC%3HkUmX9F$Ow1y2QL*)SYD;mm&kw zY_+Ws-o#-vH`@(rXfDdO&dGokw!3gw&obcMKZMS$DBaS`H#6Cy}yTcaj7g4R{c_e3>PNYAzm*J?ZS+cof0w*zYY5$pmL}vdhX5BKJUW_GZ7xnHnL3;CHd27c*{u>6j z^;2|y9x~=@GV0JyK_1GeY(E_hm3aq`mnn++o9(xtRnB=e@`5T@-dJ732t(|`^bUiO z!wb?7U|u+Av6#&zrRIu?L+2bEr1I{%@)}EJX-SR6!SVkAH_N}NmmvoM6qncr0XGs* zFisRq+;W5Q(YaL(046I2CbuC60Wl7LG&Mdx3UhRFWnpa!c-oa&30M>7x}F#&1B$4` z(3-I`(W|s+d61J$Q;IU|}R;%`^bwRsO z+gdrt>g{IL)Ak$s=iKN1A?R&u?|q&Nc_uuQnfbr(eZTMh-k*plCnriI5_v3tTfQbP zCM0%VNPOZH@5yVzBa)*-l=e>+2qT48kSix}l)#Gw#QwMEag0a5>Z2_;nYZK6!lm{i`+#o|8cQ^ zu(<8vi^IZWlcJK+d_29qr_GChi%(NVMMNeke7wDc-@YuwEaK+@w3Poag zxFRwsDSoDxS5k6>XPh#^YX`X|(JR_Vo>x$CZJ=*-N0v;#WzRgYA6YQfK5zXlL zcNjWYaMsXPW@A!gLkThzSMu+I2Mu+S3wSbJoQ1R+dF)!0z(q60BVFhii&?0YVT}ab z;4bYZ^n$+yzS%8*It<>h0=9~U&pa&5+T7x{EFRAhR?)Tj1tlsr)3LU%ZHg*BJ$RXR zgpx6sTl8l3u7dZrAD>c)S%h z$nZ+Zw~$Y}^;VLUubH9UDgxiT@Z}HU-W9M^_VvteJfw<$`(fYbm%HWnU;hHjIh$K_ zgQVM_ir{SxOoUc7i%X0aCP;w${{FW-c89NJ^H=#}n#10plN>UXp5kDn)-_F}4%=Uu(MVu1OGa&A|(Z!DegDfIp?yDSj0zXMdv9XQ& zUh<&EKasD;23hdVh2GqFNo0YsgNIcxfw|W-JA`-ZZA!NGPPU3}KNTu0;@%+0lvSY8 zEEUSn(Q$Ze9(MT~c4DXERcGN4ybB@6ZvOh@4eWV;fX_7OHC1daGVbRghuMotRfUBp zJCA#77JYcEF6XTvhwFk zj#B%soWX9w6Pq9ol;G9S`SHECu(Xa>8}#`VY!xc5-Ekzc4(VApR37)J>48ZdVB>ja} z|0DCLI%XPgfqhEJw9J^-x$&Uxug+H`Fy-Jz{Aw+4k;FkVZM*v%k^RpR`BCB~oF^rJ znreYCi~EV&=yuobY70m#8jJLiiqz@``?7=66F1}GET;O)YZEyM3g9}8;uPSzLpf{=v!G$#YS$AjN><`YxOVh0e5RVe}7BADj@;yVcpXxW9GDoCk5OtwOQrd1eg zbtmn(?*5+p(!kzED3DnhEY;B4eULGNk8We6o!_I(D1Iq}2V*ak?Ty!*g~MPEX^dHa@wkBh(xTdyqB0<$e}+TwnHrUy>+1l^Vf3J+n4%))4QhUya7h}Fm#$t&xb&x%x` zd<=;(2_=RmaaqYx>R5IK9&-qyNJmd>I{oq2zw2k|&wts`$9HTnuVe9e?12a3$#^^r z!*1Z+_DS*ic0RMbHTqeAG}B&Ga+V{5gOHSNfpCk+P7=C3e57!ljE+Tr1trIMLBbp_ zF1gNG7))s}Wu%o%A73Q*JedC1pFQtT1}Gy?b}sv#1xq0hFa6zTWN5j;%8Q~#I4FjT zuu^srA6(722@{OXy`pA&bBX~xtO<;t#&m=i8ki$S(;cqSD$zvxp>Q_L&kkF3BXh^t zFF_iWAD0jiL7&Mh8I!goUnNgVPuY{n;bgpt2Ic<~EeZ4eJs5!((iK^Mm8qHX)I@C_x#(X; zy2+~3Hj#>-jjRNwxc;`5YI>de1nf0cCYIu5>|}{_l|yh_nZv01JMNG~s<1NBMVbh2 zw4Ie64fF5AuYviqgdZbz9&H&z{13Po!NqQs#-LM8_JN|;L;1g5=-i^>+zPg$xV*fGhsxfe zdL3=M{VXks@#qbrX6@IJ<|sig+mYa@pW=RkAFoRPvv<{h3pI}iGyNgWw!J|-)SJ&j z8myE^hdTP;TSmG+32m51sOd4%(&CzvI<18#>eVt2y?=lgT>T%07;r} zpU!Qb?Ei)X zeq$=K;ks=RttvC35_=SNyzml{$^+>A+e9kwB{%kezG1ODs+_0d-b#2VgAihXeDP|$ z9#6qDh$RY!sdy@^f%Py2d|?YWrbdRV@fe@6n8N(GP!1Xx`_<#0Vf1vIPo6k5vr)*l z8z%U#9qH_4;-Ta?5eDOdFpm5VhJi4V^EJtc0UNL;Fc1&80Go*8 zlbpVPego)DcqP~8AILyG)(BUu?iZe-9^+Hk`IL;)DMG&N(WrmmQJhWmf-jOtHDsX9 zF}8g|lu7!|u}|&V^sEUW7Zx+b#k+j|lpo`<4Phrdi^CJ(fWu`xmcT582L$sD8xej0 zGAyFbXL?j5xX5nLzKIzQH~iTm{aI3`gPi$)>FX<>@xC5mXcl(oun1N=M(IrnxTD#j zqyUHFF+J~?lPzDGg%M<8-31pjq}@$3OZTgUo;+C@GO2QPYGOCT^24)~tn?a{5sR+H z@Sj?H{t3h7{~NkTtYwB0Zgy?;(k@oYsXl4Ae~O<5wXzwBz6*l5=8VKs8`;pfn4Rf= zr7Fa4stdRTo^n|6>*a1vI+g1Fbva(alLw~DblTXgNOqgHt=L>w*L2XF9Bd*%A5S(L zo8f&koVAMVq@bt6-FgrFQ&xgZ#2n|Ub)0|Rc2x?yLEQz-{&&&0wgMG#SbpjGVsQit5>az{?1aCzL)SlqQ%V!C`w7 zdp;%(N%-c5tQb{fsdTx8+zNGsPR;Jltup2FdA0e?Sy%Jga?Z?Yaqil8x${4Nt?9(jnK zwm;?O5&4Y^SMCA#{$ezk9CKgOLh5%0yYb& z(|EjvNk?i^slixY!X5pPj&1&E+fx>Xef9Vk5^r#(MQ4El#Os=?EI(L(3}*K;XJF5n zxHD2-B}|0TvMXW75x3*m9^>@-l;X&;e1r3*`aSKf@{=8>uY+_ScG*?LsSPOqSq>xn zZI&)MgDWV=$tcWsJ{lfWxt;Z1ywHE`!JU=ad}~a7;lq@3Srys?Db7;!oZ`Z!_^t9y zVe1zptc9IZaG=4qwkI|wwr$(CZQJ%2+qP{R6Wf{?lT7SPoXMGe_RW8;{;uk(uI{ex zo4)9J*ILiZXtOm2(P6D{kqzWdyLtP>kwRI23)7aUE`BQPjYjF))f-My9UVny7bl-n zSW!64blyH{Ej}sR^94^713Zu^H)u8Imw5PC5Gr)0G~bY!wF$Sp5^n6jo;| zya~HoE4(4?qB5dP<;sRB6aDgn>Xyv^ie?@Eu%=|eRxMyxFaV`gwgF6F)95JS!|lGK zHX6417evubS`cT(hWar^z{Z1Ms6U*Mru(;!_bpm zC5vz%$@HwX8H(W7J}Qlqms+3DQ_s|jpgJR-7E0)D{U3VRgX=ia_Eyh4eqOCpCfLu_gyw(Qdt zx!>F@adMh%vX&Ej`#d>%G zc{c^{5+hLSKuNrxShinQ(Uu0=SqpoFhpa=b5z|@E%V)olF*X z}4B<$!T`G`e><|8YFyv7~p zyH}Jyb2bDTKhqbr$syg*b?vFD7>J8WUfA@{*j7#@J3pU}U7?Zk;g%opBRwNk6UUx?C zs~Od7))f6@C(+BHzu8jkUqL^yLA!N?Pbh z-#KvWXjqQaThzx~P?`0EM9614)RR-jwrSd%7Gj4-xFN{-%z0gmoMphOXw4%GiqSfOV=8~f#`@ICH;{yNA&iUb3q3Gu#`UI za@qXoep%H#&>JaKGsLQxfRux`910^HWe1)wW$q^u2pkkMqadzy?ee&ett&X+^-J_c~I;;-Kgl8Bi|a@^G9&?u<&@KWn2^JYRu{T z;cV~yTbvmqMxAa!4nNBkle(5sWT_?X!_~{&m^fSdmvufP9i4ZC8Yq~l@kx?vgnacD z%+1pcT{vv*MW-fJ^nDuuclRT&vZD%zOEpS+2*FJU^W;2O4@w(ym(Yyk{Etsa|B8}d z`OxaBips#A85>Sabgabs;1|Mu+aTh91V_{icWMwm1i#*s@cj_pf&XZYbN`)Rqy`ZH z*#D1N-@nu&J=FE?f^S)kFjH9KCYB`ExW(!h;=01Lun>0WqYxY5Ft$vg_-2y(M7rP2 z_A&}pDi3A!ZPE(cYHIlO`hNl*2jzi3DwPbwo{k^`XudF5%)QeOxj`6Le^s14tU-F0gZ3_| z8JgNKod&Tt90A8Ld&N}3;ayKJZlIZ2Iw^)GhRfKSs7P{_D&an2Gz!a*!$Zc%ktTW~<@_<;2KPpLeA&xi?A z64Bt&{)4|Nmbg_laVzo!qH~baClkYE0U#`U3;Zgt1XaF2IIPvo0!dVGDibBe#pc?Q zx~GFl%9vfmBUAlo|1o_0NWnhJln?%bz*ebX5a|z?xAd$i6V#`6gI^&+C%`8%=uxFf zK!Qt|%b17m6pI5>l|r~(BuAtC%U0b zOATd%(3Za`(9!0%j%JcW<(0i=GQD`D&^quySAbF*3w=jX)jP%sb9C9 zgFME`D92_l)LB_o@3lkO%INRO^$udyPgrOZ>>muF$8N)J#zye42)1IY+p^u&*i~E~ z40fodvTw34GFItT9-71QZKIYrQVUE?e&#(|WkOLve^uD)8C__!gTpJq{B2g$TIeb{ zn?`H8+wUgAljBtaqlYP^B9uwa6Vbhs+ip9%e*b>-ltZu8S9424=XdP%CZsySoGo(e zI>{@u@x2ilv@61KmWV^JKBexF)tl1_ek2}a3lK>;t_nzWZfFIAw>y;Dp3-^R0rZY3 zWg9<$Lq^_Q)C`1L7)5Rt47S$zx`7d}QW2;58^)TqDF>Y8b;M=oYz3~7DFIu>=6*gmrL~@kK*I6>rgej?{^m3HB zp}N>gz>RNy@H;T$VZMiP&6-*UV#*a}^$4!?K)SlWsNNxdA!RgY23Z+dkL^RP;xN{o zIO!sYwC!Uepjkh@ib11?ye0#3TW9E)I3vL=mHZDz$UPDrt5t&}Jdom^c;apHwKu>W zs&xc8(aW)=5!bl%!#a0^ujbP!-4nG=`K(5zp}Z!Xkg!I=28~047z~}k ztql)dA#FCa(nRL4f}4|s4)k*al30;=m87;4PWM>cZcgc9#JvA#g8){0xSXO|A{SdS z*NllstmF1&#fyNe6I}1Jj%sYigA7BSR@GrB?PrUcEt6MQNQRDFKbLnd3gd&~%yjyY z54UI-?{OB5!8L?((MxeZ62gJW1R>aIDbS`x$)CAI_5uZuv5*Jkd=R)U!|JrKDj6E%<$?JG*{@^t!H*Hyue%{^la^2sE`$rV`W9LCwERl-SyNUsT=clk&u;t_Y zW)yht^b%4FIS(^J|V>7YN)E@6Ymxj|H=T=^=KaKgR)u!C2hnOk`77oqV0q5w&IQFB_^L`27$ zgHoM^vLK7(XZVSQa`L263LXQu<)1{}5hof^J2omK7Q;i1Ko79)tR7WG!|+!~+2sNE zN5Y&8jEtFj^A37HiX9J_Y$K}2YhEgI3S+lALYp31GN6n=j+dx}RGXw&I<5Z&@dCyT9yI{C1%}ovBq zCrP>~xSEWJjvA1eR;#C+Ji~(h3%_^a4Xwr;=o}Qwe39HkqWeX375i?d3#Z#V-x1Jc zhM(7ugfDf1!m8}eGOkrontHNES`!R*3<0%~`@vwIa8&VE7Ywgu#2{@AV{d9LjU5AT ziZzzXW`c{V%7)7Jmz4h+rsMt7)6YZS-=Ncaj30+blMeu94}V9qkTKm;`*Rlm*58rI z^W@{{4 zje&)jwrsK4d)&>L?AwIpXzc{98OYBE&ZeEd9$x5U7`hLC4%aGuy7*{h7#b36?CTmF z@#~<77KDKE_Dlm?ToCOFX&F65P9TQg-*)<)G>$ z6#?ti_t2Z}0Jf-3S_0O7L6wDwZFfywy-sk7&H0fkIV2#m7CjDBdxB&Qq>pUIv0xyV zKfBVtp4eDgISKC}kA@O*R9Yn5R5?Q6z^QwXw3{j!{@EKp<+W>W^m{@5|rtT-cCVxyWbYl17F=S$AbBqY{uAk|qm zN*3XKQytfXV4N=*G-uMK;#JGqqEk*g_H_1BPC1>ONR&CLPy0qrMg`kk!%4O@;K18t z69Q-|+0ji~A(lk%IIe97+%N|PSwu8RlKcCK^?ab9SVc!=Lr*co$HT*euYc$PxL?mkHDSq-%;;;O$ zK+F%!D!>dRIQ{&9r_&lYX^N}q3kkW%qZBk42zXdyx7;Fr$*vltllZ266wvW~V&h)D zwOwY^f`Zbr31pgI#l#gJ)uqJsNF(~ZHah#4xIX2EFt6_7g`02?VgQ591VJ({w;3Q! z?#LK$c#9RIRr{g%`E7(pCL#!KQO)S**9**9x~@OoNy{?PyW-mh%y?a?K35`nUV9`h z3+&;P7T@1@eI^L_S_3x_C=J2pw6akMbPYnw)oX-tfuATDMk&iN|173$8MZgJsjL}l zKQrm>V3K%%ALz`Pmp*jwu~&OG05zaeJP|9cNHBF5_9%5EIr}nU0H@y3VnUsXYa#I^s)<>$l)q5s; zz5*qwwj|U!m8mh0R?HNSgpX$9lj)rjm;#QWe4WK1(#*WyEb5(caj8lg6&;WvKfi&> zC|SX%Cr((w-Z*JL)m^G0`a6M<_n*y69YJk>LY#K51D!oIr<*o2wkRTWzvOyl;s?|C zhdNj zA(pN{5a;)G3eSe0Pr|t|yuN_i$Tohxk`IqbK*amySmoF~v$?JzZH^^h-#z?TQssy-5Cv4HJeXtAK?OhhM&4KC3s1V@mmQlP8`-_)z<5@T6Q8+V z@L$nkMdEg90oE2*Qa~Htc|(%+o$;An#9lfJBg|TdVVmEub;W?B3lZ>-0Ek|f1te(O z93MO!(7eV6r|;A{PXE2$wi&M8-abc;(zti}sgfFqR{d6dKTK}-li7WU9R_RHhmA?6 zvw!JBY!_AYR@<-KOYKJLbpP?u$RX~u4W;{=Dr5?uST_;Z>HUGQPS!EjYN`4Cdg)^d z79%my9w^44jT$A~T--n4qrNL=84dYO(Stl*=^$*9LQQR_;=r0Upj)lVo8ZL3|8?=> z@77)(*1|bvKGS_Xq{_sU`bXdQI}!p>sgKlj9&ax@cY!ZOjQnabGVQDCBC|f`3+}jC zLI$6^!vjU5xgpmyD=B@%i}n7dLwA~#O4Npuyr;Ng95Gh@m@6PzRHKMALfh))d7Ma3 zWPV-jiBK@0u`R`%PxSs=sm)AmKKmd|jAN?tU$TtD5>NL|X~bFDU-||mU9}Lnd_;yX z2!THp>sK!dz-%*mYA>B$-_c8Ke)k(=YK79rB?3H0>CE$Qq1q7C zF4n4<(*mEykd}5>_*{-hnr3YC8%aAR8iV)>%A{3HmDB*5jJ$&={uVt1*R!GR?AVxE zC`NRSGdZ0y1Z68NTAG9VQ{9j@k>B;Tjo;IxWo(qMgc$~2NG@r)EHWiAz4S+%y}84q z4=vQ_sBOjY!?x-BHMBO1+dRHFh4SBMjZaj6R?TDHvyN6vp^ck-$H$ZwSqil2B=F%C zpqwdt=DY!T^Zd4NZS*{<jyq0$yLpDl*)F!KEA z9OT#a12H%$_Pc*)&KRmO_5zcnSL~W~XOh}l3C0{?ymoFFHv0#hB07oIS2FvH87KA* zkB@*6^}0N(eEnq5imP8t!x`y^-?tpNW95?3O)D!*&?~JrTrDruvg&rL@GGxjf2Yu^ z*i~R?SqHDxQPDzAq7vf=yR|#^zmJOX#v>3?lSgrNn$}2!Zca*D>Sb9F%m*FX1NEEi zH%>7y^zQ-2`m5@(b9EZgFwZ+|;~KtE06r)glgI`?#E5V&Y%4v6lI)nr>>jAJ!HwT27w#91#ue zWc3KY*XyY8jH^H3P;m4MY+#9f?Z3LoI}ZpSVj|Z`&aa?{2E1>qVNe$TyU?M<3nGPM zJIv06X%Sp!A)pgCq;&q#`N2y#T<3(deQ!8Sd8io@+2r|$4!!@>8Gbk)q2G`&o~GW^ z^vbEPW=bURtO(}t)>`f*HtDQ{6%v zcaW^Xf3GFb0HK8jVat#)0Ko$&uDG51Fd&?*ck7d%eNg%`hNAt+cU(KBLOLo$PfjW@izHPJiN3#&&RPJ)#^JTV3^t9H()<*%lN z@acDD=zE6%!i0h`uZFZ=>cRV^a%K&CouWk-Y99q&?H6uOV<+{j&=t<$5m<`QQ5n$G zcadIW#9T)cn02tL;Zz)9G;EMGrA>pDbxyRN~*8Q?YDWAYT}kot9$hz_4KBbwI=t<7TP=XRudRz-1ZT)B6nqz z6#9sa342q#mZBY*UsOlrY$)M(DLeA^@4_Pdt#7EWHrDbi0fnc^^8-l7IDYqYI^+ z#?H(uqVH74rt`e#t{7PLtM#wDeY)B^Sx`CTmE^gobLsd5l6ak8N1~$!Ii zLd9%%B9dMhwZ%kH4UUj_1yZF&_r2R9PVJLXO>AK=G1f{vZ<&%bryQ)V6nUiun7)Su zUhPRe(K80y?i8kft_dNoc7_{#bDKqr44boR?$Nw%M2mQE(T7x66rHF}P?JwH;I!<2 z?&8$1!YqNR-gFx!lFuW#PQkcz^ODIP!<0xB4ZT0UdI4~FHzi`Ut8xB?s+&vDJR1!Z zBaq0noXW*>p^ct{AZO``bEtia2|}E-^r72Y7I9Hv;_Lt5^POYn9>dVi_A1U21YumC zg8n}osi4%ul5t$qCX=($(xizG=>x8fYwEOB;JX1ennIanZGTtO$}`MY@yU%YjH=08 ztd0RAz@6rEc}unZ9^qYBq@1jtrK!D?h_N9sc!WH_LoU8SAsJFlU%#-XlY2bfKl9B7 zcW*m0rlh*Awz`O!#?r2l!0?}h^cVL_9)%?wh}xIPy7F^MI(@ zk-homOYZUI8!20)wNyLGXEnxWg~KM1&<+?OQnLs(e@$;a;(6)em2d4!e_~$;im14~ z8`J@0`VrluJ*m2w$`c6Zg{f$F*KM^s(_;vX8~a?iC%C+=n_Y4&v1pjfmc*Zbgc z(ZRUjdDi4MHV|1yFZD8SD?~){KNfZnaBQk7pMKp{lqSBve19y|>|@*W{B_<|t~84^K37#iF>3 zxc^2mbwOd$p>p*Rx#c;-y=d4KTnGNL=86$kPfTX4S?IdfVo`%5$r-^+R}f{H!|xgu;W7IW`4J4Hp##+*3{w&z`w5|D14Fj=Zps6+r^ z&So~zTvC`L(c-;hOcW*ppM?UM8vUya$21x8rz_(e306%7p)=eu0_--9a)kV(@Uwm2W!Rw;eEc@^@fDR}wBI6`m6?x7o1CZ6 z!oj+@!ck8=F#wDVwAE>i`_Bmzix}`aIc2f&HGqgITfy)l@Ea2i^dL@?;ZE~;{Jo9jP?9O>s|-bwJqJ&V z7eYP59-xE}Oq_oo8ItC5^?QMx+OMz*FWB?&3OD`@$MgWx8~tIdi6UK$&4k=7L@+RI z%zvwo^9sby?MWANTj3B2IGdR-TWROwvt8q;k_nWEH(yYAzeA2$VF-#OklL8jF8q1r z(IRW)JYe}g93#XSM}^{J8jI;;y%1@Dv)unR1g{AhK;RWkN`46ktX9PjGJ;iNsOHYOcAr{sBWq9_#f0y)&W)N$xPOTq$3gw5=b}@%9lJFOxN7| zBdkkund(Y~jEkvO^@R(Q?+-C0Tc!4+^2e`_l1!0EI1XrRDrXg?ibxzq|6E7AqI* zcLDFdTs8;?Izg4staeN`NEENiRKiA!@ThmcFnKcje^b09_S&LL~3bNAoOD+1Iz{mtxzN7{OyR zi-?K@;x9VV2A$f-&J`tMc2xBXB7-Vld;z2oh|9Nz9$FzA6&ACJIqRq+l70=^69`@W z_n~|fzK>Yl$~Dm=51<+i2KY|D4nkxJ4f>vv_CAV8DD0IYKka!42)SltDE4)oDctbifp#?%@30T^btcE6xq$-Fy0cHR{l?a=#`aVT z%2dPsABc9x0Yf~#+-HmBk}H4mrvDWQmTk(J>X=yjNJAn}GV(=LS|_77^^bW{zeeW~ z_mJi-v9PGpqsSWDxei-mUVp{;d=W+v_|X~1{fvZpKV2t~dtt+H=z{6k$BN}_BKBgq zLMxqQagvMrgWq8#1}9NQ&@q8lcsk}|82>IT!zA(M3|m;Ce-ttsmeOhQbq0*;a$baZ zC|`jtszN9nTXBQ$Oe|A<5vLBepfJ^q7RS&Kf;* zI)s#QZhBB&wQR~tHqg6N+lMvTL6StPpEJAY;RI_vcCmU^Rf~Bl0V|fR7$e17#l7)mD|ps zrdn0VnKG>igRoC|XlSE#qN!JXF@@^T80kCU&dui1vdN9}c zU1|7v$aTmVzUjH&l@*Bxlf)^3n83~Sng9nOsyu;HJN{3{4e^95A}S0|@sLz8RSz-5 zHd#9HIiX{pK^qJwHT851Ie8i*0cUhRF)P;67!!9imgJj)pkAm5+}!R2gMkk*P<@LXVTV+y@G(Z+CrEmi z6wgu(mEs259K^QOGIVo{g$l$x8y?EfmHv}I?!qAVPB>-7JHSWl ztdVrA`cd|0sy-UGgtFsT_=aY3@vgYzwdMef&K9-Sm+f%ay>!F$Vn>exTI(O9XTmf? zxazXuC&NfSM;9#g!>^#6>S*)ERVy<$J&KsDDFz5f`neK^qNv<>yXB&|J+^X>TuIyN zw9sD*{{79pAb^pkKNjAoYeItGn&rD!QvySUc$?N3Qv$Gsg#w`p--#%~I^Yf)a>%TvojHg@ zAt5i$7035Oz7ScOVxKD+E`)xP*U{OSi5L> z%h?R7;}Vu9x1RF`)}9Ye>e&3^V2%iFK)N8@oE^lK`@}L$OatRw*ARK{Xxz1^i@xAr zSPqH*0&;#DkK%s?3p4J<{uP#zFJ$PbQ%JCWEW8{~e-^UlB3kObKE3^+&9~wq$9v04 zP3oTWOCyRjzf{bn+%45?^;t!?!WMxC#b66vE{|^*&BiKf4wI&#vDNE0VdA>*X4awcgtl<0>yu?;m6iy znhMkNw8oTrw^lpJvuQ?6Kt7ilr20RUFj`s*|-+`6cbJYc4V|+uBb)SR&I?PjRLxrZD+WE)g?aV zC=2;*t}G-3xLUH%aAIwsgtt#9pq)0Js)tPk?o`Oox~zmS1tK-)bj3AaNCswufd!uA zL|+8fSgh@DL3euy@7b$_jRQ&~1yL4~8QNcVz+wv_Q+Q~EJ}3V>lZ^EnA{4g!Z#AV5 z5KJOa>b#KNl?7cJp>NEmmLSNaVA7dr<;MaK@f5=!ySnh#&^(#y`@|`1KwhD)&Svbs zQ9APY_6(dXcpz73TVb@CRVlR5Q1Xux$fcCaZ67z1My@m1+%6&n?nPl}2-hXG=oQ!$ z8zTi)IJiu1XGN|F$$4WZ9Dn>L4nM7xG6N=km`e0U}09xD9(qyb0M3LRovf}H3TXSKs+)VTGhtg1G5vb}&G0 zHl=77j@Z5C-`cT51YsUsYD=d$D*Cx>(3-hmcZMOF z`)QeJ<+bC21c<=^Uf5|V;5G-({tB249rscW{ju=e+1_dN@vO@3>EZS0kbYLb;Ktc5 zY|}!0^I{Xr_y#Y|B9BVD!NT02SSx1gNq@bj|8VUR!6U8{huU#KzeeG3c76S`k9U`c zNG55Yp5MO8)Cwm~&qO17VO}x%z^ak+bF-v^LM*Bn&jHW?;4zjrQpBB`G8H5+6{8-9 zb7D-MjxKKJ7ysNotgc@^Q(j8`eP6C<`eVhzjYStt3H5~n4^nR#X~(8&7tsV9el2r3 zAv;&eBl`f2Sg{ouUfV6{KN9D{kVfs@@By|Of$58L$kB@n@vZqKz}r?Bz>Y@Y-^uu#re} zaaseuX6rb9$-d}MEmAT!76tJaJrFfMWSx-+i3rCoM)dW77h*=uRV>4aCskygb4U}^0$H4cOB_#ZK-F=hn>q^c28JdvT>Z4q5c zG|5=uAY_TzYEAt225t~bS`vsW;Jn1&NXKAau-~*w4oUY|k@-8V-Ihk!qO7KCpWu#w z&x54z&Q_PXDpUKJRz&iYI93i-h?1|hkS z6^fC#sTHpBG~pNs1Rv2YdK`uPLUG>)AP|XJD$GUkg70DR_Bw>q)l_TLJ>cxn8UWfFHDLdQIXl=+tF`% zm7vE;Igoi5pmD`YlS=1Mz!$9jw9-H-L+^6hiJ$|(7?q26x94(n20M{pUTP_PTU*9wp`=H;JPSv>Z>8lI zMvG@H3}f%cqs>fO66djZ%k1#kKNYTG=Htx6f-JzPIZ$bOWZ7N%gCXGMkN@un(&SVF zGF#1^DmU}Iw3 z*&jv$K}&bH<~O`8O=`6!SO+M3Gu7n`$@NTBvT~0~hE!8O<{Qi$bvHwH4ZXe-T7~87%z<;kUS#&!fxp>92P}gS=dY21rx~ zPlcH2oI4|l+4=ri%+b}wqX#y#-Heh~lc%f6Yp1s=uk3*=gqRI0m)9e!$Pe$v_6U)SN|1o@(t&5D&(|cLFmiLq6h()J6h@2h-(JSTazK}Yit-;+yXXzWfM(5`T z($bNVn4QVv&IzpmE&{zd))41`xR^vy#qx(k%Jz>6?W?^Om5%k0nqquG8)q&Y%twY| zJ{geW(O?5zw#q($Wf}jfr6tt=#Qd+H3p4%@nzsuhCj?{&LN& zhZ)J@>yswDiDmX5^=3`DX39I0E{_)n*Abq`zs-2E95QGDhdA21h1VWEVH{;=ap=Vn zbeon#*!$&FTIhKXvOc7;Vf%2W-XxkQjOrJUfu&%BDKtY<*c1J2nTK2bs6)=k;vUrF zkBoY(M%^GY??>Y4XU0EaW{=RDUZE6^IGlo>b1wy@kz6|k)9{*o?dqd zHI**8IPEs>n**Ugn9V3-S#+K1sAr+f)=S<73Fl9fo)+`xm*cdZ|BrDV(oUG&GxM?I%6RH zPO|6?H4`izM*n6E(xor}>{J(xw&WjoTZV>x+^@!8N2x6c5~nxFg)9xuF49n7kSnK3 z9mi7%bofHqhk;N$y{)%yE*pw;X0l+yAX=x=0-fCq77v4FnTE?b^UinP-)&iwDF+Dl zW*!EBm<_f9g=oLKb+#|Qx0Z>MQS^NrMN03xf}BCFFE(7g4`n)$n`VM{q*N%^y3iL@8sv8>)S{sWLTMJYkcHD-Ur)xjY zuRp#s?k+24oUcDUwhqoE+x0dy=uC&$;!{lkYyM-7+KThF9)e@0Ju99W#&S)~5I?<{ zlUF~!Okk4njNBRIB>Cp}{Z;rchv4kI<`dTO*W4GCy>g3~NdykZj^5RK$TQ=dVutL7 zqq8B#MO2ZOl1XG_zr7(xHO8gvecNN()a|J70fDQbLkOdL>z`gFw@EnN@S95;&LMxm zH^B)2o^LOI(d2@^Jk*UjoA7g|UK0B`cYuGg9LI2)<5ir+)ZNY`OlIgZ%So#$emBoR zyKCp}W|6cL`+6c+{2Clwd_W>hm%iYTCS6w*uOe_O-rrI{GdfCT%U+HO9t@B$bo~9B zSrKh>LH3m7Lc91fnY{wPJn7K%vXjs;S2$7 zt23J8fN;INWX9XSV|T+ru46G2RnNbY>wi zch`pN*h`gbG~<@|UOz%{?ciN4Ts-B&HrSwgDepj){pAtIFEaMM|pU5-MI%xKG9NtUP4nm zSg(3cKj?uDV(x)`zfnSCqKgec+T40S{$MZn*`46^)I9z99H@(;ut8y1K_vwXWJ55@ z8VboWJ($ox!|WyM(?vJLuvtr^cd=SELuT7p;`Ta0Y}Rx*8t^4NS%c0x#yQqTIYM81 zfO@!TSl87XSKz*4s7PklWugy%oLlr_=_d>vW0q`;c0yMPlBY6{GCly4Ucwy>4h$OF z7L`aAEoX-2n~JN7%cB{7Dth5JG~=aie0_NO@M9@wBwu{_3YG^yh*nGn6rqJILWtX> z?R18_bYs1|KAuim)+e)6z}<0~58ptV{1!l(IIf^Z+B#sga+sS62W+FNSvCvhz{cLRn=Sztjcpdc^+09&QjEU@Rij(aCnpW@HK*>jwaOQkqcR_gEO3Sv z7z)+lGsRJC9|qbIaEUOMYM&!o?%d1{1bwXLGmz%jcOC~PJDB6+18L2@I(9dIf^`J% zR6Kd_nt%SY^L6Tp%{G_8-i+EzR`%(e#Y4BIXVx!U623dr0gtnmeFnT;j3R%=)VaYhBa0%4~{0&Ciprnf*Q!AyfJ57yUeWe z`J6OpHX&rwJr8|>B1*$m|2OkN69B;xG2`GgV=?3U*NjaqShy`%*||-OIhk3FS=r54 zEzH;i`2K%8GFku-34s0oxP`JW$=b;oe;d7XJ!dvp)j&6|rW#!-*>#ht`9q}>-OlPS zkDH*SV`OS7PC+&@L>n3j@jWR~%T077I?O@LWU zBKTyH$VOh9Fshno3bq@Q7%kH2^)lk3{9b($@Qb`&kLcW5WKxjh*kEuX7zf&O0(sXa#^-clYDr1zD4 z3I?zPH-hXMMsEMB1i87G8{7Y9*{9g;^yUIZUM{9#mdACOHM8hrYHgXKS6ku A&j0`b delta 64173 zcmY(JLtrLavqh7Rla6iMwr$(C%`diX+qP}nw%y^oZ}^XEaAs9!?Y;6sz`ugP<5>aB z9Biz7e9+D=PNs&o&>ovNI+AfXZK%Cx>d%hgg6i$`vmS83NFW^(3$(L_S~Y5a#~Ns{ z$M@#m+LJ~qZx|g(6~jH|Ha+)iED6w_etTq ztNr;}x*Ed&8HC@a3$!kIKX^HDSdIcPY;FxHPL43zt{lAlyMvF$oJ&V~&&CfKy<24< zlgBWReqfJ|fZ?;pi;H}K!1CmgB0J|y$cuVP8In)vD=9nI)*|!^V$@nW!(1HYWf_ZB z)?DG$`b0Qenrw!dVTd+E0Q+yI#fAFab1h%C*sbuC{1%bN@GGN|in%^!Fbv=u>9JI; zevMlLfKi_WJdIe-#1N}F#dR6#bqy)Dk9gWm2d9}urI}x2a^zG%mkm@jOMhPY^%eE+ zc+SoojGr8)w0Lr-hNC`^+X@;FTS#g@q!x2>ZmJR;Qz9C`>(DA;-pUgAb9YJ5CUXh= zMZ0ZI%>bn-LjDfX{{aHWGz6gVP7CnsXCL$M4<$8IK*%{9f+C8yk?J7jcvL>Rnpl3s z$dS-Xu1`vsb>%eXm{o@PB}=5Hb&H+mOo#1v^Mb9JF)ENMEojTb9mMN!DdG));itF+ zfQ7s^%6M;jz2-ZdB$E_28!EMmR9LvXS~%r%`nkJ1!i=}Q9esJaf3$ht73l{e zY50ypq7ELT5oSwep^>{r3Sz~CWhW26cWPv|Q@_Z*VMiIGAv2N**9YgO(|PoO*+ zxTD)r5?-}iBmLkJtZN_#A}Rzas@g)Es{cG|^-Yol&HtuuG=|Oc%W&49)3H3o0L@M; z=W;d)l0k_-*o4dllqw55w9ahsX)34d;~27KmtDe%-N}@SO9N1OY>52~3kXWjM#w*MUabh|hpUOZCyc5W%}B$Fnnjv0r`hGfzfs&FPqS*K zDVX6Oq5~|6;xwEBXm}|flL{NaTnf2}k3d)*ijYfzP!fMB^`csjzF1yJ0{@Lb%~k3k zfX$;$Bwa<;?gI)KWs9^G9m;C?0J(O>=;j$pD~1u1$@L-YTS8tN7VEcY-;yT#h9(lp z;~Q{^AYu|`dEViHg>;xjBV7?n4J}zjsnT|vJN-s{sxLpS%ZD|NDI35-W)2^%gOXKd z*w%?Ql#`=896x%uKE!~TL&=}OS_)A-HLA8nRE|hcPe8pLLgs9bNQ}8^UsSkMhG5)W z-;4a@tk!9Q(L)!xN}H8`N{u3PR^BRj8<5@>h+bgyKDT-`kaaH2L5E;_a{;MZtpUaw zM3^+gBKVgTp^cL(4AnY=dVw>CI?jy8@|#Dp+V9iVkbKyP*Hke77X-SIa=shM%xb%r z=lf57IzX?OIk9c%m+R~1o=Vg_C;3DaAqUFe6x5a}=)`2%!aPK>)?=)AAp!fqdv;jJ zJT<44r5_J+z`Nw?ZFu$Cv5<%d4Ri8i3s>87ECwT%cG@Uo<)~rG!P}%6PaX=`Dv6_8_0v%%q3?afKZWXs3Whe9iT*;`58!k=CN2Mx_>F^4n+l_2qg-H zqtc}(l(^3xX$`gXUb-;+%sxWZn6j(i7St>NxEeyz5*yqSKPD|uUE>lX(k8pFXq@Og zURbCjS$pT`0z*=?ZpOG9u-6lY%LU zB7UHcf7O+wzsU2+sdnu-ZVk*yGlmY|2G+*qh>P7+INPm;l_<3!O!jMK!&n4Q@og}I zRaQ;TAXfJpK)F%h#+lzx^x7@-)Gy?x2b3=8cTW0OsKRLvZKrl1YO{nFRjVxQ)%!*&>W zl1VzP8@0H8i8i#P!hEIcnjRgh40v7WVW7iGCDsQ|j#7jTvnGRIiwy!*09-gFC7-p^ zLpZ0b^$Mzgn%lUHP~q6AKS2Yk%?$i-#9c8*xFgDNP?_|av{$2`OtS~j4pJ-11NoE7 z`U=h;U(~A#-YRO@hbL=fG;9nMt0u(Z?M@XNEjpE?Dqu#Go5cf~wICKnG}M)F2CZN% z{*cwU*&SV>NRI6__No(W04gqwUohvmnyOwy5~39wEgj%} z@yN_SO^zl*F@UH~qD#5ZkDWvE$g>W#kw6!V(9Tell&U}jR$*`l z?MWdWq?H=Q!a~3NkD}PRWikDtzx)HA?iUA|T*_*P+KwSqXe3fA0jywkwyV`*0Ojd< z=hBGuW4GbwkDv2nF|bI55YoZdKjcGb6rt|8T}|m~D)EkD$oBHQP?Ow);YiN|^V4i8 z<*3p-SQ$EhH0_uCO^B>vEx7;ujnpllPiF-d21dF8<8aS>J>m6f1RML;y$dl1;oPIo zr4og7TzZJp3j91g2&h4E)3u|lm`s$Ii~-Y4Al1yE<)EizJfnciHH8s+CW<8U%Y8(z zjC?Jauszj@%YZ0%h^p@->KN&ovO&j@<)nX?+Sh8%J`jm1cfv`4Ew85dw3QmmT@-JI zI=3~%EuFNGWYZ3FG2If zv)jMalcC8&0&piK9_T>YA&^TQeIUbDkj+W(fbK)?->1lw3h53xTdqQp1@;#gF@#du z?9i40j9@?Co*(yRV;=q1{H<7u98SMoeG^pi$!frzw^&=+LyL-rHTmav$xYCYL~frv zED$clQ0c=e2$xQk!4*xr^AeQTS{kyFC#LN3{_G%z188!Py43g4K=bH8LiXhCd4E#e z+kQq|SV4Qwp{dMX^5prE9^1XSZJ`4X^8=2(TJ1?H2PSv)g&qm#%>t zif#tO1E%#k;a%Ju7~S!D(|iM5g6w?9CWA`Ds}$Q7_rp!4f|U%-h86PGE}hMurz%-pV6Bpj@mP6ctWcb8g*l}4I|Z&2Y)-N-Rf428@4g)? zc7|G%U1T>#*+_|I&tTXjivdAcVzu!+fT*p2m2KB#X@~%kE`cyfX}m$%@rNVLh>C9o zf`Dmx+D2&1NI)L&CDdAdh4M@p^0xyLEEm%Iza0)WO;jX9MQn28NKF*`9R^70PSGAfMO} z<~(Lu-q3p)Z5eSilzm!n@EUd$fLQaQggpXs=V#*_m>Kxl%JfJl}}!#jn5~cXg(l@UJ*-Y!1TtNv{;eV!QOx7jq~(T?jj$H!JDss9jh{s2!LlQ9XjE zCkFaK=Yh7CMRN%rM}*AM2QGFTdQJ1OHZ9BO5^uRB46I(#+K;j8Og>8^h8QgUx8T%0 ztuC|2FA6G6W#TP#|5r>0AQv9{JLcG+xzQWSLqu~tyJ{lKk6Y@??^YJ2;uvF~z5`g3 zGfBRI9k9B);Nfs96h@w(_BI1Eg*{vlmlr&7odS9>x$x3tH>gqZAr>>AYkg}+{R(bU z443u2SS;Xw(dIwBEdKgP612@Lp8Mfcs`m6zjO*sueG|leGP^b=`j4M*zRQ*oP++m* z_0jxhRZj33=QGoix&$m7OstAgaXs4jwq@9r0Cv`m6K!J%wfv*S-zK_?f%#S z0P_nwt|#Rqa01wvIsO-htXw?jGaU5w|e+(9Yo(psNfX!#i$m?c$}k zmwp=Q2fFNGjiTZ6^=e>W{?6Tg_ZHCmK6%ZdpRJ^`wX=0}8=KV}teegtbhC@s+hrb& zhjAnyEqSH?$!&-thB_q8RBHMAy~Tg~bKiPRG(CfA=BNcz^u0YuGwE3JQNalqc2P3m zGGtf*hS?X5d%wQVs0V#B{zumbi{!W95oTs-^8`9Nscd-EPV|yntsnJ;+6jo83pd7_ zcnclk8}*jFzB>#n>WjC3f-%bs9)W*o0@R`Wb8N#f$LQDrU|sGm40b4by&001=u|5S ztq%0v{d9klEyUmTyU1#_UNgRvpv8E*8Y!5FYN#v?77zH)=)xo$`6V56s=}-(bvfD^r3KOw{jJ^rdVIhgWHibySPj+ZRIV=%1~?q-YKNG zQ9p`<-*0>#QmoH55riv-6^{BGd>kU0#O`X)At#X?7iLM`2iqAO3$wzuJm4pC^s*7v zz`qq?EN56_fg&XghkCBv>KQxy!P09x9D7?sf3W;!Hor8j_n-*7r2s;aDYJtJEGWWk z6CvgTRsKxWhlG^KC7ywxqAS}FPEp2ux?}coW(RWhp^`@Sw!|5d*kLw`QOz8(g){I) zPn#o7#x@yz7=c3`^gdAdQMV)^uV%B-Mi?}kGBV60f zW}O^!&-kMV(KK`k8!Ym+hmixtU)l68+IF!7UE}sAMoTzV z&jMpj4f3Y=uOuFlh$rCCe3m}5o9cy*(eGi@$OcYK-suw*7l1a4dojBc*NU&4vtf^? zyE8Y+@CcSB=)<0|p??y|*og*$`l%r`EW7^hu49dP($F2%&9TNJUGCT0&&)9UudN6o z+=TkHS=QIB8^vrhLfWPY?j5L_$i(>UkeH>U5-_3aW$z|J&N10|$hdn*cxYw6bfvK> zc#zG~I%)ywalaL1^{z@@=B+)1~$0xpY8tO{~5bFJ(x$`T;QkRxx)!kUozvTSZbDqk8aYp0@BKV#^%&-T3$3);L(hM47fxYe+aZK&S9NgL_xLv^t$i1 z1wc@QNQ_%7U_|5ii@)kv?w7#Vr*Blxr@zzkq*Yvp5Q55?E@O|p+otHVznL5pO;y3? zJb!c5(C!nchUvQ_xR6cw#DdHb`E7sr#3w(?81HlDs^8p;*7gseN}P^)tLNunb^Iyb zU)1P*TY)#jUyuIng64@j|UY88Sag=Q1~`7tL>u?I|n> z&3E4L%AdbG=q4Gs>d9iHM;Qz^CSafE2jv3Sz*of*Af&FS*u%=Hx?y24T3B#H1F*^>_6~dE@yX?$&uSC(lxx$XrA=IzzJ$8uoxT#$xKYl{Bd$;l zZ(QOtC<6DRYel+sApMsGGw3*?4glM%uURPy1@XjIF(ts>l3=Hlra0}(WNbU~r+H^` z<-g-EI=VSAa<4?7vgKoK;_ro}Qr$0G5XfqVB(nd6m?9IS}Hn90YX5>l$r;M z86P8n=^|>M-%m*bS6a{RJ?znFy%A$FU*Ods$-=*x5AvS6(7=dci|*7a2{__9FGrzu zzQ=@O8w(B>KBMV>q$L|xKX!9Y)r(wVC(xy5!yc2C04V$5>cG#f{^HCNHP_*?#oAC6 z+36|>?QW<<*#w1|BR;Fy%1MG+9$Vj7OFfPad#!C0Pf>6|TQ)jIiM3V$TiAMiw8U4I zLKj9Er-ZJt&4pn9fARY)WDA8B$BV6O4@GFZ(pWrpiJ{jZwc+cCQh*? z^7nIKujCq-MKzZOfbsIVfoDg1Pjm>uL`{T@KT`FXmyT%1WYUhI0LvwZQRgCv8X^rk zawuLE5s5{}Yql$|!pDLBQVkcyR-s-z11$Wy3m$`{pj9$RUXmmeY}-mT!mLkQ$mA(} zaupEZzuTRRL&&EH@|W_ZNc?kBhKMNtI8xp{)u4^(D1OfhkSxLz1D_>G&k6rw>j@4( zb6?*jV+^1;y8KZt22>GyDhHDL*XOZ&PydZAKkSY|5tE>o7*P?ATn7*Hizeq*Ez8`p zjQg9Gqv*zm?!6Q}S@U&D@z=hCPDJD~S4yym_6S?&*~ss!F< zJjEC>CUXsEdv;JakROha$K@o(87+gBD~FV!Do*f4%hVdSP_%r9GDFZaViF~0zlE}Z z0T?pvzMJB3*kP_~V+E9&f={fe%lP>OlLNAS7viF0=XAHOxt{C#eyyx(!H9vJ5Jc5@ ztAG}3m?aoHIRI7FBO{K1r@bZ{qSJJ+XD#SV5MG~6R6Q-xmZ+H)JqjaaSf*Qn;EY0y z4;vDgB)WS9gs=*iML=RLhFa8^o@z6BFe=rwrf@2bL{G;9Ss07RY z#ErSq!uQhv0g0_{ee3hkPcEH4qeUXh<3ue+4h1huFNEWU|N_b@LgAL9t#y{qt-g$Oi8x^1dT-<~`H#zli`XiV3X za;SPn1b}XPuFwDd7*_^`=nGFv$n%6U{NbiWUYb+HNP^|LSOd2*XtQ^i?ymIdqt>HdD{9abWV53z3ZkxHP;++G z0x?Bo0heVbap~Q4IVrUk2o1%;Y<&Vlb&MU12-rMzW^4F>DL=VNe(A=fCgU?!{%STt za$;Z^ilF0)Dk6Nb+R@u>LKWz&ODgsDV@FAPXLSB?-0qMOyFj$1yGT=e z0^s(5NS}&vd2gKAMV@AT+}~eK{{*V)q~-9ExO(DLMfIQ2tfy@roFUZc&;IU99f#Is z!T)xL+_ZB>#|1cbl2z0d>daO8na44EfT^b2gyZS)!ezwv0)=1!UbW?o)1pSv^;w#Y z+Z44HW_B|_Gd=}gREju0Y(lQoJTg=J0P_u(nhDfW?0YeuK=5Xfw7&P-Ozi*tt>liO z2F_bXbvZ2_fc=!! zAlpK>0v)pp&d#;$zLBYJ>`#v(H2T8*Ca1oo9NpHQGqlh1(oPUT+GK$>J@q;li z1$q5c!_LZeE4V0Js5P6d(|+>?^sTt`0OgQg3%*-MiI2|ShiyVoaa(Dl`a^JEEiQX# znW21Y4G0L&U}boq>|t8=nS|iBO9?*LSma%5Q?%K6*wm(688&|FIoilzEiu0hnw=_5 zt^HCUqir1S_>KJn1ywU!N-p&Dt61vM7e^lt_I^+LEShQMNmIC70M`HMYEQH$Vu{BP zdp@Zz*I|Q7ro(k^)Id@Vq3&fLw@Gk2Glv3zV{A7dKA&(qMCe9n8rPeIks#&P5~XSA ziS$Uy^vOA)KJ?|#$f&=shX6}Y+MTbVe0wKzS&!dWEZX*9?-zYbz~%)p{gRs-kQLX@ zZbE<(GeSXjbU?-rn=ilblhwziHYIuHkiX(m6>=XvIWtVid8zssK7R0gAm8P~nK~e+ zIj?4kj)rL`XibP<{!M63f^ZorkC%X_7$($?>H{OM_egk*BmJ3!*s5UCLacOQkm8Y( zFZ{8f7sredLH}o}s^0Y^hh^R_hYsqZGX`)bebaj5umwSBMxS#SQ+3viqcfVNF5jF@ z?Azt>TV?OdO*5{kaiHs|bM=)z{K}b~vr?S;zP^)p3n9FrnZS#Zf&avjzZE87IV**C zMy{lVCndz6syb_yu{W{cVnxib zS#{~t5=CI%*MLo|{v&t5tP+&o^2eSp8|x#+#fpQy@M$HXZ`*=+FBQ69w(F_-Z2wf2u$Nq4mEj+7p09HYsG;T*}jMxZKv?$on`5h0z*2%FlC~;ERP@ z-f0w>d4T_pI_&`)((SxkiUC9l(hJQ6{c8Yz7#vU&tJ=h&w8{1BEu&AzCEQLsP!q^Z z(NFgYlU-}@;Ay2Af&{hJc#9g*{a@BXGY7GX0I};*F}p?1L7U_i{6%_t3m-t++46!{ z&(sDGm5*Zdb(oV8c!lDgi6Ajc=?O8F{yfIB1I464Gs|Gj0mes5(27_O{HHS8RvzO| zR9jzMG^c*+cg5V`PrQ?G59n7E>X>+Q-^_yIj0z*>*NibfC4yx&diw5UDEHFmN79C_ zf+vmTw;1hLM>>Q}))@DSw*|mV^n?-S9{$bPnMF#gZS|b%gydjBUqWcLNj=c1n#JA0 zqAC$#rrv}r$$KBWU#$1w0iiKpp-lt%JV;6^_?kxvI(8gtIP5v?G(*IPJJtxqH{l(I zKNzg;j3L0+GN!Sqm4n(^eV4R6lsU0U!eChpYg{{;Mog7iL#{On(jaEU^BKxHZ` ziXGV0cJ)u$xdd?6MmGTCI*~Uv~VyZr=gBWI(@)3yM4I4E6JgOY`v* zI>9M~s^XxRiJ?W6Yef);uPvO(`q|iuRH^JSd+IWs0i3*!-#^@H$cxF$OP%TNJ1gij z%Pj)Vfr&n9d~pmHAl0Uk%w_>+cguJihu(WE^GIogD-Q^?4Ae;FWrZDn+Px6uVv;_h zSh3CupE*IHFIUB=Hip9tN%woVb3NFzR}yJZR!fNha?Yo-xx4_UfMkug&rV}zGsROp zZsw4htXvGU<<8#*m1~er;`cNzrwX4eaS6MZt07Nhk`M)}D9?U$R65U8qb3#H$8R)2 z(W-jl1PG2r-((-;g(9feC~YA%$y400sHl^_jc~M*|nB}rGTxr{zXTZ=|-D9+@OYy zPCC^?ap&LJu@B&xNyqnc1vq%TjeWujNod?j>jVDz1$UzW<9g_3vtnhq5NV)fimS_k zgp;U%((z6X1Sloj*s=F1PNtW{YJTSh)ylSNQ=ue@R_rh&EB%G|kXF{fUWi#s5u6jm zzfFng$bE{rv0Z-b11=HN7FCMPmi~J7LsL~zBc>`fROyAjc>05Ss!RoH#fSsW`oMQD ztq6#iAY_GP4-24~rEpUOX$Y(l>QmcQt!`H#sRIKbK7pyYAKd=|yFqnrL;MRx9=6#61Jb;8?_gCZZy?=|uY>pEZ~z&VHAQn@+^c>!SZ z7HtGD*{Uy{wm1~-gtdyJK{vVl?a5YbS`+_trCo0#bljbxP<|L=Gyl482KWB6^|)<( zdciqUb2>&yy!j5^pVR@0rkclG6nVsajYtG#Bdu`(uJj-d!S6FPlJ@b(17LYmIbP}7 zpw@YEp~^5FnuB>q-CQ+`ozU zJ5S4&4OyyDxtb(&Nx;2Z@yV(Y1>w3K^7GG>tVw(*DdabgyV9w@J921?gWFx4CD`+Cfl%xNvD7~WicquV6KW&0Woa0lrBHU3(;>3De zCSmyt0`d2oD5PKc1gbq$dCG63IcJ+1O27G$sKvnmz1dEQWCg3(z74>oVL%SaHlvIU zy@gYHDr!q$CmD!RXlRe9W=0V#R6wLx2nu2vsje9qo9Hj8>@^sgpNGn0kIFFuBZpsSDl)9WLNjM3{PDs4)W2 zbfiow2vU`@-ogRhg9fBVTA+$|Tx?wAH{gSHR!Rn>9@NFSS-Myv?VVBkwI)b1G9q4G zT&o^UKk(E=-$dWo>d=DUyMk`6=z!eOa)>p;)qpu4vbdBdC{Q6lfpv0UjiU7~*r_Dr zdAjOun4k^9OLV2jMh=&y-m+q^`&}QK#0He!M~ztnf%tlaRstSHIK{WdA8MpqayRz| zPuO>`R?Ekdq(t15_;5}NlfcQ_D^zUCOLMxUx+U65bx*F=bJ>=)U6`r^CqJlcW9zkB zvT8L;Q02h6GjxOC1s_H`G&lZOe~r58&dIl!%J8zdY96Bpo<}J`X>1*qpxxOz6*VSe zlgp$U(j{Jm6#?4!TK6_$TQ9^#$LL{mW)yfS)#=%bv+=*Cle9WSP1gv}(~BY&F7fD|L|3Jh^7k z@m`;$s?>5p*EBaUkL1ZoQ5NqQ;1A9ml9ke3oyAaIy8$eHM~li&*juxrcJTMsCBLBB zDEW}EK3q6xey3YFZAiS}Wo$;%ZMiVyGYu)&yD^mYcay4_N5x8=MeP#Fjw6g#%O~(4 z_1}EP`*YAxzv~re%ex$c#ooo?8?Hyo6g?u1ayX>u;u6 zy(T!9L)>E+>qn2$6W?nP;$BMxGnNMz*INU@LASGBa=kN4A1`-+H^7lG`CC#o6$gNY zfragVTQtf4TD1SyQ5o3b{b!JXk zpu`w&?1$U9tv#;Z(T>c-VYg;H@NZ%CGEU3J{%mdcj<%?K_@e69J@)b2ksVD>@0{*s z<)oXHm^T=jEiwa=%$~f6Lo%}CXH%noH@|$JUQVOVvgUeCY=Ssh)|{Tc_;}~oyfRA= z+@8Mf4>x$KFzG$NXP)}(s&k@QvW+kAar^SjZ)(AYyoKE z;2X9)nCaP>XUS!QScYJD4C_H70RMWUX~+pVsoy4j+rv&^iZ;vs2hZ>(T5%Rg&0eCKx`Sw>LLY|FWtEsIs%xZ9+OsP~%$$-R-D)KjHV~ zd?Iu6$s+*xh4;3CGIqno!AdyCCP~WBd*V*J!znm{+XjQ;geT!cKUj{F{xe2 zihW$&{dnD@LF|p4z6oMzFS0&Nn-XYu|NdE#FTcvGbC<&MMT?m_4JOmk{>wV>VvNiz zUT)HfSC6o^rk?aLyZOa0arG%eugHj&>J-`&);|K2g-v-7FF83W`AaKmb<2zio>|}K zf-w`mJ?sg;!yy~P;h~q1(7Ma?hO*QC;{px}#xzAL_~k&URD*`+*L7_Ao`YBCu5EKC zE`{W!tiqU*QKZ81)I+AS6}mVewFEa-@6asbZXkG}tb zf0EI$-flK-*9L#C|EAlix&)pXPG7WA-us6%g`q^zEt>^sV(SQkMLe3&Hj?tNuc5U! z133LDBYJ!gJD(@WE3*UceuyEfVRL26l4Al`pCa7a;COfGkdn`!+;QxE1V?Za>*jI; z^@ewk9xLVQ2mvINnTe1~UHembU1D!fG6kH^43EJmq=0&~jJru%@WK?5+0}owL0{Tk zeBa`*kpz)N=M%sdVtrlkZb5Eg<|&!x`J^ke3X9lZcX%@ZNfibw>-(R>r@N8{^P_IuGcc{%ABw$qq| zd2G>;`Cl+~>7$y|e(wzwCPFXQL zO1!Xv;*YDs#wMr2rp%$df3*7ROG0jw|5y7I@06h7kS~ov>6}O^<^*aWU%$(RuLXf5 zxG@+|XDcHXmkK#4QDl>&fg>v>WTzpy#m>B~W^J>SdcP7qb;*e6enndy1ow( zvOf`b=F-=>S?-Rp{(8ddMVqE*xaY~ikg6r8F4iTZMyx*N;07i~`cMGJ+kk>fh?lwy z(qxdwVHV^z-2S&g%FYB=z9%8ak{Y1YPjb=!Jz%4r5@!8Xvtv#(8vZuZxG-jL+-$T# z6xdFY=)+5_5nayo!UWCCK`TJwj`EQ2P&pMPn-us^zrHFAh<2|%-&mqAY*K-~U>J2A z$=@^__NBR;kaDto$S#f>b@4pcS|K3 z^t*Q;eak!qx>!0lLo{}f1_;!wo zZZAIqK2Iah_@3k1FB`=_OE1gEm)x%3?i~cjePv?nt1C69WdDHxf01eD{6{s062h)) zC6(kwHyS{bD010M9mArt927=D)KreG>y>5+fvbt98S#mOkre?wbt^?c!cu3gUUz4R zxxkr{$TtrOkMBPTfW5k9klXN@r20j-K$n2Yz~^>d$!S3x!`sZ&$Z35gKHW`Bxfk_A zNv;qI?$8~CJ=IVQc2V^?ZrQsTL=c_34DV1-|1 ziSC`_zR4_=R-@>#{~W*$iw3dO3wSQ1?j+QRyGB6*)Hd)_tCDn^%0(P`j<~rf&1kCX z@pguaAn(i*lH4cG#i>nx6x$NRv$l+(&;s&~6*;w6qyRorh?QD5^Y%wqsydbarNv5B z6@4DV5H|#Nq+V!zh5!}KZ62Z!e8x9|u8vE!32cgx;Y9eir;|S*@^f$og4x94m%C#) zQc|s#sP$hPR4pC3EvhpggoYopPHe5=Y0=sqmB>Hm;_|qm-Oj51936VnfQrjUPXqMWk>R6y-Dzyyg0k#?xex z{N@?5Wz`c;+J85P3lbwJ6i!wT+p{{|XNSg}rW-lCkTNSYi>z#|bXYjJG$#{0$K~OYFG>;m~ZcFi<#rUTi_cS@&{|5hKO&V-g!l=N@^E|Hxw9lI z=Zrn}v+zrPBzH_?oRRD$mjE>CaL3u|m7k>LwVEw%8ZrIku{DB|S{v6fgb41Bk>Eoh z^z+d?=wh(+gFj|&d5e4pa=Wbpw3;j9&S2!CMTDaI>$cy)7Az$*x2;OD<~^{16;ns5 zP^hic=IGYCPyU8>lt%;g7)>NsZ@wwLlMv=k__FA1@gbS2DZxa<$^h}Y&D!62@jG(b z6s=3)v9>jFYnys#3(&$y=#_OG-Oy8&H2GgEs6t7XOalY&gO3CSR=mqniY?X`HT}Ej zldH`%uZwzOZY+EPWJ*uq&7jayTxFTjhAc6EVGXO!oV3Q@P52Ch&&{M6H*Xum3PAA;QwR>7Oj@#SN?{w)FWgf zjYT1jRfv($Mz=K9l@;CE8=j!My3Nd+rFO|@2BDjuMnrLPq>}5oSQJz6o&a?s1cV1} z^W|e@h<=sia?XYQ97@Uol`ep8H12{jlBtV875gL89g6TD{s4j;0@0)fbb%Qt#f6#+ z;o4NzVE3?cGX)Bb=MN%^-3ZB3WDY~Rff@=zpV7y-!{KK_l6Zh_fFL!PxHEp(;R^x$ znY>{?xo9V0xQEHS>{5V*f(dfEcH{LHRCC3xIkPKc`p;a~vW$eDYAq^l??kS+2KuO{ zQ=PZJE9F87aDa^Ad8Y#sVOnS{3b@Kj_Dl>(KV09WG=?Vph`bS*yzuR`Z*rTZMI%J_ zwsw(%Gs{!ehE%2GFSBMKv45PSvby4hTPQ+yE=~_2IwYqa?(2j~%EXI6D>h&XSmUY= zmcA;zXo zmR-M(HPOpfoQt4;wIoHZ1~YHIKnY&SXfrC^Y>C}%1%#<}HAXuaoj3%)nJLz`qUNXz z-Gc3blmO&kFTEFZ`9_$tsrcWvN;&2Rpg$E(KAi8`vG0QGY?F^M#G|4cwS@&GqY)8H z6|lk5io2gt~N1-@Jx z6ZI9780nA9sAirtx7F2={YTcGBz0;}(jN@rinhLsM2}T{9*O!CI|1!NYqqp)q6_15 zI-xV@#Ixp9%1=es>~~{?1t8$mo=aNhN-O^f$W-`x1Lq}e6_^Zi3;Xnw)XUmy$LNzM z04mX=QQdiH*wpjC=!HKCDMLgdzG`HzCP;Ir8CRXCsXN`v4YMIr5D>k;hTE-SPmFZv zjq01P%@y^!4T{Cq<{N)>p ziD)e;6Iw@&RSw@r`oFh*8*`PX(57}KNkUv~00yT2NdUB^ok`kY|4RU#2_Y^RSfF}s zYv3w|YbR%7?Hu8_v1Zf`tE;l5fBgC(BI%az`cD2a>wO02;a{F9e|~Ne`8*>XKJK05hcwGwih3RtGx-7jbz@+PIB0eH zJ?u^GrQ)aY^V075)GY@Zb2ylJaxFbsISOIb3_ZGhd@oYd?%(6e`sk!M4^TXU4pYvQiNoeDjBWEBFp7+=_fd9-@unMC*viCnyyQVja}UWy$~3vMUMnH_PnOH+KSvBla`m1MzCuEtMc{ z7G&u&iK1+BtaZsCEe7X*O%zoZ?gsEfDS6bS)b#xNxbx@0A$BjYVE6gGpZw&n?6|){ zocJVJtTZ5V4X@sP%h=HKNSuln3IE-p^ytYBN|rpQPkG7@MA`JlzOO=B(26dzv%Fal zo*u0kJR0!G+T;PCqd(7YlEQP9RE%{w9ci|py6-rxTywGW7R7dxe}OvHM(SOJ(aln` zZx&O?%3pkGhAF4A-E*H%y<$a%r4$?+|KxBr&|w#1`GFDw1)H8ZaA8GV@wnXB!*+Ns5tgk6ECPON;RwYGkgo= zMg%P>fvO$11S&BZI`SM(?yh*r1K2|~pYUA%Y4?}~UFSDoh3lO?hp+bIZ)SuW8~L%8 zh_!#7O*;U@piPScLR!8m6@OU4e2J4E9a$w>1YSCkp0$J}K}l)$6AoK;2?w72jGQq< znBvXZTL$1_f+J}cZ+|#K&i@{_jHu8OXNO^ULEdl#$W^RD6t&ep>-g|<8h^ly-Db=7 zM8Op-JA&i7cxsOgNQZ61s$`;kh^9lQ=oY$tuEPKZ62W22xO24DBf&i{#RbZXMeW3pE55S_0qw$}~a5ey?mLVpyc8}-1+=ztt zKh{bkn51n`G033>RKNrY;@pp#tXN}EyelZPsNzHp z23RRW2jRGsu7s$luu;IY3_rIpZ%Chg>8=ibq+dnfYB-}P$raVwIyxZw)ZZp72&V(o zg6t?tv<kt{94pt*GCoTi?yR{X8S}6g*$EWH}X9d)yawPlSmpDzSPMg6?fA}7%lNEyI z8E>kdO%e|{9ag%G(ap;YB&NvZEjgb5kE(YJ&Mexxwma$Awr$(CZQJIZ+_7!jw$ZVj zbZpyBr{6xOzOU*zKX&c^*Q&kN9DB_%W|4iKhu5U7gU(eaZqa)p2cf2DtNAW^o9o1E zk3B~d9&<3*{sy}7UfDD0`;Q_6Azj{UeC%wRW1O)wLkv1yj%3b~678Fk%rG`$;Q4K( z5ZmxePNAdqL6}Ec`_Gj=VSCH)@2FuNd$Lh83sEy6L!ktM`Jta@av&yi0NYj-IE2da@_VwmYUhC0X_of-D_LY{$44J7vswpJI$lH0a$y5W{^+-s5XG+Cvfh@~wF zEh6#i807QfJU$rkn8XL6n%~zzfdNfQ~ zhjE&FMTp={!J}>IE!k4s0d3YlndzxPjL{_3vJ(o&OH1smR`)|?*+Lc@4NG&O5cvAg z29KEZt9?c^bChZ&aq{J4de^-so&X`wC!#xo=cFS2S?O%%g>rLaar)Anz{4Ht7)N$@ zgTs(b(2o}Qp|3LqBFgcHM&T6+a>tiX+vmy7t~`1S`OF$Hp=M9A*L+@z0?weuwr-Zk zv63(6ZPrU$^CQvQD+qlT`sb(sg`@zyQfguT-xfVGB;91Uy<2nQmstFb3e{8Z7RPbo z4=>}J+GeHd9t!KHwl?IPKp&H8Bn9{4+tBOm`j5i?)fzBfjf<6WblL%39ctOWtlL$c zz>f~`Pp$?3IFA|+{r!zX&zH3Kjtn~iGYQuEK89!gXwu`DyWh5)j|uO6s$n`?zlLJz zp>ITfm!PAK@T*(#%kDtD)ViRm zdL3p};?nyhn%46$Qz074+&T`M2=PDGY&+VH(nZ}GdL;@r=x1#ap#YQ_?Tm!DUiEG+ zX?*gUPnQCG1njkkz*>G_PYpCMF2AEG!sOaF)&sDfDQVr#uObmrfRor_ zV#=sZ02^1V45}Yls#TgTRD5#cr<4La`Q>`*sc#Um)lRSb1c)Mk2rxPB!-`8luWrO* z=vnV(YLp7?P*}P$b3ClTDCo&nAJQ`QE^qqi@ira94O6`~AqXNJyh5rQPwo!K#WEn+ z**LJgfngD(o`Q#8)2+@@3~qG%e~-d1Fl z<{#-TBNxhJ6+61RUA#lDJ>5=qfQ>C1Q7wBLpq=-ZX^FL2Sm%644Eyp?3We=La*y&V zR`gkgFw1kl5f2V&Mt<2;iKa0+O0ubLf;vAU#1tU9I-Wk+Z#pGLb$&&e7BKJ@=`cB< zuL#W>m#r+oKUxAJX!bjCOT&^qMbq(YlYlp9!`{(}`W-MuJ3JJ4#vXkl<2OLxCRWGcep)r&vhR!39*maBV=xbDn7dc=6dQb{v{ajsNL}tXBl~z|Y38?DJJ8xSZ%M>U0NrhpEGpK)kH84)v!K0OMH9%wbfaHPF6ofMfH#=rbRdop!aptunVRY4OSYtuvRu8jlT-7++n;<*?T060!st$ZvACgtM0HH z41*Rx7pdPuLev;f^~_U4`1c<1@6WZ|bru8L$&-UsieS7(TUP-X~w;!NqOOBM@ zC{KoYbU8Io(Wk(2#r{P{!KK~T^V`#h_to^J{J#hXV(~akUOt_9vx{ZU4nX8d8b6=T zuS;pDH)$yHP$o^{1cCk3e!;(=$AoO5j+RO;$_QxlFwPaqbC$AgL^B&)9Oq zy6eX;xs1x}kwMnA4sfThZ6F`-NVDM?G~*Ia4}|KkRX)iK`~$~#UXe1Ex!U`8JPP=d z1qmgwq^U5TgN!XE?0@M#9<2M`niB=&B)26O8K=ud^FFwpuXi6@`wf(tROVH3L`ekL z9>IRuK~QYFMgXMwQkIzrA^z=9@nTVN8d}Wy)|N;L^TV2H2Lf~pB%t!460;;Zt9Dko z0owG)n_XYDqjC#JRGR+^or*e`YFQ>!KzXRGzX?f*2i+fSWX#EfIMRoruNjl#@OXFudt>G+%8$WW7rt?)2L1DJGyGFzKKF4QES#+Z+y zM%5brV?WYv0EB_~J;H@@&EDH13=$*&jZABiWgy9mK>>7^C1Ye4lqCdtnwmmjHZHOm z2bFUw3z=i7YnAU~r#ID|=$5wg`*(3GXNFPse1ZB5ZcTw+jvb#yznia2n z8|hxihP*bJ_CT0ilRTNOE*77R6E+nt@f}BcJbDGobSC8t=`w9`+@;^a(-xOPH#Uj3 z5Q+se-8_j>i33q-`$v#3Bti86;Tj+TwOe_qqdhR88EB}4Vi5ae=5fzQT!duYt-66m z7c-mM(NJMapz5Qvd2e^NSYFS&n`Xs_3^zwOCVz2jkVn^7^&L zdEw8o4%~sr;_ng9grIq!Q!8gLNrtkfqF2ydL!; zrJiJ_EZ6bsnvLni%hK1U%R6bqU;T~U!MU~b1YAA&Dje+|iE#YwG(Zmi^eNF}JTzus zY$ijNqYJ8xydmFlaV>m^wt7P!rL7SsaScyKN{_am>2xsM5f z&~uR&h1ouTYQo4`;lNzrCmJh4whsJupM{78b-*wDXFb8Jej`$69euc`-*zS)f4cZ7 zY3oXY_k*`SB-gh@^-1G{=d`6ORsS)35^EWkhHKvHTP=!x#%Np>n*?9uGX zR-A2u6Ba9JK+JEB1=I<0m5Z_zBS6e(2wc|X1l-vR(dqJ(mHyzhCRVGg8i&4JoYUYg zFlGonhdD45L;>|y*MWT|)UTZtQ#W6Pmnpw*aF;RcAcrsR14NNXq2ME$fT6S8Zljg* z@>OZVN_xcmX7=U19iD@Lg8wqRE69x*>PWyM?W+(-&%>+O~tJ*&sLLi+@&=O>&edW>5G)M+++1T;6x^zZLo0C!w&L)3^yz1TQ&rNmRtmGrY29xY6lEH#!Bt7- zR|#2e%po4hpr=LW<@z1l99C^_T)V;A>}8o4FZL4HWIaIQ&((e~)$S1WU>v=>YCd)s#)oeC~q0R60znD40xN#b? zwmFo+>>=cRen!B&DH@txJv3K*a)a-(&++I*&1j!oppM~jQg(+fTEH~EXL?3!L`{zu zS_rixuWec9PiL)I55uWv-~7^c(T~}{dG1Pm`<~yO?^Gi%0z>$f{h<5D>|37Gc7-M0Th$foRX0gir#Ao z^e9twN0=xn-Ma|k)KBiR{bfn!h7NAY0{my|Fj%W4gL0&B<(A^fG|Khx-ntjeMwFOv z!WThS||=1kkqJ{k-9FILW_#3OlSZPj`gd+ zQR=Zu-93^!0UvPAKrm?RJg^Uc^KL1P4`%^Q_21=Ah|D<7r0CovWY4g`U&CnvI&`TR>w(T~1`7L<% z?uaJ!*YhVf{>9j8ZPLwv>nnr4pPw;fC|_B*l`Cwh0^J6akhHe040-$Yb-g};)yoG- zX?o4QUtZkj(8VPhd)Aq&mtRl?P_%Y<_}(aQi>PIykAaqb;`)}mLM-W=Rf*Y2Du@f| z8l;y;<{jIo_s*`zAg8&{%;$;gDtPwR8Rg1w!)OPkUXQ+hqss# z{VUsS$PH#o5IK1GF!-Gx6Fh zsQa3jAvS}+eq1HK7;K5f9aP?{L%rWw-*qah9)}E{-=O}$_2h+k)MO(wD&YSl$LJ@? z+Xf3Eh24Hf@t36>R8>Gn%u|6_FNI_Y)mz}J7S>JKAiTfkt~YW&6?|S@&UQZ*8mgI> zermr-@>bzkEIfC$GR1OM3+e4`*fK(i|2rzyJKJ$^4&O)mm`1xGh@DWbyn&y7_7Faafu_RM`#9xYRITpT{)vF#ozfi3qH-oO9)oW(O zuTwrOrV-%^EQx#hJUu|lOGYs$mZ%zH`-0M?Go$U=r}*cu`yB^RT+Ww0O?&C&?kZ9` zi0U&}hjCBf@6)`a@kZjDg)R79d?xfr#%jh&{ypj?46 z&?`D-8t4R61rY4e6tEW&i9#U-a#(PEv3+XI$m#Y*LHF>{w~V4L?dU)5%^u0c6t6f0 z{lVDq;+?0M{G0jU$;hFo$(QErKz3H<|EoG{HEq>YSKD@of9=o|B;DxNjN$uSR?5@{ z#oM;(pjWNXh=T*vyitXS_(Vt?p`?f=nFWH8SZws19o0}>9RT%K1USneCx*g&Jsz4^O z>F-GQ=U7DGCM{BV*Q`D^IR<5oRtFO|x3_QvleKUU)kCuPqki&hRIy$=bP`yRUy#%4 zKclb7Dx*jU=&JKH?5JkFmvN979a5*ySU9YhGo42HYh-5*B9v!#>G0SvjR_~RP{^9Z z#_&dzohlcYFiLX7M95F(Tqg>zVIVS)7|VqG9(o@r?S-6=avAYF3}Hfg&%G}X5Pn0_ zji?P_+?RYq-HgN^W;l>N(8moDT|-&rmgN%9$>!*`d)jL6Ot;&%Gqq1`gZIF_u3M{b z>6csn+frv@&0f3AtM zEkytsmR-juked1%h#(_eDi){b_4KW?e9);dh2RY=gk znC;h8srhbdV;$EQa2l<%Jq`Gj2qOvM@AiK$qy_p&StTbElP3?zcSHiGRN+D~EF3b9W`4#z2lx7Mx{m^vhG4ajLWqD>2o0gy>N*85X15Dtzsm%=p{6ne zvzd_Az{Usctl9I8v}HOb#raJg7u*XlSa-59MB$VgOX6Br9E%Lpk`f>65Gg2r_i4K# zZfxEPHxJpl!krT{Iv(rJ=t<7y@iX)0`-BDgV_sYFSnZONnD}I(!$=Ho(qp#7Dm&Qv7X5oDq;;hc)UzGB;dO6|RSJeC|MulWb*V zgq(b2WQClaZFz;AsV5vr&Dg;3>FZeiZpJb7zZ1D=s(~f7Ui~EuOJNTFPFVOiYz}8$KBFu_e){pBE1Cqt&;oY;|7J=yIv% z2}rOV%-{~Z01QTHpmvh;WUQUgV@0IrgQ|L}uBSCi-RFVe^pWnMsudQvAtE7zlnhRW zHAEYPpu|lKm8p}<%_N#bBI_jVMP-n+5~s!O3vrN{zw2JS4=k+0>df^QDhV9~HMpHd0kh)OOPJwVI!$yII(m4#At|JTG z(`{lB{>#{YKO~mrp=TWxwa{0Gzh|La>MP<*vQp^G&D3k}FG-)fY~}My{(&bu+6tEa z6*!pu*O#D-`SxDV4E__v4WFGuManT+R7wgwluf>aR|&-~b~eS0u2Iw;CS2>{gj3z-2qc@j!hf53wy8KIq_i z6Lm;Do#?jQZq?|5-<^Ge?~q6DqOQPs?wwl|G7;M|@;h64LWF7@+SpP^+sn7$?A-UP-jV3yp>*3H_B%1h88qdMJItV z)Q+Z>q`OK^S4B5>8*7$KG=Ywct%O=^C~76>`BHFSG(p&CD?K$42VNP?F(zOQCXjob zlAZ!UUKvlhY1VQa~ zzhLoj?rF)OPxrbzx+nKwn7cX9rAbhjzz98L6lXtC6ZhNRoYEmdF)Ki%zgL^3hOGg( zbIp@@Sn-Hhd!0T_jVExLr*gl5-+h5&{fy@UK$?@E^ZIlNfw!6?ouv^ zk545dqrgI$d6w`aWp!|i85$kJKPqxJ&0li#uzn8drC70UU0AEO`oqdm-dKHTMFw%2 zuGS!Ez}O@lZ&gG27hmQZb=myi{Tcyqd^U!E;C=_~_BSg^zin*P>Sxl#Jzrm7?&@>? zkMDXA`GAgZ%7{8mq9c{!k?mi`n@>=_%#p$iY(BRCe8$5|+f3p1(jVuLHAdV5-Z5Tq z3d6olM-g(B-YutG^bDUA zv$l1*Y{A#!i(rGUNs|GH*)>aBK<4TEGqPM@+Tp$KyIXczJCSgr?}CKm*WAkVuu{`y zPQ@MC)HIcw~j2uY7-dz5~NLaJfaSvn2C6fN)B3qLbCucLMQduL@H2ETsZKl zzGE4Dvq(Zal--k6&R%IKL%MKLlUM;t6}oneQKYbjByGj$NF}Fowmtxcicd+)!rhV{ zkM*=bd_?@3DqFziO3N{KVrx-(-PAp+0*MASbaO>pgVsmf8lyziM?s@++BbEoin7zZ zX6yi4N!we9(_aVJshrs7I5)+)X|6==3y@&Cejv_k^1Z|u5y(K6&04ldtt?bkwzLP0 zq^Jj_8JM4cdVXC!=}3{GT@;4s?8Jtad4T|mxl7n{ZI$tv@LwQS+BgTJGa)c9Z#7D3 zMcp{jM=$toxHT%OgQLQg-YCn~+{{c?v1xF+q(psPy*2_D!Ceyz3*?lcLRsZ%3(O6Q zOivPaP}a;zQB2bXw1%%cr~K!y|N29vU`dt}ZUqu}^o{sn2EKUw=vGwJ^~ z)5JU(KNPZA*R3AKjP?J@K>g&I|H$l-+i!GVJL-}6Gl!qTQgR_e{S_0X?Xe0f#Q)XV zx7D9v1T`TnnY_!Z{ITwUQKn4U-+h zse#V_Y~*ApCx6)0{xo#<%(mUU_S zmc~s}6%oz(bzhb1Y5?-Gi-dpexh+F6Q z((p+uB?-w2!CJ}LTEQY!NtNNZps9)cqz9T>=I*A~Q z#LF}w>>%VM1v?MRM)9&4v=$FdUCKL;fkPEtkV*uIHWLY2LRPG#907w2Jz{lBA-0Xo zih^cmHKSk1HxWU^ZpzDg!9m5t&;#c;lQJlR1XsBVKSKW`wjJ@8+d@v#+(Jfe(x8+T zjNo&Al$!l~YFBuk>9lV6v-qBxS>Db5)Slkf{z(7#6Z|jv`S1TK^pyzmWX@D%puBAe z)4!h{@93^uDp6gfX`w_dT`Gv(^zN84j#)7rO36rv1HfItIDwDUN#?|K_P+dNnN3MU z0sw17r-cb>0)Ezp8Ln_lSt{u|5Z0odZp-e1O?pzbd~Q1=VR?7uy|$N8vEESo!>M;! z8v!qT%|)Ai(0?ksO5ZYCl@t;jn1T{;v_w7oyQaZ+ziN!v(3cZ0Hi7i2&K#HKy6Blv zfr`GQgepi6!_y?{u!c9RESVG~s|KgdFgX&ao*q=t2}zmF1!}v<|&n>~{?AGz*8VUir`5JN6Sqj0@fMItArO&?l|RgXXO?B{;y-meqU*&*`JR*G znsob`J%k(`>V#+c+VeX4-R{%o>2b}!Ri(zk2z+T^jZItrsKn`toR^4ms4G%ow{Nhc z2&oa&-rs#juXCum^G02%MKIatGIqj3I|gqry1I%H3pW`Ea2S{Gv*>K??X?f@QB~eJ zUKQ1$J5wEUNw+!Ekz$|TC+(y>fVCM-*lN+`j5(X4r@!~wsttqKO4Qn0Il*_~vpRba z2HFcn!`t|;qeA08__LQx#H)4_w$DmY`ApB7xM&X7j|_%5rIC_J4Q>TsKgu`7;}LC_ zq-#|}Nz#dtb9PEF7_5j1wjT;A4Ys04xZ?dL(H{A*sVUu_jq5{iJ2`KjEsZ+ge9tGc z+*H{JwH;4i3uB2_?~gT19yNkvDITI!1X2f8s<;<>q4QuvauF5;Y`H6%H2P_ zR#$NOONOoxBNG-0BXY0rEV282ZNt6k+pRB+?Ml&?!mi%NXy|o$H3IU!uX@lk)O)*b zo@dXaL$|KzC;D!@$S}vb=WWt3YfToemfxjJd7edi1c9Dz8_RbEqol8=i}B6tz=2gV zk7M8|;~Q`{bC<$CTom#X5%$23jPJ6-3?+IhRn5bPu~TRSk24rPV<60iC*GJ+U`fTo z{)+sV`-N@uZIuH6?kPxR+B-bk*q$o~Muw=T6vAsbtFS|7mR|59RFUCwo#<)sbCAI( zBr++tFME1Ho;402{0CeygX#JwP|Jm2;wn2Jso@KWq7;2Re9T96s4|fbO4Il%>Jrf( zWi-5G&Is)*@*DO;n-Gfl1mYzH7NELN8ISn{c=&b})|(;M6bN-FEC!F0o)h_7Pl+MP z5ckU&34+MF18awxN1(&6S_Z9{x$~tHcGa(4JpS&} zAR|F;;qI~fp{sXj+E1dP^o#kAVUbY_(aRS5U^PqY?5346I}aL;FGxsmC-!8JY}DlM zXxKztTp%mw|8Dk*(SuC?Hv2zl++B2w{nBV91+^l^+xi+KuJ)4c1=f+zd+oa%j&}{A z82%GvjwkM;b5T7^@{C`On;N_qBc05xrM{mH=#i-UIESH$71>pmF1r^R9(nopQ#Z{$ ziFc1ZT7>bpgq!xZ8%!DB^T$AE`s=5L-4K~&H;KFN^~8vbXu!X(%#C)s6FoYVGDHy1 z_NLb_YNE+O@rk3(Z#~P-l)=Ph+{3~A^{k6_d;E#yZgE*3^Pdc;J>N-1UtiQ4*999& zjvBG>Qi+%Q<0z=i<-w!bUy468w1PwcCY`}EiASx3X2SSGyZ{P~Gzn;ZB2_2ks)V3( zpa&O}XEKQ|j%^h?4%Q+}Ms=b`M&*nqO8h%Qjp+9h(Qi|e(9GhH+nivfe3QXkXcpOj z?;zO;oDdu|DTuYrfP#XI0;C~*Xh`y{R1^gw)=Hsq@xeK$8<{tf{hpvai=e!a?x?>o zcxoNutc4KJg6KB$a$-)>B(@_yPMXaGsFwr!u^()x--#09IOx`DC5f_l9Tdk7c$@Bc z&8sQuh_>Nnp3P^Y)>mJn^R{~3ZMce^uZU_?*LMG1&&)UqAS?I(ET&SU%@7mP_A@$X zS&E5npa*i8(V>ydh6=f8^Tgsh`fyUh@xP-&iUXS`r-Qfm)$6vYs#dr7r7m*@N3!%Y>9=U&vaulDBVArS+?JXxvz&8c_ImqV~Sho^*_2>7ekllQGanL9jPvd(mG9$d}m5aNoTyioEO@ zQ)F`9&P-?5)F`$tfHWyOnn(W+BrSx0kTj{Ml(~{pW1VhI#K!Xsv(wuT?brGISbaRX zHLpO>UT}|lfLtMf`b*XBf0QwVz1=N-DR#laknn^c`h|RkAWZ?DBm!BdgRwYSd z#CUI?VM4F-q6dxqrn;!9PjablNPb!~xNh^Z zCzBE(XEK{j?*O{XEP{Lo`&2WchFz>{6C~gwfBC6JCSIJ{GtNWK4>3dky`wA8aLqzq zfR^p8MnS4gz)-E7NNHy5`v7uka0Z$?6R?1(`)s7H$d27~n@ptRdMwNK6tH39 zykaBw`;u4T>M$+6X`2`{_SV$&$Jdl-8zS#iDHzBMBjL8$bU9{XC@eD9Po~MV zn?`b)E)N{Ezij_r<)6-QJ1RWz&OIi8FQoZ5?X|+KUXjO7e8LS&6S~~S9 zG?y~rj7Mekp?O_Shvmy>ZxnO=JRMiN%`l)|p}~L}^y}p15l-$`M1xeVNk})DOs7*M zzoRtlij2zM*SGFRrGH>|+;Skv$Os$zO8k#;(ny>suCfKk-?@h`E5U zZ5A+g7}~Hyc6zqzSEefp$-M}2MiP#@D1QPp4_BQzW(TI)4H$|h_Zwuq@t*k)Ew456 zyBPu^f1QIVyuJo@Gx}JZ0Nn8AkE~b9K@)6tRc+6K@d5YZvh=a$&NoGbvK zN{R)U7(bR4E2vL0Y_y#R|Ee~5L&N474e1-G3>%Bf{cMTSSfUKt5CfQbg_r=VzrTs5 zl+@ZRJtxsUlRdfCn8CKKDg;XE`4v{jDcLzIa0Alg;}50{?+k7E^>2Q(_j@=sg&ap} z1v*2MLoq1&dfI(oXg&s*fQZKa$jTYz;0=E$Cl4$K!ZU)01ABm`f#Ln%`2*a#IT|mq zmYk~_xWAwkma*8xD!D>F{wM?6U-5Qs;CI{n286iR25wroa5~pR+U8dOb>j?o)?uD( zTt8bd_xcGEb#Y^03W05X8>f%q1~R;5&OpiXk@ljRCf?9Cr!&q9BZ<`FV>HfG1xS>{ zvLTErN|cXXP)t_zDlyb@Q6;Lc%D_{(Hri2puxLAt{ozNn+&LPwUl0P)F23maR2ZB= zzk~J(jZyCg)Yo4I37*7%6!#6(QG1S!s~ud8RC|KVbkkgbE0I`uQQ(CO%m)qoh-qvfj6ipBjXfp@Uk?Z?gsK@l-^T z_u}>X+$WnH5VqN>Cw>8}m9%e(ZIimzxxu`oi#*rW8a@`CW!T%zyKDHgP7_!D;|<;t zgyObenP7?cMP+klv0T+voa#DFegP^MJ6sk%S;w9O9+K`CsY$8Q6O^)!&|Fth(u0;D zhbc%JuwN(%J$>2lDZl~YfE5a28oN>x7DjFhIrdR@Du#@$4&9_?Q`=*FF)w{XJcZd< zok9%&Sw*U6MC%4VZjlR_-iQ4JozJGNg-NNrRmAY{fMDMKi$Z&&CsDhH`D97wIH$Eg zspV}m5}aDzgN^;RU;X!o?xVwa@>UKH2ut!n91akrV%Y#mMK*M}a9d{&GGw?wiZoJ$ z>+V)kWn&1T?dv}8+EPa?U4|C%X$)JrINZSc+=-uY-F6ia3pmT zmk`;?S70}L)6&@wuYaRV4uPl<2%0rQQle|1e_{T>rP0NiU=9`_@Y_4t6ZT4B(d`Y{ zW>;IGNYy7mUw64tW$yR)LBz&SQg4DW^10}cL@sTHd1rp239Kn%+Nqp^w7)*M#h09C z0z$9n7g)N7zr3D7xg!9!Xw9ydA8(OVbXx^%efj3G@HJMz#UO{gWSBh3WR@5#NN9H# z7`XeTX5|=WP_Q=fE4Jp2{BRl&revW>YdBG&*F4w0Yrp>2DfHylNipCp=GYL{9Q9HE zeKxuH*D&2)<0^A_t586a{4+&QD5@W(oqW~V`9}TNyVm%Y9DV{ z>hoO@QtAZ^+VwE#m~R% z{9Jz;0_j~G&Mwfa4WhM<%jXuWP28=k&aiqZh^e=4SGoVZQbqxLD3qHk)>8xn^XUNZ?4G_pGrUa^Cd?nK-VYlKMWx~%qwuNi2hGAYc_V~jNlr@UO*Prj z3%2ztuva)~i6d`xzmZ=ee-V&Jv+1wf<6q13$BM)A+HhAFgR?hj#*fjR?3UdFdW>i> zAOq+gH%@O~^0PYAsqT)!S7A)(%9Miu@ zXd*~NgSc*Bba!avb`il{2+~6kj=r_b%szgtJio8v?tt4d4_*h%6jLmv-znmfu>s|H71z6Af~waZS|B6S zAXx?(0UyLGy3sFJBRYiBFpO%*FoL&=&=Hl05NswOM%%qtl6K&>r}#+!h6(Td#9N&8Lc-JaG@oY|U4RxJxft7JdqN(PP9$ zbvY71V8iPjCJ&=JNAsAXnWOvN1N}I3uR?{VPoh(gu`}cJ#muES0b(EY3`{KA9NRF% zz?3%LkKXi>zZDA%$G(N4r21*~nKPlH{9sahNaqx19111e{$C_q}`b0!U6PKgWs|tDzmam7=+~1*2 ztQKFqog@FmmmS%673FbGal}CDS@p+s*g7TT%j5T54Jf7*{n(6Tc9*$+?Co0aKa1qc zmJtQg+7|fZr-S=I9K*J8{VcB|+^>hgN#2+k6d!OyaaoUaiCT6uT&~e*n*|Bo)#eOf zG{9V4CAN(-a5pg(pRinwbByEqC?nkae8l*JGi5(Lm>A8V=kvSP*LHKsPH(D=-3lbT z4LwuV%U#_;vAAhNPqRB+mfsa*rSVtp->;ALqnj<9zTUo4lYVgx4Hb>7Vz#R(9xo?D zQ`B%%3u=zyU!i!a^7LY>3b`VyeIGDDaEFrtd#f4}0e||JlZ;gFHQpPMIZDD}9)gm^ z>{ZtyDXYiCYAwxtp%5iKFkL8#K2!oZCkPS#Ke;dZd^Q`Sshggz6kb$+&~?HXAgZio zAK~31_}x&kv%Q;010gQzPX)LaQ~FYd?E5dL)CdZY3m1v*Lr_ih4`IVh@?@NmmT(#-rr!dpuJAYWK?6B3*9-aZA| zKPkQvL3m2jnWK}JPWHr71(Qa1^$GLajkOOMAGVHH^1~kHG;Ba;>qXw&TH1u-hJ^95 zCUVE`>Ko|fwIlX}N$>1E7nh7`*!D{Bv6fjBX>qWFYCjEg>&j{yTRT?*7sd~f_~c0{ ztVf?VZJ4Gc7Khkc^?g;4$@LwpH*FmQn7MvsYip=SOeQ2vOwEzu1H441&{xBHnm-d7 z)6=W-$`tsha4GDK78DkHg0<1lndf*)xdy2sM;6Pmks0F960cm1r55M#0*k=}g863j zKKTv4rKVf|ICC*O9YwYR&&xyo7xf5N<(xjC5Fv870y)r#L_y-C!*&M52YHjqKp!$k zpofpj7vYkrTT@sqEVG)ldv$EAznfuAf!J|cise|!-aPqw#w?qugWjz0wW9H4-o7-C zug#ydwl=SK^UPc_CLWy(tu4>RLs$F$W_F}EZ>VH#+-=+kJ7Uys$k zs~2eA8Ykw-Kv8`kBqM_FZz=;JF$r`pP|#Oiu~3}uPSclUAiTm$@D19w|F;lc)J93l zEk$S5WuPnRB22kHaS!zoZ9vu|xWP3140d4s6K6X!nMN&$l2zdiw}dv}@e`+c|B@AT_hhFy6S6-9_C44sJohFJ2t;s#OFZ}gAVu@}r87Dk`pb3GaF{3yOUmfc^=*7A?=#3Hzz zMFUx(#Ncx(ECSVT!r{7taIn#WrcG_w*Y_gW5*l2$vv3VvOpBI*i5X9rG&Yoj+*iV@ z87^FPLXsO`VK^)RUup2Q8-JN@UKKXtS9NE=tjlFu8V0l zWTVy()24b2Nqq5n452V^uA0%PpKQz%RD-kQA4$!gSBo77r9QxRA_SwxaTWOf^9KLI z2WQjHDUQ%SR&t+9*ezo0*ioJI)@rHDeqU zbQ!iuQtkS$7l}_rpb9gw<7l*g+-%+S6ydJ}Wok_&ZL3ZHI>+sW$~DU7BDuqX#zzWt zA@0BW-bRPY-lZLEHhgL=Pwr3TCSkuc;eMuA;-?evyIALzUwy zuhbQ>sQQDH?@(FGY&yANMCa@BAVvpu?{2hb0~s%rocq7S;j$MT60T>gS8Fw^;2BMU z?cf==0oI{wNkQO{M_c~vaT1YU6_)sfl2X%8nO3U(!O3GyQy4cD%c7Hpbh46$aFfOCBgfD4x6j!jv3XY{| zl&=IJ!}`$r2RiORvB|vSiKFz0qZU+ufIHYIi9h*1YGBX9cpE&+%S}W&?h$JLnT(mU zU1F6bgeo%0!1&MU(oCE$>6L=WTKxt1vbTrs;n`{_Y`4I6qr{143-|s?gKzaBJj&|A zJ%4FFv=bsSy?z9drWeNaG7kg~7^gU=iK16;PfmIrecH^yVeoZgUx?7qrWu=N%qWv> zBi-{h?v(o6(ck#&#D&i|`JG&+AUqj6rkoNFemXc&K~!A71nF4al3^8ZrrRv2tc7nc zHJdV&L*r@u+_bgW1Ft9rMcU*%Na9DEQDStKU zSD_R(c`dS}^6HOiSUKb+$+B8Xn%Vkl%KnX}Of%Tk59)(aZyB>?Adga=La62n(Jy2tl zavpAxI!|}xIojy~qS5W;=JIm0#avxgd&|?ax8js-kHuP5Zn0K)juAe5X0S!^FTIXf zOOuhw6q7C4ntw4fK0XR_baG{3Z3=kWrFsWg6xaIxEX(d-LuFi}GiGKjF~;7FNsNNM z_l{DR-eD;su#{!X*5{aCaEngEF>=80);(xd`3SVWAM}+n20%3^oCg{xy zEFo|rL2f4u6$``c(a7H7eZB3wT+sd@65Vqod6OVS>3#9L-SGYb=sgVoZ^Y)_Bdiyg z-fs~Oc7%xNMf4*E5pNO?iQh$1QSUz<4+QL&M@C{C)kR z4O`IsPxuzPhj?G$Pm|h`?VAF@zXK z3@1htV~DZDIAT08ftW;0CZ-V6i5bMZ#7yEn;(g*nVm2{{a3)-cxx_qTKCysUNGu|j z5PwUFWyEq~6|tIFL#!p%5$lN!#3o`hv6XNo+z5BVgYYE02ya3`_z-@CKM_C#5>v_|MA0A-Bl=B@Hw}L+C6-#Z&z^`8k)ljHCp)#>J@Fgj zHR9v;^X(5yiM*nXRmX;1Nu1h@#$0UIN9+| z-z|Nw_8ZzSrQfCgv-+QMqMgE>GMw&Ei>T`Z3WCbP5IkGbL8ZF#U94s{v2acC)QbnHw4JlxC++hRnW9)AP* zIC5n|cB6_zL7fswVY)6mo{8706Vf^4)%9cCCa<0A^@D@RpG}a|N_cW}gr&x0$TJwZ zL+}j=1k#geoW!6psx@@*S`fwtk*4NrU=M=^z+4$pN})ZxC2DiJhwMzmT5@%SS{0xMNL6^ z<2D1=RC_ePfjN5Cdot%kZqm-zE(>;;;~%lnjiyxZJ&d^AerVs}a+Y%O%Bb}?%v4@1 z`ievEAD2;vv2*>mZ}Dt8cDn5@oSDTsc5cEDw432x-U)Jd;kkz{PpE^P9DiIY`*!B% zNL;`^EWdi?Gy2;JchEq#%Py!|def-#<+`d#U!CehHZn*UBBi3>;QJl_;7~vKwQT8@ z8A!~cx1e6yXv{v$!hTZf=EJ7x8@Ot8S2l0s%n>Z=FLl*rH*&BBt|BS>$8F;magM@C zmHA>DJa2|RZBEbH7vF`(yMNSSp$rDeAob2;B%S5%8W+#PYa|+>k_3_0a?-YNT|ZdJ z=1Y>O;$BqwKlH|i| zn6e*EgEQ{5oy@(g_kV5TJhkfJL^`}Irmm)VCXZ^y=P*$PH>dfM^=Y~L`H^$R7cj=S6A1w=$8FouHoN8bE4t;XCk=REQW3d z*_FsvFTc33@bJ>G=HjBPynNR9LFfuMR|mCPuh-CNnVQUk+kdwqi9;z9@SP%F1lfB> zpTNW?;?9k*Lk8C%22Mh=UmrYnQuqO6;V8I@dwU=f$FB6r4XtI^5p!~?4z9U3rUsyDO6Ap z*&O0tvz|dchkw0;WQck+3QDgW+}&W|7L(K*D6@|PI@J!A+PgJ)yV9N6?pw1bk_&Iu zo^7RDj2G$;vV7&9##2mF+s^Ggm%jt(;&8~!!r60yzsJKF*zX}7aEk?wv6w94PC3k% zMM%;3h$t5{he3m1M0~}4@?n19bj||HBBaw)L8IA8Fn{euqR6ChT9J0j)o+3JP5_*++sLbhLWZ5ovg1dyP2nXi1Egusa4i>_O*!G zj#VCm+J9qWVH^BF)~q_HXk%k5+D@i~4;j*|tlwyOh58QQP$!S2oUNE4ySe z=Q$f^PMdq_?zx65-(V>E&XUx{9u8iC5gv*WK7Y)^L%2Q->G0~f-4S7d!!1;c@SCtz zW`pU_Uty9`22;~5u$O6l0JO~V#<0_1mXvy@dkcC? z;b_ZR#FlW5@I2rJ2@j`!onsb3`}Xg@6~oTXzOojd#trkQZy3rV@`Iq;=q9;c1s36A z4}Ya5$-cutyN~0eNoQStGY3D1QXchy4&--7jvO!*gS28cNfVbCK}VIu)YcbQRI@ge zq1~e~cj0C;eBbvQvB%LJ(;q`w2Q{FdupVxc!Co_;AY!X~yq_-%HNdo{)?HQE}QsX&v zl#BzZMcGDERt8&=-*W0J=7eQiTn<;_WAxrk`+=2_9;|nyPxv;*WzEU+J2-029KFe; z&u1D-E!EXKECC6AY1=rea?rG^qj5`f`1zlZvigCyiRJ}mj0m2*E*1jeOd0>Suz!`T z09o-VT5B~#7rOinqT{CFk)nq?JOg&_m+bS)=4( z0x!H}tIE?m#GR2aUB;8hQ7iG94otpWPy>12r1IgGlE@xWDWOZymL^+$-rRWMw+EBZ z+ZF8Jl;?)P^`}NLXa$;%CL<5z0e`2WDR9k=FKf>4=9Go|yz-e`kWh>|;g14hMXxG(Cwtn8`cSbHa ze)7cmfr_b_p*3I5`KuDE-jx zDpXvhZU_yy_HT>;yYw7fOWp%~P%~Sa5Kyt2nc=q5H=fhOpQO~oKYz1_(pSkVF+qDP zsavE12RXQm{N?(YE0^=vuVGQbpVmkm;XSzWoG2fbiD5llDLaSCR+EnK_ZN+CNQW5S z8_0)~A8217WXiwI)>z&D0K4z~fzACI^0w)P`&!ipY`E38+t|0gw{5=RzHKc$L2Xd< z%;}ro-fO2U5Ma+1YkxBLa)Owo&LX+hg^zO8xT~38nP+E`G&Sk;yE$smI^ic-ZB$LL zKOR}as={~euBfx%;LTT>0kmHa?UiOBns;i2_DiAtPxq+Jf}&Gy%_^Bz;xPr76kJd< z`3NNXab2$Dgw1O=u+%X1LU7KPqeh%4*j3XUS`o@o^OXJxZhssFoUgC!`1!`Q&+}PB z4lwSfg;iv05x9)0#ChalF8NM~w0Yw7{qf%xw^KhtpUpBsMWV4*A8zJaw?R&BPnotf zJqi?tv&9(|MYVKITt#STTuk`JGofcczjWmROAY88X8%{0I?Mk+zB3Agv_3lHKrn~m z;SX>Tbk4~+d4K#X!(g7&h6jQ$#&-=c`Exc634lum67q>Rb}D zE<3mwRZ6l_aubv2U~dp(<^N4q{_iMQx;sg@HCUguf8M^gow_UJU~7Gp;G}{>6kI6L z8`HB)bjI}yc{%Lm<88)Dg9A1@BoTx%zRQh<1Z;M>K7UV@NXH~3SjhNQjsHvT%A~en#v5eKQgJ)g z07(2`-mb%gc4Jxs8N6_XL*a4jb^8%U`B-A?1LB zU6;Pv?v_t5iP-xw(6(TtKC|tk%KW0@;C};#P=7}|PyJ2VQS9KpO>oZN&bYX`FAwE@ zC8@8_&_6h7<$o+d)X~?Mp#QB*Su1yUUepOQ>u}lb&hH*t)Wd`y{uH=|ab2@X^h_v` z6r|`4u?+e!s9suJq1P94Ra%2SmhLM555Sd@ndt_7hFTcg^>9#9u0fy86oKq=PB==| zf`7wL+3w=}C!DDI|6G}0!E*I~gr)vJx?HZT>3r8>0ZOtxP#RS+QylL7XAvROktmig&70+Kys2czl9Ud zk=4K$#V%bw1q3odWW{+M%HX5Y8AeqozI_SC@3yi?OtaDUJA z6s~M?y9+Q<}%Kp=hVLZNb8#KvosbHUVP=e=@m4DMdN@mm= zjamzI<#F7jQD^a7ub|OvAI!M=AtyF53Fx%6Izy9{cmESeooA0jy8x9vtB3^&ENXy_(jrhE69Z!LPmQRNOo#;8H(hZg>6pD<(pQWh8&s*Ge45(C3r8J86ucyR1>?e1q^woToD9~A_i z{TvMsKiK-<6SLEq?<=2edA8=Ikgp) z&22Q2cNM)3!YIsdczDtbPjsgOcsjt5D(?*YPL>7ov5u3Hrf27={o*2+Evd(UNZ=B` zjJvX>ab5Y~ht;<(-hZP1iKJhnw^{V@pVEa2d6h8D*R8GKx^13(nJ`(9Bum)wLWsMtB-#jI)BHL)Tqa89baS{f!@e! z^)X4|bZF&HbD5=}!CIT~KsBF-v-yiWe6LP)=E}!c;Ib=XK?7IFo-DpIqk=^Pc**i& z?^EaKYo{*Wg72HryW6u^wE?K}nL?0R&cSWAAu3I;PfBC`)=27uJzFO;IGv^pefOlx ztuQWMb3EzVj(^&;U1@bb4wd0~0Ri;9&2wjj%+5Tzfn9w{dqv;ukf%=tS|$;s#c{}$ z+yT<^vkk`lY!;X@8c(wR{1x902KM;!5wyebuqv~$To_vJ)OZAX9*H_ay$ieG0NKgl zT}cIOK!bOVM~=hV{6&T5=%$+1d(bX(Qjn1~q~znd*MDa5Pd07UmLaja{`Guu}Y?fvk z4CUpl(FlxXfKtAuf#gJeh}MUJ|{at#+D4M3g}i_}z- zd%3W{AuIb9C}g0Q6c5CcC6f>nfVj8lWU&3vA#gB{Jx-oBoz6MMW>gj3J8%gyaLu4x zTplZFn-(DqtGfGOpT()|K6JhxdY|gTvX471>VK2}iNp(iHf@bzt4SR7{hE(gx2gTj z>`pJRJ*ax8xnx)F-V1I=-bDitgT=LQQ~@34{lHPDq4>vtz(L$3gKR6P(YG3&+%Gu#i4suDEG$yyfu z<9}Z+lJrE74ror4`X|121Z}z?=@4V;L}{(^=Rg zi?wP+P#TMr7%MofsJIc1VajTNE|06!096!?)7>8=3}d~l%EZs$5Oe6MO_}f_hk>Hq zR@PYoipnxrV}30tXW*FP05q-0+Agph_kVs2Oy%LS7^h#ZVttCXP|x8vu%GN~#5&SB zSmCn>JSb%$4~O$;xRu3BjgabR2Fu^kI4DVq{|41dQQseuny=SgztXJtwGny;eQKmGjY-v~>62+yHYjBeNbj^PdH9xouLZe8dyLv#29OZA4|kW_2;r?-H1GKWT! zv%o|+_RN90OZDs|tTkiD5L+X%zRPNUPYEaS1$}hqbY4{6xmDa5VslX!(ne-+7*q~3V%k4x(H+dy8bM%uy|7RU4R`$ly&t-vBfZ1eHW@(r24qyY}Sg8RbU*gzt4_{o6&{TGk3*OU{<<&)du3YJzbR@x{ETtkYTC zxwacAWr!YYde(b4>&NEr6WNI?n=r;NC%)>=HR8^IfqX2vF*=iv=8!M^6ut?!nHzRDbXcVjB*xO| zR$%h72_a@>6rHTorKwmXMYE6$u9xUc={Xs6fw^?Q!HT4xixtU%*s^l+ba`4R?*GdW zkKRS;l9+_xZGR(~?uotSw2c^%+YCEu%jjHFMs_|6he01W8Kp{$8k0s#Cnus{g zjhC{B?eqljP6dL2(h7w*!KhX1R@V&Xn_G(yTjda4crZ?h;0;DY zA~v3qSL$*S@T{Je{Si>7?m>;dv2sKA(N-Hd;&Gn5JKV`poTau$Hj91esDLcdT zET!$1O=|b*b-Qc~+rwP;H82Zqyi@tt)WN@|5w9=V5kV*Gb?Hg$NTkQ?&n(g8=`u3t znif$0e}B^GNICGsNjthr6tPOOP8qsjDMQ`Fl2}hq$)HGx51yY%3Giv1Eo^b(A7thf{SlDoqv#xt%G;*LDQGr!c`_d05P=#{KBo& z0iS_1`|V!4%1xn3XInznW`;3ik;6tra7g;^Cn91K6#p^h-Gn~><+Oz4QBQnY(sAVQ z!PlOaprm?Pyf!8=lvW!eYExKkB1lQ15sCbvgnW3X?W3c0vEY^d0I7~HBY=Ndkm9G8ABSq~8x;9Dn+ZrMz=%K}sE$qKQuqr{i*Vma6fPG}$(V z-p8*2VXqxU;iiHA(>2HEJ0U(A1JC2d&*Nc^%>1u-CDy`uVQTmJ@iKgV>?z!nU5GsB z{*vle6y;@Qu?C~j0Dv(;LkFhKUACM}N_s_i?|uP$vX0h2AL=JCy+`6d0wSha|2H5g z*a>Uab4y`JewJCv_OxFY+Peqn-HY#ewU^mowB9h=Xf$2y-P_V?$}(n|Os34NjNX0z zACtiLxqmS-HakcYnKtLiOO4wJG$Y#O`L3U77P*4PgDoe|VZC$Fh*8Xl!tGCNvt9|O)T3h=K9(wOR z!KL-~xqqG~ljO{qIp2A|_xs*=qM+>TD1}1dG=G1=Qp1MuxYvBVd=2X(ZA1RDoBRrL z5M+UpMTM*#AP-l{E_T@6L3zf(t}hcGbz}V9>^9J^P?RFo$#y(H1RNbkkZ;bvIE=In zeC#laVh2zP%ARth)Rcz0O@V@Xy2y-3Lxg^Aq&_Y=COKn*w~x=HnTGg`#F(h)WbFhW zpMQztY{xgWzTVn-;cH?IX-Tm$+VHqY?L2So0&nd?LmC;4(Rvu-v=RE~@bzo8hPB#2 zeULUKNuQXcjY>46#3y-rYXhTWlC)`t#8@r)Ow_N}hbQSHwJC9t`b2GVv|j7KEFe%j z+YpzmogcGCAD5)pdU{a>XXA`)+c$ti0SiR6nXKK8}%wRh8jzGQ{$=Es0q~T z)SJ{aY6j&?`BQVKdDJ`90%|d}gbJVnsnt{%6;4G^YpG}|hFVW;pyH@_DuGI*l7FdG zDqTqcCNB@l!v?h-WgsALQ2ab#<$!jjhY~6OuvFL0x?N4P zd)0e3qs_uX3##6ufmrtDtk1CLQmooKadm6@zUG#?&I9SSrXrzae~YDx|9_%2e3}@; z&NAWB96U=_S|Q=Wj)Jo@_*p@@COh@?Bc3JfeDd9L|lk5miLQ1zI}@b#cb=oqfIAx zP$BH-C#rfc8zuR%q&Np&vZ+VeTMB9F-FcSG%{eHuP{5iz?6}aKpZfaVPgaP!Q`ukq zpgCy;>eKVW`Om(u{E!Fg&NR%`i|JO2c@N)cDQXl6RJ%}lMvmE>mwzkFT}&TaFlVPf zkBhJu_Qc^h9LOgW!6Kl+30P4jm*uz1PVLTz!3DvP8W{2#4B>9ci7-%Ivk7gqcqj{E+>)5_ziNMui{~4IY_ITLsao`RA zH~{vgqUte&B>ASKfPc~8CJ%+t%HEIR2Q|dwF`&l5LZ7b=sJ@$+(#bO!SP5fDeCtF! z+M>pa?B%hcg8!eWs(&Y@!Z9tlPuG2cAI{dX;B2K2 zpF^fLQJMq~ft9f10~Q8+S>CX18#0#)8;!Z?>6);?iQy5T~_bJPR*ggjimC$lwVqs|n)THqSvt1`mtX7G4q;8eee z-4QLxqJJSee_g6(V{veMtgzYSn>t?;bYeQ_!0+1WiwAN)opVUw7RT=oScJy%2_eW* zAg(So9^R>`DBfODCU6e%N!=fq8+aIb;@U+i@!-ZC;(;A?MU@tNbAdT0U&9TGPpr*o zY^>e2SHQ93)S+n3M1lJ_ej@t({rA!5H*TOgp?@OKeU9p=$Yt0SI|O%kpZOHk2r!Uo z1{y6O)ZoycYy?}Q3nO&AB4bx+HdL{{Eju!LY|s*Yicko*81CL*`^m8dZ;dSnt=B_#at!Nd0t2(l=5g-2{WW77>PASc%l$K#1mN~L>(}sm>3;v?YNJ+ z`G4;F$kajddzs|-0SW8Bz{mL;@WCpexCS`lDlL#lv)n1{YYXLce~Fy6t*5fm+{+i_ z2kNfq*08YX=&-QX=&r8T)-KX9{}KkAk(9kd`*}Y*Hxp05{`^Q16E88Q)mVXo`PJ^o zsu!PE_(irq6mq!@A$tE&7O!hhfJa3Y(2d=*KnCHXq3Hjj@Ed?YuMHr4UC zjclGG3sDkIU{j1J(J11)zDArUH_{D8BTC_MqHWPL3^zTcu+U%ppKC~D^`Xi2)EM4TlM^; zrJ?gZ% zRsYVv-rtO!y-a(*Xw3jGWp6wDsDFm_codAli8up`*aNS{Ye0)%g*3 z{kBdBpDp-?`lT%67yIu2GD2K;HuutX%{l9L@BK^oq~r63)BL#$hPmzH;dhGP_ScY} zLDY%sZ+}n4T^Hh&&Defj!~WTA)U%BknH*Nc6Y7t*kh<$R6NUGHt_;9<-! zPu&F`?zdvzCLKB+Dj2JgWtRq`S#G#-m&H<3BXqQXSW?e_esUppCh^#sdUG2z_bZhf z0=Hj?y1sJEL%{ZA`XB}-7foGAyhz+L&OQPPfhr4BoQJ$nwo`w! ztZ9Ycnw9gD1)y2(3V)vRH^%-cMKjrp4WRm2KL9@R{*DGxaGiKc7+*`L395QE+J2ua1rT*vQ4+xCd?PQR z?FP?WVP_%Qmcl2Y{A3Z&W7AR2&hnz7(lVjDn+`sIE1x9xjk7>MAQ%Max5T!|e2eqp zOOSnuOP42tR^64>8oghLY>F+JT5Z3!bY7*TIeQE3`r>e0;5K2S#ZvGz&ae8z+-gSY zkwSGsbjcci1Aq2Lc<7ql@w+p{_UOa;S2x+W<$tkxv1a;slrlWwN>{c}Q&505J&jG# zA6sb=+FmDcrp8)2w);c0n{Nhh1n#0u!!uGJZ%k91lb0&x(%v=d-C32JQVTY1$`|^a z{@a@_$VPccleWWHQ@5>pdx;=BJ@%%zRiRQus;+}{{C{hbqEmuAA{n_vGR*R9b*jzz zOlDGXJ}ogomsflZBl)X+*RU7i)^XwtPV=Y9(R7@!%S*oYB@3@syos3#5={W{NB<*1 zK$`PIV001;lDK=aGYnRLqzhvGW0MR^(78PVG?8Z+><$4sws3T?hlj~Nig4%~ypPd- zpK|HT_kT}a5mb){5+i<(D9jTx_Zg8GPTPCEms3l`51RB#Q6$fKgsaV#DN6js0*So$3? zSx$mHgiNfrSoT{5PHWv;g!c0`OZyQ#IH7pfynmIp-u=|eQY?PwPyPq6Zv-v(ecRM< zT!7mzoY}-`zfSq7@iMyBElg#}Lml=!s<0i^NB`(u#j50AYg6S>lA=-mKq+4(wo78o ziE)LAypNUKV^@1@5{Kdg6 z`F2UIrKs6DtY~)~+ChN5+`nn(2H1#az<-PYST`KjAy_v8W^jJJ-A_6N&dg0Rylr9Z4SN9c(W1SFCH3KPHO< zOUyO2rzC|gsIO&;|M34cHK{2zWS>KmTCie8lx>pzR%WVTpy#q#1kHLaS1 zNqb`qap{pkQpB07AD=z7Pk>lCgsv&I+L$Akp?2AhD8B5glp`RNI7;`NtcCbxeli0G z;19a2vZ$zBQ@06a2z^Q>8_y(N<6%j2+e7|42DkK4bdC|3b2KT{XqO-lW=mAIf3^n8 zyn{RWdy4u8IdHVIu~J@K4xx1wm5eaRF3{jG5IMXg4Fcw+gI24>QdDdyEj@I`!9gnN ztS+gvmK7IOS{)q!7e^}sKa;V+6PHy70XPy6FisRq+;W5Q(YaR*043`NR+F*86SuMl z0Ui#2GBiFu3UhRFWnpa!c-oa&30PCtwmvbO98g3hhSnT=POxh0fFO#tUQltKYn`rE zK@kv8#t;yYAwYm6I|*}`6pRFrsHotjXdSAx_NjHCcA&Pk(rfi~vew&n9lCSh_jU;S z+S>QN?}dCPd?)8*ul4_H{p;U}h;nhEL?V%Y*NT;E65_(*=Z7UGPxbRz6B(5f8>V!; zvq%^%ynx&}fujUoBq#<6-eO^l3m)$-?r?XpYjEdi`itN0(l1by$UMw(gb^^reGK{U z`IGKr9VdQtA4j#QT?xCBcf}~e;v*ExrYKfUQLIYXP0q$BUQUQt zY>$i%i`}6}*r8Y#xlxgl9H~rJL@5(e5|bxPQLKxONmlGmQ0`KYS7l^uWLR=!gd!zA zB2uYPMMo+YuV1@Pu_z&4rC1RY9vPp19I5c}Q7DolBNfpqRpKmPUsX!flmumz?+$WJ zvTv;OqGVskeLjm`TeZ$-#nSl;SFK$*MU|#f>_|{5A|h2`F|o;0{u1IpoLrTlj0=mU z$b%&~jiFwoCQ_5ADU>fYmGY-%P&27n)ZeH%R4_GredgFR4f%oB~nS0l1ipjRH~SeKpsAnkAn^uDvJWqZqcJb;e)!x zv&Bs=a+eBv1f3~)PSVNvGiv6H>n7JuH<4S_VBf(_Lwtth47usP!u{CLNDtoQytpq&a4(%cIL68-`-Yq%;ey|FEwu*&MysXUH zyppzT9?un4({%-frD`_IxpshksyZ ziDfNazPmc6K%NkVGB_pki-|tcuG2U2cq?p>;nyVJKmqO1S4C33W`TB_2m5-Fm$Q2Cuxjr6eV<-`?vdYn`7>ja=k{+Wvinlj1Q9AW(E;&}1C;{%fd*ATb z6TXx!TpfgI4*P*#a@bgQii6RTpH95GmT$s$WWlcmDmctZw&=&1KNAa0z{%cUeqp+q`=It>*Sqp73$J1khmRY^f)lh6HqHhpC0staRTya@;5>3A#N2ILjq zfS_MsGz=lR{xi!esI8*2y@H2TEF6P{vWllmdTRn+o5^m%lbRs`l;GRg_2J#WV`)9F zF&YXg*=kf$&qEMXhstvc^t#-9E@C5XNqytQTozBlGCTx-PsWphyuu{#y8W-ONP^wS zcFHDn5MrnI3R8X*_i5pt4A$Zo!4EI`YdSq7umHxv%dnb%$!mQ2{4MLbknPKN1+kcR zGAA5ooYc~icRtOS{Vw0N+B|)Z+@sG0Sx;Go(S${}n{e@tSm=gQ8E|7^7#{mq+#O-S z`xkJ97x-6yyk5-=OWnn#?Af--m!0h-O7LL%dF+w&c5X?p;VlW4u<>M>3{tJ{*?Mg- zlceQ;kCY7rSc|QWqBYBu66~W)4#o=!Bm;%k@B{O(CT==!g?&oN^sKm-xe1^dsLmH9 zF!j(z{9+w%l_Wq4ZNKvrk^N5*`9b0#Tp%TyW`ziUt7qpebgTPTjTIzTtyTI^O=@+6 zW7(k@$(!*=7E=Rdwz8Y~sA~5Tvg~H1WJ%8MU~VD2>a5J4B+}y)xHd&&k(n%<20^s_ z^(XL%2jCGx7%8Rp=nFx%gcdkugOfH-l5Y4t@U_BZt8`Ztt0PB9|;=xR@NW(qCbaC`q2c#99krOI5!iW;UVxk^N~4qsS}8$ z>eOIv3Cwo_@vQ^~9o&UoDoLq6NU=edO&Di?^Ca!K?%tkz(gl4@P$;u8SgNJB`y*2n zAKS)AyS_tNG5m4{55>MH#}BVL2S?y2tULY1FF(DCeGhO_7iW7)yZ*94TVq#W@@Da7 zJPXgjAvgqH!82j&?cSzKZG1{~N4S}-K;^YO%p>@{X-1bhk3LzJ73NsswAJ%$FLd^Q zhTM`a5FWrXnU&G)4A&>IQLiIYG_P!6J}Fj*^Km3b6-o|QaoH&`ns{~<9(x#KNJmd< zKK5nXJD*k28v7(bnrSaCJ;#y3 zK}gE5LZnsXAPLFn#{MkCYwYX9f%^? zh(HHIu}#cv(}3D&)qz}6ti;rIn@qy?+eq8SdjRP&A%f@!b4v_$B!vZqC7rx~;KFp4 zlziX_(?yG29ASP?Qn-z`yD-}d3l}&-Atychh9iggocbLK56DOCOf{X|!Q)`ahSy>4 zenY2e6roL(saLXze>WPkcqL=jl@_Sw=^3eevN)WAH_@Q{e*z<6zPk&f@M5|$yDBY9 zo|df3Cl~$8L^s>?x@J<`bC8XHz?3l1!BTCXOTSjVq|U-pyn^+SNLM?xwUs%7YQE(T zOQZ@LBVD46@9L5Qe*7AlKdbd|qD__ksP%WK^rjHkFGqjN&-KR&cEx~{GR&D; z^syr$(wfq{So~~Gf-mFoTaqJABC9i6;9v_Be=hzlz{{4*%+?K%YBcwMaE&$`BvENm zT0Zea_G>s(xK4ag0ZL_Y&{HAOGKpx*SYma+%>-@^S2Px#YIe*Hg8?f3?MCMnm*iEl zl_eDw#XMB?4L9g%`>iKwNleFX5aDXSj!`RMHfyx{JL?KS(h zKgPC3h;qc{_k?=FYTHqr{xIKf75mZhZxL1!v#9S=IMm8bi``6gEb|zJ#PRlE#?A=! zbak%Y&<~KLYH7qRv_Zw=GYoDOWOQ|&4yCc!-GN49 z3(_?3_7VR#9Pk@cnFH7DOK5eu36(mcsON>}h$+w{7Jd?PeP?(0N!5UZ(Qy~DhaARv_xCW2)ABQQ-e*+bug>hdz`WeQ| zzy;(Jhi5ejISy|G|J5rNzJPKWOdxJ%IDVO^;b1(R94En0JQ&84zo9S~CUF5~8Sz?! z)+|_n2VI0s#N0_vU%vtLCj1)LZxzVwJl+IXY@Qb%qh8a0W7zqaOwOr7f$X98SJ<1g zi=MGS5~-HV&bg+xkBBl!-#q@YLz|v70p!6_h8T9YPapGRy|y9jf@gDhA{=lUiboQd zjqspQ-su=34?w0>)b&JtvNR_!{Nq1yP-cz$_$V*KYn@T6F$Hz0?o#r92UXr z&Z&7*0-k7pPBwCn(k;^cYN0n@R*uZ-JiUfE%ZP%= zY$YqbMrFpMYjOO?w%&ijNcsPU?onI0v6P!rSF^mEm2&Ej8tJRxppMwuG%o9crv!4x^H`t@=TC!I$1{JIiUi5#S$$0I%Z4*XMAip<0Z=V|m@Q2usxD!W0`1+MA5y1U4_ z>rkp?SAB%p72FSAy>>(XJ@&YRCvmvvS^p|A9sR%|9Ut^No+qr8!U*R4q1}7-^2c`| z7xizl?BXRGF>T=M{0QG*NmDyF0JiT)^ko6|`hu&2P&+JNKY# z_rA+r{~_=8e;*I!u;v-=e=+^~Kg`ejnz1&2>0@pD{uGqFRlXLbAByGD_jP1;u~%y^ zH}H4`T$M!>&c|NdG*~3D8Y?SHI8#|O+Q)u$SF0d#LW)h;WrJBQqFYzt{#CKi1}kNs zZT=PyYUc3h!}Rq1sW*?xZ(O``7r3{_kE`Lbji}H(u)hfDw@VtNCw6JaYPvZwq-zp? zi(UPvc`rW_TAj(CdZSfyXLnuEUTs~`rL2ZRTa0U|rZr~0e8GnKeql?>JJ)f`&T2o@ zA8{?y7Z?lKY@|u&@fIcnY0PCtQ$;Cv>^(ZZ<^64sSs3xfqhCn8p;=bF6$TMoYpJ$= zZ#7yx&z^-nXA{m!d9^SJ#>lQjoIu=vj^lewGa6D$qRR`6uA3V6w71GncAUNr(tX%% zS23qCqJk$mj2>`Q`jkwrurN2XsKE7DWJuL^)^F+JptXl~R^{-maScTeQqN~s>JFs3 zN-c9sikcI*$~Q%RTget)^=wqs|BYyu{u;n6*4tQGeq2^)!DPNDz(oFomW&{hcq z)-*K1RJQ&g(pB;nEy_{JNfF1OtT9|TDdH19p_A;%$4GmEw}mrfP|Xp?{5^oGEGEua zc@WjJ;I}^s2i7>Yl*pQ55kkp-NZ1FPZ^0Y;q-nxABF9+{gPtSF!aA zgot&bnD8RfE~#Xnpmb8{1CB zww+XL+ZEgP|DAJNr*#{cR?8ES~^X>WVoa=x4yk%{hgay?zaKN3TcydgdE6W&@ zmVhJIdOG{$_!v^qTE^y4kBQ_wv&Im~|DD*sb6^>37#~g;vYf+w8L32a%$AWcJJtbU zR=S2K6mwGq-(gN<2mBR7yo|9oYDO*k3C3{v-jR8U*UWVR_690MaABj3*|{sO;6rvn z+UetS)xuJ`7XKUQ8(}N1N8+@IKfgwG~hZl@53@35WsX^3e zkkbQYObS*zbZrU+Qx)WyvC+nD>^B;h*gJHy4a<*jvHkMo0eF15eMmUbN7GVs291Z< z47ttQ-glv@(=)h)n(JC~>m`y39D1hoH8d23q_13tXRIr@lHQ*lhIzDZn+Y}B$285` zAuekq+xMHp0Y>i54e#75kyqY53K&dG;++F_h)Qiz2Ry-HD_FLZQAagcDr}af%_`B* z-CpGf=W@#%05>+nEL$C2aw+Mm>4RIj=MHY^u@bnd#CdW@f2bDj>qvvs_R&G-_{SmN zC+fniaO$pHO)R_z+cg`t(9Dcl2vH3BX062J(r%m9tV3nd6q(oVLTkqpD)R7N!r-j* zmKC2YD`(=@fbAc0*Tc_WfimEK&k!+pmNbUU$Cf2*0IIgy!9pX0|BM0RCHFx4@|{S7 z4NWSa(di9=RKJc+N^qVe>In)ptUf6RZ*f$nO5!$r0O)J79&c6D{uIA_Q=L0_w&B%S zev;WgiU<*lvi~B-@8lWy5t!tQS+3<494P^g_^SjZ&hWSWUR@?V+$I?fzlfAH*v>r) z=%P9|z&6Q4_)?d)+>Q2XADG2=!ESbrq}6;m8xKGH?8r`ibRXL%bemZlikAoBf*^%BGrXx!nWmx|-wt=%@iUaBszk3`R{k_d@0KdbTj1pe0d-JlE<+&k3-ipQl=!Kn|<+Tn8Hw;{>I^3I) zfE57#c_4|`@xyRv>40BHDYJkGSG}73<=lncXEM0BQO_GRWpwBV*iq7LS3uM0gv;|7 zn@olc)Za;FD7~`*Rt58t?mgm4msajv?CY3EeT^WnU7q%gk$a)(hhd2l$0S*!u~lon z_&-AWi;7;JpX@C9M%$ywMA+rOM&l7yx%ne~vTXy|(sjwVjMCjwzq3a5I&|UlPlWS%v^}xcVVL%i_%qSLD17-XhHZ8Uyi0FM=iHd05&GE*xS$#3Xd z7x4gRY=jVb_SXVc(MLT%ZS#;>pvCpap_(@;WSP$x;z`xF=1%r?~wO1VJv( zig_u2PcPwBQPME8VD=a9;#>JUlj<8zB9gWG>16ca@5DmfXA-ZD;{0yABTp(TN1=EM zkGX;NQR75&W;W zx_gXgm$tyGt`wH@-z4{e5dTY4P??3tJJ}VHxAl26rL)QR)^Ol zn?Tb4o~=7vk6hf96Z8w_0ph$f*=3Qity-1ACs*YDOAiBKycH*ByV)0wBa61RT^AD? ze=fAvZB01tb7fQgq!K8J$>r^T`hN2*JgZj@fDKT))DJNvQ}fxdasabrrVB09Txoo5 z2s!d}rF&#}ROKq#Xe@0Yo)}xQT*~;%Un*Lbcx?n)f;Ob>OFyXr0NOxRWs?7|Lh;Z))K>cMO2w)+4!c)HPO(DTi3$58@aRwdi^jUim&0WfWqArD*huQKq|=k(+=s zB|}wK#*H6^Uy4j-r>06)8lRHiCWCX7)Yx+E>_lnt9LhK9-Q!}RFU6yb2AV2sdy!Qh z=PhbXm*%r*h!bYn3{%&~wOnD(Kb7FlELEobdhh@XH}X_b(cnqIE%Tj`-+|KLAlP zmkpEM5Fn?QO3fjNS5oTIg)TxvLSJeD0^z~@kbctFN~rW?_wU7=u1ajZ#38yKTTkD! z+x=dKiQ7;tA#k_9?x4X7)F5^jf{q{M5UG!6ccdYD>Vg z?-{WYT(@%UOF(|!dY*A0%1)np-fG}ly;r+J1mN@6Svm1W!gJ-$Qrw6g7|c?p*kw&X zm!T?;Fw9orLF!;H0=X;s?)voOCPmmf;*HJ*?PZR$xc1Y#L77s zVx!z`t-D>W!*{S2smAy!29oABBtRRh)*IYD6xZ}R8Qnkp~3w{N%)(#MpE#Kt;>(0=%oxYi@%y-L#>w#)lf}<$aW6RTtQ)NyD=< z_sS8?2rzpBb(kOeqrc_y>|NJZ0@ctov=@09XU{iq-Wmv!~ zWfyM{5WU&2hTOo~s1eZ9&TMK;QTV))lizSCOvg<|B~oY(X}Jpu!AZWWqb4;M`tp`j zM#ll0RJGfoE%qYu5%D99APj{P0O;mcBE$%2ygU#see*r`i<@qFa1HmQ7$MRRA|bLr4OgVEP@(=Dy_J(>J%TAM*;5Q1^PNag)+7- zJz7{U_P&<1gx~D;H%4*EROTl5IjGnPCuE-(p!fIP6m>Xo8~<0VV76d*dt?m>%ulQ(=3j5XqqtxE$EqD09JM_9xXq-@*Vbkx zrDZQ>PAA1muroKnQd2GkHBEkvlTd&pM;gum-+o%sAajqI7h8Sc0uVJ>G`?Q#+qvp2 z>{rJi>Z_yfcuO6MpOJ-;Io!b5qyyEqe!|3{vw3^!YbMzHsAPO}J^Fx>&4Y&KsN+dmv%ap|NGLsdw}@x_d(N{d&k~cn zrL1MR&Bfh376R&Co-7)d^|?s{!Q7sI&(idaS4ijv0fZ=T50GISdJ_L1qNR1!k_-xc zL{*6D=WH=Ij^ce0)#C-uCU=jd$)>?T7}jO0@wR{Nc2s+R2(Dk%=sqdP$S$DD(aYh9#;x^gXc1=f_LOe&T@*o&4;#zA*tMu$gkL& z!6%JPh+0nn8J8WbNE~?~m8H=$^NZnfW|#?n+%2fa;V7`wj%hnzod6^zbPgHGV{*o4Tyj<>|4=59 z(yFoQfCTBE#B|wxbCS z{J3=%fH)Pydw?s{N343kE3XyFwd!98nn%et6QoNNg2?#&olfc7UoY>Nr$_p-#e}E1 zDF|EdDckdKYh3E?d6W<1Wplt;v-J7&5~}uSDBg+dhPJMSZrWE?qQgTPJVxIW_qk%E zFazG0CV(t#apV^~KO&pZ36 zAgh3!F_T_JScr-VZf8Y||FXc72S;0PBK#-vETE;MtFKJ*L0skG3I06V+77Olld;>9 zspowB{4UwrQU46aIutHG{)vcHV$^U|u&-9@ShUE)Q8f`GqeMSVJ#}mhgQ@XP!lChu zk0o9QijjpwfKDaN0DW-@^4kcGA_s#VV1Oku+>X8h0I-{o7#E8#BIX>q-u2f(b+I+G6C^A z)DO=9K&FeF$fO5jzPY2LLyNVBgToV}W}TIn9i5DD<@0;t{R}GOXFca0fEKQltM7t{ zq?XA|D5398E`**$jhEC?o0nZcAjOcxo5pSC?@$oAV`M+fOqiEfGO012Ec#jECk2Vd z{s9E}I9Fh7Gww2tsB6&yPyCj91@5!GTW4fAyEZ9HN*%?^X*%m=9UOwmF<*|t>^E!s z&B|`v^Tko5FtWEE6ph&-fOhY?o-;8HlsrIw@^?=US7+`w6jb~s1-UlC6|mo@WK|7U zPblN%avezYHw##Hl_YYwZ#RWRw#ERow(FrXUdk4xiuS&V zlHl>mQK7Et#O>|m5Y8D%xI%31!rgtUDhnZ8YsyhLcY)I*qOHv4?c7;7fpfB^D7~=~ zPI_Csvf&vx%A1M+Ah-}}cuFGb^a^`Eu!P)kuc6&P!(_1!g|wNoq@Ec8)8nt`abmrp zq43kBE@u>sm5`s%HrpVdmJDW^_cB!LA!a6L!fYKoCkLcN$^ANx;2PEBAbUFFRLwZ| zq&J+)9kq3{{7qr~t)}|G$vhX7&F~a~gfSVdsho|Maiya3hY-q)T2S?OFeMp=dS~`0^?l*Z4 zP9EyXf^YUqj2~Zi?1I90f_6x?*C!1g!(eMwk&*<pNC&|K68e1*AMWMRFFQoseWL_Hm7mI{k#M z`WGgR?)K_a+x`J#)eIxn~EdzB4k1HBeNeND4^IX;iy$@~l{iBqzy+~q-xz=`X z@P2nz_m?HobsIj}Qv6T&`9Sy(Q`R1)@BY4NA^u{cg3|uq z)TE7&A4Ce*)|Cm`#h7x<=D28?8op#EezNa@Wec{fpW%nYk2HWI0^3(>R&_7B-Sd2{ zssH6-P-WQNu9~3MQ$q)uS%6&xUoaLXkA>CWZNvyj;k8chuITIaY92l9a1@}a)4jaM zwtj5*HTFa09VBOvZfpR8-gc-7LJJMTk&a{tf)6-uJ{fw_MIS10_g(QX0yP=7{auNS ziu#v4A*3EWq#jy+jSFPmTx#g3uveAgMzfw{wFEa^)6uZTqhV!Zb9Lqp!M~3$T;lZV zUNiv4b!N)!d-vx9@G!lH0t(W9aJR+z@pjhSNnxOuYFs+@&+2+Z@|oh<{GV(Sdp|3` zrXV0Lcfhg?)~=TS@NA;*k{0_F?)_Bf@n<812@RadoAl+ag%7<;arj5ZvP9YRM^Xb=be5+)>xIH4D(Yu46l>!hNI9)g&xbhmxv18-norx^UKyXO~`hiM{)TisEJXB#7ik~N#<0?Vc@Dvy+;snbV#mSwT``? z#tl`qRi5bSK6C-9(rDquzL&V3;3^pJbe`6C&~A+~bGei!8#8jv*!lX4fvc|x%E z6s9PS(cmH_hFDE_8iKWDsww{muo)Y|bO+gxc}oTwSu?~*8nLE$NxVb)j6xC;tc+qK zDkhHn0mD#06(bIhGCDeUu*z>j2uKG9aSnVHG8pH6appPbJ@LI;{(O0@HVPtEyKQnR zs}Dm@0ATfy)1v3P`^;DPeWu*OS_ZjhPs*p$rgzmE)K1r4hA5ydp(#dEMJvf4D(MS0 z`0dX+ShPW1lfE#&I0{7ZUk_-Q82xO_%rHM>>2TyJ*rt1k2Z8g(iO<-j0_r&!O0>mF z60%a3#@Hcwkknxgq@(}CxpD=|Lf(%>QO@lcgW8ln+mbv|+V+Ilph3r5fntz(;xwZ= zB$IkJqD1lLd+xwc;bofAuba!cqFSzjT{^{m$#sqLAzHaedo-mIEO-b=*AE2_Xyb-O zUW0Fm+zYVv%i90e7cXGr}|#Hu*m_s^?!{ zg6B#otUp(iJ*IDlq=n(*YbZ#Va{yI%Y{T7Ofu5Z|eRsR$EViHF&F(_dk(1yuk(OwyrZssqm1jcPV^hT+9&xh-?9W_fO2OSwxR^c9k>v~^)C*dN~E-{y| z*q|ShE4AObo*})B^#X)k`Bs9G^(WStKA+wTWG7Og%iyJjAq&2_(b6>K(sM`@*1w5nW4B<1*&J@`++O2a|B~!N1 zlqA_yR#j zd`6jB0wAsJ{K~r#0U1SJF+1_!52U~zK#1EZ+G=Sdm_|cL?pFGtKyr;+Q=$%39TBzQz zcmC(vFxZNbWD5Q z0>E;`l8FtQQfZo>5=5`#1MkLS)tXNSPrmq{g?KPf)*Cd1?UqF+02JS{-R81IOiP~E z5dN9+Ip+~b@%R3H!Qb_)%_180aZvUH`)P(8tpz*BBF>G$%o|SAoS>eW&6o*a2pTM$ z0yvUK;i%u-vhcpO5u#h!+0dPfQnV)L1UQrUk-O0k-emG#BM2aR<)Mru&ufIZB)3kk zuX7guggb;!>;`RAl`r*k5 z;{;>qNlVF5QUL9!kRK3HPMgw>W)NFLwXbV5Sd~b1l;@chHDxi}&#Q zNMuWUOxgdnsC>Q*XdJe+@%s8qAm~|{MIgsgPP;}_n`ryGl83-89MC!iN~VvSQCLV~ zQJelVZ+zsom_}y9-}M!KdZEKlioO)==FV~Qq3sq>XRsQ{ct@q;OWOKur_aYXVCGd_-Het_wY>NeIw zl_|twM(z|2s&^gp`Bw#h55URo1B$!Ov6J|*rlcwqncvL>TCZl1-r+DL{pK3qXe+aw_>L`!3^766KEs zIT+6puR#Ff#f8F!=u!_n0n#-ss{(TadMbMO(ScVl+R zJ7Ql~Zmc2deFU&9I~S}JG8(hcQ8d%QNL{|?+Hqz|z9DLV`05eKUf`i}ly*U5HzvEX zf<~(vysa&AED4#Ul`?ca*}FYOg1-HXApZTNedP;Mi15Na;77VY!TOa6s(R#d(aBv^TNDJH-v@sCR3mi38>C9B4zd02w!Z|a*5G5#@!JR`*@JrIn-d`( z3E$$zuQjInmjcRN#Y(XouS>3I-I!aWS$5;|07;<2xM++iaoS_95C^GxR{a-#&P=B$WnY zMHE+|mk9KX$gIug%H8rbxy?uM%Q9QdI6%a?qME!%(orA1{Z_%}WB>Kd)*Ns0-1E5iNRWxq%v{`7(mkp3d2(h09-BPyfo%nL@v#2_bGCUGm)-Hor`4A2=PF z-2DC*kU6Z!g%|8GZ%;bDOk#~1W3x|H?zx^Zoo}Q5hD_9%fcmyLnI{sgdBj;pPO z#O1owWfC58QJ#+>Rrt_Dn-dJjnd&i)yP4hhN~&erINzB= zsVpvYxE7_w=8<^FILc}|64Wf;soQ}zq-GTh7?x`LDlVW;<-uLdxtj$q8vzng^-x4F zRa$<~*?dg7DQxts5|n^SvK^2${_^*g@HTcusR?)iEid@D5XPj$Og7^_^rZMru$mp; zn_{p;goL@WQdM{neF0gFV{z~GU`OH8Msq_V&3ierAHW(seu& z@E8({?k3ZEMx?jUuV^ru5390925VX>LUG=v&IZfyW>xaIpSwF5sfAHy^3j_CdKp=1 zX%>mjOGSGpv+-ge_>m1!`mQ-1(xju*=U2PTAGE3qO{Y&_S*s#)cnB8(_U+1qXYM@b z9w+0UD-M_6FDJ`^v^Hzs(TOJtgjJFQLQZO_EcC6o2-SaPEgIE2O8(UhdoMAtCoLxW z^ESibjV{GKSShq5IN^6Psi!Ujz%DD}15LfIoo`)@C)XI#Ms>qEv_%Rdod^&6&jylB z?P4b9zL$Q*G2`{D0x0aPuv>M>Jlk_G$Rl$n-2A>XsMHgRaIaX|<58?K;|mrA4#{I! zKko@42khz^n}R$H+JMonxSpS}WhgQu2i!@psC0;|I&a+gHdOaZJz3R&ab{Dpq!klE zBKPUFJWb>FLv?5HLv-fwdpJ2dGcR-Jt$*O~s&+W7&t6V&VFbM~0mvJT_ep{9*y%dW z%`ycFaWk~lf3`9)expjA?VCJBwfdjbJq6tAz=0VWgN24?izo`JrKlNbQ=w69>KL8u zorn;4rTSE-%mE<%P+$@OPLzUl@1#FB^U+=hXo8`vr*d_)sf9QJR-gAr0dqKt;cT^j zu3zsMO6<9qNP?8+G^?4f4U`QrzC#@YWF0BL)nL=pPd21Cbm|pOcnRA(*K!FMN9p8l z3%ICeJ-ExpmRqU>sknrg$`&-Y;I3-X);eW`WPTuCyb-AWCn8CYbOfmbu>Bv}&9L64 z%cdl%|E$(O`zu+Rj1P0VBii5edWlEbW_&DzP`V-J4%Di0s&PGrePx!yCc>bN#F1Sp z#W=rQ&jkHT%Y1*yr8UT(%R?v1q|4N?QLx$Tmr7XNRk&NfeSbI~T_V5M2An(52x zIsaR{BE zYq&M?Yl2pC*k7=t;F4(ug5)3}hzY{OCWL(vtx^D3%Ca!(-Ng1m$dT1J&4c>56Eym4 z`h5Lt5X!&E>QRux+4SY8O;}B(52v(M|Lnz$vRjQY$Uv0vWF4&1N)4K?3<%Ek^O}xB zXPQP${o>{~g2IOdbLkI^4xhuGqDiqjCE!5*1r6p37kC>!h;2Y8f6R8n3=(d#vuGBV zpDYYW&QC5TDEO89%ckwS675KoGHv1|#$8=%EaX<3E zDTp%6B?)K+3o@GWw$d!ldVZZD8%1$a9(U^!2(H8c+h4H>g_^p-1|gUs3+r`#$Ki|P!LY0> zlRL-ZsFDa68Kj!@>kLc$IJ8S+pyp7lPV1rV){{2O!qvJ1e3}F#gq7_Ob0(>1^$oJo zh1Ig4teO}g5$DNNZx`#2pUv8j_p4ChZ#sAOiA28<8Kb_mFNgQbe-6i$B0W-JEA%4) zh^Y~gajBeiAT}lxSz-Y+t8U#@^}i2gOBF(_g=-;zsTi%4c4U02FmxDBa`vc4KjfaG zM;2yi9dxu>1F%XLY>YIDlPsDPHN6P-s6Gs<1HUokLabLPbz&n;BL9n^Kc%9rAr@LS zZA(^jt#4a1l3);*UUv^}4uE{AIe?Vc|6 z>FsTXuFc|)$|rn;+Iih-7=M1;Lb<;Y!C9q|sdkwet5lkW4PBUR7q#A=9b&n`wIh+6 zx0tqQZEl`zmpcTG1jr@%w_$%;msr?iB^g=hBu-DLhMm~gvI8#W)x-*tv5uSoA5A9p z88d_oQ4hmzth@mEVj_b)tR|`5FheP=!P)fPMj@gG!`HlcqS2M2vqPhLYC?ifw4iKb zO|R<=bGz^cz`vhS=YPces@cSk-$pFhN=&!ymJC58DC!GLp-yfD0QRbY^ws5fXokh- zZNNUZ35R3`0=+VNM7p`V=XxAK7AjBTC2f{EU#dQyJNgk?DLAjN@Ac{jysGK4SBRbM zD`|!OLDjeFES=Oeq`43Q1#Kl8A#F~vWd^lS@P0}ANLT%(q>wr({RKbtG(jYljD}vy zoLWN|4wiV(-=2otY|b$3C`zThjyyQJ&@D8TOuE_Cz|5{nNY!iVAe9tQRsU`WyB;3= zXyhuMmMKXNl%Uy+?)w@$EU~XlyZUU#v!`)1I?6lj)a=-fXE_dHDN04w8hk(3`8JaH z1+nKN`_Zc|P^4iP`3Hmoz34LRDK&KV*nirmeQ~vuiXN882Ya!IQgcWYOliKY0eDal zskS$P=OpvgBewS;jWr0scY|BkvklPkx>JZ2vf!@ipCUQ*&)Qyc#Y*MopEH!9GKCKj=;% zTnq{UAW=cJ-B=lpj*(gCMvf7Cs69(Fv0}bE{s&^&8e4)!5Z8O#bxrAvnQTjw7JG32 zGKi4OI9!@HA|$6RyhE8b2|EmgJRwWHLEzrN4PrrCa%K^nk3^Dc1crCUWyP9omPI0a|RZU9JSa;g4pl1 z$H*-{XnZoj8hU}K-Wb&LxOMu6l%V5AZ--5pLp@To+?RkRBut!y6~7ZT)G1KHGm@7( zM>#Fdhnj-;Jc1cSC51TRI_r__sn|eZ3JHLkZA2+))Gbd;(dzMnveztiGai}p)M3j{pSc3t`+2V|R z_bP1e(3SogtLfR(UZUIPxQrrYf*qsoX;K@JGCp+mx1QXqt5;?iU`_zJ5|T4vm>K{tC`LHeINb|m50Tf2MK3y_BVcn$d_TO-VA?0 zGha>sE#XO>#h?jimOei(E~!5uL$ANFrPku(;tfbbM+aMhDgbWfu z@UK?WrWr_=y{5qZ_8U*^QO3*}Kje5Py8aeS*zA;^!#x8n#Ie-aQM8_ z9mGc$g4mVHc@J=!c(eanm?=!(i)d~(J^>|u#*+%`fYkJD&&!Er7JlSQkWmy?Aaf?# zaQ(kJu+5JEjk_;TJN?F85oUOiGsq}lav~X>SeO13LK3(^$F{v7$S1ev`R7E5>SU2b zK9TB$)MyYSBqK~M!Hd+H(?5JL?5zIpj!QE7wAuQH*W>r$fh=}|LSwmmV}@SRaIu*R zg-%Ru;xM^cB1a?fJDAQucGm6vm>{{1?w#Ct;e}ZP!Po+4fL7xgLSy;08P7Z5%jsp1 z589PASH5i$qirPB>-bZwmN>b*Js7K6LBD{Lu9jFrp(7aDUTPs*g?G~;Vhpds#4$h0 z@r91zv_$@nqtt0)z` zf%H`yKr8;drFkq}QRSr2^NRv7gWh=St?xTH(N7~&>hCqqU~C7`#R5IeWz8&OX-oMu zzP;YBx2wKVsqLZ_OZ;-v>W8;&!E7ktdcCB_NojNoZm0&RP>{MoTK3abhgT~0`IJlS zKWFR`@$4G5*BgC5{eBLdNvG5BqqULjLfg|#_bQhgq!V3+6HWWmRrmpfi~pow3kc-c zLSXd71zItFn{1Bs^QoD$HM$EW1(nqtFyh|kTDJP>31m%UeHcQr<=`AML7EdxXbuqd zeSZyCCFi{;?YOk|Lw|vRsvi%N%q1a0!k5n%I83Gz>F@@!4}&1Nc{%Q#-8SZFkEKBd zgVxU#`8j!OOdSKqnfm~RSa}Eg52Py=W?~GO>j_3Rf^H& zHus8Z-8U;buWBWEVz{ypA0Ez*j);k{9=rvKkd6)S;e5pO|F#_Zz6As%^v&Eodz#0p zeAj)}Euh+alp8Nj&-V&>{ajvNxsZG?Tu+z?FrdH4!lBHzT}?IM3lz!!>N_4Kigc=xf&GX$d}MhE2BlQ)4nG>GFNu)Mr=xWFI@L9djGdo$*HcZpPUEZyI!_)(;>H|*Em`ETY>B3gTOsXcc_n= zAoy{0_;yEF>a%fLFlAq0Dg^3qmX%^ZM)Vi=Qgd&2Vy@4n!F_5<7a6x#O`rZ&$01lp zAyE&g>3Zp~-SDA%ol2jUE@k;q<|tae@t3P>nVJx=VoSQcP(`+_`*7|sT?L>mx|8{4 z=x5S6YrXYO8#sBx^`i|^qB`x_-1K#QPC~7PyxnR=)!IyHCyirpjyBvbZueFhWl)e^ zY%WThB@O`Ij4=rsuZ#zELMms>>x(cC?qe{ihyDy$QljOou zmC6FtWMB+_-urZ8|BmfDIVoCyYln^$qDW!sXL=a-5@}C`YnrI@>j0P+C@6 z98ULB)ek{RHC2tOi<3G5-kEid3B{K!>Tokcx2)ZuJ z^3W$4sfPc%)L73YNt?Z!Tiv~Ce7b7(b-Zr@;FT*nS9-Pul6tNcXvc3^h}tWz8(21O5ICWirw@cnNg0c zTa1HPnW|wPg{}Hg1`Y?)od_w3-(gyCpZ|SV`qm=MN3vgYa_&5jHiSKFT3e7>`kuAj ze(sGQ&l)t7oJvy5@7C$PtKrC+?*Hw`^u5=*yCAw#D4$Fjw~sQaydP(Xv>Fhy=wAkY zKuPQo7XKGX^9=yO6=h~+Vdvsz=V9SwXESFu<2K`9G3MrBH)Z8CHs|0pHx}go|NA3d zHxNV;!1;e%qa1T`cJju*hVNWu%c4ByR+>!tN-&3sH=XDeP=W`X|Ux!Q)UO@l8dJ!gwN9It# z9n$L1tWGR9>eeRX9YJ2FsEU-PS4f%**W3f3wOJf4|p`H)dGjJ|{69h)+T= zvGAxVrb@U#&TFFhLu~9q@?8Mewfbwnjb5a0m#BowQn+*BCOak#CL#bk`}L0{z^{gX z%e#WP3i?$DlTTdzcy&FqBo2t-M=MZFu8~1P#vKBu=7i2CTH4nXJyTTIT;WIkLqvMQ z{QPKXqn<#ZX+PCZu+o2XRNDP@3QwN9#?>Cm^Vw@~mZcK|q9@N6(f#d&t}rjdYem%E zh3jb*Ar~f6Ny+V&AuEG$UlyE?^Bob)rS|lM5~i)Lu}0Jv)kc^7uk6C_Qv=D;ku9pR zPR$pWMwVTF)^i6>h)kJwzQW#90}+e7E?2YKxt^rUxBc4m%_9SEVZxN~BNgwQ?j(?J au!bRH#&pkM5JFfMHg;GF3JE1i*#8Hxk>k+- diff --git a/tinygrad_repo/spec/tinyspec.tex b/tinygrad_repo/spec/tinyspec.tex index 4444d8456..346022bc8 100644 --- a/tinygrad_repo/spec/tinyspec.tex +++ b/tinygrad_repo/spec/tinyspec.tex @@ -23,7 +23,6 @@ \definecolor{axblue}{HTML}{1565C0} % GLOBAL \definecolor{axcyan}{HTML}{00838F} % LOCAL \definecolor{axbrcyan}{HTML}{00ACC1} % WARP -\definecolor{axbrblue}{HTML}{42A5F5} % THREAD \definecolor{axwhite}{HTML}{616161} % LOOP (gray on white paper) \definecolor{axred}{HTML}{C62828} % REDUCE \definecolor{axbrred}{HTML}{E53935} % GROUP_REDUCE @@ -50,10 +49,12 @@ All nodes in the tinygrad graph are \textbf{UOps}. A UOp is a tuple $(\mathrm{op \toprule \textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ \midrule -\op{Param} & $(\mathbf{s})$ & slot, dtype, device?, addrspace? & - Placeholder with shape $\mathbf{s}$. Substituted in \op{Function}. \\[4pt] -\op{Buffer} & $(\mathbf{s})$ & slot, dtype, device, addrspace & - Concrete buffer slot with shape $\mathbf{s}$. If device is a tuple, it creates the fully sized buffer across multiple devices. \\ +\op{Param} & () & \texttt{ParamArg} & + Placeholder with flat storage of $\mathrm{size}$ elements. Substituted in \op{Call}. \\[4pt] +\op{Returned} & () & \texttt{ParamArg} & + Call output buffer placeholder; slot is its call argument index. \\[4pt] +\op{Buffer} & () & \texttt{ParamArg} & + Concrete buffer slot with flat storage of $\mathrm{size}$ elements. \\ \op{Const} & () & value, dtype & A scalar constant with shape $(\ )$. \\ & & & Form vector consts with \op{Stack} \\ @@ -62,7 +63,21 @@ All nodes in the tinygrad graph are \textbf{UOps}. A UOp is a tuple $(\mathrm{op \end{tabular} \smallskip -\textbf{addrspace} is \texttt{GLOBAL}, \texttt{LOCAL}, or \texttt{REG}. +\texttt{ParamArg} contains slot, dtype, concrete size (or \textsc{null} for a scalar), value bounds, alignment, name, addrspace, device, volatility, and optional image shape. \textbf{addrspace} is \texttt{GLOBAL}, \texttt{LOCAL}, or \texttt{REG}. + +%% ============================================================ +\subsection*{{\color{callblue}Call Ops} \normalfont\small--- function abstraction, like the lambda calculus} + +\begin{tabular}{@{}l l l l@{}} +\toprule +\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ +\midrule +\op{Call} & (body, $a_0$, $a_1$, \ldots) & --- & Substitute each \op{Param} $k$ in body with $a_k$. \\ +\bottomrule +\end{tabular} + +\smallskip +A value \op{Call} is void: its \op{Sink} body stores to \op{Param}s bound positionally to \op{Returned} arguments; output $a_k$ is \op{After}$(a_k, \op{Call})$. %% ============================================================ \subsection*{{\color{movgreen}Movement Ops} \normalfont\small--- no arithmetic; view, indexing, and reinterpretation only} @@ -95,20 +110,6 @@ All nodes in the tinygrad graph are \textbf{UOps}. A UOp is a tuple $(\mathrm{op \bottomrule \end{tabular} -%% ============================================================ -\subsection*{{\color{callblue}Call Ops} \normalfont\small--- function abstraction, like the lambda calculus} - -\begin{tabular}{@{}l l l l@{}} -\toprule -\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ -\midrule -\op{Function} & (body, $a_0$, $a_1$, \ldots) & --- & Substitute each \op{Param} $k$ in \op{Tuple} body with $a_k$. Gradient-able. \\ -\op{Call} & (body, $a_0$, $a_1$, \ldots) & --- & Opaque invocation of a compiled kernel or custom function. \\ -\op{Tuple} & $(v_0, v_1, \ldots)$ & --- & Pack values; required as \op{Function} body to return a value. \\ -\op{GetTuple} & $(T,)$ & idx & Extract element at idx from a \op{Tuple}. \\ -\bottomrule -\end{tabular} - %% ============================================================ \subsection*{{\color{loadred}Load Ops} \normalfont\small--- can change device or addrspace} @@ -256,9 +257,10 @@ Every UOp has a \textbf{dtype}, \textbf{shape}, \textbf{device}, \textbf{addrspa \toprule \textbf{Op} & \textbf{dtype} & \textbf{shape} & \textbf{device} & \textbf{min\_max} \\ \midrule -\op{Buffer} & from arg & from $\mathrm{src}[0]$ & from arg & dtype range \\ +\op{Buffer} & from arg & from arg ($\mathrm{size}$) & from arg & dtype range \\ \op{Const} & from arg & $()$ & \textsc{null} & $[v, v]$ \\ -\op{Param} & from arg & from $\mathrm{src}[0]$ & from arg & from src or dtype range \\[3pt] +\op{Param} & from arg & from arg ($\mathrm{size}$) & from arg & from src or dtype range \\ +\op{Returned} & from arg & from arg ($\mathrm{size}$) & from arg & dtype range \\[3pt] Movement ops & $\mathrm{src}[0].\mathrm{dtype}$ & (see op) & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Unshard} & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0]$, each $a_k \times n_k$ & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Reduce} & $\mathrm{src}[0].\mathrm{dtype}$ & remove first $n$ axes & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\[3pt] @@ -272,7 +274,7 @@ ALU unary & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0].\mathrm{shape}$ Other binary & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\ \op{CmpLt}, \op{CmpNe} & bool & broadcast & $\mathrm{src}[0].\mathrm{device}$ & from intervals \\ \op{Where} & $\mathrm{src}[1].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\min(b,c),\, \max(B,C)]$ \\[3pt] -\op{Function}, \op{Call} & $\mathrm{src}[0].\mathrm{dtype}$ & substitute \op{Param} shapes & $\mathrm{src}[1].\mathrm{device}$ & dtype range \\ +\op{Call} & void & --- & first non-null src device & --- \\ \op{Range} & index & $()$ & \textsc{null} & $[0,\, n{-}1]$ \\ \op{Index} & $\mathrm{src}[0].\mathrm{dtype}$ & remaining dims & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Store} & void & $()$ & $\mathrm{src}[0].\mathrm{device}$ & --- \\ @@ -304,7 +306,6 @@ Each kernel's iteration space is a set of \op{Range} axes. Every range has an \t {\color{axblue}\texttt{GLOBAL}} & \texttt{g} & --- & --- & GPU global workgroup dimension. \\ {\color{axcyan}\texttt{LOCAL}} & \texttt{l} & g, L & inner & Workgroup local dimension (shared memory). \\ {\color{axbrcyan}\texttt{WARP}} & \texttt{w} & \multicolumn{2}{l}{(created by \op{TC})} & Warp-level lanes for tensor cores. \\ -{\color{axbrblue}\texttt{THREAD}} & \texttt{t} & g & outer & CPU thread parallelism. \\ {\color{axwhite}\texttt{LOOP}} & \texttt{L} & --- & --- & Generic sequential loop (initial state). \\ {\color{axred}\texttt{REDUCE}} & \texttt{R} & --- & --- & Reduction axis. \\ {\color{axbrred}\texttt{GROUP\_REDUCE}} & \texttt{G} & R & inner/outer & Shared-memory group reduction. \\ @@ -327,8 +328,6 @@ An optimization is a triple $(\mathrm{op},\;\mathrm{axis},\;\mathrm{arg})$: Pad axis to next multiple of $m$ with validity masks. \\[4pt] \op{Swap} & axis$_i$ & axis$_j$ & Swap two axes $i \leftrightarrow j$. \\ -\op{Nolocals} & --- & --- & - Disable local memory; no workgroup dims emitted. \\ \op{TC} & reduce idx & (tc, opt, mode) & Apply tensor core \op{Wmma}: split reduce/output axes into \texttt{WARP}, \texttt{UPCAST}, and \texttt{UNROLL} dims. \\ \bottomrule @@ -421,7 +420,7 @@ def allreduce(T): %% ============================================================ \subsection*{{\color{callblue}The \texttt{@function} Decorator} \normalfont\small--- graph capture via tracing} -The \texttt{@function} decorator transforms a Python function on Tensors into a single \op{Function} node. +The \texttt{@function} decorator transforms a Python function on Tensors into a single \op{Call} node. \begin{lstlisting} @function @@ -433,14 +432,15 @@ When \texttt{f(x, y)} is called, the decorator: \begin{enumerate}[leftmargin=1.5em, itemsep=2pt] \item \textbf{Extracts inputs}: walks all arguments to find every Tensor, deduplicates by identity. - \item \textbf{Runs the function} lazily (no device execution), building a UOp graph from the result. - \item \textbf{Parameterizes}: replaces each input UOp with a \op{Param}$(k)$ placeholder. - \item \textbf{Wraps the body} in a \op{Tuple} (even for single returns) and creates\\ - \op{Function}(\op{Tuple}(body), $x$, $y$). - \item \textbf{Returns} the result via \op{GetTuple}$(0)$, or one \op{GetTuple} per element for tuple returns. + \item \textbf{Runs the function} lazily (no device execution), building a UOp graph from each returned value. + \item \textbf{Parameterizes inputs}: replaces each input UOp with a positional \op{Param}$(k)$ placeholder. + \item \textbf{Parameterizes outputs}: for each returned value $v_i$, creates an output \op{Param}$(m+i)$ and a matching \op{Returned}$(m+i)$, where $m$ is the number of inputs. + \item \textbf{Builds the call}: stores every $v_i$ into its output parameter and creates\\ + \op{Call}(\op{Sink}(\op{Store}(\op{Param}$(m)$, $v_0$), \ldots), $x$, $y$, \op{Returned}$(m)$, \ldots). + \item \textbf{Returns values}: exposes each result as \op{After}(\op{Returned}$(m+i)$, \op{Call}). \end{enumerate} -The result is a reusable graph fragment: the body contains only \op{Param} references, not concrete buffers. At schedule time, the \op{Function} is resolved by substituting each \op{Param}$(k)$ back with its corresponding argument $a_k$, or lowered into an opaque \op{Call} if it is to be compiled as a reusable kernel. +The result is a reusable graph fragment: the body contains only \op{Param} references, not concrete buffers, and a single call can return any number of values. At schedule time, an ordinary value-producing \op{Call} is inlined by positional \op{Param} substitution and each output \op{After} resolves to the value stored in the body. A precompiled call instead materializes real output buffers in the \op{Returned} slots and lowers the body to an opaque call that writes them. %% ============================================================ \subsection*{Lowering Pipeline \normalfont\small--- from Tensor graph to machine code} diff --git a/tinygrad_repo/test/amd/hw/test_sop.py b/tinygrad_repo/test/amd/hw/test_sop.py index ba9ea6691..f6a5006f2 100644 --- a/tinygrad_repo/test/amd/hw/test_sop.py +++ b/tinygrad_repo/test/amd/hw/test_sop.py @@ -1002,6 +1002,39 @@ class TestBarrier(unittest.TestCase): for tid in range(64): self.assertEqual(st.vgpr[tid][0], tid + 100 + 1000, f"tid={tid}") +class TestSMaxMinSCCRegressions(unittest.TestCase): + """Regression test: S_MAX sets SCC only on strict inequality (equal operands -> SCC=0).""" + + def test_s_max_i32_equal_scc(self): + st = run_program([s_mov_b32(s[4], 64), s_mov_b32(s[5], 64), s_max_i32(s[6], s[4], s[5])], n_lanes=1) + self.assertEqual(st.scc, 0) + self.assertEqual(st.sgpr[6], 64) + st = run_program([s_mov_b32(s[4], 65), s_mov_b32(s[5], 64), s_max_i32(s[6], s[4], s[5])], n_lanes=1) + self.assertEqual(st.scc, 1) # still set when strictly greater + + def test_s_max_u32_equal_scc(self): + st = run_program([s_mov_b32(s[4], 64), s_mov_b32(s[5], 64), s_max_u32(s[6], s[4], s[5])], n_lanes=1) + self.assertEqual(st.scc, 0) + +class TestAbsdiffOverflowRegressions(unittest.TestCase): + """Regression test: S_ABSDIFF_I32 computes abs on the WRAPPED 32-bit difference (found by random difftest vs hardware).""" + + def test_s_absdiff_wrapped(self): + # |45 - (-2147483647)| overflows int32; hardware takes abs of the wrapped 32-bit difference + instructions = [s_mov_b32(s[4], 45), s_mov_b32(s[5], 0x80000001), s_absdiff_i32(s[6], s[4], s[5])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.sgpr[6], 0x7FFFFFD4) + self.assertEqual(st.scc, 1) + # INT_MIN - 1 wraps to +2147483647, already positive + instructions = [s_mov_b32(s[4], 0x80000000), s_mov_b32(s[5], 1), s_absdiff_i32(s[6], s[4], s[5])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.sgpr[6], 0x7FFFFFFF) + # equality -> 0 and SCC=0 + instructions = [s_mov_b32(s[4], 7), s_mov_b32(s[5], 7), s_absdiff_i32(s[6], s[4], s[5])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.sgpr[6], 0) + self.assertEqual(st.scc, 0) + if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/amd/hw/test_vop1.py b/tinygrad_repo/test/amd/hw/test_vop1.py index d1f9b6557..ced25512e 100644 --- a/tinygrad_repo/test/amd/hw/test_vop1.py +++ b/tinygrad_repo/test/amd/hw/test_vop1.py @@ -1629,5 +1629,66 @@ class TestSwap(unittest.TestCase): self.assertEqual(st.vgpr[0][1], 0x55555555) +class TestCvtFrexpRegressions(unittest.TestCase): + """Regression tests for float<->int conversion and FREXP corner cases (found by random difftest vs hardware).""" + + def test_cvt_i32_f32_nan_is_zero(self): + """v_cvt_i32_f32 of NaN is 0, not INT_MIN (x86 cvttss2si returns INT_MIN).""" + for nan in (0x7FC00000, 0xFFC00000, 0x7F800001): + st = run_program([v_mov_b32_e32(v[0], nan), v_cvt_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0, f"nan=0x{nan:08x}") + + def test_cvt_i32_f32_positive_overflow(self): + """v_cvt_i32_f32 saturates positive overflow/inf to INT_MAX, not INT_MIN.""" + for bits in (0x7F800000, 0x4F000000, 0x4F800000): # +inf, 2^31, ~2^32 + st = run_program([v_mov_b32_e32(v[0], bits), v_cvt_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0x7FFFFFFF, f"bits=0x{bits:08x}") + + def test_cvt_i32_f32_negative_overflow(self): + """v_cvt_i32_f32 saturates negative overflow/-inf to INT_MIN.""" + for bits in (0xFF800000, 0xCF000001): # -inf, below -2^31 + st = run_program([v_mov_b32_e32(v[0], bits), v_cvt_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0x80000000, f"bits=0x{bits:08x}") + + def test_cvt_u32_f32_nan_is_zero(self): + """v_cvt_u32_f32 of NaN is 0, not UINT_MAX.""" + for nan in (0x7FC00000, 0xFFC00000, 0x7F800001): + st = run_program([v_mov_b32_e32(v[0], nan), v_cvt_u32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0, f"nan=0x{nan:08x}") + + def test_cvt_i32_f64_nan_and_overflow(self): + """v_cvt_i32_f64: NaN -> 0, positive overflow/+inf -> INT_MAX.""" + st = run_program([v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0x7FF80000), v_cvt_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0) + st = run_program([v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0x41F00000), v_cvt_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x7FFFFFFF) # 2^32 -> INT_MAX + + def test_frexp_f32_denormal(self): + """v_frexp_exp/mant_f32 of denormal/zero inputs is (0, signed zero) on hardware.""" + for bits in (0x00000001, 0x007FFFFF, 0x00000000): + st = run_program([v_mov_b32_e32(v[0], bits), v_frexp_exp_i32_f32_e32(v[1], v[0]), v_frexp_mant_f32_e32(v[2], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1] & 0xFFFFFFFF, 0, f"exp bits=0x{bits:08x}") + self.assertEqual(st.vgpr[0][2], bits & 0x80000000, f"mant bits=0x{bits:08x}") + # negative denormal: mant is -0.0 + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_frexp_mant_f32_e32(v[2], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x80000000) + + def test_frexp_f64_denormal(self): + """v_frexp_exp_f64 of a denormal returns the normalized exponent (-1073 for min-denormal); zero -> 0.""" + st = run_program([v_mov_b32_e32(v[0], 1), v_mov_b32_e32(v[1], 0), v_frexp_exp_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2] & 0xFFFFFFFF, 0xFFFFFBCF) # -1073 + st = run_program([v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0), v_frexp_exp_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0) + + def test_frexp_exp_inf_nan(self): + """v_frexp_exp of +/-inf and NaN is 0 on hardware (host frexp gives 129/1024), for both f32 and f64.""" + for bits in (0x7F800000, 0xFF800000, 0x7FC00000): + st = run_program([v_mov_b32_e32(v[0], bits), v_frexp_exp_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1] & 0xFFFFFFFF, 0, f"f32 bits=0x{bits:08x}") + for lo, hi in ((0, 0x7FF00000), (0, 0xFFF00000), (0, 0x7FF80000), (1, 0x7FF00000)): + st = run_program([v_mov_b32_e32(v[0], lo), v_mov_b32_e32(v[1], hi), v_frexp_exp_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2] & 0xFFFFFFFF, 0, f"f64 bits=0x{hi:08x}{lo:08x}") + + if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/amd/hw/test_vop2.py b/tinygrad_repo/test/amd/hw/test_vop2.py index a341cb47b..c69b96fc4 100644 --- a/tinygrad_repo/test/amd/hw/test_vop2.py +++ b/tinygrad_repo/test/amd/hw/test_vop2.py @@ -989,6 +989,53 @@ class TestCarryOps(unittest.TestCase): self.assertEqual(st.vgpr[0][0], 0) # 0xFFFFFFFF + 1 + 0 = 0 (overflow) self.assertEqual(st.vcc, 0xDEADBEEF) # VCC unchanged - carry was discarded +class TestSelectFlushRegressions(unittest.TestCase): + """Regression tests: f32 MIN/MAX flush denormal inputs to signed zero (select-style ops propagate inputs bitwise).""" + + def test_v_min_f32_denormal_flush(self): + """min(denormal, 1.0) is +0, min(-denormal, -1.0) is -0.""" + st = run_program([v_mov_b32_e32(v[0], 0x00000001), v_mov_b32_e32(v[1], 0x3F800000), v_min_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + # flush(-denormal) = -0.0 > -1.0, so the result is -1.0 (both operand orders) + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0xBF800000), v_min_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0xBF800000) + st = run_program([v_mov_b32_e32(v[1], 0xBF800000), v_mov_b32_e32(v[2], 0x80000001), v_min_f32_e32(v[3], v[1], v[2])], n_lanes=1) + self.assertEqual(st.vgpr[0][3], 0xBF800000) + + def test_v_max_f32_denormal_flush(self): + """max(-denormal, -1.0) is -0; max(+denormal, -0) is +0.""" + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0xBF800000), v_max_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x80000000) + st = run_program([v_mov_b32_e32(v[0], 0x00000001), v_mov_b32_e32(v[1], 0x80000000), v_max_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + + +class TestCarryExecRegressions(unittest.TestCase): + """Regression tests: per-lane VCC writes (carry ops) zero inactive lane bits - VCC = mask & EXEC, never preserved.""" + + def test_co_ci_e32_vcc_masked_by_exec(self): + """v_sub_co_ci_u32_e32 with EXEC=0xFFFF0000: hw clears inactive VCC bits instead of preserving them.""" + instructions = [ + s_mov_b32(EXEC_LO, 0xFFFF0000), + s_mov_b32(VCC_LO, 0xFFFFFFFF), # preset all bits + v_mov_b32_e32(v[0], 0xFFFFFFFE), v_mov_b32_e32(v[1], 0x80000000), + v_sub_co_ci_u32_e32(v[2], v[0], v[1]), # active lanes: no borrow + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.vcc, 0x00000000) + + def test_co_ci_e32_vcc_masked_by_exec_ones(self): + """Same with all-ones carry: VCC = borrow_mask & EXEC.""" + instructions = [ + s_mov_b32(EXEC_LO, 0x0F0F0F0F), + s_mov_b32(VCC_LO, 0), + v_mov_b32_e32(v[0], 0xFFFFFFFF), v_mov_b32_e32(v[1], 1), + v_add_co_ci_u32_e32(v[2], v[0], v[1]), # all lanes would carry if active + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.vcc, 0x0F0F0F0F) + self.assertEqual(st.vgpr[31][2], 0) # 0xFFFFFFFF + 1 wraps to 0 in active lanes + if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/amd/hw/test_vop3.py b/tinygrad_repo/test/amd/hw/test_vop3.py index e828e3574..52ed3c73e 100644 --- a/tinygrad_repo/test/amd/hw/test_vop3.py +++ b/tinygrad_repo/test/amd/hw/test_vop3.py @@ -4,6 +4,7 @@ Includes: v_fma_f32, v_div_scale_f32, v_div_fmas_f32, v_div_fixup_f32, v_alignbit_b32, v_bfe_i32, v_mad_u64_u32, v_readlane_b32, v_writelane_b32 """ import unittest +from tinygrad.helpers import OSX from test.amd.hw.helpers import * class TestFMA(unittest.TestCase): @@ -3264,6 +3265,23 @@ class TestVOP3ClampMAD(unittest.TestCase): # 0xFFFF * 2 = 0x1FFFE, low 16 bits = 0xFFFE self.assertEqual(st.vgpr[0][3] & 0xFFFF, 0xFFFE, f"expected 0xFFFE, got 0x{st.vgpr[0][3] & 0xFFFF:04x}") +class TestMadNarrowClampRegressions(unittest.TestCase): + """Regression tests: mad i16/i24 with clamp saturate to narrow output range (found by random difftest vs hardware).""" + + def test_mad_i16_clamp_sat_max(self): + # neg/src-floggled 16-bit mul operands are sign-extended after toggling bit15; sum > INT_MAX saturates + instructions = [s_mov_b32(s[4], 1232348160), v_mov_b32_e32(v[3], 0x80000000), + v_mov_b32_e32(v[1], 0x7F7FFFFF), v_mad_i32_i16(v[0], s[4], v[3], v[1], 0, 3, 5, 1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][0], 0x7FFFFFFF) + + def test_mad_i24_clamp_sat_min(self): + # sext24(-6344704) * sext24(+4210688) << -2^31 saturates to INT_MIN + instructions = [s_mov_b32(s[7], 4290772992), v_mov_b32_e32(v[1], 1077936128), + v_mad_i32_i24(v[0], s[7], v[1], v[1], 1, 0, 0, 1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][0], 0x80000000) + class TestCvtPkF16(unittest.TestCase): """Tests for V_CVT_PK_RTZ_F16_F32 - pack two f32 to f16 with round toward zero.""" @@ -3651,6 +3669,80 @@ class TestPermlane(unittest.TestCase): self.assertEqual(st.vgpr[21][1], 5) self.assertEqual(st.vgpr[31][1], 15) +class TestClampLdExpRegressions(unittest.TestCase): + """Regression tests for f32 clamp (-0 -> +0) and ldexp input passthrough.""" + + def test_clamp_negative_zero(self): + """clmp=1 maps -0.0 to +0.0 (found by random difftest vs hardware).""" + instructions = [ + v_mov_b32_e32(v[0], 0x80000000), v_mov_b32_e32(v[1], 0x80000000), + v_add_f32_e64(v[2], v[0], v[1], clmp=1), # -0 + -0 = -0, clamp -> +0 + ] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + instructions = [ + v_mov_b32_e32(v[0], 0x3F800000), v_mov_b32_e32(v[1], 0x80000000), + v_min_f32_e64(v[2], v[0], v[1], clmp=1), # min(1.0, -0) = -0, clamp -> +0 + ] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + + def test_ldexp_special_inputs(self): + """v_ldexp_f32 of 0/-0/inf/NaN propagates the input instead of computing val * 2**exp (0*inf = NaN on host).""" + # -0.0 * 2^INT_MIN = -0.0 (src1 as integer exponent; huge negative) + instructions = [v_mov_b32_e32(v[0], 0x80000000), v_mov_b32_e32(v[1], 0x80000000), v_ldexp_f32(v[2], v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x80000000) + # inf stays inf even with negative exponent + instructions = [v_mov_b32_e32(v[0], 0x7F800000), v_mov_b32_e32(v[1], 0xFFFFFF80), v_ldexp_f32(v[2], v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x7F800000) + + def test_ldexp_denormal_flush(self): + """v_ldexp_f32/f64 flush denormal inputs to signed zero (found by random difftest vs hardware).""" + # ldexp(+denorm, 1) = +0, ldexp(-denorm, 250) = -0 + for src, exp_val, want in [(0x00000001, 1, 0x00000000), (0x80000001, 250, 0x80000000)]: + st = run_program([v_mov_b32_e32(v[0], src), v_mov_b32_e32(v[1], exp_val), v_ldexp_f32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], want) + + def test_v_mul_neg_modifier_nan_sign(self): + """neg modifier is a pure sign-bit toggle on a NaN operand; result keeps that sign (found by random difftest).""" + # mul(normal, NEG(ABS(qNaN))): NaN payload negated in the operand stays negative qNaN + instructions = [v_mov_b32_e32(v[0], 0xC96CF47F), v_mov_b32_e32(v[1], 0x7FC00000), + v_mul_f32_e64(v[2], v[0], v[1], s[0], 0, 7, 6)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0xFFC00000) + # plain neg modifier still applies to non-NaN values: mul(-1.0, NEG(2.0)) = +2.0 + st = run_program([v_mov_b32_e32(v[0], 0xBF800000), v_mov_b32_e32(v[1], 0x40000000), + v_mul_f32_e64(v[2], v[0], v[1], s[0], 0, 2, 0)], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x40000000) + + +class TestNaNPropagationRegressions(unittest.TestCase): + """Regression tests: float arithmetic propagates a NaN from the FIRST NaN operand, quieted with its own sign/payload.""" + + @unittest.skipIf(OSX, "broken on mac, TODO: why?") + def test_mul_nan_priority(self): + # first NaN operand wins (sign+payload), not x86's second-source propagation + for a, b, want in [(0x7FC00001, 0x7F800003, 0x7FC00001), (0xFFC00005, 0x7F800003, 0xFFC00005), + (0x7F800001, 0xFFC00005, 0x7FC00001), (0xFF9F1800, 0x7F800001, 0xFFDF1800)]: + st = run_program([v_mov_b32_e32(v[0], a), v_mov_b32_e32(v[1], b), + v_mul_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], want, f"mul({a:#x}, {b:#x})") + +class TestMinMaxFlushE64Regressions(unittest.TestCase): + """Regression tests: f32 min/max/median flush denormal inputs to signed zero (e64 forms).""" + + def test_v_min3_f32_denormal_flush(self): + st = run_program([v_mov_b32_e32(v[0], 0x00000001), v_mov_b32_e32(v[1], 0x3F800000), v_mov_b32_e32(v[2], 0x40000000), + v_min3_f32(v[3], v[0], v[1], v[2])], n_lanes=1) + self.assertEqual(st.vgpr[0][3], 0x00000000) # min(+denorm, 1, 2) = +0 + + def test_v_med3_f32_denormal_flush(self): + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0x3F800000), v_mov_b32_e32(v[2], 0x40000000), + v_med3_f32(v[3], v[0], v[1], v[2])], n_lanes=1) + self.assertEqual(st.vgpr[0][3], 0x3F800000) # med(-0, 1, 2) = 1 + if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/amd/hw/test_vopc.py b/tinygrad_repo/test/amd/hw/test_vopc.py index 5a7ed5e93..7e0f1d443 100644 --- a/tinygrad_repo/test/amd/hw/test_vopc.py +++ b/tinygrad_repo/test/amd/hw/test_vopc.py @@ -973,6 +973,71 @@ class TestCmpxPartialWavefront(unittest.TestCase): self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x4, "Only lane 2 should be active after v_cmpx_eq_u32_e64") +class TestClassDenormalRegressions(unittest.TestCase): + """Regression tests: V_CMP_CLASS classifies denormals as DENORMAL (raw bits), not as zero class.""" + + def test_class_pos_denormal(self): + for bits in (0x00000001, 0x007FFFFF): + instructions = [v_mov_b32_e32(v[0], bits), v_mov_b32_e32(v[1], 0x80), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 1, f"bits=0x{bits:08x}") # n_lanes=1 + # ...and it is not the zero class + instructions = [v_mov_b32_e32(v[0], bits), v_mov_b32_e32(v[1], 0x40), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0, f"bits=0x{bits:08x}") + + def test_class_neg_denormal(self): + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0x10), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 1) # n_lanes=1 + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0x20), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0) # not the negative-zero class + + +class TestIntCmpModRegressions(unittest.TestCase): + """Regression tests: int compares (i32/u32) honor abs/neg as bit-level sign clear/flip (not integer abs/negate).""" + + def test_cmp_i32_abs_neg_bit_level(self): + # abs(0x80000001) = 1 -> 1 > 1 is false (integer abs would give 2147483647 > 1) + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 1), v_cmp_gt_i32_e64(VCC_LO, v[0], v[1], abs=1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0) + # neg(0x80000001) flips the sign bit -> 1 > 2 is false (integer negate would give 2147483647 > 2) + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 2), v_cmp_gt_i32_e64(VCC_LO, v[0], v[1], neg=1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0) + + def test_cmp_u32_abs_bit_level(self): + # abs(0x80000000) = 0 -> 0 < 1 is true + instructions = [v_mov_b32_e32(v[0], 0x80000000), v_mov_b32_e32(v[1], 1), v_cmp_lt_u32_e64(VCC_LO, v[0], v[1], abs=1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 1) # n_lanes=1 + + +class TestCmpxSdstRegressions(unittest.TestCase): + """Regression tests: V_CMPX_*_E64 writes EXEC only, never SDST (hardware verified).""" + + def test_cmpx_e64_no_sdst(self): + instructions = [ + s_mov_b32(VCC_LO, 0), # preset VCC to 0 + v_mov_b32_e32(v[0], 0x3F800000), v_mov_b32_e32(v[1], 0x40000000), + v_cmpx_lt_f32_e64(VCC_LO, v[0], v[1]), # 1.0 < 2.0 + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.sgpr[EXEC_LO.offset], 0xFFFFFFFF) # EXEC updated + self.assertEqual(st.vcc, 0) # but VCC untouched + + def test_cmpx_e64_partial_exec(self): + instructions = [ + s_mov_b32(EXEC_LO, 0x0F0F0F0F), + s_mov_b32(VCC_LO, 0xFFFFFFFF), + v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0x3F800000), + v_cmpx_lt_f32_e64(VCC_LO, v[0], v[1]), + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.sgpr[EXEC_LO.offset], 0x0F0F0F0F) # EXEC = computed & old EXEC + if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/amd/test_asm_kernel.py b/tinygrad_repo/test/amd/test_asm_kernel.py index c7b6b036c..9bbb81e33 100644 --- a/tinygrad_repo/test/amd/test_asm_kernel.py +++ b/tinygrad_repo/test/amd/test_asm_kernel.py @@ -30,7 +30,7 @@ def custom_add_one(A:UOp) -> UOp: s_endpgm(), ] sink = UOp.sink(A.base, threads, arg=KernelInfo(f"custom_add_one_{A.numel()}", estimates=Estimates(ops=A.numel(), mem=A.numel()*4*2))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_add_var(A:UOp, B:UOp) -> UOp: A,B = A.flatten(), B.flatten() @@ -49,7 +49,7 @@ def custom_add_var(A:UOp, B:UOp) -> UOp: s_endpgm(), ] sink = UOp.sink(A.base, B.base, var, threads, arg=KernelInfo(f"custom_add_var_{A.numel()}")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_wave_sync(A:UOp, arch:str) -> UOp: # 4 waves across 1024 WG — enough to saturate a SIMD with many concurrent WGs @@ -63,7 +63,7 @@ def custom_wave_sync(A:UOp, arch:str) -> UOp: insts += [s_nop(0)]*4 insts.append(s_endpgm()) sink = UOp.sink(A.base, threads, wg, arg=KernelInfo("custom_wave_sync")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_lds_sync(A:UOp, arch:str) -> UOp: A = A.flatten() @@ -97,7 +97,7 @@ def custom_lds_sync(A:UOp, arch:str) -> UOp: isa.s_endpgm(), ] sink = UOp.sink(A.base, lds, threads, wg, arg=KernelInfo("custom_lds_sync")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_handwritten(A:UOp) -> UOp: A = A.flatten() @@ -143,7 +143,7 @@ def custom_handwritten(A:UOp) -> UOp: k.emit(r4.s_endpgm()) insts = k.finalize() sink = UOp.sink(A.base, threads, wg, lds, arg=KernelInfo("custom_handwritten")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_data_deps(A:UOp) -> UOp: A = A.flatten() @@ -159,7 +159,7 @@ def custom_data_deps(A:UOp) -> UOp: k.emit(s_endpgm()) insts = k.finalize() sink = UOp.sink(A.base, threads, arg=KernelInfo("custom_data_deps")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) @unittest.skipUnless(Device.DEFAULT == "AMD", "requires AMD device") class TestAsmKernel(unittest.TestCase): diff --git a/tinygrad_repo/test/amd/test_emu2_pcode.py b/tinygrad_repo/test/amd/test_emu2_pcode.py index 29dcfefdf..ff6366372 100644 --- a/tinygrad_repo/test/amd/test_emu2_pcode.py +++ b/tinygrad_repo/test/amd/test_emu2_pcode.py @@ -67,32 +67,26 @@ class TestParseExpr(unittest.TestCase): def test_integer_literals(self): """Test parsing integer literals.""" - self.assertEqual(parse_expr('0', {}).val, 0) - self.assertEqual(parse_expr('42', {}).val, 42) - self.assertEqual(parse_expr('42U', {}).val, 42) + self.assertIs(parse_expr('0', {}), UOp.const(0, dtypes.uint32)) + self.assertIs(parse_expr('42', {}), UOp.const(42, dtypes.uint32)) + self.assertIs(parse_expr('42U', {}), UOp.const(42, dtypes.uint32)) def test_negative_integers(self): """Test parsing negative integer literals.""" - result = parse_expr('-1', {}) - self.assertEqual(result.val, -1) - self.assertEqual(result.dtype, dtypes.int) + self.assertIs(parse_expr('-1', {}), UOp.const(-1, dtypes.int)) def test_float_literals(self): """Test parsing float literals.""" - result = parse_expr('1.0F', {}) - self.assertEqual(result.val, 1.0) - self.assertEqual(result.dtype, dtypes.float32) + self.assertIs(parse_expr('1.0F', {}), UOp.const(1.0, dtypes.float32)) def test_hex_literals(self): """Test parsing hex literals.""" - result = parse_expr('0xFF', {}) - self.assertEqual(result.val, 255) + self.assertIs(parse_expr('0xFF', {}), UOp.const(255, dtypes.uint32)) def test_variable_lookup(self): """Test variable lookup in parse_expr.""" vrs = {'x': UOp.const(42, dtypes.uint32)} - result = parse_expr('x', vrs) - self.assertEqual(result.val, 42) + self.assertIs(parse_expr('x', vrs), vrs['x']) def test_binary_ops(self): """Test parsing binary operations.""" @@ -103,9 +97,7 @@ class TestParseExpr(unittest.TestCase): self.assertEqual(result.op, Ops.ADD) # Subtraction with constant folding - result = parse_expr('10 - 5', {}) - self.assertEqual(result.op, Ops.CONST) - self.assertEqual(result.val, 5) + self.assertIs(parse_expr('10 - 5', {}), UOp.const(5, dtypes.uint32)) def test_ternary(self): """Test parsing ternary expressions.""" @@ -142,15 +134,8 @@ class TestForLoopParsing(unittest.TestCase): S0 = UOp.const(0, dtypes.uint32) _vrs, assigns = parse_pcode(pcode, {'S0': S0}) - # Check that the innermost value (default) is -1 (may be wrapped in CAST) - val = assigns[0][1] - # Traverse to innermost WHERE - while val.op == Ops.WHERE: - val = val.src[2] # false branch - # Unwrap CAST if present - while val.op == Ops.CAST: - val = val.src[0] - self.assertEqual(val.val, -1) + # every cond folds (S0 is a const), leaving the default branch: -1 in the destination dtype + self.assertIs(assigns[0][1].simplify(), UOp.const(-1, dtypes.uint32)) def test_ctz_parsing(self): """Test CTZ pcode parsing.""" @@ -167,7 +152,7 @@ class TestDSPcodePatterns(unittest.TestCase): def test_global_atomic_add_f32_parsing(self): """Test GLOBAL_ATOMIC_ADD_F32 keeps memory values in float dtype.""" - vmem = UOp.param(2, dtypes.uint32, (1024,)) + vmem = UOp.param(2, dtypes.uint32, 1024) srcs = { 'ADDR': UOp.const(0, dtypes.uint64), 'DATA': UOp.const(0x3f800000, dtypes.uint32), @@ -198,7 +183,7 @@ class TestDSPcodePatterns(unittest.TestCase): def test_mem_read_parsing(self): """Test MEM[addr].type read expression parsing.""" # Create a mock LDS buffer - lds = UOp.param(3, dtypes.uint32, (16384,)) + lds = UOp.param(3, dtypes.uint32, 16384) addr = UOp.const(0, dtypes.uint32) vrs = {'_lds': lds, 'ADDR': addr, 'OFFSET': UOp.const(0, dtypes.uint32)} @@ -233,7 +218,7 @@ class TestDSPcodePatterns(unittest.TestCase): pcode = PCODE.get(DSOp.DS_LOAD_2ADDR_B32) self.assertIsNotNone(pcode) assert pcode is not None - lds = UOp.param(3, dtypes.uint32, (16384,)) + lds = UOp.param(3, dtypes.uint32, 16384) srcs = { 'ADDR': UOp.const(0, dtypes.uint32), 'OFFSET0': UOp.const(0, dtypes.uint32), @@ -262,8 +247,8 @@ class TestDSPcodePatterns(unittest.TestCase): _, assigns = parse_pcode(pcode, srcs) # Check addresses: 100 + 2*4 = 108, 100 + 5*4 = 120 # assigns[i][1] is (addr, val) tuple for MEM writes; mypy sees UOp - self.assertEqual(assigns[0][1][0].simplify().val, 108) # type: ignore[index] - self.assertEqual(assigns[1][1][0].simplify().val, 120) # type: ignore[index] + self.assertIs(assigns[0][1][0].simplify(), UOp.const(108, dtypes.uint32)) # type: ignore[index] + self.assertIs(assigns[1][1][0].simplify(), UOp.const(120, dtypes.uint32)) # type: ignore[index] def test_ds_store_data_values(self): """Test DS_STORE_2ADDR_B32 uses correct data values.""" @@ -280,8 +265,8 @@ class TestDSPcodePatterns(unittest.TestCase): _, assigns = parse_pcode(pcode, srcs) # assigns[i][1] is (addr, val) tuple for MEM writes; mypy sees UOp # DATA[31:0] should preserve the value - self.assertEqual(assigns[0][1][1].simplify().val, 0xAAAAAAAA) # type: ignore[index] - self.assertEqual(assigns[1][1][1].simplify().val, 0xBBBBBBBB) # type: ignore[index] + self.assertIs(assigns[0][1][1].simplify(), UOp.const(0xAAAAAAAA, dtypes.uint32)) # type: ignore[index] + self.assertIs(assigns[1][1][1].simplify(), UOp.const(0xBBBBBBBB, dtypes.uint32)) # type: ignore[index] class TestConditionalParsing(unittest.TestCase): """Test conditional (if/elsif/else) pcode parsing.""" @@ -306,15 +291,15 @@ class TestConcatWidthParsing(unittest.TestCase): def test_permlanex16_altrow_concat(self): for row, expected in [(0, 1), (1, 0), (2, 3), (3, 2)]: parsed = parse_expr('{ row[1], ~row[0] }', {'row': UOp.const(row, dtypes.uint32)}) - self.assertEqual(parsed.simplify().val, expected) + self.assertIs(parsed.simplify(), UOp.const(expected, dtypes.uint32)) def test_permlane64_altlane_concat(self): for lane, expected in [(0, 32), (1, 33), (31, 63), (32, 0), (63, 31)]: parsed = parse_expr('{ ~lane[5], lane[4:0] }', {'lane': UOp.const(lane, dtypes.uint32)}) - self.assertEqual(parsed.simplify().val, expected) + self.assertIs(parsed.simplify(), UOp.const(expected, dtypes.uint32)) def test_permlane64_wave64_pcode_indices(self): - vgpr = UOp.param(0, dtypes.uint32, (256,)) + vgpr = UOp.param(0, dtypes.uint32, 256) srcs = { 'SRC0': UOp.const(0, dtypes.uint32), 'VDST': UOp.const(1, dtypes.uint32), @@ -327,19 +312,17 @@ class TestConcatWidthParsing(unittest.TestCase): 'S2': UOp.const(0, dtypes.uint32), } - def load_idx(v: UOp) -> int: + def check_load_idx(v: UOp, expected: int): simp = v.simplify() self.assertEqual(simp.op, Ops.LOAD) self.assertEqual(simp.src[0].op, Ops.INDEX) - idx = simp.src[0].src[1].simplify() - self.assertEqual(idx.op, Ops.CONST) - return idx.val + self.assertIs(simp.src[0].src[1].simplify(), UOp.const(expected, dtypes.uint32)) _, assigns = parse_pcode(PCODE[VOP1Op.V_PERMLANE64_B32_E32], srcs) self.assertEqual(len(assigns), 64) for lane, (dst_idx, src_idx) in {0: (64, 32), 31: (95, 63), 32: (96, 0), 63: (127, 31)}.items(): - self.assertEqual(assigns[lane][1][0].simplify().val, dst_idx) # type: ignore[index] - self.assertEqual(load_idx(assigns[lane][1][1]), src_idx) # type: ignore[index] + self.assertIs(assigns[lane][1][0].simplify(), UOp.const(dst_idx, dtypes.uint32)) # type: ignore[index] + check_load_idx(assigns[lane][1][1], src_idx) # type: ignore[index] class TestAllPcode(unittest.TestCase): """Test that all pcode from all architectures can be parsed.""" @@ -347,7 +330,7 @@ class TestAllPcode(unittest.TestCase): def _make_srcs(self): """Create dummy source variables for pcode parsing.""" u32, u64 = lambda v=0: UOp.const(v, dtypes.uint32), lambda v=0: UOp.const(v, dtypes.uint64) - lds = UOp.param(3, dtypes.uint32, (16384,)) + lds = UOp.param(3, dtypes.uint32, 16384) return {'laneId': u32(), 'laneID': u32(), 'S0': u32(), 'S1': u32(), 'S2': u32(), 'S3': u32(), 'SRC0': u32(), 'D0': u32(), 'D1': u32(), 'DST': u32(), 'VDST': u32(), 'SDST': u32(), 'VCC': u64(), 'VCCZ': u32(), 'EXEC': u64(), 'EXEC_LO': u32(), 'EXECZ': u32(), 'SCC': u32(), diff --git a/tinygrad_repo/test/amd/test_sqtt_examples.py b/tinygrad_repo/test/amd/test_sqtt_examples.py index fd1820e11..f8326c2f9 100644 --- a/tinygrad_repo/test/amd/test_sqtt_examples.py +++ b/tinygrad_repo/test/amd/test_sqtt_examples.py @@ -88,7 +88,6 @@ def run_rocprof_decoder(blobs: list[bytes], lib: bytes, base: int, target: str): if t.is_alive(): raise RuntimeError("rocprof decoder timeout") return occupancy_records, wave_insts -@unittest.skip("TODO: fix to not require unpickling UOps.") class SQTTExamplesTestBase(unittest.TestCase): target: str examples: dict diff --git a/tinygrad_repo/test/amd/test_sqtt_profiler.py b/tinygrad_repo/test/amd/test_sqtt_profiler.py index 3738bcfcc..45c23664f 100644 --- a/tinygrad_repo/test/amd/test_sqtt_profiler.py +++ b/tinygrad_repo/test/amd/test_sqtt_profiler.py @@ -1,30 +1,28 @@ import unittest, contextlib from tinygrad import Device, Tensor, Context, TinyJit -from tinygrad.device import Compiled, ProfileProgramEvent, ProfileDeviceEvent +from tinygrad.device import Compiled, ProfileProgramEvent from tinygrad.engine.realize import run_linear from tinygrad.codegen import to_program from tinygrad.viz.serve import load_amd_counters, VizData @contextlib.contextmanager def save_sqtt(): + Device[Device.DEFAULT].synchronize() + profile_start = len(Compiled.profile_events) data = VizData() yield data.ctxs Device[Device.DEFAULT].synchronize() Device[Device.DEFAULT]._at_profile_finalize() - load_amd_counters(data, Compiled.profile_events) + load_amd_counters(data, [e for e in Compiled.profile_events[:profile_start] if isinstance(e, ProfileProgramEvent)] + + Compiled.profile_events[profile_start:]) data.ctxs[:] = [r for r in data.ctxs if r["name"].startswith("SQTT")] @unittest.skipUnless(Device.DEFAULT == "AMD", "only runs on AMD") class TestSQTTProfiler(unittest.TestCase): - # TODO: can we enable SQTT profiling in context? @classmethod def setUpClass(cls): if not Device[Device.DEFAULT].sqtt_enabled: raise unittest.SkipTest("device must be in SQTT profiling mode") - def setUp(self): - Device[Device.DEFAULT].synchronize() - Compiled.profile_events[:] = [e for e in Compiled.profile_events if isinstance(e, (ProfileProgramEvent, ProfileDeviceEvent))] - def test_simple(self): t = Tensor.empty(1) + 1 with save_sqtt() as sqtt: diff --git a/tinygrad_repo/test/backend/test_asm_gemm.py b/tinygrad_repo/test/backend/test_asm_gemm.py index e18458f80..594de42b2 100644 --- a/tinygrad_repo/test/backend/test_asm_gemm.py +++ b/tinygrad_repo/test/backend/test_asm_gemm.py @@ -1,4 +1,5 @@ import unittest +import functools from tinygrad import Tensor, Device, dtypes, Context from tinygrad.helpers import getenv, system, DEV from extra.gemm.cdna_asm_gemm import asm_gemm, hk_bf16_atb_gemm @@ -9,6 +10,7 @@ from examples.mlperf.models.flat_llama import FP8_DTYPE, quantize_fp8, FP8_MAX # Use DEV=NULL:HIP:gfx950 to also test the assembly def is_cdna4(): return Device[Device.DEFAULT].renderer.target.arch.startswith("gfx950") +@functools.cache def has_hipcc(): try: system("hipcc --version") except Exception: return False @@ -186,7 +188,7 @@ class TestMXFP4(unittest.TestCase): M, N, K = getenv("M", 16384), getenv("N", 4096), getenv("K", 14336) a = Tensor.empty(M, K, dtype=dtypes.bfloat16) b = Tensor.empty(N, K, dtype=dtypes.bfloat16) - asm_gemm(a, b.T, mxfp4=True).realize() + for _ in range(getenv("CNT", 1)): asm_gemm(a, b.T, mxfp4=True).realize() # test the Asm GEMM with Llama shapes, only run on the real machine for speed diff --git a/tinygrad_repo/test/unit/test_assign.py b/tinygrad_repo/test/backend/test_assign.py similarity index 91% rename from tinygrad_repo/test/unit/test_assign.py rename to tinygrad_repo/test/backend/test_assign.py index 7847ee95b..d10d8888c 100644 --- a/tinygrad_repo/test/unit/test_assign.py +++ b/tinygrad_repo/test/backend/test_assign.py @@ -1,10 +1,10 @@ #!/usr/bin/env python import unittest import numpy as np -from tinygrad import dtypes, Tensor, TinyJit, GlobalCounters, Variable +from tinygrad import Device, dtypes, Tensor, TinyJit, GlobalCounters, Variable from tinygrad.uop.ops import Ops, UOp from tinygrad.helpers import temp, DEV, Context -from test.helpers import assert_kernel_count +from test.helpers import assert_kernel_count, needs_second_gpu N = 200 # has to be bigger than the cache to fail @@ -540,6 +540,10 @@ class TestAssign(unittest.TestCase): c = Tensor([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32).realize() c[0:2].bitcast(dtypes.uint32).assign(Tensor([0x40800000, 0x40400000], dtype=dtypes.uint32)).realize() np.testing.assert_allclose(c.numpy(), [4.0, 3.0, 3.0, 4.0]) + # without .realize() + a = Tensor([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32).realize() + a.bitcast(dtypes.uint32).assign(Tensor([0x40800000, 0x40400000, 0x40000000, 0x3f800000], dtype=dtypes.uint32)) + np.testing.assert_allclose(a.numpy(), [4.0, 3.0, 2.0, 1.0]) def test_assign_bitcast_different_size(self): # assign to a shape-changing bitcast view (only works on DISK currently) @@ -1075,5 +1079,80 @@ class TestBatchNormRunningStats(unittest.TestCase): with Context(TRAINING=1): bn(x).realize() self.assertTrue(bn.running_mean.uop.base.is_realized) +class TestMultiAssign(unittest.TestCase): + device = tuple(f"{Device.DEFAULT}:{i}" for i in range(2)) + + @needs_second_gpu + def setUp(self): pass + + def test_multi_assign_realized(self): + out = Tensor.zeros(4).shard(self.device, 0).contiguous().realize() + ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(ones).realize() + self.assertListEqual(out.tolist(), [1,1,1,1]) + + def test_multi_assign_unrealized(self): + out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) + ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(ones).realize() + self.assertListEqual(out.tolist(), [1,1,1,1]) + + def test_multi_assign_both_unrealized(self): + out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) + ones = Tensor.ones(4).contiguous().realize().shard(self.device, 0) + out.assign(ones).realize() + self.assertListEqual(out.tolist(), [1,1,1,1]) + + def test_multi_assign_scalar(self): + out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(0).realize() + self.assertListEqual(out.tolist(), [0,0,0,0]) + + def test_multi_assign_const_like(self): + out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(out.const_like(7)).realize() + self.assertListEqual(out.tolist(), [7,7,7,7]) + + def test_multi_assign_piece(self): + out = Tensor.zeros(4,4).shard(self.device, 0).contiguous().realize() + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + out[:, 2:3].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + def test_multi_assign_piece_noncontig(self): + out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + out[:, 2:3].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + @unittest.expectedFailure + def test_multi_assign_piece_unrealized(self): + out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0) + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + out[:, 2:3].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + def test_multi_assign_var_offset(self): + out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + vi = Variable("i", 0, 3).bind(2) + out[:, vi:vi+1].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + def test_multi_assign_var_offset_jit_none(self): self.test_multi_assign_var_offset_jit(None) + def test_multi_assign_var_offset_jit(self, shard_axis=0): + out = Tensor.zeros(4,6).contiguous().realize().shard(self.device, shard_axis).realize() + ones = Tensor.ones(4,1).shard(self.device, shard_axis).contiguous().realize() + + @TinyJit + def f(out:Tensor, vi): + out[:, vi:vi+1].assign(ones).realize() + ones.assign(ones+1).realize() + + vi = Variable("i", 0, 5) + for i in range(1,5): + GlobalCounters.reset() + f(out, vi.bind(i)) + self.assertListEqual(out.tolist(), [[0,1,2,3,4,0]]*4) if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/backend/test_const_folding.py b/tinygrad_repo/test/backend/test_const_folding.py index 0e4239ecf..fc8ad9f0a 100644 --- a/tinygrad_repo/test/backend/test_const_folding.py +++ b/tinygrad_repo/test/backend/test_const_folding.py @@ -1,7 +1,7 @@ import unittest, math from tinygrad import Tensor, Device, dtypes from tinygrad.dtype import DTYPES_DICT -from tinygrad.uop.ops import Ops, UOp +from tinygrad.uop.ops import Ops, UOp, GroupOp from tinygrad.codegen.decomp.op import threefry2x32 import numpy as np from test.helpers import not_support_multi_device @@ -17,7 +17,7 @@ def _check_ast_count(desired_count:int, t:Tensor): class TestMovedConstFolding(unittest.TestCase): def test_contiguous_deviceless_const(self): t = Tensor(UOp.const(2.0, dtypes.float)).contiguous() - self.assertIs(t.uop.op, Ops.CONST) + self.assertIs(t.uop, UOp.const(2.0, dtypes.float)) self.assertIsNone(t.uop.device) def test_add_shrunk_zero(self): @@ -169,8 +169,8 @@ class TestMultiConstFolding(unittest.TestCase): class TestThreefryConstFolding(unittest.TestCase): def test_threefry(self): # THREEFRY(const,const) folds to a const once decomposed - x = threefry2x32(UOp.const(5, dtypes.uint64), UOp.const(10, dtypes.uint64)) - self.assertIs(x.simplify().op, Ops.CONST) + x = threefry2x32(UOp.const(5, dtypes.uint64), UOp.const(10, dtypes.uint64)).simplify() + self.assertEqual([u.op for u in x.toposort() if u.op in GroupOp.ALU], []) class TestTautologicalCompare(unittest.TestCase): # without const folding, these would have triggered -Wtautological-compare in clang @@ -188,7 +188,6 @@ class TestTautologicalCompare(unittest.TestCase): np.testing.assert_equal((Tensor(True) < Tensor(False)).numpy(), False) np.testing.assert_equal((Tensor(True) < Tensor(True)).numpy(), False) - @unittest.skipIf(Device.DEFAULT == "WEBGPU", "WEBGPU doesn't support NaN comparison correctly") def test_a_eq_a(self): # self eq is always true for int or bool a = Tensor([1, 2, 3]) diff --git a/tinygrad_repo/test/backend/test_custom_kernel.py b/tinygrad_repo/test/backend/test_custom_kernel.py index f92a253a7..396508578 100644 --- a/tinygrad_repo/test/backend/test_custom_kernel.py +++ b/tinygrad_repo/test/backend/test_custom_kernel.py @@ -4,7 +4,7 @@ import numpy as np from tinygrad.dtype import AddrSpace, dtypes, Invalid from tinygrad.uop.ops import KernelInfo, AxisType, Ops from tinygrad.renderer.ptx import PTXRenderer -from test.helpers import assert_kernel_count +from test.helpers import assert_kernel_count, KernelCountException # **** kernels **** @@ -347,7 +347,7 @@ class TestCustomKernel(unittest.TestCase): self.assertTrue((c == 2).all().item()) def test_partial_invalid_store_keeps_uncovered_reads(self): - x = Tensor([10., 20., 30., 40.]) + x = Tensor([10., 20., 30., 40.]).realize() after = x.uop.after(x.uop.shrink(((0, 2),)).store(Invalid)) self.assertEqual(Tensor(after).contiguous().tolist(), [10., 20., 30., 40.]) @@ -422,9 +422,8 @@ class TestCustomKernel(unittest.TestCase): return Tensor.custom_kernel(y, x, fxn=custom_add_one_kernel)[0] GlobalCounters.reset() y = run(x[0]).realize() - # it's copying the input and the output - # TODO: subbuffer usage has runtime specific behavior, this will be fixed after the removal of SLICE. - assert_kernel_count(2 if y.device in ("CL", "WEBGPU") else 1) + # backends that support contiguous views don't launch extra kernels + assert_kernel_count(2 if x[0].uop.contiguous_view() is None else 1) self.assertEqual(y.tolist(), [1, 2, 3, 4]) @Context(DEV="CPU") @@ -475,7 +474,7 @@ class TestCustomKernelInput(unittest.TestCase): y.realize() kernel_count = GlobalCounters.kernel_count self.assertEqual(y.tolist(), x.add(1).tolist()) - self.assertLessEqual(kernel_count, max_kernels) + if kernel_count > max_kernels: raise KernelCountException(max_kernels, kernel_count) # same test with @function, input is PARAM from tinygrad import function x0 = Tensor.arange(32).clone("CPU").realize() @@ -488,7 +487,7 @@ class TestCustomKernelInput(unittest.TestCase): y = run(x0).realize() kernel_count = GlobalCounters.kernel_count self.assertEqual(y.tolist(), mop_fxn(x0).add(1).tolist()) - self.assertLessEqual(kernel_count, max_kernels) + if kernel_count > max_kernels: raise KernelCountException(max_kernels, kernel_count) def test_reshape(self): self._test_mop(lambda x: x.reshape(16, 2), max_kernels=2) def test_permute(self): self._test_mop(lambda x: x.reshape(4, 8).T, max_kernels=3) diff --git a/tinygrad_repo/test/backend/test_dtype.py b/tinygrad_repo/test/backend/test_dtype.py index 594ebd0e2..27de82721 100644 --- a/tinygrad_repo/test/backend/test_dtype.py +++ b/tinygrad_repo/test/backend/test_dtype.py @@ -258,6 +258,11 @@ class TestDoubleDType(TestDType): a = [2, 3, 4] np.testing.assert_allclose(func(Tensor(a, dtype=self.DTYPE)).numpy(), func(torch.tensor(a, dtype=torch.float64)), rtol=1e-12, atol=1e-12) + def test_float32_compare_selecting_float64(self): + a = Tensor([1.0, 2.0, 5.0, 9.0], dtype=dtypes.float32) + p, q = Tensor([10., 20., 30., 40.], dtype=self.DTYPE), Tensor([50., 60., 70., 80.], dtype=self.DTYPE) + _test_op(lambda: (a < 3.0).where(p, q), self.DTYPE, [10., 20., 70., 80.]) + def test_float64_to_float32_cast_inf(self): _test_op(lambda: Tensor([3.4e40, 3.4e38, 1, 0], dtype=dtypes.float64).cast(dtypes.float32), dtypes.float32, [float('inf'), 3.4e38, 1, 0]) @@ -323,7 +328,10 @@ class TestUint16DType(TestDType): class TestInt32DType(TestDType): DTYPE = dtypes.int32 class TestUint32DType(TestDType): DTYPE = dtypes.uint32 -class TestInt64DType(TestDType): DTYPE = dtypes.int64 +class TestInt64DType(TestDType): + DTYPE = dtypes.int64 + def test_int64_to_uint32_to_int64(self): + _test_op(lambda: Tensor([0x12345678ABCDEF01], dtype=dtypes.int64).cast(dtypes.uint32).cast(dtypes.int64), dtypes.int64, [2882400001]) @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "PTX does indexing math with longs") class TestEmulatedInt64DType(TestInt64DType): @@ -340,6 +348,9 @@ class TestUint64DType(TestDType): DTYPE = dtypes.uint64 def test_uint64_load(self): assert Tensor(2**64 - 1, dtype=dtypes.uint64).numpy() == 2**64 - 1 + @unittest.skipIf(dtypes.double not in supported_dtypes, "needs float64") + def test_uint64_cast_double(self): + assert Tensor([2**32 + 1], dtype=dtypes.uint64).cast(dtypes.double).numpy() == 2**32 + 1 @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "PTX does indexing math with longs") class TestEmulatedUInt64DType(TestUint64DType): @@ -420,6 +431,11 @@ class TestDtypeUsage(unittest.TestCase): t = Tensor([[1, 2], [3, 4]], dtype=d) (t*t).max().item() + def test_where_float16_compare_to_const(self): + # t > 0 is CMPLT(0, t): the float16 operand is on the right + t = Tensor([-1.0, 1.0], dtype=dtypes.float16) + np.testing.assert_equal((t > 0).where(Tensor.ones(2, dtype=dtypes.float16), Tensor.zeros(2, dtype=dtypes.float16)).numpy(), [0.0, 1.0]) + @unittest.skipUnless(dtypes.bfloat16 in supported_dtypes, f"no bfloat16 on {Device.DEFAULT}") class TestOpsBFloat16(unittest.TestCase): def test_cast(self): diff --git a/tinygrad_repo/test/backend/test_dtype_alu.py b/tinygrad_repo/test/backend/test_dtype_alu.py index 6a182616c..d55e51807 100644 --- a/tinygrad_repo/test/backend/test_dtype_alu.py +++ b/tinygrad_repo/test/backend/test_dtype_alu.py @@ -6,6 +6,8 @@ from tinygrad.tensor import _to_np_dtype from tinygrad.runtime.ops_python import from_storage_scalar from tinygrad.renderer.ptx import PTXRenderer from tinygrad.renderer.nir import NIRRenderer +from tinygrad.renderer.llvmir import CPULLVMRenderer +from tinygrad.renderer.isa.x86 import X86Renderer from tinygrad.uop import Ops import numpy as np import pytest @@ -64,6 +66,8 @@ ht.fp8e5m2fnuz = ht.uint8 def universal_test(a, b, dtype, op): if not isinstance(op, tuple): op = (op, op) if op[0] == operator.mod and b == 0: return + # TODO: throws floating point exception + if isinstance(Device[Device.DEFAULT].renderer, (X86Renderer, CPULLVMRenderer)) and op[0] == operator.mod and a == dtype.min and b == -1: return # lt and max with nan is undefined in tinygrad if op[0] in (operator.lt, Tensor.maximum) and (math.isnan(a) or math.isnan(b)): return ta, tb = Tensor([a], dtype=dtype), Tensor([b], dtype=dtype) diff --git a/tinygrad_repo/test/backend/test_encodings.py b/tinygrad_repo/test/backend/test_encodings.py index 38e1373be..01423a0ee 100644 --- a/tinygrad_repo/test/backend/test_encodings.py +++ b/tinygrad_repo/test/backend/test_encodings.py @@ -4,7 +4,7 @@ from tinygrad.uop.ops import UOp, Ops from tinygrad.dtype import dtypes from tinygrad.renderer.isa.x86 import X86Ops, X86Renderer, RBP, RDI, RSP, RSI, RAX, RDX, XMM, GPR, imm, def_reg -def ins(op, dt, src, tag=None): return UOp(Ops.INS, arg=op, dtype=dt, src=src, tag=tag) +def ins(op, dt, src, tag=None): return UOp(Ops.INS, arg=(op, dt), src=src, tag=tag) @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, X86Renderer), "only on x86") class TestEncodingsX86(unittest.TestCase): @@ -100,13 +100,6 @@ class TestEncodingsX86(unittest.TestCase): # vaddss xmm0, xmm0, xmm8 self.assertEqual(bytes.fromhex(self.encode(add)), bytes.fromhex("C4 C1 7A 58 C0")) - # test ymm encoding - def test_ymm_encoding(self): - xmm0, xmm1 = def_reg(dtypes._uint256, XMM[0]), def_reg(dtypes._uint256, XMM[1]) - add = ins(X86Ops.VADDPS, dtypes._uint256, (xmm0, xmm1), XMM[0]) - # vaddps ymm0, ymm0, ymm1 - self.assertEqual(bytes.fromhex(self.encode(add)), bytes.fromhex("C5 FC 58 C1")) - # test encoding where register is in the immediate field def test_reg_in_imm_field(self): xmm0, xmm1, xmm2 = def_reg(dtypes.float32, XMM[0]), def_reg(dtypes.float32, XMM[1]), def_reg(dtypes.float32, XMM[2]) @@ -143,7 +136,7 @@ class TestEncodingsX86(unittest.TestCase): # cmoves have the cmp as the last src even though it is not explicitly used, the cmp doesn't define a reg and is ignored in the encoding def test_cmove_ignore_cmp(self): - cmove = ins(X86Ops.CMOVE, dtypes.int32, (def_reg(dtypes.int32, RAX), UOp(Ops.INS, arg=X86Ops.CMP)), RDX) + cmove = ins(X86Ops.CMOVE, dtypes.int32, (def_reg(dtypes.int32, RAX), UOp(Ops.INS, arg=(X86Ops.CMP, dtypes.void))), RDX) # cmove edx, eax self.assertEqual(bytes.fromhex(self.encode(cmove)), bytes.fromhex("0F 44 D0")) diff --git a/tinygrad_repo/test/backend/test_isel.py b/tinygrad_repo/test/backend/test_isel.py index 6965a5db1..edb037d90 100644 --- a/tinygrad_repo/test/backend/test_isel.py +++ b/tinygrad_repo/test/backend/test_isel.py @@ -7,7 +7,7 @@ from tinygrad.renderer.isa.x86 import X86Renderer, X86Ops from tinygrad.renderer.isa import IselContext # INDEX on a register value with a constant index extracts a single element (the old GEP) -def lane(y:UOp, i:int) -> UOp: return y.index(UOp.const(i, dtypes.int), dtype=y.dtype.scalar()) +def lane(y:UOp, i:int) -> UOp: return y.index(UOp.cconst(i, dtypes.int)) @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, X86Renderer), "only x86") class TestIselX86(unittest.TestCase): @@ -20,7 +20,7 @@ class TestIselX86(unittest.TestCase): with self.subTest(dtype=dt): v = [UOp.variable(str(i), 0, 0, dt) for i in range(nargs)] n = self.isel_rewrite(expr(*v)) - self.assertIs(n.arg, op) + self.assertIs(n.arg[0], op) def test_cmove(self): a = UOp.variable("a", 0, 0, dtypes.int32) @@ -29,9 +29,9 @@ class TestIselX86(unittest.TestCase): d = (a != b).where(a, b) f = c + d n = self.isel_rewrite(f) - self.assertTrue(n.src[0].arg is X86Ops.CMOVL and n.src[1].arg is X86Ops.CMOVNE) + self.assertTrue(n.src[0].arg[0] is X86Ops.CMOVL and n.src[1].arg[0] is X86Ops.CMOVNE) # both comparisons become the same instruction - self.assertTrue(n.src[0].src[2] == n.src[1].src[2] and n.src[0].src[2].arg is X86Ops.CMP) + self.assertTrue(n.src[0].src[2] == n.src[1].src[2] and n.src[0].src[2].arg[0] is X86Ops.CMP) def test_vinsertps(self): a = UOp.variable("a", 0, 0, dtypes.float32) @@ -41,15 +41,15 @@ class TestIselX86(unittest.TestCase): valid = [UOp.stack(lane(a, 0), lane(b, 1), lane(a, 2), lane(b, 3)), UOp.stack(lane(a, 3), lane(b, 2), lane(c, 1), d)] - for shuf in valid: self.assertIs(self.isel_rewrite(shuf).arg, X86Ops.VINSERTPS) + for shuf in valid: self.assertIs(self.isel_rewrite(shuf).arg[0], X86Ops.VINSERTPS) # complex address is [base + index*scale + displacement] def test_complex_address(self): a = UOp.variable("a", 0, 0, dtypes.int32) - load = UOp.param(0, dtypes.int32, (16,)).index(a + 1).load() + load = UOp.param(0, dtypes.int32, 16).index(a + UOp.cconst(1, dtypes.int32)).load() n = self.isel_rewrite(load) # displacement is the constant in "a" scaled to the buffer element size, dtype is int8 when the value fits otherwise int32 - self.assertTrue(n.src[2].op is Ops.CONST and n.src[2].dtype is dtypes.int8 and n.src[2].val == 4) + self.assertTrue(n.src[2].dtype is dtypes.int8 and n.src[2].src[0].op is Ops.CONST and n.src[2].src[0].val == 4) if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/backend/test_jit.py b/tinygrad_repo/test/backend/test_jit.py index 997d176ee..920cfa2f3 100644 --- a/tinygrad_repo/test/backend/test_jit.py +++ b/tinygrad_repo/test/backend/test_jit.py @@ -4,9 +4,9 @@ import numpy as np from test.helpers import assert_jit_cache_len, call_is_graph, not_support_multi_device, needs_second_gpu, KernelCountException from test.unit.test_jit import _simple_test -from tinygrad import Tensor, Variable, TinyJit, Device, dtypes +from tinygrad import Tensor, TinyJit, Device, dtypes from tinygrad.engine.jit import graph_class -from tinygrad.helpers import JIT, DEV, GlobalCounters +from tinygrad.helpers import JIT, DEV, GlobalCounters, HCQ2 from tinygrad.uop.ops import Ops from tinygrad.renderer.isa.x86 import X86Renderer @@ -16,19 +16,6 @@ class TestJit(unittest.TestCase): def add(a, b): return (a+b).realize() _simple_test(add) - @unittest.skipUnless(Device.DEFAULT == "CPU", "core_id is a CPU runtimevar") - def test_hcq_core_id_runtimevar_merge(self): - N = 262144 - @TinyJit - def f(x, st): - y = (x + 1).contiguous().realize() - z = x.shrink(((st, st + N),)).contiguous().realize() - return y, z - x = Tensor.arange(2*N).clone().realize() - for _ in range(3): y, z = f(x, Variable("a", 0, N).bind(0)) - self.assertEqual(y.shape, (2*N,)) - self.assertEqual(z.shape, (N,)) - def test_jit_input_view(self): @TinyJit def f(x): return (x[2:5].contiguous() + 1).realize() @@ -235,6 +222,7 @@ class TestJitPrune(unittest.TestCase): assert_jit_cache_len(w2_prune, 1) class TestJitFree(unittest.TestCase): + @unittest.skipIf(HCQ2, "hcq2 keeps refs to intermediate buffers") def test_free_intermediates(self): ext_tensor = Tensor([1,24,23,45,1]) @TinyJit @@ -360,7 +348,7 @@ class TestJitGraphSplit(unittest.TestCase): self.expect(f, inp, inp_cpu, graph=[self.ji_graph(2), self.ji_comp(), self.ji_comp()], multigraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp()], - hcqgraph=[self.ji_graph(4)]) + hcqgraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp()]) # cpu is hcq2 now, it does not join hcq graphs def test_jit_cpu_several(self): if Device.DEFAULT == "CPU": raise unittest.SkipTest("CPU is not a valid default device for this test") @@ -377,9 +365,9 @@ class TestJitGraphSplit(unittest.TestCase): inp = Tensor.randn(10, 10, device=Device.DEFAULT).realize() inp_cpu = Tensor.randn(10, 10, device="CPU").realize() self.expect(f, inp, inp_cpu, - graph=[self.ji_graph(2), self.ji_graph(2), self.ji_comp()], - multigraph=[self.ji_graph(2), self.ji_graph(2), self.ji_comp()], - hcqgraph=[self.ji_graph(5)]) + graph=[self.ji_graph(2), self.ji_comp(), self.ji_comp(), self.ji_comp()], + multigraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp(), self.ji_comp()], + hcqgraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp(), self.ji_comp()]) def test_jit_multidev(self): if Device.DEFAULT == "CPU": raise unittest.SkipTest("CPU is not a valid default device for this test") diff --git a/tinygrad_repo/test/backend/test_linearizer.py b/tinygrad_repo/test/backend/test_linearizer.py index 958a9be84..82e79d860 100644 --- a/tinygrad_repo/test/backend/test_linearizer.py +++ b/tinygrad_repo/test/backend/test_linearizer.py @@ -7,7 +7,7 @@ from tinygrad.device import Device, Buffer from tinygrad.tensor import Tensor, _to_np_dtype from tinygrad.engine.realize import run_linear from tinygrad.codegen import to_program -from tinygrad.helpers import Context, flatten, dedup, TC_SELECT, TC_OPT, DEV +from tinygrad.helpers import Context, dedup, TC_SELECT, TC_OPT, DEV from tinygrad.dtype import DType, dtypes, AddrSpace from tinygrad.renderer.ptx import PTXRenderer from tinygrad.renderer.cstyle import CUDARenderer @@ -16,8 +16,6 @@ from test.helpers import replace_opts, check_schedule from test.backend.test_softmax_fusion import single_kernel_softmax MOCKGPU = DEV.interface.startswith("MOCK") -from tinygrad.uop.render import print_uops # noqa: F401 # pylint: disable=unused-import - @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, ISARenderer), "isa backends don't preserve the op spec when lowering") class TestLinearizer(unittest.TestCase): def test_arg_dedup(self): @@ -30,7 +28,7 @@ class TestLinearizer(unittest.TestCase): c = ((a.shrink(((0, 2),)) - a.shrink(((2, 4),))) - (b.shrink(((0, 2),)) - b.shrink(((2, 4),)))) linear = c.schedule_linear() run_linear(linear) - rawbufs = [s.buffer for s in linear.src[-1].src[1:] if s.op is not Ops.BIND] + rawbufs = [s.buffer for s in linear.src[-1].src[1:] if not s.is_bound_var] assert len(rawbufs) == 3 and set(rawbufs[1:]) == {a.uop.base.realized, b.uop.base.realized} np_c = (np_a[:2] - np_a[2:]) - (np_b[:2] - np_b[2:]) np.testing.assert_allclose(np_c, c.numpy(), atol=1e-4, rtol=1e-4) @@ -75,14 +73,6 @@ class TestLinearizer(unittest.TestCase): # assert that there is a global load after the reduce ends assert any(u.addrspace == AddrSpace.GLOBAL for u in load_idxs) - def _test_no_nested_ranges(self, lins, skip=None): - for l in lins: - range_in_acc = flatten([[x for x in u.src if x.op is Ops.RANGE] for u in l.uops if u.op is Ops.BUFFER and u.addrspace is AddrSpace.REG]) - ranges = [u.op for u in l.uops if (u.op is Ops.RANGE and u in range_in_acc) or (u.op is Ops.END and u.src[0] in range_in_acc)] - for i,u in enumerate(ranges): - if skip and i in skip: continue - assert ranges[i-1] != u, f"multireduce nested the ranges! {ranges[i-1], {u}}" - def test_two_nested_range(self): a = Tensor.randn(2, ).realize() out = a.reshape(2, 1).expand(2, 3).sum() @@ -137,7 +127,7 @@ class TestLinearizer(unittest.TestCase): # these are of size 3 to avoid float4 coalesce r = a[:-1] + a[1:] - uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=0)]), + uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(0, AxisType.UPCAST))]), renderer=Device[Device.DEFAULT].renderer).src[1].src) num_loads = len([uop for uop in uops if uop.op is Ops.LOAD]) assert num_loads <= 4, "more load uops than needed" @@ -150,7 +140,7 @@ class TestLinearizer(unittest.TestCase): a, b = Tensor.randn(1).realize(), Tensor.randn(1).realize() r = a.expand([2]) + b.expand([2]) - uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=0)]), + uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(0, AxisType.UPCAST))]), renderer=Device[Device.DEFAULT].renderer).src[1].src) num_ops = len([uop for uop in uops if uop.op in GroupOp.ALU]) assert num_ops <= 1, "more alu uops than needed" @@ -161,7 +151,8 @@ class TestLinearizer(unittest.TestCase): r = Tensor.conv2d(x,w,padding=1).relu() uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], - [Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.UNROLL, axis=0, arg=0)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + [Opt(op=OptOps.SPLIT, axis=0, arg=(0, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=1, arg=(0, AxisType.UNROLL))]), renderer=Device[Device.DEFAULT].renderer).src[1].src) accs = [u for u in uops if u.op is Ops.BUFFER and u.addrspace is AddrSpace.REG] stores = [u for u in uops if u.op is Ops.STORE] assert len(accs) == 0 # it's removed now @@ -172,7 +163,7 @@ class TestLinearizer(unittest.TestCase): @unittest.skipUnless(Device.DEFAULT == "CPU", "test only for CPU") def test_upcast_with_locals_cpu(self): out = Tensor.ones(64,64).contiguous() @ Tensor.ones(64,64).contiguous() - prg = to_program(replace_opts(out.schedule_linear().src[-1].src[0], [Opt(OptOps.LOCAL, axis=0, arg=4)]), + prg = to_program(replace_opts(out.schedule_linear().src[-1].src[0], [Opt(OptOps.SPLIT, axis=0, arg=(4, AxisType.LOCAL))]), renderer=Device[Device.DEFAULT].renderer) self.assertEqual(len(prg.src[2].arg.split("for")), 5) @@ -183,7 +174,8 @@ class TestLinearizer(unittest.TestCase): def test_upcast_with_locals(self): x, y = Tensor.rand(1,128), Tensor.rand(128, 128) r = (x@y).relu() - opts_to_apply = [Opt(op=OptOps.GROUP, axis=0, arg=8), Opt(op=OptOps.LOCAL, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=0, arg=4)] + opts_to_apply = [Opt(op=OptOps.SPLIT, axis=1, arg=(8, AxisType.GROUP_REDUCE)), Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.LOCAL)), + Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))] program = to_program(replace_opts(r.schedule_linear().src[-1].src[0], opts_to_apply), renderer=Device[Device.DEFAULT].renderer) stores = [u for u in tuple(program.src[1].src) if u.op is Ops.STORE and u.src[0].addrspace != AddrSpace.REG] @@ -199,7 +191,7 @@ class TestLinearizer(unittest.TestCase): def test_zero_fold(self): a, b = Tensor.randn(1).realize(), Tensor.randn(1).realize() r = Tensor.stack(a, b) - uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=0)]), + uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(0, AxisType.UPCAST))]), renderer=Device[Device.DEFAULT].renderer).src[1].src) num_ops = len([uop for uop in uops if uop.op in GroupOp.ALU]) assert num_ops == 0, "more alu uops than needed" @@ -230,7 +222,7 @@ class TestLinearizer(unittest.TestCase): (dtypes.float, dtypes.float16, dtypes.float16), ) for tensor_dtype, acc_dtype, expected_dtype in tests: - if tensor_dtype in (dts:=Device[Device.DEFAULT].renderer.supported_dtypes()) and acc_dtype in dts and expected_dtype in dts: + if tensor_dtype in (dts:=Device[Device.DEFAULT].renderer.supported_dtypes()) and acc_dtype in dts|{None} and expected_dtype in dts: a, b = Tensor.rand(8, 8, dtype=tensor_dtype), Tensor.rand(8, 8, dtype=tensor_dtype) helper_arg_acc_dtype(a.sum(dtype=acc_dtype), expected_dtype) helper_arg_acc_dtype(a.matmul(b, dtype=acc_dtype), expected_dtype) @@ -242,23 +234,19 @@ class TestLinearizer(unittest.TestCase): def test_simple_unroll_no_between_phi_dependencies(self): x, y = Tensor.empty(64, 64), Tensor.empty(64, 64) r = (x@y).relu() - opt = [Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 4)] + opt = [Opt(OptOps.SPLIT, 2, (4, AxisType.UNROLL)), Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))] ast = helper_linearizer_opt(r, [opt]) # the uops graph is reg BUFFER -> 4x STORE 0.0 -> RANGE -> 4x ALU -> 4x STORE -> ENDRANGE uops = tuple(to_program(replace_opts(ast, opt), renderer=Device[Device.DEFAULT].renderer).src[1].src) begin_range = [i for i, x in enumerate(uops) if x.op is Ops.RANGE][-1] end_range = [i for i, x in enumerate(uops) if x.op is Ops.END][0] - for i,u in enumerate(uops): print(i, u.op, [uops.index(s) for s in u.src], u.arg, u.dtype) for u in uops: if u.op is Ops.STORE and u.src[0].addrspace is AddrSpace.REG: if uops.index(u) < begin_range: - assert u.src[1].op is Ops.CONST + assert u.src[1].op not in GroupOp.ALU else: assert u.src[1].op in GroupOp.ALU assert begin_range < uops.index(u) < end_range - # children of END are placed after ENDRANGE - if any(x.op is Ops.END and x.src[1].op in GroupOp.ALU for x in u.src): - assert end_range < uops.index(u) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") def test_default_global_reversed(self): @@ -268,9 +256,9 @@ class TestLinearizer(unittest.TestCase): uops = tuple(to_program(replace_opts(ast, []), renderer=Device[Device.DEFAULT].renderer).src[1].src) idxs = dedup([uop for uop in uops if uop.op is Ops.SPECIAL]) idxs = sorted(idxs, key=lambda uop: uop.arg) - assert (idxs[0].arg, idxs[0].src[0].val) == ('gidx0', 6), idxs[0] - assert (idxs[1].arg, idxs[1].src[0].val) == ('gidx1', 5), idxs[1].arg - assert (idxs[2].arg, idxs[2].src[0].val) == ('gidx2', 4), idxs[2].arg + assert (idxs[0].arg, idxs[0].src[0].src[0].val) == ('gidx0', 6), idxs[0] + assert (idxs[1].arg, idxs[1].src[0].src[0].val) == ('gidx1', 5), idxs[1].arg + assert (idxs[2].arg, idxs[2].src[0].src[0].val) == ('gidx2', 4), idxs[2].arg def test_sum_collapse(self): t = Tensor([2]).reshape(1, 1).expand(256, 256).sum() @@ -356,8 +344,9 @@ class TestLinearizer(unittest.TestCase): def test_grouped_store_locals_and_globals(self): x, y = Tensor.empty(64, 64), Tensor.empty(64, 64) out = x@y - opt = [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), - Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 2)] # upcast accs in both reduces + opt = [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 3, (8, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 3, (4, AxisType.UNROLL)), Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), + Opt(OptOps.SPLIT, 1, (2, AxisType.UPCAST))] # upcast accs in both reduces ast = helper_linearizer_opt(out, opts=[opt]) def get_recursive(uop): return set.union(set(uop.src), [uop], *[get_recursive(v) for v in uop.src]) uops = tuple(to_program(replace_opts(ast, opt), renderer=Device[Device.DEFAULT].renderer).src[1].src) @@ -395,9 +384,9 @@ class TestLinearizer(unittest.TestCase): @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_shared, "test requires shared") def test_two_grouped_stores_local(self): - # GROUP on both reduces puts two LOCAL buffers in one kernel, and the store to each needs its own barrier + # GROUP_REDUCE on both reduces puts two LOCAL buffers in one kernel, and the store to each needs its own barrier a = Tensor.rand(32, 32).realize() - opts = [Opt(OptOps.GROUP, 1, 4), Opt(OptOps.GROUP, 2, 4)] + opts = [Opt(OptOps.SPLIT, 3, (4, AxisType.GROUP_REDUCE)), Opt(OptOps.SPLIT, 5, (4, AxisType.GROUP_REDUCE))] ast = helper_linearizer_opt(single_kernel_softmax(a), [opts]) uops = to_program(replace_opts(ast, opts), renderer=Device[Device.DEFAULT].renderer).src[1].src self.assertEqual(len([u for u in uops if u.op is Ops.BARRIER]), 2) @@ -411,7 +400,7 @@ def helper_realized_ast(r:Tensor|list[Tensor]) -> tuple[UOp, list[Buffer]]: last_call = linear.src[-1] ast = last_call.src[0] assert ast.op is Ops.SINK, f"helper_realized_ast expects a SINK {last_call}" - last_bufs = [s.buffer for s in last_call.src[1:] if s.op is not Ops.BIND] + last_bufs = [s.buffer for s in last_call.src[1:] if not s.is_bound_var] # now all input buffers in last_call should be realized # create fresh buffers for the outputs bufs = [Buffer(x.device, x.size, x.dtype).allocate() if i < len(ast.src) else x for i,x in enumerate(last_bufs)] @@ -419,12 +408,6 @@ def helper_realized_ast(r:Tensor|list[Tensor]) -> tuple[UOp, list[Buffer]]: for b in bufs: b.ensure_allocated() return ast, bufs -def helper_linearizer_ast(ast:UOp, inputs:list[Tensor], *args, **kwargs): - assert isinstance(ast, UOp), "ast must be UOp" - inbufs = [x.uop.base.buffer for x in inputs] - outbufs = [Buffer(inbufs[-1].device if inbufs else Device.DEFAULT, out.size, out.src[1].dtype).allocate() for out in ast.src] - _helper_linearizer_opt_ast(ast, outbufs+inbufs, *args, **kwargs) - def helper_linearizer_opt(r:Tensor|list[Tensor], *args, **kwargs): realized_ast, real_bufs = helper_realized_ast(r) _helper_linearizer_opt_ast(realized_ast, real_bufs, *args, **kwargs) @@ -437,7 +420,7 @@ def reset_bufs(bufs:list[Buffer]): for buf in bufs: buf.copy_from(Buffer("PYTHON", buf.size, buf.dtype, opaque=memoryview(bytearray(buf.nbytes)))) def _helper_linearizer_opt_ast(realized_ast:UOp, real_bufs:list[Buffer], opts=[], - apply_tc=False, atol=1e-4, rtol=1e-4, color_sizes=[], wanna_output=[]): + apply_tc=False, atol=1e-4, rtol=1e-4, color_sizes=[], wanna_output=[], check_default_opt=True): outbufs = real_bufs[:len(realized_ast.src)] wanna_output = [np.array(x).flatten() for x in wanna_output] buf_uops = [UOp.new_buffer(b.device, b.size, b.dtype) for b in real_bufs] @@ -459,9 +442,7 @@ def _helper_linearizer_opt_ast(realized_ast:UOp, real_bufs:list[Buffer], opts=[] for buf,want in zip(copyout_outputs(outbufs), wanna_output): np.testing.assert_allclose(buf, want, atol=atol, rtol=rtol) # Check correctness of handcoded optimiztions. - reset_bufs(outbufs) - run_prg(opts=None) - for buf,want in zip(copyout_outputs(outbufs), wanna_output): np.testing.assert_allclose(buf, want, atol=atol, rtol=rtol) + if check_default_opt: check_opt(None) for x in opts: # Check custom transformations if any. check_opt(([Opt(OptOps.TC, 0, (TC_SELECT.value, TC_OPT.value, 1))] if apply_tc else [])+x) diff --git a/tinygrad_repo/test/backend/test_linearizer_dumb.py b/tinygrad_repo/test/backend/test_linearizer_dumb.py index 317f0abd3..164f9df19 100644 --- a/tinygrad_repo/test/backend/test_linearizer_dumb.py +++ b/tinygrad_repo/test/backend/test_linearizer_dumb.py @@ -11,20 +11,20 @@ from tinygrad.codegen import to_program class TestLinearizerFailure(unittest.TestCase): @unittest.skipUnless(Device.DEFAULT == "METAL", "only tested on METAL") def test_failure_beam_mnist(self): - c0 = UOp.param(0, dtypes.uchar, (4014080,)) + c0 = UOp.param(0, dtypes.uchar, 4014080) c1 = UOp.range(UOp.const(512), 0, AxisType.GLOBAL) c2 = UOp.range(UOp.const(784), 1, AxisType.GLOBAL) c3 = UOp.range(UOp.const(10), 3, AxisType.GLOBAL) - c4 = UOp.param(1, dtypes.int, (512,)) + c4 = UOp.param(1, dtypes.int, 512) c5 = c4.index(c1.valid(UOp.const(True))) c6 = UOp.range(UOp.const(6000), 1004, AxisType.REDUCE) c7 = UOp.range(UOp.const(3750), 2006, AxisType.REDUCE) c8 = UOp.range(UOp.const(16), 2007, AxisType.GROUP_REDUCE) - c9 = UOp.param(2, dtypes.uchar, (47040000,)) + c9 = UOp.param(2, dtypes.uchar, 47040000) c10 = c9.index((((c3*UOp.const(4704000))+c2)+(c6*UOp.const(784))).valid(UOp.const(True))) c11 = c5.alu(Ops.CMPNE, ((((c3*UOp.const(6000))+c6)+((c7*UOp.const(16))+c8)).alu(Ops.CMPLT, UOp.const(59999)).where(UOp.const(0).cast(dtypes.int), UOp.const(1).cast(dtypes.int)).reduce(c7, c8, arg=Ops.ADD)+UOp.const(-1).cast(dtypes.int))).where(UOp.const(0).cast(dtypes.uchar), c10).reduce(c6, arg=Ops.ADD) c12 = c0.index((((c1*UOp.const(7840))+(c2*UOp.const(10)))+c3).valid(UOp.const(True))).store(c11).end(c1, c2, c3) - ast = c12.sink(arg=KernelInfo(name='test', axis_types=(), dont_use_locals=False, applied_opts=(Opt(op=OptOps.GROUP, axis=1, arg=16),), opts_to_apply=None)) + ast = c12.sink(arg=KernelInfo(name='test', applied_opts=(Opt(op=OptOps.SPLIT, axis=4, arg=(16, AxisType.GROUP_REDUCE)),), opts_to_apply=None)) _ = to_program(ast, Device["METAL"].renderer) if __name__ == '__main__': diff --git a/tinygrad_repo/test/backend/test_llama_kernels.py b/tinygrad_repo/test/backend/test_llama_kernels.py index f03a57b39..81f98eb7e 100644 --- a/tinygrad_repo/test/backend/test_llama_kernels.py +++ b/tinygrad_repo/test/backend/test_llama_kernels.py @@ -9,7 +9,7 @@ from extra.llama_kernels.swiglu import swiglu from extra.models.llama import apply_rotary_emb, precompute_freqs_cis from extra.thunder.amd.fa import custom_fused_qkv_rope_backward, fused_qkv_rope from test.helpers import needs_second_gpu, assert_kernel_count -from test.backend.test_asm_gemm import has_hipcc +from test.backend.test_asm_gemm import has_hipcc, is_cdna4 def run_fused_ce(bs:int, seqlen:int, vocab:int, label_smoothing:float=0.0) -> None: Tensor.manual_seed(0) @@ -99,22 +99,20 @@ class TestLocalAmax(unittest.TestCase): assert_kernel_count(2) self.assertEqual(out.tolist(), [[0., 7., 14., 21.], [28., 35., 42., 49.], [120., 135., 150., 165.], [180., 195., 210., 225.]]) -@unittest.skipUnless(has_hipcc() and Device.DEFAULT == "AMD", "requires hipcc to compile and amd device to run") class TestFusedQKVRoPE(unittest.TestCase): SHAPE = (2, 8192, 32, 8, 128) + def setUp(self): + if dtypes.bfloat16 not in Device[Device.DEFAULT].renderer.supported_dtypes(): self.skipTest("test uses bf16 inputs") + def rand_bf16(self, *shape:int) -> Tensor: return (Tensor.randn(*shape) * 0.1).cast(dtypes.bfloat16).contiguous().realize() - def freqs_cis(self) -> Tensor: - _, N, _, _, D = self.SHAPE - return precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() - - def test_llama31_8b_forward(self): + def test_forward(self): Tensor.manual_seed(0) - B, N, H, H_KV, D = self.SHAPE + B, N, H, H_KV, D = 1, 32, 8, 2, 16 GROUP = H // H_KV - freqs_cis = self.freqs_cis() + freqs_cis = (Tensor.randn(1, N * 2, 1, D // 2, 2) * 0.1).cast(dtypes.bfloat16).contiguous().realize() x = self.rand_bf16(B, N, H_KV * (GROUP + 2) * D) q, k, v = fused_qkv_rope(x, freqs_cis, H, H_KV, D) @@ -131,12 +129,13 @@ class TestFusedQKVRoPE(unittest.TestCase): self.assertTrue(k.allclose(k_ref, atol=2e-2, rtol=0).item(), "K forward mismatch") self.assertTrue(v.allclose(v_ref, atol=0, rtol=0).item(), "V forward mismatch") - def test_llama31_8b_backward(self): + @unittest.skipUnless(has_hipcc() and is_cdna4(), "backward kernel requires hipcc to compile") + def test_llama31_8b(self): Tensor.manual_seed(1) B, N, H, H_KV, D = self.SHAPE PARTIALS = 2 GROUP = H // H_KV - freqs_cis = self.freqs_cis() + freqs_cis = precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() dq = self.rand_bf16(B, N, H, D) dk_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) dv_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) diff --git a/tinygrad_repo/test/backend/test_multitensor.py b/tinygrad_repo/test/backend/test_multitensor.py index 1b5d4896f..360ee2679 100644 --- a/tinygrad_repo/test/backend/test_multitensor.py +++ b/tinygrad_repo/test/backend/test_multitensor.py @@ -1,12 +1,12 @@ import unittest, random from tinygrad import Tensor, Device, nn, GlobalCounters, TinyJit, dtypes, Variable -from tinygrad.uop.ops import Ops, UOp, AxisType +from tinygrad.uop.ops import Ops, UOp, AxisType, graph_rewrite from tinygrad.helpers import getenv, prod, Context from tinygrad.nn.state import get_parameters -from tinygrad.engine.realize import run_linear, compile_linear +from tinygrad.engine.realize import run_linear, compile_linear, lower_and_compile, pm_beam import numpy as np from hypothesis import given, strategies as strat, settings -from test.helpers import not_support_multi_device, needs_second_gpu, slow, call_is_graph, check_schedule, assert_kernel_count +from test.helpers import not_support_multi_device, needs_second_gpu, slow, call_is_graph, check_schedule, assert_kernel_count, KernelCountException settings.register_profile("my_profile", max_examples=200, deadline=None, derandomize=getenv("DERANDOMIZE_CI", False)) settings.load_profile("my_profile") @@ -58,6 +58,11 @@ class TestMultiTensor(unittest.TestCase): assert X.uop.ended_ranges == X.uop.src[1:] (X + X).realize() + def test_shard_invalids_contiguous(self): + # every store is Invalid, so none of them should become a (empty) kernel + t = Tensor.invalids(8).shard(devices_2, axis=0).contiguous() + self.assertEqual(len([c for c in t.schedule_linear().src if c.src[0].op is Ops.SINK]), 1) + @unittest.expectedFailure # TODO: fix def test_shard_empty(self): GlobalCounters.reset() @@ -72,16 +77,16 @@ class TestMultiTensor(unittest.TestCase): X.shard_(devices_2, 0) out = (X + X) linear = compile_linear(out.schedule_linear()) - names = [call.src[0].src[0].arg.name for call in linear.src if call.src[0].op is Ops.PROGRAM] + uops = [call.src[0].src[0] for call in linear.src if call.src[0].op is Ops.PROGRAM] run_linear(linear) - self.assertEqual(len(set(names)), 1, "function was relinearized") + self.assertEqual(len(set(uops)), 1, "function was relinearized") def test_shard_beam(self): cpu_2 = ("CPU:1", "CPU:2") src = Tensor.ones(16).shard(cpu_2, 0).realize() - pad = src.to(cpu_2[::-1]).schedule_linear().src[0] - with Context(BEAM=1, IGNORE_BEAM_CACHE=1): prg = compile_linear(UOp(Ops.LINEAR, src=(pad,))).src[0].src[0] - self.assertNotEqual(prg.src[0].arg.applied_opts, ()) + lin = UOp(Ops.LINEAR, src=(src.to(cpu_2[::-1]).schedule_linear().src[0],)) + with Context(BEAM=1, IGNORE_BEAM_CACHE=1): call = lower_and_compile(graph_rewrite(lin, pm_beam, ctx=1, walk=True)).src[0] + self.assertNotEqual(call.src[0].src[0].arg.applied_opts, ()) def test_shard_same_device(self): X = Tensor.ones(256).contiguous().realize() @@ -182,6 +187,13 @@ class TestMultiTensor(unittest.TestCase): a,b = jit_allreduce(Tensor.rand(256, 256)) np.testing.assert_almost_equal(a.numpy(), b.numpy(), decimal=5) + def test_allreduce_all2all_jit(self): + with Context(ALL2ALL=2): + jit_allreduce = TinyJit(_test_allreduce) + for _ in range(5): + a,b = jit_allreduce(Tensor.rand(256, 256)) + np.testing.assert_almost_equal(a.numpy(), b.numpy(), decimal=5) + def test_multitensor_jit_input(self): @TinyJit def f(x): return (x+1).contiguous().sum() @@ -395,7 +407,7 @@ class TestMultiBufferView(unittest.TestCase): linear, var_vals = b_multi.linear_with_vars() if all(not d.startswith(("WEBGPU", "CL")) for d in b_multi.device): compiled = [call for call in linear.src if call.src[0].op is Ops.SINK] - self.assertEqual(len(compiled), 0, f"expected zero compiled kernels, got {len(compiled)}") + if len(compiled) != 0: raise KernelCountException(0, len(compiled)) run_linear(linear, var_vals) np.testing.assert_equal(b_multi.numpy(), b_ref.numpy()) diff --git a/tinygrad_repo/test/backend/test_nn.py b/tinygrad_repo/test/backend/test_nn.py index 75195d88e..c12ec9126 100644 --- a/tinygrad_repo/test/backend/test_nn.py +++ b/tinygrad_repo/test/backend/test_nn.py @@ -135,7 +135,7 @@ class TestNN(unittest.TestCase): def test_conv2d_same_padding_large_kernel(self): self._test_conv(Conv2d, torch.nn.Conv2d, BS=16, C1=16, DIMS=[28, 33], C2=32, K=9, S=1, P='same') def test_conv2d_same_padding_with_dilation(self): - self._test_conv(Conv2d, torch.nn.Conv2d, BS=16, C1=3, DIMS=[28, 28], C2=32, K=3, S=1, P='same', D=3) + self._test_conv(Conv2d, torch.nn.Conv2d, BS=16, C1=3, DIMS=[28, 31], C2=32, K=(3,5), S=1, P='same', D=(2,3)) def test_conv2d_same_padding_invalid_stride(self): self.assertRaises(ValueError, Conv2d, in_channels=16, out_channels=32, kernel_size=2, stride=2, padding='same') diff --git a/tinygrad_repo/test/backend/test_ops.py b/tinygrad_repo/test/backend/test_ops.py index 9eacc5b52..5483cb35d 100644 --- a/tinygrad_repo/test/backend/test_ops.py +++ b/tinygrad_repo/test/backend/test_ops.py @@ -6,6 +6,7 @@ from tinygrad.helpers import getenv, DEBUG, DEV, IMAGE, Context from tinygrad import Tensor, Device, dtypes from tinygrad.tensor import _to_np_dtype from tinygrad.renderer.nir import NIRRenderer +from tinygrad.renderer.isa.x86 import X86Renderer TINY_BACKEND = getenv("TINY_BACKEND") if TINY_BACKEND: @@ -358,6 +359,13 @@ class TestOps(unittest.TestCase): lambda x: torch.where(x > 0.5, 4, 2).type(torch.int32).permute((1, 0)), lambda x: (x > 0.5).where(4, 2).clone().permute((1, 0)), forward_only=True) + @unittest.skipIf(Device.DEFAULT == "WEBGPU", "software vulkan evaluates a NaN != x as false") + def test_where_nan_cond(self): + # a NaN compares false against everything except !=. + for fxn in (lambda x: x<1, lambda x: x>1, lambda x: x!=1, lambda x: x==1): + helper_test_op(None, lambda x,a,b: torch.where(fxn(x), a, b), lambda x,a,b: fxn(x).where(a, b), forward_only=True, + vals=[[math.nan, 1.0, 2.0, -1.0], [10, 20, 30, 40], [-1, -2, -3, -4]]) + def _test_cmp(self, fxn, reverse=True): # test different dtypes helper_test_op(None, fxn, fxn, forward_only=True, vals=[[0.,1,2], [2.,1,0]]) @@ -720,10 +728,11 @@ class TestOps(unittest.TestCase): return torch.autograd.grad(t ** c, t)[0].item() for x in [-math.inf, 0, 1, math.inf]: for c in [-1, 0, 0.3, 1, 2]: - tiny_out = get_tiny_gradient(x, c) torch_out = get_torch_gradient(x, c) + # the pow backward routes through exp2/log2, whose 0/inf behavior is undefined on WEBGPU + if Device.DEFAULT == "WEBGPU" and not math.isfinite(torch_out): continue + tiny_out = get_tiny_gradient(x, c) if math.isnan(tiny_out): - if Device.DEFAULT == "WEBGPU": continue # TODO: WEBGPU issue with nan assert math.isnan(torch_out) else: self.assertAlmostEqual(tiny_out, torch_out, msg=f"{x}, {c}") @@ -749,6 +758,7 @@ class TestOps(unittest.TestCase): def test_exp2_log2_zero_times_negative(self): # gallivm's exp2/log2 have "undefined behavior with infs, 0s and nans", so exp2(log2(0)*y) returns 0 instead of inf helper_test_op(None, lambda x,y: (x.log2()*y).exp2(), lambda x,y: (x.log2()*y).exp2(), vals=[[0.0], [-0.7]], forward_only=True) + @unittest.skipIf(Device.DEFAULT == "WEBGPU", "pow at 0 routes through exp2/log2, whose 0/inf behavior is undefined on WEBGPU") def test_pow_zero_const(self): helper_test_op(None, lambda x: x**0.3, vals=[[0.0]]) helper_test_op(None, lambda x: x**0.0, vals=[[0.0]]) @@ -806,6 +816,8 @@ class TestOps(unittest.TestCase): helper_test_op([], lambda: tor^0x1337, lambda: ten^0x1337, forward_only=True) helper_test_op([], lambda: 0x1337^tor, lambda: 0x1337^ten, forward_only=True) + # TODO: x86 PARAM dtype fails SPEC=2 + @Context(SPEC=1 if isinstance(Device[Device.DEFAULT].renderer, X86Renderer) else 2) def test_and(self): data = [[1,-8,1],[32,1,6]] tor = torch.tensor(data, dtype=torch.int) @@ -820,6 +832,10 @@ class TestOps(unittest.TestCase): helper_test_op([], lambda: tor0&tor1, lambda: ten0&ten1, forward_only=True) helper_test_op(None, lambda x: (1 < x) & (x < 2), forward_only=True, vals=[[1.2, 1.2, 1.2, 3.2]]) + helper_test_op([(3000,)]*10, lambda *xs: (sum(xs[1:], xs[0]) > 5) & (xs[0] < 0.9), forward_only=True) + + if not COMPILE_ONLY: + np.testing.assert_equal((Tensor(2**64-1, dtype=dtypes.uint64) & 0xFFFFFFFF).numpy(), 0xFFFFFFFF) def test_or(self): data = [[1,-8,1],[32,1,6]] @@ -859,9 +875,9 @@ class TestOps(unittest.TestCase): lambda: (ten << Tensor([0,2,4], dtype=dtypes.uint32)).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.__lshift__(2), lambda: ten.__lshift__(2).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.bitwise_left_shift(2), lambda: ten.lshift(2).cast(dtypes.int32), forward_only=True) - self.helper_test_exception([], lambda: torch.tensor([1.0]) << 2, lambda: Tensor([1.0]) << 2, expected=RuntimeError) - self.helper_test_exception([], lambda: tor << torch.tensor([1.0]), lambda: ten << Tensor([1.0]), expected=RuntimeError) - self.helper_test_exception([], lambda: tor << 1.0, lambda: ten << 1.0, expected=RuntimeError) + self.helper_test_exception([], lambda: torch.tensor([1.0]) << 2, lambda: (Tensor([1.0]) << 2).realize(), expected=RuntimeError) + self.helper_test_exception([], lambda: tor << torch.tensor([1.0]), lambda: (ten << Tensor([1.0])).realize(), expected=RuntimeError) + self.helper_test_exception([], lambda: tor << 1.0, lambda: (ten << 1.0).realize(), expected=RuntimeError) def test_rshift(self): data = [[0,1,2],[1<<8,1<<16,1<<31-1]] @@ -875,8 +891,8 @@ class TestOps(unittest.TestCase): lambda: (ten >> Tensor([0,2,4], dtype=dtypes.uint32)).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.__rshift__(2), lambda: ten.__rshift__(2).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.bitwise_right_shift(2), lambda: ten.rshift(2).cast(dtypes.int32), forward_only=True) - self.helper_test_exception([], lambda: torch.tensor([4.0]) >> 1, lambda: Tensor([4.0]) >> 1, expected=RuntimeError) - self.helper_test_exception([], lambda: tor >> torch.tensor([1.0]), lambda: ten >> Tensor([1.0]), expected=RuntimeError) + self.helper_test_exception([], lambda: torch.tensor([4.0]) >> 1, lambda: (Tensor([4.0]) >> 1).realize(), expected=RuntimeError) + self.helper_test_exception([], lambda: tor >> torch.tensor([1.0]), lambda: (ten >> Tensor([1.0])).realize(), expected=RuntimeError) def test_lshift_signed(self): data = [[-1, -3, 1, 7], [0, -2147483648, 2147483647, -1]] @@ -939,15 +955,18 @@ class TestOps(unittest.TestCase): helper_test_op([(45,65)], lambda x: x.asin(), low=-1, high=1) helper_test_op([(45,65)], lambda x: x.asin(), low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.asin(), low=300, high=303) + helper_test_op(None, lambda x: x.asin(), vals=[[-0.5, 0., 0.5]]) def test_acos(self): # high grad atol helper_test_op([(45,65)], lambda x: x.acos(), low=-1, high=1) helper_test_op([(45,65)], lambda x: x.acos(), low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.acos(), low=300, high=303) + helper_test_op(None, lambda x: x.acos(), vals=[[-0.5, 0., 0.5]]) def test_atan(self): helper_test_op([(45,65)], lambda x: x.atan()) helper_test_op([(45,65)], lambda x: x.atan(), low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.atan(), low=300, high=303) + helper_test_op(None, lambda x: x.atan(), vals=[[-0.5, 0., 0.5]]) def test_relu(self): helper_test_op([(64,64)], lambda x: x.relu()) @@ -962,9 +981,12 @@ class TestOps(unittest.TestCase): def test_celu(self): for val in range(1, 5): helper_test_op([(45,65)], lambda x: torch.nn.functional.celu(x,val), lambda x: x.celu(val)) + helper_test_op([(3,3)], lambda x: torch.nn.functional.celu(x,val), lambda x: x.celu(val), low=300, high=400) helper_test_op([()], lambda x: torch.nn.functional.celu(x,val), lambda x: x.celu(val)) def test_selu(self): helper_test_op([(45,65)], torch.nn.functional.selu, Tensor.selu) + helper_test_op([(3,3)], torch.nn.functional.selu, Tensor.selu, low=300, high=400) + helper_test_op(None, torch.nn.functional.selu, Tensor.selu, vals=[[-1.,0.,1.]]) helper_test_op([()], torch.nn.functional.selu, Tensor.selu) def test_silu(self): helper_test_op([(45,65)], torch.nn.functional.silu, Tensor.silu) @@ -1031,6 +1053,7 @@ class TestOps(unittest.TestCase): helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[-1.], [-1.0, 2, 3]]) helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[-100.0, -200, -300], [-1.0, 2, 3]]) helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[1.0, 2000, 30000], [-1.0, 2, 3]]) + helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[-math.inf, math.inf, 1.0, -math.inf], [-math.inf, math.inf, -math.inf, 1.0]]) def test_softsign(self): helper_test_op([(45,65)], torch.nn.functional.softsign, Tensor.softsign) @@ -1072,11 +1095,13 @@ class TestOps(unittest.TestCase): helper_test_op([(45,65)], torch.nn.functional.softplus, Tensor.softplus, grad_atol=1e-6, low=300, high=400) helper_test_op([(45,65)], torch.nn.functional.softplus, Tensor.softplus, grad_atol=1e-6, low=-400, high=-300) helper_test_op([()], torch.nn.functional.softplus, Tensor.softplus, grad_atol=1e-6) + helper_test_op(None, torch.nn.functional.softplus, Tensor.softplus, vals=[[-math.inf, math.inf, 0.0]], forward_only=True) def test_erf(self): helper_test_op([(45,65)], torch.erf, Tensor.erf) helper_test_op([(45,65)], torch.erf, Tensor.erf, low=300, high=400) helper_test_op([(45,65)], torch.erf, Tensor.erf, low=-400, high=-300) + helper_test_op(None, torch.erf, Tensor.erf, vals=[[-1., 0., 1.]]) helper_test_op([()], torch.erf, Tensor.erf) def test_gelu(self): @@ -1101,6 +1126,7 @@ class TestOps(unittest.TestCase): def test_elu(self): helper_test_op([(45,65)], torch.nn.functional.elu, Tensor.elu) helper_test_op([(45,65)], lambda x: torch.nn.functional.elu(x, alpha=0.1), lambda x: Tensor.elu(x, alpha=0.1)) + helper_test_op([(3,3)], torch.nn.functional.elu, Tensor.elu, low=300, high=400) helper_test_op([()], torch.nn.functional.elu, Tensor.elu) def test_relu6(self): helper_test_op([(45,65)], torch.nn.functional.relu6, Tensor.relu6) @@ -1752,6 +1778,9 @@ class TestOps(unittest.TestCase): helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=3, dim=0), lambda x: x.normalize(p=3, dim=0), atol=1e-7, grad_atol=1e-7) helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=0), lambda x: x.normalize(p=0), atol=1e-7, grad_atol=1e-7) helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=-1), lambda x: x.normalize(p=-1), atol=1e-7, grad_atol=1e-7) + def test_normalize_int(self): + helper_test_op(None, lambda x: torch.nn.functional.normalize(x.float(), p=2), lambda x: x.normalize(p=2), forward_only=True, + vals=[[[3, 4], [6, 8]]]) def test_logsumexp(self): helper_test_op([(45,65)], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) @@ -1764,6 +1793,7 @@ class TestOps(unittest.TestCase): helper_test_op([(45)], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) helper_test_op([()], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) helper_test_op([()], lambda x: torch.logsumexp(x, dim=-1), lambda x: x.logsumexp(-1), atol=1e-7, grad_atol=1e-7) + helper_test_op(None, lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), vals=[[-math.inf, -math.inf]], forward_only=True) @slow_test def test_logcumsumexp(self): @@ -1779,6 +1809,7 @@ class TestOps(unittest.TestCase): def test_logcumsumexp_numerical(self): helper_test_op(None, lambda x: torch.logcumsumexp(x, dim=0), lambda x: x.logcumsumexp(), atol=1e-7, grad_atol=1e-7, vals=[[0.0, 100.0]]) + helper_test_op(None, lambda x: torch.logcumsumexp(x, dim=0), lambda x: x.logcumsumexp(), vals=[[-math.inf, 0.0, 1.0]], forward_only=True) def test_sinh(self): helper_test_op([(45,65)], lambda x: x.sinh(), grad_atol=1e-6) @@ -1801,9 +1832,10 @@ class TestOps(unittest.TestCase): helper_test_op([()], lambda x: torch.nn.functional.hardtanh(x, -val, val), lambda x: x.hardtanh(-val, val), grad_atol=1e-6) def test_asinh(self): helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6) - # TODO: this one has larger tol? - helper_test_op([(45,65)], lambda x: x.asinh(), atol=1e-2, rtol=2e-2, grad_rtol=2e-2, low=-300, high=-297) + helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=300, high=303) + helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=-1e10, high=-1e9) + helper_test_op(None, lambda x: x.asinh(), grad_atol=1e-6, vals=[[-1.0, 0.0, 1.0]]) def test_acosh(self): helper_test_op([(45,65)], lambda x: x.acosh(), grad_atol=1e-6) helper_test_op([(45,65)], lambda x: x.acosh(), grad_atol=1e-3, grad_rtol=1e-2, low=-300, high=-297) @@ -2162,6 +2194,10 @@ class TestOps(unittest.TestCase): def test_roll(self): helper_test_op([(2, 4)], lambda x: x.roll(1)) helper_test_op([(2, 4)], lambda x: x.roll((1,))) + helper_test_op([(0,)], lambda x: x.roll(1, 0)) + helper_test_op([(2, 0, 3)], lambda x: x.roll(1, 0)) + helper_test_op([(2, 0, 3)], lambda x: x.roll(1, 1)) + helper_test_op([(2, 0, 3)], lambda x: x.roll(1)) self.helper_test_exception([(2, 4)], lambda x: x.roll((1, 2)), expected=RuntimeError) helper_test_op([(2, 4)], lambda x: x.roll(1, 0)) helper_test_op([(2, 4)], lambda x: x.roll(-1, 0)) @@ -2796,7 +2832,7 @@ class TestOps(unittest.TestCase): lambda x: Tensor.interpolate(x, size=out_sz, mode="linear")) def test_interpolate_linear_corners_aligned(self): - for in_sz, out_sz in [((52,),(29,)), ((29,),(52,))]: + for in_sz, out_sz in [((52,),(29,)), ((29,),(52,)), ((29,),(1,))]: helper_test_op([(2,3)+in_sz], lambda x: torch.nn.functional.interpolate(x, size=out_sz, mode="linear", align_corners=True), lambda x: Tensor.interpolate(x, size=out_sz, mode="linear", align_corners=True)) @@ -2949,6 +2985,10 @@ class TestOps(unittest.TestCase): data = [math.inf, -math.inf, math.nan] helper_test_op((), lambda: torch.tensor(data)[torch.tensor([0, 1, 2])], lambda: Tensor(data)[Tensor([0, 1, 2])]) + def test_fancy_indexing_index_dtypes(self): + helper_test_op((), lambda: torch.tensor([10., 20., 30., 40.])[torch.tensor([1, 2, 3, 0])], + lambda: Tensor([10., 20., 30., 40.])[Tensor([1, 2, 3, 0], dtype=dtypes.uint8)]) + @slow_test def test_slice_fancy_indexing_no_dim_collapse(self): a,b,c,d,e,i,j,k,o,p = self._get_index_randoms() diff --git a/tinygrad_repo/test/backend/test_opt_gemm.py b/tinygrad_repo/test/backend/test_opt_gemm.py index 244e3df88..c24825455 100644 --- a/tinygrad_repo/test/backend/test_opt_gemm.py +++ b/tinygrad_repo/test/backend/test_opt_gemm.py @@ -4,7 +4,7 @@ from tinygrad import Tensor from tinygrad.helpers import get_single_element from tinygrad.codegen.opt import Opt, OptOps from tinygrad.engine.realize import run_linear -from tinygrad.uop.ops import Ops, UOp +from tinygrad.uop.ops import Ops, UOp, AxisType from test.helpers import replace_opts class TestOptGemm(unittest.TestCase): @@ -26,20 +26,21 @@ class TestOptGemm(unittest.TestCase): np.testing.assert_allclose(self.res, test, atol=1e-4) def test_gemm_unrolled_permute_l_44(self): - opts = [Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=1, arg=4)] + opts = [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=1, arg=(4, AxisType.UPCAST))] self._test_gemm_unrolled_permute_l(opts) def test_gemm_unrolled_permute_l_424(self): # was failing with LLVM - opts = [Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=1, arg=2), Opt(op=OptOps.UPCAST, axis=0, arg=4)] + opts = [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=1, arg=(2, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))] self._test_gemm_unrolled_permute_l(opts) def test_gemm_unrolled_permute_l_42(self): - opts = [Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=1, arg=2)] + opts = [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=1, arg=(2, AxisType.UPCAST))] self._test_gemm_unrolled_permute_l(opts) def test_gemm_unrolled_permute_l_22(self): - opts = [Opt(op=OptOps.UPCAST, axis=0, arg=2), Opt(op=OptOps.UPCAST, axis=1, arg=2)] + opts = [Opt(op=OptOps.SPLIT, axis=0, arg=(2, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=1, arg=(2, AxisType.UPCAST))] self._test_gemm_unrolled_permute_l(opts) if __name__ == '__main__': diff --git a/tinygrad_repo/test/backend/test_optim.py b/tinygrad_repo/test/backend/test_optim.py index 299ec7a44..74ce1d1f2 100644 --- a/tinygrad_repo/test/backend/test_optim.py +++ b/tinygrad_repo/test/backend/test_optim.py @@ -87,7 +87,8 @@ class TestOptim(unittest.TestCase): def test_muon(self): self._test_muon(1, {'lr': 0.001}, 1e-3, 0) # TODO: disabled due to big atol # def test_muon_high_lr(self): self._test_muon(1, {'lr': 10}, 1e-6, 3e-4) - def test_muon_wd(self): self._test_muon(1, {'lr': 0.001, 'weight_decay': 0.01}, 1e-3, 3e-4) + # NOTE: big weight_decay so a missing wd would be way over atol + def test_muon_wd(self): self._test_muon(1, {'lr': 0.001, 'weight_decay': 10}, 1e-3, 3e-4) # TODO: disabled due to big atol # def test_muon_high_lr_wd(self): self._test_muon(1, {'lr': 10, 'weight_decay': 0.01}, 1e-6, 5e-4) diff --git a/tinygrad_repo/test/backend/test_pickle.py b/tinygrad_repo/test/backend/test_pickle.py index cbb0aa378..75af07e71 100644 --- a/tinygrad_repo/test/backend/test_pickle.py +++ b/tinygrad_repo/test/backend/test_pickle.py @@ -2,7 +2,8 @@ import unittest, pickle, types, tracemalloc import numpy as np from tinygrad import Tensor, Device, TinyJit, Variable, dtypes from tinygrad.helpers import GlobalCounters, ContextVar, Context, DEV -from tinygrad.uop.ops import PatternMatcher, UPat, UOp +from tinygrad.uop.ops import PatternMatcher, UPat, UOp, deconstruct_function +from test.helpers import KernelCountException class TestPickle(unittest.TestCase): def test_pickle_code_object(self): @@ -11,6 +12,11 @@ class TestPickle(unittest.TestCase): fxn = types.FunctionType(pickle.loads(code_str), globals()) self.assertEqual(fxn(2), 4) + def test_deconstruct_function_nested_comprehension(self): + # pre PEP 709, each comprehension is its own code object, so dtypes here is referenced two code objects deep + def fxn(): return [[dtypes.int for _ in range(2)] for _ in range(2)] + self.assertEqual(types.FunctionType(*deconstruct_function(fxn))(), fxn()) + def test_pickle_pattern_matcher(self): pm = PatternMatcher([(UPat.cvar('x'), lambda x: x*2)]) sink = UOp.const(2) @@ -36,7 +42,7 @@ class TestPickle(unittest.TestCase): t2:Tensor = pickle.loads(st) np.testing.assert_equal(t_values, t2.numpy()) # expect at most one COPY kernel - self.assertLessEqual(GlobalCounters.kernel_count, 1) + if GlobalCounters.kernel_count > 1: raise KernelCountException(1, GlobalCounters.kernel_count) def test_pickle_realized_tensor_alt(self): print("** init") diff --git a/tinygrad_repo/test/backend/test_quantize_onnx.py b/tinygrad_repo/test/backend/test_quantize_onnx.py index 587b03f88..6848190b5 100644 --- a/tinygrad_repo/test/backend/test_quantize_onnx.py +++ b/tinygrad_repo/test/backend/test_quantize_onnx.py @@ -2,7 +2,7 @@ import numpy as np import tempfile, unittest from tinygrad import Tensor, Context, Device, dtypes, UOp -from tinygrad.uop.ops import Ops +from tinygrad.uop.ops import Ops, AxisType from tinygrad.dtype import AddrSpace from tinygrad.codegen.opt import Opt, OptOps from tinygrad.engine.realize import run_linear @@ -82,7 +82,7 @@ class TestQuantizeOnnxCPU(unittest.TestCase): linear = run_onnx({"input":inp})["output"].schedule_linear() prg = to_program(linear.src[-2].src[0], renderer=Device[Device.DEFAULT].renderer) daccs = [u for u in tuple(prg.src[1].src) if u.op is Ops.BUFFER and u.addrspace is AddrSpace.REG] - assert all(u.dtype.scalar() is dtypes.int for u in daccs) + assert all(u.dtype is dtypes.int for u in daccs) @unittest.skipIf(Device.DEFAULT != "DSP", "only tests for DSP") class TestQuantizeOnnx(unittest.TestCase): @@ -98,7 +98,7 @@ class TestQuantizeOnnx(unittest.TestCase): X = Tensor(np.random.uniform(0, 255, size=(1, 32, 128, 128)).astype(np.uint8)) W = Tensor(np.random.uniform(0, 255, size=(64, 32, 1, 1)).astype(np.uint8)) out = X.conv2d(W, dtype=X.dtype) - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.SPLIT, axis=1, arg=(128, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=3, arg=(4, AxisType.UNROLL))] sexec(out, opts) def test_prequant_gemm(self): @@ -106,7 +106,7 @@ class TestQuantizeOnnx(unittest.TestCase): X = Tensor(np.random.uniform(0, 255, size=(N,N)).astype(np.uint8)) W = Tensor(np.random.uniform(0, 255, size=(N,N)).astype(np.uint8)) out = X.matmul(W, dtype=X.dtype) - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.SPLIT, axis=1, arg=(128, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=3, arg=(4, AxisType.UNROLL))] sexec(out, opts) # TODO: this has to work @@ -116,7 +116,7 @@ class TestQuantizeOnnx(unittest.TestCase): W = Tensor(np.random.uniform(0, 255, size=(N,N)).astype(wi)) # this divide is interesting and forces the accumulator to actually be an int out = (X.cast("int").matmul(W.cast("int"))//1000).cast("int8") - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.SPLIT, axis=1, arg=(128, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=3, arg=(4, AxisType.UNROLL))] sexec(out, opts) def test_prequant_gemm_handcode(self): @@ -200,9 +200,11 @@ class TestQuantizeOnnx(unittest.TestCase): self.test_prequant_gemm_intacc(np.uint8, np.int8, src) def test_prequant_gemm_intacc_32(self): - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=0), Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.UNROLL, axis=0, arg=0)] + opts = [Opt(op=OptOps.SPLIT, axis=1, arg=(0, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=3, arg=(0, AxisType.UNROLL))] self.test_prequant_gemm_intacc(np.uint8, np.int8, N=32, opts=opts) - def test_prequant_gemm_intacc_128(self): self.test_prequant_gemm_intacc(np.uint8, np.int8, N=128) + def test_prequant_gemm_intacc_128(self): self.test_prequant_gemm_intacc(np.uint8, np.int8, N=128, + opts=[Opt(op=OptOps.SPLIT, axis=1, arg=(128, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=2, arg=(4, AxisType.UNROLL))]) def test_prequant_gemm_intacc_256(self): self.test_prequant_gemm_intacc(np.uint8, np.int8, N=256) def test_prequant_gemm_intacc(self, xi=np.uint8, wi=np.uint8, replace_src=None, N=512, clip=True, opts=None): X = Tensor(m1:=(np.random.uniform(0, 255, size=(N,N)).astype(xi))).realize() @@ -211,7 +213,8 @@ class TestQuantizeOnnx(unittest.TestCase): out = (X.int().matmul(W.int())//1000) if clip: out = out.clip(tg_dtype.min, tg_dtype.max) out = out.cast(tg_dtype) - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] if opts is None else opts + opts = [Opt(op=OptOps.SPLIT, axis=1, arg=(128, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=3, arg=(4, AxisType.UNROLL))] if opts is None else opts sexec(out, opts, replace_src, run_count=1) tout = out.numpy() mout = ((m1.astype(np.int32) @ m2.astype(np.int32)) // 1000) @@ -232,7 +235,7 @@ class TestQuantizeOnnx(unittest.TestCase): #out = X.cast(dtypes.int) @ W.cast(dtypes.int) #out = X @ W out = X.matmul(W, dtype=X.dtype) - opts = [Opt(op=OptOps.UPCAST, axis=0, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.SPLIT, axis=0, arg=(128, AxisType.UPCAST)), Opt(op=OptOps.SPLIT, axis=2, arg=(4, AxisType.UNROLL))] sexec(out, opts) if __name__ == "__main__": diff --git a/tinygrad_repo/test/backend/test_rangeify.py b/tinygrad_repo/test/backend/test_rangeify.py index 011af5bf9..6743275ba 100644 --- a/tinygrad_repo/test/backend/test_rangeify.py +++ b/tinygrad_repo/test/backend/test_rangeify.py @@ -1,55 +1,16 @@ import unittest -from tinygrad import Tensor, nn, Device, dtypes, Variable -from tinygrad.helpers import Context, GlobalCounters, getenv, PCONTIG, DEBUG +from tinygrad import Tensor, dtypes, Variable +from tinygrad.helpers import Context, GlobalCounters, getenv, DEBUG from tinygrad.uop.ops import graph_rewrite, PatternMatcher, UPat, Ops, UOp -from tinygrad.codegen.opt import OptOps, Opt -from tinygrad.renderer.ptx import PTXRenderer -from tinygrad.renderer.nir import NIRRenderer -@unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, (NIRRenderer, PTXRenderer)), "broken in LVP and PTX") class TestDoubleMatmul(unittest.TestCase): - def setUp(self): + def test_double_matmul(self): with Context(DEBUG=0): - self.a, self.b, self.c = [Tensor.randn(16, 16).contiguous().realize() for _ in range(3)] - self.ref = (self.a @ self.b @ self.c).realize() - - def _test(self, opts): - with Context(PCONTIG=2, DEBUG=max(2, DEBUG.value)): - out = (self.a @ self.b @ self.c).contiguous(arg=opts).realize() - - with Context(DEBUG=0): - err = (out-self.ref).square() - self.assertLess(err.max().item(), 1e-4) - self.assertLess(err.mean().item(), 1e-6) - - def test_baseline(self): self._test(()) - def test_upcast_0(self): self._test((Opt(OptOps.UPCAST, 0, 4),)) - def test_upcast_1(self): self._test((Opt(OptOps.UPCAST, 1, 4),)) - def test_upcast_2(self): self._test((Opt(OptOps.UPCAST, 2, 4),)) - def test_upcast_01(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4))) - def test_upcast_01_mismatch(self): self._test((Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.UPCAST, 1, 4))) - def test_upcast_02(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 2, 4))) - def test_upcast_12(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 2, 4))) - - def test_unroll_0(self): self._test((Opt(OptOps.UNROLL, 0, 4),)) - def test_unroll_1(self): self._test((Opt(OptOps.UNROLL, 1, 4),)) - def test_unroll_01(self): self._test((Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4))) - - def test_upcast_0_unroll_0(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 4))) - def test_upcast_1_unroll_0(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4))) - def test_upcast_2_unroll_0(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UNROLL, 0, 4))) - - def test_upcast_0_unroll_1(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 1, 4))) - def test_upcast_1_unroll_1(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 1, 4))) - def test_upcast_2_unroll_1(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UNROLL, 1, 4))) - - def test_upcast_1_unroll_1_small(self): self._test((Opt(OptOps.UPCAST, 1, 2), Opt(OptOps.UNROLL, 1, 2))) - def test_upcast_1_unroll_1_rev(self): self._test((Opt(OptOps.UNROLL, 1, 2), Opt(OptOps.UPCAST, 1, 2))) - - def test_upcast_01_unroll_01(self): - self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4))) - def test_upcast_12_unroll_01(self): - self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4))) + a, b, c = [Tensor.randn(16, 16).contiguous().realize() for _ in range(3)] + ref = a.numpy() @ b.numpy() @ c.numpy() + with Context(DEBUG=max(2, DEBUG.value)): + out = (a @ b @ c).numpy() + self.assertLess(abs(out-ref).max(), 1e-3) class TestRangeifyAssign(unittest.TestCase): def test_assign_permuted(self): @@ -88,16 +49,15 @@ class TestRangeifyEdgeCase(unittest.TestCase): res = Tensor.cat(a, c, dim=0) self.assertEqual(res.numpy()[-1, :16].tolist(), [512] * 16) - def test_pcontig_multi_gather(self): + def test_multi_gather(self): # regression test: local bufferize must have device set for const_like to work - with Context(PCONTIG=2): - # NOTE: with uint type, this will become a long and fail on WEBGPU - forest = Tensor(list(range(8)), dtype='int') - idx = Tensor([0, 0], dtype='int') - node_val = forest.gather(0, idx) - idx2 = idx * 2 + 1 - node_val2 = forest.gather(0, idx2) - result = (node_val + node_val2).numpy() + # NOTE: with uint type, this will become a long and fail on WEBGPU + forest = Tensor(list(range(8)), dtype='int') + idx = Tensor([0, 0], dtype='int') + node_val = forest.gather(0, idx) + idx2 = idx * 2 + 1 + node_val2 = forest.gather(0, idx2) + result = (node_val + node_val2).numpy() self.assertEqual(result.tolist(), [1, 1]) if getenv("BIG") > 2: @@ -118,65 +78,6 @@ def fa(): GlobalCounters.reset() return q.scaled_dot_product_attention(k, v) -def fa_bw(): - Tensor.manual_seed(1337) - with Context(DEBUG=0): - q,k,v = [Tensor.rand(BS, HEADS, SEQLEN, EMB).contiguous().realize() for _ in range(3)] - attn_output = nn.Linear(HEADS*EMB, HEADS*EMB, bias=False) - attn_output.weight.realize() - target = Tensor.rand(BS, SEQLEN, HEADS*EMB).contiguous().realize() - - GlobalCounters.reset() - attn = q.scaled_dot_product_attention(k, v).contiguous().contiguous_backward() - attn = attn.transpose(1, 2).reshape(BS, SEQLEN, -1) - out = attn_output(attn) - loss = (out - target).square().mean() - loss.backward() - #ret = [out, Tensor.stack(q.grad, k.grad, v.grad, dim=-1)] - #ret = [out, Tensor.stack(q.grad, k.grad, dim=-1), v.grad] - ret = [out, q.grad, k.grad, v.grad] - Tensor.realize(*ret) - return ret - -@unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, (NIRRenderer, PTXRenderer)), "broken in LVP and PTX") -class TestPcontig(unittest.TestCase): - def test_flash_attention_bw(self): - with Context(PCONTIG=max(2, PCONTIG.value), DEBUG=2): - grads = fa_bw() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - - with Context(PCONTIG=0, DEBUG=2): - cmp_grads = fa_bw() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - - with Context(DEBUG=0): - mses = [((x-y)**2).sum().item() for x,y in zip(grads, cmp_grads)] - mse = sum(mses) - print(f"mse: {mse}") - self.assertLessEqual(mse, 1e-6) - - def test_flash_attention(self, opts=None): - with Context(PCONTIG=2, DEBUG=max(2, DEBUG.value)): - ret = fa().realize() if opts is None else fa().contiguous(arg=opts).realize() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - with Context(DEBUG=2): - cmp = fa().realize() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - with Context(DEBUG=0): - mse = ((cmp-ret)**2).sum().item() - print(f"mse: {mse}") - self.assertLessEqual(mse, 1e-6) - - def test_flash_attention_opt(self): - opts = () - # columns in top matrix - opts += (Opt(OptOps.UPCAST, 0, 4),) - # columns in bottom matrix - opts += (Opt(OptOps.UPCAST, 3, 4),) - # rows in all the matrix - opts += (Opt(OptOps.UPCAST, 4, 4),) - self.test_flash_attention(opts) - # contiguous + reduce can support ranges? @unittest.skip("pm_rangeify no longer exists. test this in a different way") diff --git a/tinygrad_repo/test/backend/test_renderer_failures.py b/tinygrad_repo/test/backend/test_renderer_failures.py index d1f7a145f..8f5460d1b 100644 --- a/tinygrad_repo/test/backend/test_renderer_failures.py +++ b/tinygrad_repo/test/backend/test_renderer_failures.py @@ -23,8 +23,8 @@ def _test_uop_result(inputs:list[Tensor], sink:UOp, local_size=None): def _setup_and_test_alu(alu_op:Ops, input_val:ConstType, *alu_src_uops:UOp): dtype = alu_src_uops[0].dtype - a = UOp.param(0, dtype, (1,)) - b = UOp.param(1, dtype, (1,)) + a = UOp.param(0, dtype, 1) + b = UOp.param(1, dtype, 1) idx = UOp.const(0) ld = b.index(idx).load() alu = ld.alu(alu_op, *alu_src_uops) @@ -34,7 +34,7 @@ def _setup_and_test_alu(alu_op:Ops, input_val:ConstType, *alu_src_uops:UOp): class TestRendererFailures(unittest.TestCase): @unittest.skipIf(not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, PythonRenderer)), "test is for ptx or python renderer") def test_gated_store_with_alu(self): - a = UOp.param(0, dtypes.int, (4,)) + a = UOp.param(0, dtypes.int, 4) gate_alu = (lidx0:=UOp.special(4, 'lidx0')).ne(0) gated_alu_store = UOp(Ops.STORE, src=(a.index(lidx0.valid(gate_alu)), UOp.const(1).cast(dtypes.int))) sink = UOp(Ops.SINK, src=(gated_alu_store,), arg=KernelInfo()) @@ -43,7 +43,7 @@ class TestRendererFailures(unittest.TestCase): @unittest.skipIf(not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, PythonRenderer)), "test is for ptx or python renderer") def test_gated_store_with_alu_2d(self): - a = UOp.param(0, dtypes.int, (8,)) + a = UOp.param(0, dtypes.int, 8) gate_alu_0 = (lidx0:=UOp.special(4, 'lidx0')).ne(0) gate_alu_1 = (lidx1:=UOp.special(2, 'lidx1')).ne(0) gated_alu_store = UOp(Ops.STORE, src=(a.index((lidx0+lidx1*4).valid(gate_alu_0&gate_alu_1)), UOp.const(1).cast(dtypes.int))) @@ -77,6 +77,14 @@ class TestCStyleFailures(unittest.TestCase): @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, WGSLRenderer), "tests for wgsl renderer") class TestWGSLFailures(unittest.TestCase): + def test_folded_packed_store(self): + b = UOp.param(0, dtypes.char, 4) + idx = b.index(UOp.const(0).cast(dtypes.int)) + store = UOp.store(idx, idx.cast(dtypes.uint32).load() & UOp.const(0xffffff00).cast(dtypes.uint32)) + src = Device[Device.DEFAULT].renderer.render(UOp.sink(store, arg=KernelInfo()).toposort()) + self.assertIn("atomicAnd(&data0_4[0],4294967040u);", src) + self.assertNotIn("atomicAdd", src) + def test_multiply_infinity(self): # multiplying a positive constant by infinity should return infinity # WGSL pipelines do not handle this reliably, some of which return zero, unless infinity always comes from a read on a dynamic buffer @@ -85,9 +93,9 @@ class TestWGSLFailures(unittest.TestCase): # WGSL has a specific select(alt, val, gate) ternary operator instead of gate?val:alt def test_gated_load(self): - a = UOp.param(0, dtypes.int, (4,)) - b = UOp.param(1, dtypes.int, (4,)) - c = UOp.param(2, dtypes.int, (4,)) + a = UOp.param(0, dtypes.int, 4) + b = UOp.param(1, dtypes.int, 4) + c = UOp.param(2, dtypes.int, 4) lidx0 = UOp.special(4, "lidx0") gate = lidx0.ne(0) alt = c.index(lidx0).load() @@ -102,7 +110,7 @@ class TestWGSLFailures(unittest.TestCase): class TestPTXFailures(unittest.TestCase): @unittest.skip("INDEX can only have a gate ALU parent, not an IF") def test_gated_store_with_if(self): - a = UOp.param(0, dtypes.int, (4,)) + a = UOp.param(0, dtypes.int, 4) gate_alu = (lidx0:=UOp.special(4, 'lidx0')).ne(0) val = UOp.const(1).cast(dtypes.int) if_uop = UOp(Ops.IF, src=(gate_alu,)) diff --git a/tinygrad_repo/test/backend/test_schedule.py b/tinygrad_repo/test/backend/test_schedule.py index dd2b9e5d7..0f7738d6b 100644 --- a/tinygrad_repo/test/backend/test_schedule.py +++ b/tinygrad_repo/test/backend/test_schedule.py @@ -147,7 +147,7 @@ class TestSchedule(unittest.TestCase): devs = ("CPU:0", "CPU:1") x = Tensor.ones(2, device="CPU").shard(devs, axis=0).realize() out = (x.sum()*2).reshape(1).to("CPU") - run_linear(*check_schedule(out, 5)) + run_linear(*check_schedule(out, 3)) np.testing.assert_equal(out.numpy(), [4.]) class TestLimitBufs(unittest.TestCase): @@ -176,7 +176,7 @@ class TestLimitBufs(unittest.TestCase): def test_limit_bufs_linear_scaling(self): def sched_time(n): - with Context(TRACK_MATCH_STATS=0, DEBUG=0): + with Context(TRACK_MATCH_STATS=0, DEBUG=0, PARALLEL=0): bufs = [Tensor.ones(16).contiguous().realize() for _ in range(4)] root = bufs[0] for i in range(n): root = root + bufs[i % 4] @@ -365,16 +365,6 @@ class TestCopyFolding(unittest.TestCase): b = a.to("CPU") self.assertListEqual(b.tolist(), [2.]) - def test_copy_to_same_device(self): - a = Tensor.empty(4).uop - b = a.copy_to_device(a.device) - check_schedule(b, 1, filter_sink=False) # TODO: 0? - - def test_copy_to_same_device_alt(self): - a = Tensor.empty(4, 4).uop - b = a.copy_to_device(a.device) - check_schedule(b, 1, filter_sink=False) # TODO: 0? - def test_copy_to_same_device_sched(self): a = Tensor.ones(4).contiguous().realize().uop.buf_uop t = Tensor(a.copy_to_device(a.device)) @@ -384,40 +374,35 @@ class TestCopyFolding(unittest.TestCase): assert t.uop.is_realized, f"didn't realize Tensor {t}" self.assertListEqual(t.tolist(), [1.,1.,1.,1.]) - @unittest.skip("same-device copies are no-ops") - def test_self_assign_same_device_copy(self): - a = Tensor.ones(4, 4).contiguous().realize() - # use copy_to_device to bypass Tensor.to() shortcircuit and force a real same-device COPY in the graph - a.assign(Tensor(a.uop.copy_to_device(a.device), a.device)) - run_linear(*check_schedule(a, 2, filter_sink=False)) - self.assertListEqual(a.tolist(), [[1.]*4]*4) - def test_clone(self): a = Tensor.empty(4) check_schedule(a.clone(), 1, filter_sink=False) def test_shrink_copy(self): - a = Tensor.arange(4) - view = a.shrink(((0, 2),)) - b = view.clone() - run_linear(*check_schedule(b, 1, filter_sink=False)) - self.assertEqual(b.uop.base.buffer.size, 2) - self.assertEqual(b.uop.numel(), 2) - self.assertListEqual(b.tolist(), [0, 1]) + a = Tensor.arange(4).clone("CPU:1").realize() + b = a.to("CPU:2").shrink(((1, 3),)).to("CPU:3") + GlobalCounters.reset() + run_linear(*check_schedule(b, 3, filter_sink=False)) + # extra E kernel, copy exactly 4 bytes + self.assertEqual(GlobalCounters.global_mem, 4*4 + 2*4*2 + 2*4) + self.assertListEqual(b.tolist(), [1, 2]) def test_expanded_copy(self): - a = Tensor.arange(2) - view = a.reshape(2, 1).expand(2, 2) - b = view.clone() - run_linear(*check_schedule(b, 1, filter_sink=False)) - self.assertEqual(b.uop.base.buffer.size, 4) - self.assertEqual(b.uop.numel(), 4) - self.assertListEqual(b.tolist(), [[0, 0], [1, 1]]) + a = Tensor.arange(4).clone("CPU:1").realize() + b = a.to("CPU:2").reshape(4, 1).expand(4, 2).to("CPU:3") + GlobalCounters.reset() + run_linear(*check_schedule(b, 3, filter_sink=False)) + # TODO: expands before copy + self.assertEqual(GlobalCounters.global_mem, 4*4 + (4*4 + 8*4) + 8*4) + self.assertListEqual(b.tolist(), [[0, 0], [1, 1], [2, 2], [3, 3]]) def test_permuted_copy(self): - a = Tensor.arange(4) - b = a.reshape(2, 2).permute(1, 0) - b.realize() + a = Tensor.arange(4).clone("CPU:1").realize() + b = a.to("CPU:2").reshape(2, 2).permute(1, 0).to("CPU:3") + GlobalCounters.reset() + run_linear(*check_schedule(b, 3, filter_sink=False)) + # permutes before copy + self.assertEqual(GlobalCounters.global_mem, 4*4 + (4*4 + 4*4) + 4*4) self.assertListEqual(b.tolist(), [[0, 2], [1, 3]]) def test_permute_on_disk(self): diff --git a/tinygrad_repo/test/backend/test_setitem.py b/tinygrad_repo/test/backend/test_setitem.py index 2017756da..4ee3290b5 100644 --- a/tinygrad_repo/test/backend/test_setitem.py +++ b/tinygrad_repo/test/backend/test_setitem.py @@ -18,13 +18,16 @@ class TestSetitem(unittest.TestCase): ((4,4,4,4), (slice(1,3), slice(None), slice(None), slice(0,3)), 4), ((6,6), (slice(1,5,2), slice(0,5,3)), 1.0), ((6,6), (slice(5,1,-2), slice(5,0,-3)), 1.0), + ((6,6), (slice(None), slice(0,6,2)), 1.0), ) for shp, slc, val in cases: - t = Tensor.zeros(shp).contiguous() - t[slc] = val - n = np.zeros(shp) - n[slc] = val.numpy() if isinstance(val, Tensor) else val - np.testing.assert_allclose(t.numpy(), n) + for realize in (False, True): + t = Tensor.zeros(shp).contiguous() + if realize: t.realize() + t[slc] = val + n = np.zeros(shp) + n[slc] = val.numpy() if isinstance(val, Tensor) else val + np.testing.assert_allclose(t.numpy(), n) def test_padded_setitem(self): t = Tensor.arange(10) @@ -301,6 +304,14 @@ class TestSetitem(unittest.TestCase): self.assertListEqual(z[2:5].tolist(), [2, 2, 2]) self.assertListEqual(z[6:7].tolist(), [3]) +class TestAssignBitcast(unittest.TestCase): + def test_assign_through_bitcast(self): + # the dest is unrealized, so callify cannot fold the BITCAST into a buffer view and the STORE keeps a + # BITCAST dest; the bitcast has to move to the value side or the store never reaches the buffer + a = Tensor.full((4,), 1.0, dtype=dtypes.float32).contiguous() + a.bitcast(dtypes.uint32).assign(Tensor([0x40800000, 0x40400000, 0x40000000, 0x3f800000], dtype=dtypes.uint32)).realize() + np.testing.assert_allclose(a.numpy(), [4.0, 3.0, 2.0, 1.0]) + class TestWithGrad(unittest.TestCase): def test_basic_setitem_works(self): z = Tensor.rand(8, 8) diff --git a/tinygrad_repo/test/backend/test_tensor.py b/tinygrad_repo/test/backend/test_tensor.py index c44a2b9e0..c77b8d649 100644 --- a/tinygrad_repo/test/backend/test_tensor.py +++ b/tinygrad_repo/test/backend/test_tensor.py @@ -653,9 +653,19 @@ class TestZeroShapeTensor(unittest.TestCase): np.testing.assert_equal(Tensor([[1, 2]]).pad_to(2, 3).numpy(), [[1, 2, 0], [0, 0, 0]]) np.testing.assert_equal(Tensor([[1, 2]]).pad_to(1, 3).numpy(), [[1, 2, 0]]) np.testing.assert_equal(Tensor([[1, 2]]).pad_to(None, 3).numpy(), [[1, 2, 0]]) + np.testing.assert_equal(Tensor([1, 2]).pad_to(4, value=2).numpy(), [1, 2, 2, 2]) + np.testing.assert_equal(Tensor([[1, 2]]).pad_to(2, 3, value=-1).numpy(), [[1, 2, -1], [-1, -1, -1]]) + np.testing.assert_equal(Tensor([1, 2]).pad_to(None, value=5).numpy(), [1, 2]) # no-op pad ignores the fill with self.assertRaises(ValueError): Tensor([1, 2]).pad_to(2, 3) with self.assertRaises(ValueError): Tensor([[1, 2]]).pad_to(3) + def test_max_shape(self): + from tinygrad import UOp + t = Tensor.empty(2, UOp.variable('v', 1, 32), 4) + self.assertEqual(t.max_shape, (2, 32, 4)) + self.assertEqual(t.max_numel(), 2*32*4) + self.assertEqual(Tensor.empty(2, 3).max_shape, (2, 3)) + def test_shrink_into_zero(self): t = Tensor.rand(3, 4).realize() assert t.shrink((None, (2, 2))).realize().shape == (3, 0) diff --git a/tinygrad_repo/test/backend/test_tensor_variable.py b/tinygrad_repo/test/backend/test_tensor_variable.py index 8544d04e2..834d8c998 100644 --- a/tinygrad_repo/test/backend/test_tensor_variable.py +++ b/tinygrad_repo/test/backend/test_tensor_variable.py @@ -1,7 +1,7 @@ import unittest import numpy as np from tinygrad import Device, Tensor, Variable, TinyJit, dtypes -from tinygrad.helpers import CHECK_OOB +from tinygrad.helpers import CHECK_OOB, Context class TestTensorVariable(unittest.TestCase): def test_add_tvar(self): @@ -35,7 +35,14 @@ class TestTensorVariable(unittest.TestCase): vv = Variable("a", 1, 10).bind(2) self.assertEqual(Tensor(vv).dtype, dtypes.weakint) self.assertEqual((Tensor(vv) + Tensor([1], dtype=dtypes.int8)).dtype, dtypes.int8) # takes the concrete side, no widening - self.assertEqual(Tensor(vv).item(), 2) # a read commits at default_int + self.assertEqual(Tensor(vv).item(), 2) # a read commits by bounds, like a kernel + + def test_weak_read_widens_by_bounds(self): + self.assertEqual(Tensor(2**40).item(), 2**40) + self.assertEqual(Tensor(Variable("b", 0, 2**40).bind(2**35+3)).item(), 2**35+3) + + def test_long_variable_emulated_raises(self): + with Context(EMULATED_DTYPES="long"), self.assertRaises(RuntimeError): Tensor(Variable("c", 0, 2**40).bind(2**35+3)).item() def test_variable_tensor_dtype_arg(self): vv = Variable("a", 1, 10).bind(2) @@ -50,6 +57,8 @@ class TestTensorVariable(unittest.TestCase): # bound variables in an expression are fine self.assertEqual(Tensor(Variable("u", 1, 10).bind(2) + 1).item(), 3) + def test_negative_variable_on_device(self): self.assertEqual(Tensor(Variable("n", -10, 10).bind(-3)).clone().item(), -3) + def test_shrink_beyond_buffer_variable(self): # TODO: shrink by a variable whose vmax exceeds the dim should fail at build, today only CHECK_OOB=1 rejects it t = Tensor.ones(3).contiguous()[:Variable("a", 1, 10).bind(5)] diff --git a/tinygrad_repo/test/backend/test_uops.py b/tinygrad_repo/test/backend/test_uops.py index d2b51ae7c..e5d0c8c9a 100644 --- a/tinygrad_repo/test/backend/test_uops.py +++ b/tinygrad_repo/test/backend/test_uops.py @@ -2,7 +2,7 @@ from typing import Optional, Any import unittest, math import numpy as np from tinygrad.tensor import Tensor, _to_np_dtype -from tinygrad.helpers import Context +from tinygrad.helpers import Context, ceildiv from tinygrad.dtype import dtypes, DType, AddrSpace, ConstFloat # noqa: F401 from tinygrad.device import Buffer, Device from tinygrad.uop.ops import Ops, UOp, KernelInfo, AxisType, buffers @@ -20,8 +20,8 @@ def run_uops(uops_list:list[UOp], bufs:list[Buffer]): def uop(uops:list[UOp], op:Ops, dtype:Optional[DType], src:tuple[UOp, ...], arg:Any=None) -> UOp: if op is Ops.CONST: uops.append(UOp.const(arg).cast(dtype)) - elif op is Ops.PARAM: uops.append(UOp.param(arg, dtype, shape=(1,))) - else: uops.append(UOp(op, dtype, tuple(src), arg)) + elif op is Ops.PARAM: uops.append(UOp.param(arg, dtype, 1)) + else: uops.append(UOp(op, tuple(src), arg)) return uops[-1] def _test_single_value(vals, op, dts): @@ -57,6 +57,35 @@ def _test_uops_result(output_dtype, uops, res): run_uops([out], [buf]) return np.frombuffer(buf.as_memoryview(), _to_np_dtype(output_dtype))[0] +@unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, CStyleLanguage) and + dtypes.uint64 in Device[Device.DEFAULT].renderer.supported_dtypes(), "requires C-style pointer bitcast and 64-bit ints") +class TestBitcastBufferView(unittest.TestCase): + @Context(SPEC=2) + def test_render(self): + buf = UOp.param(0, dtypes.uint32, 4) + uops = to_uops_list([buf.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0).store(1)], ren=Device[Device.DEFAULT].renderer) + idx = next(u for u in uops if u.op is Ops.INDEX and u.src[0].op is Ops.BITCAST) + self.assertEqual(idx.src[0].src[0].op, Ops.SHRINK) + Device[Device.DEFAULT].renderer.render(uops) + + @Context(SPEC=2) + def test_load(self): + val = 0x1122334455667788 + src, out = UOp.param(0, dtypes.uint32, 4), UOp.param(1, dtypes.uint64, 1) + ibuf = Buffer(Device.DEFAULT, 4, dtypes.uint32, initial_value=np.array([0, 0x55667788, 0x11223344, 0], dtype=np.uint32).tobytes()) + obuf = Buffer(Device.DEFAULT, 1, dtypes.uint64).allocate() + run_uops([out.index(0).store(src.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0))], [ibuf, obuf]) + self.assertEqual(np.frombuffer(obuf.as_memoryview(), dtype=np.uint64)[0], val) + + @Context(SPEC=2) + def test_store(self): + val = 0x1122334455667788 + dst = UOp.param(0, dtypes.uint32, 6) + buf = Buffer(Device.DEFAULT, 6, dtypes.uint32, initial_value=bytes(24)) + view = dst.shrink(((1, 5),)).bitcast(dtypes.uint64) # two stores through one view: it must inline, not get a declared vector-pointer + run_uops([view.index(0).store(val ^ 0xff), view.index(1).store(val)], [buf]) + self.assertEqual(np.frombuffer(buf.as_memoryview(), dtype=np.uint64, count=2, offset=4).tolist(), [val ^ 0xff, val]) + class TestUOps(unittest.TestCase): def _equal(self, v1, v2): assert isinstance(v2, (float, int, bool)) @@ -193,15 +222,16 @@ class TestLocalAccess(unittest.TestCase): @unittest.skipUnless(Device.DEFAULT == "WEBGPU", "Test local memory size for packed data types") def test_packed_smem_size(self): _dtypes = [dtypes.char, dtypes.uchar, dtypes.short, dtypes.ushort, dtypes.half] - size = 16 - for dtype in _dtypes: - temp = UOp.placeholder((size,), dtype, slot=0, addrspace=AddrSpace.LOCAL) - uops = to_uops_list([temp], ren=Device[Device.DEFAULT].renderer) - out = Device[Device.DEFAULT].renderer.render(uops) - # half is supported in wgsl, so it doesn't have to be packed - corrected_size = size//(4//dtype.itemsize) if dtype != dtypes.half else size - # temp0: array<{Device[Device.DEFAULT].renderer.buf_map(dtype)},{corrected_size}>; - self.assertIn(f",{corrected_size}>;", out) + # a partial word still needs a whole word, so sizes that don't fill one must round up + for size in (16, 5): + for dtype in _dtypes: + temp = UOp.placeholder((size,), dtype, slot=0, addrspace=AddrSpace.LOCAL) + uops = to_uops_list([temp], ren=Device[Device.DEFAULT].renderer) + out = Device[Device.DEFAULT].renderer.render(uops) + # half is supported in wgsl, so it doesn't have to be packed + corrected_size = ceildiv(size, 4//dtype.itemsize) if dtype != dtypes.half else size + # temp0: array<{Device[Device.DEFAULT].renderer.buf_map(dtype)},{corrected_size}>; + self.assertIn(f",{corrected_size}>;", out) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_shared, "test requires shared memory") @unittest.skip("tinygrad doesn't support this behavior") @@ -219,8 +249,8 @@ class TestLocalAccess(unittest.TestCase): @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "This only tests assembly backends") class TestAssembly(unittest.TestCase): def test_bitshift_left(self): - g1 = UOp.param(0, dtypes.int32, shape=(3,)) - out = UOp.param(1, dtypes.int32, shape=(2,)) + g1 = UOp.param(0, dtypes.int32, 3) + out = UOp.param(1, dtypes.int32, 2) c1 = UOp.const(2) c2 = UOp.const(3) l1 = g1.index(c1) @@ -241,14 +271,14 @@ class TestAssembly(unittest.TestCase): b = Tensor.empty(1024) c = (a*b).sum() ast = c.schedule_linear().src[-1].src[0] - opts_to_apply = [Opt(OptOps.UNROLL, 0, 4)] + opts_to_apply = [Opt(OptOps.SPLIT, 0, (4, AxisType.UNROLL))] ast = ast.replace(arg=KernelInfo(opts_to_apply=tuple(opts_to_apply))) program = to_program(ast, Device[Device.DEFAULT].renderer) uops = tuple(program.src[1].src) self.assertGreaterEqual(len([x.op for x in uops if x.op is Ops.MULACC]), 4) def test_mulacc_shl(self): - g1 = UOp.param(0, dtypes.int32, shape=(2,)) + g1 = UOp.param(0, dtypes.int32, 2) c1 = UOp.const(0) c2 = UOp.const(1) expr = g1.index(c1) * UOp.const(4096) + g1.index(c2) @@ -257,7 +287,7 @@ class TestAssembly(unittest.TestCase): self.assertIn(Ops.MULACC, [x.op for x in uops]) def test_use_cmpeq(self): - g = UOp.param(0, dtypes.uint32, shape=(8,)) + g = UOp.param(0, dtypes.uint32, 8) c = UOp.const(7) comp = g.index(c).ne(c).ne(True) uops = to_uops_list([comp], ren=Device[Device.DEFAULT].renderer) diff --git a/tinygrad_repo/test/backend/test_wait_loop.py b/tinygrad_repo/test/backend/test_wait_loop.py index 83e0543e6..6073c9422 100644 --- a/tinygrad_repo/test/backend/test_wait_loop.py +++ b/tinygrad_repo/test/backend/test_wait_loop.py @@ -43,7 +43,7 @@ def nested_loop_kernel(C:UOp) -> UOp: return C[0].store(i[0].load()).sink(arg=KernelInfo(name="nested_loop", opts_to_apply=())) def wait_ext_kernel() -> UOp: - sig = UOp.param(0, dtypes.int, (1,), volatile=True) + sig = UOp.param(0, dtypes.int, 1, volatile=True) l = UOp.loop(0) v = sig.after(l)[0].load() e = v.end(l, v < 1) diff --git a/tinygrad_repo/test/device/test_hcq.py b/tinygrad_repo/test/device/test_hcq.py index b8390d0b8..55c481266 100644 --- a/tinygrad_repo/test/device/test_hcq.py +++ b/tinygrad_repo/test/device/test_hcq.py @@ -9,6 +9,7 @@ from tinygrad.runtime.support.system import PCIIfaceBase from tinygrad.engine.realize import get_runtime from tinygrad.codegen import to_program from tinygrad.codegen.opt import Opt, OptOps +from tinygrad.uop.ops import AxisType from tinygrad import Variable MOCKGPU = DEV.interface.startswith("MOCK") @@ -167,7 +168,7 @@ class TestHCQ(unittest.TestCase): b = a + 1 si = b.schedule_linear().src[-1] - prg = to_program(replace_opts(si.src[0], [Opt(op=OptOps.LOCAL, axis=0, arg=3) for _ in range(3)]), TestHCQ.d0.renderer) + prg = to_program(replace_opts(si.src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(3, AxisType.LOCAL)) for _ in range(3)]), TestHCQ.d0.renderer) runtime = get_runtime(Device.DEFAULT, prg) zb = Buffer(Device.DEFAULT, 3 * 3 * 3, dtypes.int, options=BufferSpec(cpu_access=True, nolru=True)).ensure_allocated() diff --git a/tinygrad_repo/test/device/test_hcq2.py b/tinygrad_repo/test/device/test_hcq2.py new file mode 100644 index 000000000..002be5457 --- /dev/null +++ b/tinygrad_repo/test/device/test_hcq2.py @@ -0,0 +1,64 @@ +import unittest, numpy as np +from unittest.mock import patch +from tinygrad import Device, Tensor +from tinygrad.device import Buffer +from tinygrad.dtype import dtypes +from tinygrad.helpers import HCQ2 +from tinygrad.runtime.support.hcq2 import HCQ_DEVS, all_devices_in, hcq_compile_cache, link_linear_cache + +@unittest.skipUnless(HCQ2 and all_devices_in(Device.DEFAULT, HCQ_DEVS), "hcq2 device required") +class TestHCQ2(unittest.TestCase): + def test_copy_without_copy_queue(self): + with patch.object(Device[Device.DEFAULT], "has_copy_queue", False): + np.testing.assert_equal(Tensor(np.arange(61, dtype=np.float32)).to(Device.DEFAULT).contiguous().realize().numpy(), np.arange(61)) + + @unittest.skipIf(Device.DEFAULT == "CPU", "ping-pong needs a non-CPU hcq2 device") + def test_cpu_device_ping_pong(self): + # CPU submits run inline, so alternating dependencies must be submitted in schedule order to avoid blocking the host submitter. + x = Tensor.ones(16, device="CPU").contiguous().realize() + a = (x + 1).contiguous() + b = (a.to(Device.DEFAULT).contiguous() + 1).contiguous() + c = (b.to("CPU").contiguous() + 1).contiguous() + out = (c.to(Device.DEFAULT).contiguous() + 1).contiguous().realize() + np.testing.assert_equal(out.numpy(), np.full(16, 5)) + + @unittest.skipIf(Device.DEFAULT == "CPU", "staged copies need a non-CPU hcq2 device") + def test_staged_copy_slot_reuse(self): + # chunks of a staged copy rotate through the staging buffer slots, many rotations must stay bit-exact in both directions + import tinygrad.runtime.support.hcq2 as hcq2 + buf = Buffer("CPU", 1 << 20, dtypes.uint8, preallocate=True) + data = np.random.default_rng(42).integers(0, 256, (5 << 20) + 123, dtype=np.uint8) + with patch.object(hcq2, "STAGING_SIZE", 1 << 20), patch.object(hcq2, "STAGING_SLOTS", 4), patch.object(hcq2, "_staging", lambda: buf): + np.testing.assert_equal(Tensor(data).to(Device.DEFAULT).realize().numpy(), data) + + def test_overlapping_device_tuples(self): + # an op on a wide device tuple followed by an op on an overlapping smaller tuple used to MMU-fault the smaller one + d4, d2 = tuple(f"{Device.DEFAULT}:{i}" for i in range(4)), tuple(f"{Device.DEFAULT}:{i}" for i in range(2)) + ref = Tensor.arange(16).contiguous().realize() + Tensor(ref.uop.copy_to_device(d4)).realize() + out = Tensor.ones(8).shard(d2, axis=0).contiguous().realize() + np.testing.assert_equal(out.numpy(), np.ones(8)) + + def relowers(self, t:Tensor) -> tuple[int, int]: + # a miss in either cache relowers the whole submit, a hit costs ~0.1ms + before = (len(hcq_compile_cache), len(link_linear_cache)) + t.realize() + return (len(hcq_compile_cache) - before[0], len(link_linear_cache) - before[1]) + + def test_relower_only_on_new_kernel(self): + a, b = (Tensor.empty(64, 64).contiguous().realize() for _ in range(2)) + self.relowers(a.sin()) + self.assertEqual(self.relowers(a.sin()), (0, 0)) # nothing changed + self.assertEqual(self.relowers(b.sin()), (0, 0)) # new buffers, patched in at link time + self.assertEqual(self.relowers(a.cos()), (1, 1)) # new kernel, though only the code address moved + self.assertEqual(self.relowers(a.cos()), (0, 0)) + self.assertEqual(self.relowers(Tensor.empty(32, 32).contiguous().realize().sin()), (1, 1)) # new shape + + def test_dtype_sweep_relowers_every_dtype(self): + # test_dtype sweeps dtypes at one shape, so nearly every kernel is new: this is where hcq2 ci time goes + src = Tensor.empty(64, 64).contiguous().realize() + dts = (dtypes.int8, dtypes.uint8, dtypes.int16, dtypes.uint16, dtypes.int32) + self.assertEqual([self.relowers(src.cast(dt).contiguous())[0] for dt in dts], [1] * len(dts)) + +if __name__ == "__main__": + unittest.main() diff --git a/tinygrad_repo/test/external/external_benchmark_all2all.py b/tinygrad_repo/test/external/external_benchmark_all2all.py new file mode 100644 index 000000000..e278dfad7 --- /dev/null +++ b/tinygrad_repo/test/external/external_benchmark_all2all.py @@ -0,0 +1,17 @@ +from tinygrad import Device, Tensor, TinyJit, dtypes +from tinygrad.helpers import Timing, Context + +GPUS, DEPTH, SZ = 8, 4, 128 * 2**20 +WARMUP, ITERS = 3, 5 +devs = tuple(f"{Device.DEFAULT}:{i}" for i in range(GPUS)) +bufs = tuple(Tensor.empty(SZ, dtype=dtypes.uint8, device=dev).contiguous().realize() for _ in range(DEPTH) for dev in devs) + +@TinyJit +def all_to_all(*srcs:Tensor): return Tensor.realize(*(src.to(dst) for i,src in enumerate(srcs) for j,dst in enumerate(devs) if i % GPUS != j)) + +if __name__ == "__main__": + with Context(ALL2ALL=1, JIT_BATCH_SIZE=0): + for i in range(-WARMUP, ITERS): + with Timing("ALL2ALL ", lambda ns: f" {SZ*GPUS*(GPUS-1)*DEPTH/ns:.2f} GB/s", enabled=i>=0): + all_to_all(*bufs) + for dev in devs: Device[dev].synchronize() diff --git a/tinygrad_repo/test/external/external_benchmark_op_conv.py b/tinygrad_repo/test/external/external_benchmark_op_conv.py index 4076facd5..af512ae7d 100644 --- a/tinygrad_repo/test/external/external_benchmark_op_conv.py +++ b/tinygrad_repo/test/external/external_benchmark_op_conv.py @@ -9,7 +9,7 @@ from tinygrad.helpers import dedup, getenv from tinygrad.device import Buffer from tinygrad.dtype import Invalid -# PYTHONPATH="." DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx +# PYTHONPATH="." DEV=QCOM FLOAT16=1 IMAGE=2 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx def vision_conv_143(): c0 = UOp.param(0, dtypes.half, shape=(16, 1024, 4)) @@ -28,13 +28,12 @@ def vision_conv_143(): c48 = (c24&c32).where(c34.index(c45), UOp.const(0.0, dtypes.float)) c49 = UOp.param(2, dtypes.half, shape=(64, 49, 4)) c61 = c48*c49.index((c26*4+c5%2+c16*28+c38*196)) - c63 = UOp.param(3, dtypes.float, (128,)) + c63 = UOp.param(3, dtypes.float, 128) c65 = c61.reduce(c16, c26, arg=Ops.ADD)+c63.index(c5) c67 = c0.index((c2*128+c5+c8*4096)).store(c65).end(c8, c2, c5) opts = None # JITBEAM=2 - # (Opt(op=OptOps.UPCAST, axis=2, arg=4), Opt(op=OptOps.NOLOCALS, axis=None, arg=None), Opt(op=OptOps.UPCAST, axis=2, arg=2), Opt(op=OptOps.UPCAST, axis=1, arg=4), Opt(op=OptOps.SWAP, axis=1, arg=2)) return c67.sink(arg=KernelInfo(name="conv", opts_to_apply=opts)) def vision_conv_153(): @@ -54,13 +53,12 @@ def vision_conv_153(): c48 = (c24&c32).where(c34.index(c45), UOp.const(0.0, dtypes.float)) c49 = UOp.param(2, dtypes.half, shape=(128, 49, 4)) c61 = c48*c49.index((c26*4+c5%2+c16*28+c38*196)) - c63 = UOp.param(3, dtypes.float, (256,)) + c63 = UOp.param(3, dtypes.float, 256) c65 = c61.reduce(c16, c26, arg=Ops.ADD)+c63.index(c5) c67 = c0.index((c2*256+c5+c8*4096)).store(c65).end(c8, c2, c5) opts = None # JITBEAM=2 - # (Opt(op=OptOps.UPCAST, axis=2, arg=4), Opt(op=OptOps.NOLOCALS, axis=None, arg=None), Opt(op=OptOps.UPCAST, axis=2, arg=2), Opt(op=OptOps.SWAP, axis=1, arg=2)) return c67.sink(arg=KernelInfo(name="conv", opts_to_apply=opts)) def dm_conv_172(): @@ -73,7 +71,7 @@ def dm_conv_172(): c18 = UOp.range(8, 2, AxisType.REDUCE) c23 = UOp.param(2, dtypes.half, shape=(240, 128, 4)) c35 = c5.index((c7*4+c10+c13*128+c18*1536))*c23.index((c10*4+c2%4+c7*16+c2//4*512)) - c37 = UOp.param(3, dtypes.float, (960,)) + c37 = UOp.param(3, dtypes.float, 960) c39 = c35.reduce(c7, c10, arg=Ops.ADD)+c37.index(c2) c50 = (1.0+((c39+0.044708251953125*(c39*(c39*c39)))*-2.3021129851685216).exp2()).reciprocal()*c39 c53 = c50.reduce(c18, c13, arg=Ops.ADD)*0.010416666666666666 @@ -81,7 +79,6 @@ def dm_conv_172(): opts = None # JITBEAM=2 - # (Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.GROUPTOP, axis=1, arg=32), Opt(op=OptOps.UNROLL, axis=1, arg=4), Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UNROLL, axis=0, arg=4), Opt(op=OptOps.GROUP, axis=1, arg=0)) return c55.sink(arg=KernelInfo(name="conv", opts_to_apply=opts)) ast = {143: vision_conv_143, 153: vision_conv_153, 172: dm_conv_172}[getenv("NUM", 143)]() diff --git a/tinygrad_repo/test/external/external_benchmark_pyrender.py b/tinygrad_repo/test/external/external_benchmark_pyrender.py index 2183e2a99..48238e529 100644 --- a/tinygrad_repo/test/external/external_benchmark_pyrender.py +++ b/tinygrad_repo/test/external/external_benchmark_pyrender.py @@ -7,8 +7,8 @@ BENCHMARK_OPS = {Ops.INDEX, Ops.STAGE} @functools.cache def create_uop(a:int) -> UOp: - op, dtype, src, arg, *rest = trace.uop_fields[a] - return UOp(op, dtype, tuple(create_uop(s) for s in src), arg, *rest) + op, src, arg, *rest = trace.uop_fields[a] + return UOp(op, tuple(create_uop(s) for s in src), arg, *rest) if __name__ == "__main__": # load rewrite trace diff --git a/tinygrad_repo/test/external/external_test_gpu_crash.py b/tinygrad_repo/test/external/external_test_gpu_crash.py index ab31ae356..16e3c8fe6 100644 --- a/tinygrad_repo/test/external/external_test_gpu_crash.py +++ b/tinygrad_repo/test/external/external_test_gpu_crash.py @@ -23,7 +23,7 @@ class TestGPUCrash(unittest.TestCase): cls.is_cdna = cls.dev.target[0] < 10 ins = importlib.import_module('tinygrad.runtime.autogen.amd.' + ('cdna' if cls.is_cdna else 'rdna3') + '.ins') for rdna3_name, cdna3_name in RDNA3_CDNA3_MAP.items(): - setattr(cls, rdna3_name, getattr(ins, cdna3_name if cls.is_cdna else rdna3_name)) + setattr(cls, rdna3_name, staticmethod(getattr(ins, cdna3_name if cls.is_cdna else rdna3_name))) def setUp(self): # Verify device works before each test @@ -36,8 +36,8 @@ class TestGPUCrash(unittest.TestCase): def _run_insts(self, insts: list[Inst]): buf = UOp.new_buffer("AMD", 64, dtypes.uint8) - sink = UOp.sink(UOp.param(0, dtypes.uint8, (64,), device="AMD"), UOp.special(1, "lidx0"), arg=KernelInfo("test")) - prg = UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=i) for i in insts)))) + sink = UOp.sink(UOp.param(0, dtypes.uint8, 64, device="AMD"), UOp.special(1, "lidx0"), arg=KernelInfo("test")) + prg = UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(i, dtypes.void)) for i in insts)))) run_linear(UOp(Ops.LINEAR, src=(prg.call(buf),)), wait=True) def _assert_gpu_fault(self, func): diff --git a/tinygrad_repo/test/external/external_test_onnx_ops.py b/tinygrad_repo/test/external/external_test_onnx_ops.py index e3ac1fe69..02071da69 100644 --- a/tinygrad_repo/test/external/external_test_onnx_ops.py +++ b/tinygrad_repo/test/external/external_test_onnx_ops.py @@ -54,6 +54,12 @@ class TestMainOnnxOps(TestOnnxOps): outputs = ["squeezed"] self.helper_test_single_op("Squeeze", inputs, attributes, outputs) + def test_mean_variance_normalization_axes(self): + inputs = {"x": np.random.randn(2, 3, 4, 5).astype(np.float32)} + attributes = {"axes": [2, 3]} + outputs = ["out"] + self.helper_test_single_op("MeanVarianceNormalization", inputs, attributes, outputs) + def test_conv(self): # test VALID auto_pad inputs = { @@ -235,6 +241,15 @@ class TestMainOnnxOps(TestOnnxOps): outputs = ["y"] self.helper_test_single_op("MaxUnpool", inputs, attributes, outputs) + def test_maxunpool_pads(self): + # per-axis pads shrink the output: spatial dim is (i-1)*stride + kernel - pad_begin - pad_end -> (2, 4), and indices index into that output + # NOTE: indices must be in bounds of that output; ORT aborts the process on out-of-bounds indices + xT = np.array([[[[5, 6], [7, 8]]]], dtype=np.float32) + xI = np.array([[[[0, 3], [4, 7]]]], dtype=np.int64) + inputs = {"x": xT, "indices": xI} + attributes = {"kernel_shape": [2, 2], "strides": [2, 2], "pads": [1, 0, 1, 0]} + self.helper_test_single_op("MaxUnpool", inputs, attributes, ["y"]) + def test_averagepool_3d_dilations_large_count_include_pad_is_1_ceil_mode_is_True(self): # https://github.com/onnx/onnx/blob/main/docs/Operators.md#examples-13 inputs = {"x": np.random.randn(1, 1, 32, 32, 32).astype(np.float32)} diff --git a/tinygrad_repo/test/external/external_test_onnx_runner.py b/tinygrad_repo/test/external/external_test_onnx_runner.py index 193a30b88..96b73505a 100644 --- a/tinygrad_repo/test/external/external_test_onnx_runner.py +++ b/tinygrad_repo/test/external/external_test_onnx_runner.py @@ -13,10 +13,10 @@ def _check_ast_count(desired_count:int, t:Tensor): asts = [call for call in linear.src if call.src[0].op is Ops.SINK] assert len(asts) == desired_count, f"{len(asts)} != {desired_count}" -def build_onnx(nodes, from_disk:bool=True, **kwargs): +def build_onnx(nodes, from_disk:bool=True, opset_imports=None, **kwargs): """Helper to build and return an OnnxRunner from ONNX nodes.""" graph = onnx.helper.make_graph(nodes, 'test', kwargs.get('inputs', []), kwargs.get('outputs', []), kwargs.get('initializers', [])) - model = onnx.helper.make_model(graph) + model = onnx.helper.make_model(graph) if opset_imports is None else onnx.helper.make_model(graph, opset_imports=opset_imports) if from_disk: with tempfile.TemporaryDirectory() as tmpdir: tmp_path = pathlib.Path(tmpdir) @@ -29,6 +29,23 @@ def build_onnx(nodes, from_disk:bool=True, **kwargs): return runner class TestOnnxRunner(unittest.TestCase): + def test_tinygrad_contiguous(self): + runner = build_onnx( + nodes=[ + onnx.helper.make_node('Add', ['inp', 'one'], ['added']), + onnx.helper.make_node('Contiguous', ['added'], ['materialized'], domain='org.tinygrad'), + onnx.helper.make_node('Mul', ['materialized', 'two'], ['output']) + ], + inputs=[onnx.helper.make_tensor_value_info('inp', onnx.TensorProto.FLOAT, (4,))], + outputs=[onnx.helper.make_tensor_value_info('output', onnx.TensorProto.FLOAT, (4,))], + initializers=[ + onnx.helper.make_tensor('one', onnx.TensorProto.FLOAT, (), [1.0]), + onnx.helper.make_tensor('two', onnx.TensorProto.FLOAT, (), [2.0]) + ], + opset_imports=[onnx.helper.make_opsetid('', 13), onnx.helper.make_opsetid('org.tinygrad', 1)], + from_disk=False).to('PYTHON') + _check_ast_count(2, runner({'inp': Tensor.empty(4, device='PYTHON')})['output']) + def _test_const_fold_unary_op(self, from_disk:bool): runner = build_onnx( nodes=[ @@ -162,4 +179,4 @@ class TestOnnxMetadata(unittest.TestCase): self.assertEqual(parsed["metadata_props"][1]["value"], "dGVzdA==") if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tinygrad_repo/test/external/external_test_schedule_scaling.py b/tinygrad_repo/test/external/external_test_schedule_scaling.py index e7026508e..2e62a809c 100644 --- a/tinygrad_repo/test/external/external_test_schedule_scaling.py +++ b/tinygrad_repo/test/external/external_test_schedule_scaling.py @@ -1,5 +1,5 @@ -import unittest, time -from tinygrad import Tensor +import unittest, time, itertools +from tinygrad import Tensor, Context, dtypes class TestScheduleScaling(unittest.TestCase): """Test that .schedule() scales linearly with graph size (no O(n^2) behavior).""" @@ -130,5 +130,18 @@ class TestScheduleScaling(unittest.TestCase): return parts[0].cat(*parts[1:]) self._assert_linear(concat_chain) + @Context(DEV="NULL:HIP:gfx1100") + def test_custom_kernel_assign_scaling(self): + from tinygrad.uop.ops import UOp, Ops, KernelInfo + from tinygrad.runtime.autogen.amd.rdna3.ins import s_nop + count = itertools.count(0) + def custom_kernel_assign(n): + def custom_asm(out): + return UOp(Ops.PROGRAM, src=(UOp.sink(out, arg=KernelInfo(f"fxn_{next(count)}")), + UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(s_nop(i), dtypes.void)) for i in range(n*8))))) + call = Tensor.custom_kernel(Tensor.empty(1), fxn=custom_asm)[0] + return Tensor.cat(*[Tensor.empty(1).assign(call+i) for i in range(n)]) + self._assert_linear(custom_kernel_assign, n_small=50, n_large=500) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/tinygrad_repo/test/external/external_test_usb_asm24.py b/tinygrad_repo/test/external/external_test_usb_asm24.py index 7aec4adac..fe1314d34 100644 --- a/tinygrad_repo/test/external/external_test_usb_asm24.py +++ b/tinygrad_repo/test/external/external_test_usb_asm24.py @@ -6,7 +6,7 @@ import numpy as np class TestDevCopySpeeds(unittest.TestCase): @classmethod def setUpClass(cls): - cls.sz = getenv("SIZE", 2e6) + cls.sz = getenv("SIZE", 2000000) cls.dev = Device["AMD"] if not cls.dev.is_usb(): raise unittest.SkipTest("only test this on USB devices") diff --git a/tinygrad_repo/test/external/external_uop_gc.py b/tinygrad_repo/test/external/external_uop_gc.py index e52500fc1..35f58c148 100644 --- a/tinygrad_repo/test/external/external_uop_gc.py +++ b/tinygrad_repo/test/external/external_uop_gc.py @@ -44,7 +44,7 @@ def realized_matmul(): z = y.matmul(x) Tensor.realize(z) def realized_gradient(): - x = Tensor.eye(3) + x = Tensor.eye(3).clone() y = Tensor([[2.0,0,-2.0]]) z = y.matmul(x).sum() z.backward() diff --git a/tinygrad_repo/test/external/speed_v_theoretical.py b/tinygrad_repo/test/external/speed_v_theoretical.py index 4f0975994..1ee30ccbd 100644 --- a/tinygrad_repo/test/external/speed_v_theoretical.py +++ b/tinygrad_repo/test/external/speed_v_theoretical.py @@ -86,10 +86,10 @@ class TestKernelSpeed(unittest.TestCase): self._compare(tm, tflops, gbs, nv_tflops, nv_gbs, amd_tflops, amd_gbs) # TODO: why are convs so slow?!? - def test_conv_3x3_256_32_32_256_256(self): self._test_conv_3x3(256, 32, 32, 256, 256, nv_tflops=27, amd_tflops=14) + def test_conv_3x3_256_32_32_256_256(self): self._test_conv_3x3(256, 32, 32, 256, 256, nv_tflops=27, amd_tflops=13) # theoretical is nv_tflops=165, amd_tflops=123 - def test_gemm_4096(self): self._test_matmul(4096, nv_tflops=110, amd_tflops=65) + def test_gemm_4096(self): self._test_matmul(4096, nv_tflops=109, amd_tflops=65) def test_gemm_8192(self): self._test_matmul(8192, nv_tflops=115, amd_tflops=60) # theoretical is nv_gbs=1008, amd_gbs=960 diff --git a/tinygrad_repo/test/helpers.py b/tinygrad_repo/test/helpers.py index ea900d505..3dadc0a7a 100644 --- a/tinygrad_repo/test/helpers.py +++ b/tinygrad_repo/test/helpers.py @@ -48,6 +48,8 @@ def check_schedule(t:Tensor|list[Tensor]|UOp, allowed:int, to_prerealize:list[Te else: assert isinstance(t, UOp), f"can't schedule {t}" linear, var_vals = Tensor(t).linear_with_vars() + # test compiling the linear + compile_linear(linear) kernel_cnt = sum((len(call.device) if isinstance(call.device, tuple) else 1) for call in linear.src if call.src[0].op is Ops.SINK or not filter_sink) if kernel_cnt != allowed: @@ -57,8 +59,6 @@ def check_schedule(t:Tensor|list[Tensor]|UOp, allowed:int, to_prerealize:list[Te print("kernel", i+1) print(call.src[0]) raise KernelCountException(allowed, kernel_cnt) - # test compiling the linear - compile_linear(linear) return linear, var_vals def assert_kernel_count(expected:int): @@ -86,7 +86,10 @@ def assert_jit_cache_len(fxn, expected_len): if linear is None or not linear.src: if expected_len != 0: raise KernelCountException(expected_len, 0) return - if expected_len and all(call_is_hcq(call) for call in linear.src): expected_len = 3 # HCQ2: merged same-queue calls + finalizer + bumps + if expected_len and any(call_is_hcq(call) for call in linear.src): # HCQ2: kernels batch into submits, the finalizers carry the batch's kernels + count = sum(len(call.arg.aux.kernels) if call_is_hcq(call) else 1 for call in linear.src) + if count != expected_len: raise KernelCountException(expected_len, count) + return if call_is_graph(linear.src[0]): if len(linear.src) != 1: raise KernelCountException(1, len(linear.src)) inner = linear.src[0].src[0].src[0] # LINEAR UOp inside CUSTOM_FUNCTION @@ -120,7 +123,7 @@ def eval_uop(uop:UOp, inputs:list[tuple[DType, list[Any]]]|None=None, vals:tuple for buf_dt, data in inputs or []: bufs.append(buf:=allocator.alloc(len(data) * buf_dt.itemsize)) allocator._copyin(buf, memoryview(struct.pack(str(len(data)) + (buf_dt.fmt or ""), *data))) - g = UOp.param(0, uop.dtype, (1,)) + g = UOp.param(0, uop.dtype, 1) prg = to_program(UOp.store(g.index(UOp.const(0)), uop).sink(arg=KernelInfo()), PythonRenderer(Target("PYTHON"))) prog = dev.runtime(prg.to_elf()) prog(out_buf:=allocator.alloc(uop.dtype.itemsize), *bufs, vals=vals) diff --git a/tinygrad_repo/test/mockgpu/amd/amddriver.py b/tinygrad_repo/test/mockgpu/amd/amddriver.py index 6d619bff6..0ec6d9814 100644 --- a/tinygrad_repo/test/mockgpu/amd/amddriver.py +++ b/tinygrad_repo/test/mockgpu/amd/amddriver.py @@ -1,20 +1,15 @@ -import pathlib, re, ctypes, mmap, collections, functools, copy, os +import ctypes, mmap, collections, functools, copy, os from tinygrad.runtime.autogen import kfd, amdgpu_drm, libc import tinygrad.runtime.autogen.am.am as am from tinygrad.helpers import from_mv from test.mockgpu.driver import VirtDriver, VirtFileDesc, TextFileDesc, DirFileDesc, VirtFile from test.mockgpu.amd.amdgpu import AMDGPU, gpu_props, GFX_TARGET_VERSION, MOCKGPU_ARCH -def ioctls_from_header(): - # hdrpy = (pathlib.Path(__file__).parent.parent.parent.parent / "tinygrad" / "runtime" / "autogen" / "kfd.py").read_text() - # pattern = r'# (AMDKFD_IOC_[A-Z0-9_]+)\s=\s_(IOW?R?).*\(( 0x[0-9a-fA-F]+) ,\s+struct\s([A-Za-z0-9_]+)\s+\)' - # matches = re.findall(pattern, hdrpy, re.MULTILINE) - hdr = (pathlib.Path(__file__).parent.parent.parent.parent / "extra" / "hip_gpu_driver" / "kfd_ioctl.h").read_text().replace("\\\n", "") - pattern = r'#define\s+(AMDKFD_IOC_[A-Z0-9_]+)\s+AMDKFD_(IOW?R?)\((0x[0-9a-fA-F]+),\s+struct\s([A-Za-z0-9_]+)\)' - matches = re.findall(pattern, hdr, re.MULTILINE) - return type("KFD_IOCTLS", (object, ), {name: int(nr, 0x10) for name, _, nr, _ in matches}), \ - {int(nr, 0x10): getattr(kfd, "struct_"+sname, None) for name, idir, nr, sname in matches} -kfd_ioctls, kfd_headers = ioctls_from_header() +def _ioctl_nr(ioctl: functools.partial) -> int: return ioctl.args[2] + +kfd_ioctl_info = { + _ioctl_nr(ioctl): (name, ioctl.args[3]) for name, ioctl in vars(kfd).items() + if name.startswith("AMDKFD_IOC_") and isinstance(ioctl, functools.partial)} class KFDFileDesc(VirtFileDesc): def __init__(self, fd, driver): @@ -116,42 +111,44 @@ class AMDDriver(VirtDriver): def kfd_ioctl(self, req, argp): nr = req & 0xFF - struct = kfd_headers[nr].from_address(argp) + if nr not in kfd_ioctl_info: raise RuntimeError(f"unknown kfd ioctl, {nr} unknown") + name, struct_type = kfd_ioctl_info[nr] + struct = struct_type.from_address(argp) - if nr == kfd_ioctls.AMDKFD_IOC_ACQUIRE_VM: pass - elif nr == kfd_ioctls.AMDKFD_IOC_RUNTIME_ENABLE: pass - elif nr == kfd_ioctls.AMDKFD_IOC_GET_VERSION: + if nr == _ioctl_nr(kfd.AMDKFD_IOC_ACQUIRE_VM): pass + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_RUNTIME_ENABLE): pass + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_GET_VERSION): struct.major_version = 1 struct.minor_version = 14 - elif nr == kfd_ioctls.AMDKFD_IOC_ALLOC_MEMORY_OF_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_ALLOC_MEMORY_OF_GPU): if struct.gpu_id not in self.gpus: return -1 struct.handle = self._alloc_handle() self.object_by_handle[struct.handle] = copy.deepcopy(struct) # save memory struct to know what mem it is # Track signal memory (uncached + coherent) - progress queues when written to if struct.flags & kfd.KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED: self.track_address(struct.va_addr, struct.va_addr + struct.size, lambda mv,off: None, lambda mv, off: self._emulate_execute()) - elif nr == kfd_ioctls.AMDKFD_IOC_FREE_MEMORY_OF_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU): self.object_by_handle.pop(struct.handle) - elif nr == kfd_ioctls.AMDKFD_IOC_MAP_MEMORY_TO_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_MAP_MEMORY_TO_GPU): dev_ids = (ctypes.c_int32 * struct.n_devices).from_address(struct.device_ids_array_ptr) for i in range(struct.n_devices): gpu = self.gpus[dev_ids[i]] mem_obj = self.object_by_handle[struct.handle] gpu.map_range(mem_obj.va_addr, mem_obj.size) struct.n_success = i + 1 - elif nr == kfd_ioctls.AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU): dev_ids = (ctypes.c_int32 * struct.n_devices).from_address(struct.device_ids_array_ptr) for i in range(struct.n_devices): gpu = self.gpus[dev_ids[i]] mem_obj = self.object_by_handle[struct.handle] gpu.unmap_range(mem_obj.va_addr, mem_obj.size) struct.n_success = i + 1 - elif nr == kfd_ioctls.AMDKFD_IOC_CREATE_EVENT: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_CREATE_EVENT): struct.event_slot_index = self._alloc_next_event_slot() struct.event_id = struct.event_slot_index if struct.event_type == kfd.KFD_IOC_EVENT_MEMORY: self.mmu_event_ids.append(struct.event_id) - elif nr == kfd_ioctls.AMDKFD_IOC_CREATE_QUEUE: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_CREATE_QUEUE): gpu = self.gpus[struct.gpu_id] if struct.queue_type == kfd.KFD_IOC_QUEUE_TYPE_SDMA: gpu.add_sdma_queue(struct.ring_base_address, struct.ring_size, struct.read_pointer_address, struct.write_pointer_address) @@ -162,7 +159,7 @@ class AMDDriver(VirtDriver): # Track writes to doorbell, calling callback struct.doorbell_offset = self._alloc_doorbell(struct.gpu_id) self.track_address(struct.doorbell_offset, struct.doorbell_offset + 8, lambda mv,off: None, lambda mv, off: self._emulate_execute()) - elif nr == kfd_ioctls.AMDKFD_IOC_WAIT_EVENTS: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_WAIT_EVENTS): evs = (kfd.struct_kfd_event_data * struct.num_events).from_address(struct.events_ptr) for ev in evs: if ev.event_id in self.mmu_event_ids and "MOCKGPU_EMU_FAULTADDR" in os.environ: @@ -170,11 +167,7 @@ class AMDDriver(VirtDriver): ev.memory_exception_data.va = int(os.environ["MOCKGPU_EMU_FAULTADDR"], 16) ev.memory_exception_data.failure.NotPresent = 1 else: - name = "unknown" - for k,v in kfd_ioctls.__dict__.items(): - if nr == v: name = k - assert False, f"unknown kfd ioctl, {nr} {name}" - exit(1) + raise RuntimeError(f"unsupported kfd ioctl, {nr} {name}") return 0 def _emulate_execute(self): diff --git a/tinygrad_repo/test/mockgpu/amd/emu.py b/tinygrad_repo/test/mockgpu/amd/emu.py index 3f36e331e..9d0ac9ef6 100644 --- a/tinygrad_repo/test/mockgpu/amd/emu.py +++ b/tinygrad_repo/test/mockgpu/amd/emu.py @@ -6,7 +6,7 @@ # arg=3: lds - local data share # arg=4: scratch - per-lane scratch memory from __future__ import annotations -import ctypes, functools, re, platform, subprocess, tempfile +import ctypes, functools, itertools, re, platform, subprocess, tempfile from typing import Callable # Set/restore DAZ+FTZ (denormals-are-zero + flush-to-zero) to match RDNA3 default float mode @@ -66,132 +66,17 @@ from tinygrad.runtime.autogen.amd.cdna.str_pcode import PCODE as PCODE_CDNA from tinygrad.runtime.autogen.amd.rdna3 import ins as ir3 from tinygrad.runtime.autogen.amd.rdna4 import ins as ir4 from tinygrad.runtime.autogen.amd.cdna import ins as irc -from tinygrad.renderer.amd.dsl import VCC_LO, EXEC_LO, SCC, ttmp +from tinygrad.renderer.amd.dsl import VCC_LO, EXEC_LO, SCC, ttmp, Inst from tinygrad.runtime.autogen.amd.common import Fmt, OpType from test.amd.helpers import decode_dpp16 -from test.mockgpu.amd.pcode import parse_block, _FUNCS, _set_bits, _to_bool, _val_to_bits +from test.mockgpu.amd.pcode import parse_pcode, _FUNCS, _set_bits, _to_bool, _to_u32, _val_to_bits, _ftz_f32 MASK32 = 0xFFFFFFFF -# ═══════════════════════════════════════════════════════════════════════════════ -# SQTT TRACE COLLECTION -# ═══════════════════════════════════════════════════════════════════════════════ - -# Global trace storage: populated by run_asm as raw SQTT blobs, consumed by amdgpu.py +# SQTT encoder lives in sqtt_enc.py; traces are consumed by amdgpu.py +from test.mockgpu.amd.sqtt_enc import make_encoder as _make_sqtt_encoder sqtt_traces: list[bytes] = [] -# Encoder primitives -from tinygrad.renderer.amd.sqtt import _build_decode_tables, PACKET_TYPES_RDNA3, LAYOUT_HEADER, WAVESTART, WAVEEND, INST, IMMEDIATE, VALUINST, InstOp - -_NIB_COUNTS: dict = {cls: nc for _, (cls, nc, *_) in _build_decode_tables(PACKET_TYPES_RDNA3)[0].items()} - -def _encode_raw(pkt_cls, **kwargs) -> tuple[int, int]: - raw = pkt_cls.encoding.default - for k, v in kwargs.items(): raw = pkt_cls.__dict__[k].set(raw, v) - return raw, _NIB_COUNTS[pkt_cls] - -def _emit_nibbles(nibbles: list[int], pkt_cls, **kwargs): - raw, nc = _encode_raw(pkt_cls, **kwargs) - for i in range(nc): nibbles.append((raw >> (i * 4)) & 0xF) - -def _nibbles_to_bytes(nibbles: list[int]) -> bytes: - result = bytearray() - for i in range(0, len(nibbles), 2): result.append(nibbles[i] | ((nibbles[i + 1] if i + 1 < len(nibbles) else 0) << 4)) - return bytes(result) - -def _init_sqtt_encoder(): - """Initialize and return SQTT encoder state. Called once per dispatch with tracing enabled.""" - from tinygrad.runtime.autogen.amd.rdna3.enum import SOPPOp as SOPPOp3 - from tinygrad.runtime.autogen.amd.rdna4.enum import SOPPOp as SOPPOp4 - import re - - _SOPP = (ir3.SOPP, ir4.SOPP, irc.SOPP) - _SMEM = (ir3.SMEM, ir4.SMEM, irc.SMEM) - _VALU = (ir3.VOP1, ir3.VOP2, ir3.VOP3, ir3.VOP3P, ir3.VOPC, ir3.VOPD, ir3.VOP3SD, ir3.VOP3_SDST, ir3.VOP1_SDST, - ir4.VOP1, ir4.VOP2, ir4.VOP3, ir4.VOP3P, ir4.VOPC, ir4.VOPD, ir4.VOP3SD, ir4.VOP3_SDST, ir4.VOP1_SDST, - irc.VOP1, irc.VOP2, irc.VOP3, irc.VOP3P, irc.VOP3PX2, irc.VOPC, irc.VOP3SD, irc.VOP3_SDST) - _DS = (ir3.DS, ir4.DS, irc.DS) - _GLOBAL = (ir3.GLOBAL, ir4.VGLOBAL, irc.GLOBAL) - _FLAT = (ir3.FLAT, ir4.VFLAT, irc.FLAT) - _SCRATCH = (ir3.SCRATCH, ir4.VSCRATCH, irc.SCRATCH) - - # SOPP classification sets - _SOPP_SKIP = {SOPPOp3.S_ENDPGM.value, SOPPOp3.S_ENDPGM_SAVED.value, SOPPOp3.S_ENDPGM_ORDERED_PS_DONE.value, - SOPPOp3.S_DELAY_ALU.value} - _SOPP_IMMEDIATE = {SOPPOp3.S_NOP.value, SOPPOp3.S_CLAUSE.value, SOPPOp3.S_WAITCNT.value, SOPPOp3.S_WAITCNT_DEPCTR.value, - SOPPOp3.S_WAIT_IDLE.value, SOPPOp3.S_WAIT_EVENT.value, SOPPOp3.S_SLEEP.value, - SOPPOp3.S_SET_INST_PREFETCH_DISTANCE.value} - for _op in (SOPPOp4.S_WAIT_ALU, SOPPOp4.S_WAIT_LOADCNT, SOPPOp4.S_WAIT_STORECNT, SOPPOp4.S_WAIT_SAMPLECNT, - SOPPOp4.S_WAIT_BVHCNT, SOPPOp4.S_WAIT_EXPCNT, SOPPOp4.S_WAIT_DSCNT, SOPPOp4.S_WAIT_KMCNT, - SOPPOp4.S_WAIT_LOADCNT_DSCNT, SOPPOp4.S_WAIT_STORECNT_DSCNT): - _SOPP_IMMEDIATE.add(_op.value) - _SOPP_BARRIER = {SOPPOp3.S_BARRIER.value} - if hasattr(SOPPOp4, 'S_BARRIER_WAIT'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_WAIT.value) - if hasattr(SOPPOp4, 'S_BARRIER_LEAVE'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_LEAVE.value) - _SOPP_BRANCH = {SOPPOp3.S_BRANCH.value, SOPPOp3.S_CBRANCH_SCC0.value, SOPPOp3.S_CBRANCH_SCC1.value, - SOPPOp3.S_CBRANCH_VCCZ.value, SOPPOp3.S_CBRANCH_VCCNZ.value, - SOPPOp3.S_CBRANCH_EXECZ.value, SOPPOp3.S_CBRANCH_EXECNZ.value} - - # VALU sub-classification patterns - _VALUT_4_RE = re.compile(r'V_(EXP|LOG|RCP|RSQ|SQRT|SIN|COS|CEIL|FLOOR|TRUNC|RNDNE|FRACT|FREXP)_') - _VALUB_2_RE = re.compile(r'V_(LSHLREV|LSHRREV|ASHRREV)_(B|I)64') - _VALUB_4_RE = re.compile(r'V_MAD_(U|I)64') - _VALUB_16_RE = re.compile(r'V_\w+_F64') - - def _valu_op(op_name: str) -> InstOp|None: - if 'CMPX' in op_name: return InstOp.VALU1_WR_EXEC - if _VALUB_2_RE.search(op_name): return InstOp.VALUB_2 - if _VALUB_4_RE.search(op_name): return InstOp.VALUB_4 - if _VALUB_16_RE.search(op_name): return InstOp.VALUB_16 - if _VALUT_4_RE.search(op_name): return InstOp.VALUT_4 - return None - - def _mem_op(t, op_name: str) -> InstOp: - is_store = "STORE" in op_name - if issubclass(t, _DS): return InstOp.LDS_WR_2 if is_store else InstOp.LDS_RD - if issubclass(t, _GLOBAL): return InstOp.SGMEM_WR_2 if is_store else InstOp.SGMEM_RD_1 - if issubclass(t, _FLAT): return InstOp.FLAT_WR_3 if is_store else InstOp.FLAT_RD_2 - if issubclass(t, _SCRATCH): return InstOp.FLAT_WR_3 if is_store else InstOp.FLAT_RD_2 - return InstOp.SALU - - nibbles: list[int] = [] - started: set[int] = set() - _emit_nibbles(nibbles, LAYOUT_HEADER, layout=3, sel_a=6) - - def emit(wave_id: int, inst, branch_taken: bool|None): - """Emit an SQTT packet for one executed instruction.""" - w = wave_id & 0x1F - if wave_id not in started: - _emit_nibbles(nibbles, WAVESTART, delta=1, simd=0, wgp=0, wave=w, id7=wave_id) - started.add(wave_id) - inst_type, inst_op, op_name = type(inst), inst.op.value if hasattr(inst, 'op') else 0, inst.op.name if hasattr(inst, 'op') else "" - if issubclass(inst_type, _SOPP): - if inst_op in _SOPP_SKIP: return - elif inst_op in _SOPP_IMMEDIATE: _emit_nibbles(nibbles, IMMEDIATE, delta=1, wave=w) - elif inst_op in _SOPP_BARRIER: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.BARRIER) - elif inst_op in _SOPP_BRANCH: - _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.JUMP if branch_taken else InstOp.JUMP_NO) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SALU) - elif issubclass(inst_type, _VALU): - op = _valu_op(op_name) - if op is None: _emit_nibbles(nibbles, VALUINST, delta=1, wave=w) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=op) - elif issubclass(inst_type, _SMEM): _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SMEM_RD) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=_mem_op(inst_type, op_name)) - - def finish(wave_id: int): - """Emit WAVEEND for a completed wave.""" - if wave_id in started: _emit_nibbles(nibbles, WAVEEND, delta=1, simd=0, wgp=0, wave=wave_id & 0x1F) - - def finalize() -> bytes: - """Pad and return the encoded SQTT blob.""" - while len(nibbles) % 2 != 0: nibbles.append(0) - nibbles.extend([0] * 32) - while len(nibbles) % 64 != 0: nibbles.append(0) - return _nibbles_to_bytes(nibbles) - - return emit, finish, finalize - def _c(val, dtype=dtypes.uint32): return UOp.const(val, dtype) def _u64(lo: UOp, hi: UOp) -> UOp: @@ -211,30 +96,29 @@ def _apply_src_mods(val: UOp, mod_bit: int, abs_bits: int, neg_bits: int, bits: ut, ft, mask = _SRC_MOD_TYPES[bits] fv = val.cast(ut).bitcast(ft) if bits == 16 else val.bitcast(ft) if val.dtype == ut else val if abs_bits & (1 << mod_bit): fv = (fv.bitcast(ut) & UOp.const(mask, ut)).bitcast(ft) - if neg_bits & (1 << mod_bit): fv = fv.neg() + # neg modifier is a pure sign-bit toggle (preserves NaN payloads), not an arithmetic negate + if neg_bits & (1 << mod_bit): fv = (fv.bitcast(ut) ^ UOp.const((mask + 1) & (1 << (bits - 1)), ut)).bitcast(ft) return fv.bitcast(ut).cast(dtypes.uint32) if bits == 16 else fv.bitcast(ut) -# Map VOPD ops to VOP2 ops for pcode lookup (both RDNA3 and RDNA4) -VOPD_TO_VOP2 = { - ir3.VOPDOp.V_DUAL_FMAC_F32: ir3.VOP2Op.V_FMAC_F32_E32, ir3.VOPDOp.V_DUAL_MUL_F32: ir3.VOP2Op.V_MUL_F32_E32, - ir3.VOPDOp.V_DUAL_ADD_F32: ir3.VOP2Op.V_ADD_F32_E32, ir3.VOPDOp.V_DUAL_SUB_F32: ir3.VOP2Op.V_SUB_F32_E32, - ir3.VOPDOp.V_DUAL_SUBREV_F32: ir3.VOP2Op.V_SUBREV_F32_E32, ir3.VOPDOp.V_DUAL_MAX_F32: ir3.VOP2Op.V_MAX_F32_E32, - ir3.VOPDOp.V_DUAL_MIN_F32: ir3.VOP2Op.V_MIN_F32_E32, ir3.VOPDOp.V_DUAL_ADD_NC_U32: ir3.VOP2Op.V_ADD_NC_U32_E32, - ir3.VOPDOp.V_DUAL_LSHLREV_B32: ir3.VOP2Op.V_LSHLREV_B32_E32, ir3.VOPDOp.V_DUAL_AND_B32: ir3.VOP2Op.V_AND_B32_E32, - ir3.VOPDOp.V_DUAL_MOV_B32: ir3.VOP1Op.V_MOV_B32_E32, ir3.VOPDOp.V_DUAL_CNDMASK_B32: ir3.VOP2Op.V_CNDMASK_B32_E32, - ir3.VOPDOp.V_DUAL_FMAAK_F32: ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOPDOp.V_DUAL_FMAMK_F32: ir3.VOP2Op.V_FMAMK_F32_E32, - ir3.VOPDOp.V_DUAL_DOT2ACC_F32_F16: ir3.VOP2Op.V_DOT2ACC_F32_F16_E32, - # RDNA4 mappings (same VOP1/VOP2 targets, RDNA4 uses _NUM_ suffix for min/max) - ir4.VOPDOp.V_DUAL_FMAC_F32: ir3.VOP2Op.V_FMAC_F32_E32, ir4.VOPDOp.V_DUAL_MUL_F32: ir3.VOP2Op.V_MUL_F32_E32, - ir4.VOPDOp.V_DUAL_ADD_F32: ir3.VOP2Op.V_ADD_F32_E32, ir4.VOPDOp.V_DUAL_SUB_F32: ir3.VOP2Op.V_SUB_F32_E32, - ir4.VOPDOp.V_DUAL_SUBREV_F32: ir3.VOP2Op.V_SUBREV_F32_E32, ir4.VOPDOp.V_DUAL_MAX_NUM_F32: ir3.VOP2Op.V_MAX_F32_E32, - ir4.VOPDOp.V_DUAL_MIN_NUM_F32: ir3.VOP2Op.V_MIN_F32_E32, ir4.VOPDOp.V_DUAL_ADD_NC_U32: ir3.VOP2Op.V_ADD_NC_U32_E32, - ir4.VOPDOp.V_DUAL_LSHLREV_B32: ir3.VOP2Op.V_LSHLREV_B32_E32, ir4.VOPDOp.V_DUAL_AND_B32: ir3.VOP2Op.V_AND_B32_E32, - ir4.VOPDOp.V_DUAL_MOV_B32: ir3.VOP1Op.V_MOV_B32_E32, ir4.VOPDOp.V_DUAL_CNDMASK_B32: ir3.VOP2Op.V_CNDMASK_B32_E32, - ir4.VOPDOp.V_DUAL_FMAAK_F32: ir3.VOP2Op.V_FMAAK_F32_E32, ir4.VOPDOp.V_DUAL_FMAMK_F32: ir3.VOP2Op.V_FMAMK_F32_E32, - ir4.VOPDOp.V_DUAL_DOT2ACC_F32_F16: ir3.VOP2Op.V_DOT2ACC_F32_F16_E32, +# Map VOPD ops to VOP2/VOP1 ops for pcode lookup (both RDNA3 and RDNA4 share these targets) +_VOPD_TARGETS = { + 'V_DUAL_FMAC_F32': ir3.VOP2Op.V_FMAC_F32_E32, 'V_DUAL_MUL_F32': ir3.VOP2Op.V_MUL_F32_E32, + 'V_DUAL_ADD_F32': ir3.VOP2Op.V_ADD_F32_E32, 'V_DUAL_SUB_F32': ir3.VOP2Op.V_SUB_F32_E32, + 'V_DUAL_SUBREV_F32': ir3.VOP2Op.V_SUBREV_F32_E32, 'V_DUAL_MAX_F32': ir3.VOP2Op.V_MAX_F32_E32, + 'V_DUAL_MIN_F32': ir3.VOP2Op.V_MIN_F32_E32, 'V_DUAL_ADD_NC_U32': ir3.VOP2Op.V_ADD_NC_U32_E32, + 'V_DUAL_LSHLREV_B32': ir3.VOP2Op.V_LSHLREV_B32_E32, 'V_DUAL_AND_B32': ir3.VOP2Op.V_AND_B32_E32, + 'V_DUAL_MOV_B32': ir3.VOP1Op.V_MOV_B32_E32, 'V_DUAL_CNDMASK_B32': ir3.VOP2Op.V_CNDMASK_B32_E32, + 'V_DUAL_FMAAK_F32': ir3.VOP2Op.V_FMAAK_F32_E32, 'V_DUAL_FMAMK_F32': ir3.VOP2Op.V_FMAMK_F32_E32, + 'V_DUAL_DOT2ACC_F32_F16': ir3.VOP2Op.V_DOT2ACC_F32_F16_E32, } +# RDNA4 uses a _NUM_ suffix for min/max +VOPD_TO_VOP2 = {getattr(ir3.VOPDOp, n): t for n, t in _VOPD_TARGETS.items()} +VOPD_TO_VOP2.update({getattr(ir4.VOPDOp, n.replace('_MAX_', '_MAX_NUM_').replace('_MIN_', '_MIN_NUM_')): t for n, t in _VOPD_TARGETS.items()}) def _wave_size(arch: str) -> int: return 64 if arch.startswith("cdna") else 32 +def _iattr(inst, name: str, default: int = 0) -> int: + """Optional integer attribute of a decoded instruction (None/missing -> default).""" + v = getattr(inst, name, default) + return default if v is None else v # Special registers stored after inline constants (256-259) PC_LO_IDX, PC_HI_IDX, SCRATCH_STRIDE_IDX = 256, 257, 259 # SGPR buffer: 0-127 = SGPRs, 128-255 = inline constants, 256-259 = special registers @@ -246,10 +130,6 @@ def _op_name(inst) -> str: if hasattr(inst, 'opx'): return f"{inst.opx.name}_{inst.opy.name}" # VOPD has opx/opy not op return inst.op.name if hasattr(inst.op, 'name') else str(inst.op) -def _to_u32(val: UOp) -> UOp: - if val.dtype == dtypes.uint32: return val - if val.dtype.itemsize == 4: return val.bitcast(dtypes.uint32) # same size: bitcast (float32->uint32) - return val.cast(dtypes.uint32) # different size: cast (bool, int16, etc) def _lane_active(exec_mask: UOp, lane: UOp) -> UOp: if exec_mask.dtype == dtypes.uint64: return ((exec_mask >> lane.cast(dtypes.uint64)) & UOp.const(1, dtypes.uint64)).ne(UOp.const(0, dtypes.uint64)) return ((exec_mask >> lane.cast(dtypes.uint32)) & _c(1)).ne(_c(0)) @@ -260,19 +140,6 @@ def _cond(cond, if_true, if_false): def _cond_hi16(cond, val: UOp) -> UOp: return _cond(cond, _hi16(val), val) def _apply_opsel(val: UOp, sel_bit: int, opsel: int) -> UOp: return _hi16(val) if opsel & (1 << sel_bit) else val -def _set_lane_bit(old: UOp, lane: UOp, val: UOp, exec_mask: UOp) -> UOp: - """Set/clear a single bit in a mask based on lane index, respecting exec mask.""" - if old.dtype in (dtypes.uint64, dtypes.int64): - dt = dtypes.uint64 - mask = UOp.const(1, dt) << lane.cast(dt) - new_bit = _to_u32(val).cast(dt) << lane.cast(dt) - cleared = old.cast(dt) & (mask ^ UOp.const(0xFFFFFFFFFFFFFFFF, dt)) - return _lane_active(exec_mask, lane).where(cleared | new_bit, old.cast(dt)) - mask = _c(1) << lane.cast(dtypes.uint32) - new_bit = _to_u32(val) << lane.cast(dtypes.uint32) - cleared = old & (mask ^ _c(MASK32)) - return _lane_active(exec_mask, lane).where(cleared | new_bit, old) - def _val_to_u32(val: UOp) -> UOp: """Convert any value to uint32 for storage (bitcast floats, cast ints).""" if val.dtype == dtypes.uint32: return val @@ -291,13 +158,30 @@ _pcode_fixes = { 'V_DIV_FIXUP_F64': ('D0.f64 = sign_out ? -abs(S0.f64) : abs(S0.f64)', 'D0.f64 = isNAN(S0.f64) ? (sign_out ? -INF : +INF) : (sign_out ? -abs(S0.f64) : abs(S0.f64))'), 'V_TRIG_PREOP_F64': ("result = 64'F((1201'B(2.0 / PI)[1200 : 0] << shift.u32) & 1201'0x1fffffffffffff)", "result = trig_preop_result(shift)"), + # exponent() returns 0 for denormals; frexp_exp handles them per hardware (f32: 0, f64: normalized) + 'V_FREXP_EXP_I32_F32': ('D0.i32 = exponent(S0.f32) - 127 + 1', 'D0.i32 = frexp_exp(S0.f32)'), + 'V_FREXP_EXP_I32_F64': ('D0.i32 = exponent(S0.f64) - 1023 + 1', 'D0.i32 = frexp_exp(S0.f64)'), + # route through ldexp() which propagates 0/inf/NaN inputs instead of computing val * 2**exp (0*inf = NaN on the host) + 'V_LDEXP_F32': ('D0.f32 = S0.f32 * 2.0F ** S1.i32', 'D0.f32 = ldexp(S0.f32, S1.i32)'), + 'V_LDEXP_F64': ('D0.f64 = S0.f64 * 2.0 ** S1.i32', 'D0.f64 = ldexp(S0.f64, S1.i32)'), + # hardware sets SCC only on STRICT inequality for S_MAX (equal operands -> SCC=0) + 'S_MAX_I32': ('SCC = S0.i32 >= S1.i32', 'SCC = S0.i32 > S1.i32'), + 'S_MAX_U32': ('SCC = S0.u32 >= S1.u32', 'SCC = S0.u32 > S1.u32'), + # hardware computes abs on the WRAPPED 32-bit difference; the i32 pcode overflows into UB on the host (e.g. |45 - -2147483647|), + # so compute in u32 with a UB-free two's-complement negate + 'S_ABSDIFF_I32': ('D0.i32 = S0.i32 - S1.i32;\nif D0.i32 < 0 then\nD0.i32 = -D0.i32\nendif', + 'D0.u32 = S0.u32 - S1.u32;\nif D0.i32 < 0 then\nD0.u32 = -D0.u32\nendif'), + # CLASS denormal test uses abs(x) > 0.0, which the host's DAZ flushes; use bit-domain test instead + 'V_CMP_CLASS_F32': ('64\'F(abs(S0.f32)) > 0.0', '(64\'U(S0.u32 & 0x7FFFFFFF) != 0)'), + 'V_CMP_CLASS_F16': ('64\'F(abs(S0.f16)) > 0.0', '(64\'U(S0.u32 & 0x7FFF) != 0)'), + 'V_CMP_CLASS_F64': ('64\'F(abs(S0.f64)) > 0.0', '(64\'U(S0.u64 & 0x7FFFFFFFFFFFFFFF) != 0)'), } def _get_pcode_dict(op) -> dict: """Return the PCODE dictionary for the given opcode based on its architecture.""" return PCODE_CDNA if 'cdna' in type(op).__module__ else PCODE_RDNA4 if 'rdna4' in type(op).__module__ else PCODE_RDNA3 -# Pcode parser +# Pcode lookup with hardware errata fixes (the AMD-pdf pcode for these ops is subtly wrong) @functools.cache def get_pcode(op) -> str: op_name = op.name @@ -306,60 +190,41 @@ def get_pcode(op) -> str: # VOP3 ops ending in _E64 may share pcode with VOP1 _E32 equivalents import importlib enum_mod = importlib.import_module(type(op).__module__) - vop1_cls = getattr(enum_mod, 'VOP1Op', None) - e32_name = op_name.replace('_E64', '_E32') + vop1_cls, e32_name = getattr(enum_mod, 'VOP1Op', None), op_name.replace('_E64', '_E32') if vop1_cls and hasattr(vop1_cls, e32_name): op = vop1_cls[e32_name] pcode = pcode_dict[op] fix_name = op_name.replace('_E64', '').replace('_E32', '') if fix_name in _pcode_fixes: pcode = pcode.replace(*_pcode_fixes[fix_name]) - if 'V_DIV_SCALE' in op_name: - dt, exp_lim, ldexp_val = ('f32', '23', '64') if 'F32' in op_name else ('f64', '52', '128') - for old, new in [(f'S2.{dt} / S1.{dt} == DENORM.{dt}', f'divWouldBeDenorm(S2.{dt}, S1.{dt})'), (f"1.0 / 64'F(S1.{dt}) == DENORM.f64", '0'), - (f'1.0 / S1.{dt} == DENORM.{dt}', '0'), (f'S1.{dt} == DENORM.{dt}', f'isDENORM(S1.{dt})'), - (f'D0.{dt} = NAN.{dt}', f'VCC = 0x1LL;\nD0.{dt} = NAN.{dt}'), - (f'elsif isDENORM(S1.{dt}) then\nD0.{dt} = ldexp(S0.{dt}, {ldexp_val})', f'elsif 1 == 0 then\nD0.{dt} = S0.{dt}'), - (f'elsif exponent(S2.{dt}) <= {exp_lim} then\n// Numerator is tiny\n' - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})', - f'elsif exponent(S2.{dt}) <= {exp_lim} then\nVCC = 0x1LL;\n' - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})'), - (f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\nVCC = 0x1LL;\n' - f'if S0.{dt} == S2.{dt} then\n// Only scale the numerator\n' - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif', - f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\n' - f'VCC = 0x1LL;\nD0.{dt} = S0.{dt}'), - (f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif\nelsif', - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nelse\n' - f'D0.{dt} = S0.{dt}\nendif\nelsif')]: - pcode = pcode.replace(old, new) - lines = pcode.rstrip().split('\n') - for i in range(len(lines) - 1, -1, -1): - if lines[i].strip() == 'endif': - lines.insert(i, f'else\nD0.{dt} = S0.{dt}') - break - pcode = '\n'.join(lines) + f';\nif isDENORM(S1.{dt}) then\nD0.{dt} = NAN.{dt}\nendif' - pcode = pcode.replace('VCC = 0x0LL', 'VCC.u64[laneId] = 0').replace('VCC = 0x1LL', 'VCC.u64[laneId] = 1') - return pcode + return _fix_div_scale(pcode, 'f32' if 'F32' in op_name else 'f64') if 'V_DIV_SCALE' in op_name else pcode -def parse_pcode(pcode: str, srcs: dict[str, UOp | int] | None = None) -> tuple[dict, list]: - env: dict = srcs.copy() if srcs else {} - assigns: list[tuple[str, UOp]] = [] - raw_lines = [l.strip().rstrip(';') for l in pcode.split('\n') if l.strip() and not l.strip().startswith('//')] - # TODO: pcode.py should tokenize full pcode string instead of line-by-line, then this hack can be removed - lines: list[str] = [] - for l in raw_lines: - if lines and re.search(r'(&&|\|\||[&|+\-*/^])\s*$', lines[-1]): lines[-1] = lines[-1] + ' ' + l - else: lines.append(l) - _, final, _ = parse_block(lines, 0, env, assigns=assigns) - sliced = set(d.split('[')[0] for d, _ in assigns if '[' in d) - for var, val in final.items(): - if var in ['D0', 'S0', 'SCC', 'VCC', 'EXEC', 'PC', 'RETURN_DATA', 'VDATA'] and isinstance(val, UOp): - if var in sliced and not any(re.match(rf'{var}\.\w+\s*=', l) for l in lines): continue - for l in lines: - if (m := re.match(rf'{var}\.(\w+(?:\[\w+\])?)', l)): - assigns.append((f'{var}.{m.group(1)}', val)) - break - else: assigns.append((var, val)) - return env, assigns +def _fix_div_scale(pcode: str, dt: str) -> str: + """V_DIV_SCALE fixes: only the divWouldBeDenorm/exponent-overflow path may return the scaled value; + all other paths must return S0 unchanged, and VCC is set exactly when scaling happened.""" + exp_lim, ldexp_val = ('23', '64') if dt == 'f32' else ('52', '128') + for old, new in [(f'S2.{dt} / S1.{dt} == DENORM.{dt}', f'divWouldBeDenorm(S2.{dt}, S1.{dt})'), (f"1.0 / 64'F(S1.{dt}) == DENORM.f64", '0'), + (f'1.0 / S1.{dt} == DENORM.{dt}', '0'), (f'S1.{dt} == DENORM.{dt}', f'isDENORM(S1.{dt})'), + (f'D0.{dt} = NAN.{dt}', f'VCC = 0x1LL;\nD0.{dt} = NAN.{dt}'), + (f'elsif isDENORM(S1.{dt}) then\nD0.{dt} = ldexp(S0.{dt}, {ldexp_val})', f'elsif 1 == 0 then\nD0.{dt} = S0.{dt}'), + (f'elsif exponent(S2.{dt}) <= {exp_lim} then\n// Numerator is tiny\n' + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})', + f'elsif exponent(S2.{dt}) <= {exp_lim} then\nVCC = 0x1LL;\n' + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})'), + (f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\nVCC = 0x1LL;\n' + f'if S0.{dt} == S2.{dt} then\n// Only scale the numerator\n' + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif', + f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\n' + f'VCC = 0x1LL;\nD0.{dt} = S0.{dt}'), + (f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif\nelsif', + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nelse\n' + f'D0.{dt} = S0.{dt}\nendif\nelsif')]: + pcode = pcode.replace(old, new) + lines = pcode.rstrip().split('\n') + for i in range(len(lines) - 1, -1, -1): + if lines[i].strip() == 'endif': + lines.insert(i, f'else\nD0.{dt} = S0.{dt}') + break + pcode = '\n'.join(lines) + f';\nif isDENORM(S1.{dt}) then\nD0.{dt} = NAN.{dt}\nendif' + return pcode.replace('VCC = 0x0LL', 'VCC.u64[laneId] = 0').replace('VCC = 0x1LL', 'VCC.u64[laneId] = 1') def _write_64bit(val: UOp, wfn, reg_or_addr, is_mem: bool, *args) -> list[UOp]: """Write a 64-bit value as two 32-bit writes. args passed to wfn after reg/addr and lo/hi value.""" @@ -420,13 +285,46 @@ def _collect_data_slices(assigns: list[tuple[str, UOp]], data_prefix: str, pcode # INSTRUCTION COMPILER - converts decoded instruction to UOp SINK # ═══════════════════════════════════════════════════════════════════════════════ +def _int_clamp(op_name: str, srcs: dict) -> UOp | None: + """Integer clamp for V_*ADD/SUB/MAD* ops: compute in wide arithmetic and saturate to the narrow range. None if not applicable. + NOTE: MUL_LO ops don't saturate - they always return the low bits.""" + s0, s1, s2 = srcs.get('S0'), srcs.get('S1'), srcs.get('S2') + if not isinstance(s0, UOp) or not isinstance(s1, UOp): return None + is_signed, is_16bit = '_I' in op_name and '_U' not in op_name, '16' in op_name + if any(p in op_name for p in ('_NC_U', '_MAD_U', '_NC_I', '_MAD_I')): + op_bits = 16 if '16' in op_name else (24 if '24' in op_name else 32) + # D0 range: 16 for the *_U16/*_I16 result-narrow ops, else 32 (mad*32* D0 is u32/i32; mul operands have op-fmt width) + narrow_dt = dtypes.uint16 if is_16bit and '32' not in op_name else (dtypes.int32 if is_signed else dtypes.uint32) + wide_dt = dtypes.int64 + narrow_max, narrow_min = ((0xFFFF, 0) if narrow_dt == dtypes.uint16 else + ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0))) + def to_mulin(x: UOp) -> UOp: # mul-source: extract the op-fmt-width suboperand with sext for signed + mask = (1 << op_bits) - 1 + if op_bits == 32: return x.bitcast(narrow_dt) if x.dtype.itemsize == 4 else x.cast(narrow_dt) + m = (x & _c(mask)).cast(dtypes.int) + if not is_signed: return m.cast(wide_dt) + sign = (m >> _c(op_bits - 1)) & _c(1) + return sign.ne(_c(0)).where(m - _c(1 << op_bits), m).cast(wide_dt) + def to_wide(x: UOp) -> UOp: return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.itemsize else x.cast(narrow_dt)).cast(wide_dt) + if isinstance(s2, UOp) and 'MAD' in op_name: full = to_mulin(s0) * to_mulin(s1) + to_wide(s2) + elif 'SUBREV' in op_name: full = to_wide(s1) - to_wide(s0) + elif 'SUB' in op_name: full = to_wide(s0) - to_wide(s1) + else: full = to_wide(s0) + to_wide(s1) + return full.clamp(narrow_min, narrow_max).cast(narrow_dt) + # V_SUB_U32 / V_ADD_U32 with clamp: unsigned saturate (SUB underflow->0, ADD overflow->0xFFFFFFFF) + if any(p in op_name for p in ('_SUB_U32', '_ADD_U32', '_SUB_U16', '_ADD_U16')): + a, b = (s1.cast(dtypes.uint32), s0.cast(dtypes.uint32)) if 'SUBREV' in op_name else (s0.cast(dtypes.uint32), s1.cast(dtypes.uint32)) + if 'SUB' in op_name: return (a < b).where(_c(0), a - b) # underflow -> 0 + return (a + b < a).where(_c(0xFFFFFFFF), a + b) # overflow -> MAX + return None + class _Ctx: """Context for instruction compilation - holds buffers and helpers.""" __slots__ = ('inst_size', 'dyn_fields', '_axis_id', 'wave_size', 'vgpr', 'accvgpr') - sgpr = UOp.param(0, dtypes.uint32, (SGPR_COUNT,)) - vmem = UOp.param(2, dtypes.uint32, (1 << 46,)) - lds = UOp.param(3, dtypes.uint32, (16384,)) - scratch = UOp.param(4, dtypes.uint8, (1 << 30,)) + sgpr = UOp.param(0, dtypes.uint32, SGPR_COUNT) + vmem = UOp.param(2, dtypes.uint32, 1 << 46) + lds = UOp.param(3, dtypes.uint32, 16384) + scratch = UOp.param(4, dtypes.uint8, 1 << 30) # Cache PARAM UOps by wave_size so all _Ctx instances with same wave_size share identical UOp references _vgpr_cache: dict[int, UOp] = {} _accvgpr_cache: dict[int, UOp] = {} @@ -434,10 +332,10 @@ class _Ctx: def __init__(self, inst_size: int, wave_size: int = 32): self.inst_size, self._axis_id, self.wave_size = inst_size, 0, wave_size self.dyn_fields: list[tuple[int, int]] = [] # (lo, hi) of fields read dynamically - if wave_size not in _Ctx._vgpr_cache: _Ctx._vgpr_cache[wave_size] = UOp.param(1, dtypes.uint32, (256 * wave_size,)) + if wave_size not in _Ctx._vgpr_cache: _Ctx._vgpr_cache[wave_size] = UOp.param(1, dtypes.uint32, 256 * wave_size) self.vgpr = _Ctx._vgpr_cache[wave_size] if wave_size == 64: - if wave_size not in _Ctx._accvgpr_cache: _Ctx._accvgpr_cache[wave_size] = UOp.param(5, dtypes.uint32, (256 * wave_size,)) + if wave_size not in _Ctx._accvgpr_cache: _Ctx._accvgpr_cache[wave_size] = UOp.param(5, dtypes.uint32, 256 * wave_size) self.accvgpr = _Ctx._accvgpr_cache[wave_size] else: self.accvgpr = self.vgpr @@ -485,6 +383,10 @@ class _Ctx: hi_part = self.inst_word(dword_idx + 1) & UOp.const(hi_mask, dtypes.uint32) return lo_part | (hi_part << UOp.const(lo_bits, dtypes.uint32)) + def optional_field(self, inst: Inst, name: str) -> UOp | None: + """Extract a field that only exists on some instruction formats (e.g. 'literal', 'saddr'). None if absent.""" + return self.inst_field(getattr(type(inst), name)) if hasattr(type(inst), name) else None + def inst_field_signed(self, field) -> UOp: """Extract field and sign-extend based on field width.""" val = self.inst_field(field) @@ -532,6 +434,19 @@ class _Ctx: return [self.wsgpr_dyn(reg, lo), self.wsgpr_dyn(reg + _c(1), hi)] return [self.wsgpr_dyn(reg, val)] + def wmask_lane_bit(self, reg: UOp, lane: UOp, val: UOp, exec_mask: UOp) -> list[UOp]: + """Set/clear bit `lane` of the mask at `reg` from val for exec-active lanes, preserving memory for inactive lanes""" + active, bit = _lane_active(exec_mask, lane), _to_u32(val) + if self.wave_size <= 32: + old = self.rsgpr_dyn(reg) + mask = _c(1) << lane.cast(dtypes.uint32) + return [self.wsgpr_dyn(reg, active.where((old & (mask ^ _c(MASK32))) | (bit << lane.cast(dtypes.uint32)), old))] + off = (lane & _c(31, dtypes.int)).cast(dtypes.uint32) + mask = _c(1) << off + def half(old: UOp, sel: UOp) -> UOp: return sel.where(active.where((old & (mask ^ _c(MASK32))) | (bit << off), old), old) + return [self.wsgpr_dyn(reg, half(self.rsgpr_dyn(reg), lane < _c(32, dtypes.int))), + self.wsgpr_dyn(reg + _c(1), half(self.rsgpr_dyn(reg + _c(1)), _c(32, dtypes.int) <= lane))] + def rmask(self, reg: UOp) -> UOp: """Read a lane mask (VCC/EXEC). Combines lo/hi for wave64.""" if self.wave_size > 32: return _u64(self.rsgpr_dyn(reg), self.rsgpr_dyn(reg + _c(1))) @@ -627,8 +542,7 @@ class _Ctx: def compile_sop_pcode(self, op, srcs: dict[str, UOp | int], sdst_reg: UOp, sdst_size: int) -> UOp: """Compile a scalar instruction with dynamic destination register.""" pcode = get_pcode(op) - srcs.update({'VCC': self.rmask(_c(VCC_LO.offset)), 'EXEC': self.rexec(), 'SCC': self.rsgpr_dyn(_c(SCC.offset)), - '_wave_size': self.wave_size}) + srcs.update(self.base_srcs(self.rexec()), VCC=self.rmask(_c(VCC_LO.offset))) if 'D0' not in srcs: srcs['D0'] = self.rsgpr_dyn(sdst_reg) # D0 is current dest value for read-modify-write ops _, assigns = parse_pcode(pcode, srcs) return UOp.sink(*self.scalar_stores(assigns, sdst_reg, sdst_size), *self.inc_pc()) @@ -639,8 +553,7 @@ class _Ctx: op_name = op.name if hasattr(op, 'name') else str(op) src0_off, vdst_off = self.inst_field(type(inst).src0), self.inst_field(type(inst).vdst) src0_reg = (src0_off >= _c(256)).where(src0_off - _c(256), _c(0)) # VGPR index or 0 - src1_off = self.inst_field(type(inst).src1) if hasattr(type(inst), 'src1') else None - src2_off = self.inst_field(type(inst).src2) if hasattr(type(inst), 'src2') else None + src1_off, src2_off = self.optional_field(inst, 'src1'), self.optional_field(inst, 'src2') src1_reg = (src1_off >= _c(256)).where(src1_off - _c(256), src1_off) if src1_off is not None else _c(0) src2_reg = (src2_off >= _c(256)).where(src2_off - _c(256), src2_off) if src2_off is not None else _c(0) exec_val = self.rexec() @@ -659,6 +572,16 @@ class _Ctx: elif dest.startswith('VGPR['): stores.append(self.vgpr.index(val[0]).store(val[1].cast(dtypes.uint32))) return UOp.sink(*stores, *self.inc_pc()) + def base_srcs(self, exec_mask: UOp, lane: UOp | None = None) -> dict[str, UOp | int]: + """Pcode environment entries shared by all instructions: EXEC/SCC, rounding mode (emulator always rounds-nearest-even), + and CDNA SDWA byte/word select defaults (E32 encodings always use BYTE0/WORD0).""" + srcs: dict[str, UOp | int] = {'EXEC': exec_mask, 'SCC': self.rsgpr_dyn(_c(SCC.offset)), '_vgpr': self.vgpr, '_wave_size': self.wave_size, + 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0), 'ROUND_NEAREST_EVEN': _c(0), + 'SDWA_SRC0_SEL': _c(0), 'BYTE0': _c(0), 'BYTE1': _c(1), 'BYTE2': _c(2), 'BYTE3': _c(3), + 'WORD0': _c(0), 'WORD1': _c(1)} + if lane is not None: srcs['laneId'] = lane + return srcs + def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: UOp, exec_mask: UOp, opsel_dst_hi: bool | UOp = False, sdst_reg: int | None = None, clmp: int = 0, src0_off: UOp | None = None) -> UOp: @@ -666,84 +589,40 @@ class _Ctx: pcode = get_pcode(op) vcc_reg = sdst_reg if sdst_reg is not None else VCC_LO.offset if 'VCC' not in srcs: srcs['VCC'] = self.rmask(_c(vcc_reg)) - srcs.update({'EXEC': exec_mask, 'SCC': self.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane, 'VDST': vdst_reg, - 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0), 'ROUND_NEAREST_EVEN': _c(0), '_vgpr': self.vgpr, '_wave_size': self.wave_size, - 'MAX_FLOAT_F32': UOp.const(3.4028234663852886e38, dtypes.float32), - # CDNA SDWA byte/word select constants (E32 always uses BYTE0/WORD0 defaults) - 'SDWA_SRC0_SEL': _c(0), 'BYTE0': _c(0), 'BYTE1': _c(1), 'BYTE2': _c(2), 'BYTE3': _c(3), - 'WORD0': _c(0), 'WORD1': _c(1)}) # rounding mode and SDWA constants + srcs.update(self.base_srcs(exec_mask, lane), VDST=vdst_reg, MAX_FLOAT_F32=UOp.const(3.4028234663852886e38, dtypes.float32)) + # f32 min/max/median ops flush denormal inputs to signed zero (select-style ops: results propagate inputs bitwise) + # (RDNA4 calls them _NUM_: V_MIN_NUM_F32 etc.) + if any(p in op.name for p in ('MIN_F32', 'MAX_F32', 'MIN3_F32', 'MAX3_F32', 'MED3_F32', 'MIN_NUM_F32', 'MAX_NUM_F32')): + srcs = {k: _ftz_f32(v) if k in ('S0', 'S1', 'S2') and isinstance(v, UOp) else v for k, v in srcs.items()} _, assigns = parse_pcode(pcode, srcs) - # For integer ops with clamp, compute overflow using wide arithmetic - # NOTE: MUL_LO ops don't saturate - they always return the low bits - int_saturate = None - if clmp and any(p in op.name for p in ('_NC_U', '_MAD_U', '_NC_I', '_MAD_I')): - is_signed, is_16bit = '_I' in op.name and '_U' not in op.name, '16' in op.name - if not (is_16bit and is_signed): # Skip 16-bit signed ops due to codegen issues - s0, s1, s2 = srcs.get('S0'), srcs.get('S1'), srcs.get('S2') - if s0 is not None and s1 is not None: - narrow_dt = dtypes.uint16 if is_16bit else (dtypes.int32 if is_signed else dtypes.uint32) - wide_dt = dtypes.int32 if is_16bit else dtypes.int64 - narrow_max, narrow_min = (0xFFFF, 0) if is_16bit else ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0)) - def to_wide(x): return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.itemsize else x.cast(narrow_dt)).cast(wide_dt) - is_sub, is_mad = 'SUB' in op.name, 'MAD' in op.name - full = (to_wide(s0) * to_wide(s1) + to_wide(s2)) if is_mad and s2 is not None else \ - (to_wide(s1) - to_wide(s0)) if is_sub and 'SUBREV' in op.name else \ - (to_wide(s0) - to_wide(s1)) if is_sub else (to_wide(s0) + to_wide(s1)) - int_saturate = full.clamp(narrow_min, narrow_max).cast(narrow_dt) - # V_SUB_U32 / V_ADD_U32 with clamp: unsigned saturate (SUB underflow->0, ADD overflow->0xFFFFFFFF) - if clmp and int_saturate is None and any(p in op.name for p in ('_SUB_U32', '_ADD_U32', '_SUB_U16', '_ADD_U16')): - s0, s1 = srcs.get('S0'), srcs.get('S1') - if s0 is not None and s1 is not None: - assert isinstance(s0, UOp) and isinstance(s1, UOp) - a, b = (s1.cast(dtypes.uint32), s0.cast(dtypes.uint32)) if 'SUBREV' in op.name else (s0.cast(dtypes.uint32), s1.cast(dtypes.uint32)) - if 'SUB' in op.name: - int_saturate = (a < b).where(_c(0), a - b) # underflow -> 0 - else: - raw_sum = a + b - int_saturate = (raw_sum < a).where(_c(0xFFFFFFFF), raw_sum) # overflow -> MAX + # For integer ops with clamp, pre-compute the saturated result; floats clamp to [0,1] at write time + int_saturate = _int_clamp(op.name, srcs) if clmp else None - raw_stores: list = [] + lane_stores, scalar_stores, slice_stores = [], [], [] vcc_val, exec_val = None, None for dest, val in assigns: - # VGPR bit-slice assignment: VGPR[lane][reg][hi:lo] = (vgpr_idx, rhs_val, hi, lo[, cond]) -> read-modify-write + # VGPR bit-slice assignment: VGPR[lane][reg][hi:lo] -> read-modify-write with optional condition if dest.startswith('VGPR[') and re.search(r'\[\d+:\d+\]', dest): - # VGPR bit-slice: (vgpr_idx, rhs_val, hi_bit, lo_bit) - hi/lo are UOp constants - hi_bit, lo_bit = int(val[2].val), int(val[3].val) - width = hi_bit - lo_bit + 1 - old = self.vgpr.index(val[0]).load() - new_val = _set_bits(old, _val_to_bits(val[1]), width, lo_bit).cast(dtypes.uint32) - active = _lane_active(exec_mask, lane) - if len(val) > 4: active = active & _to_bool(val[4]) - raw_stores.append(('vgpr_direct', self.vgpr.index(val[0].valid(active)).store(new_val))) - continue - if 'D0' in dest and '[laneId]' in dest: - old_vcc = self.rmask(_c(VCC_LO.offset)) - new_vcc = _set_lane_bit(old_vcc, lane, val, exec_mask) - raw_stores.extend([('vcc', s) for s in self.wmask(_c(VCC_LO.offset), new_vcc)]) + hi_bit, lo_bit = int(val[2]), int(val[3]) + new_val = _set_bits(self.vgpr.index(val[0]).load(), _val_to_bits(val[1]), hi_bit - lo_bit + 1, lo_bit).cast(dtypes.uint32) + active = _lane_active(exec_mask, lane) & _to_bool(val[4]) if len(val) > 4 else _lane_active(exec_mask, lane) + lane_stores.append(self.vgpr.index(val[0].valid(active)).store(new_val)) + elif 'D0' in dest and '[laneId]' in dest: continue # per-lane mask bits are written via VCC/EXEC assigns instead elif dest.startswith('D0'): - dest_suffix = re.match(r'D0\.(\w+)', dest) - if dest_suffix is not None: + if (dest_suffix := re.match(r'D0\.(\w+)', dest)) is not None: target_dt = {'u16': dtypes.uint16, 'i16': dtypes.int16, 'f16': dtypes.half}.get(dest_suffix.group(1)) if target_dt is not None and val.dtype != target_dt: val = val.cast(target_dt) - if (slice_match := re.match(r'D0\[(\d+)\s*:\s*(\d+)\]', dest)): - d0_hi_bit, d0_lo_bit = int(slice_match.group(1)), int(slice_match.group(2)) - if d0_hi_bit != 31 or d0_lo_bit != 0: - d0_width, slice_mask = d0_hi_bit - d0_lo_bit + 1, (1 << (d0_hi_bit - d0_lo_bit + 1)) - 1 - val_bits = val.bitcast(dtypes.uint16).cast(dtypes.uint32) if val.dtype == dtypes.half else \ - val.cast(dtypes.uint32) if val.dtype in (dtypes.uint16, dtypes.int16) else \ - val.cast(dtypes.uint32) & UOp.const(slice_mask, dtypes.uint32) - raw_stores.append(('vgpr_slice', (d0_lo_bit, d0_width, val_bits))) - continue - # For integer ops with clamp, use pre-computed saturated value; for floats, clamp to [0,1] + if (slice_match := re.match(r'D0\[(\d+)\s*:\s*(\d+)\]', dest)) and (int(slice_match.group(1)), int(slice_match.group(2))) != (31, 0): + slice_stores.append((int(slice_match.group(2)), int(slice_match.group(1)) - int(slice_match.group(2)) + 1, _val_to_bits(val))) + continue if int_saturate is not None: val = int_saturate elif clmp and val.dtype in (dtypes.float32, dtypes.half, dtypes.float64): - clamped = val.maximum(UOp.const(0.0, val.dtype)).minimum(UOp.const(1.0, val.dtype)) - val = _FUNCS['isNAN'](val).where(UOp.const(0.0, val.dtype), clamped) + # hardware clamp: -0 becomes +0 and NaN becomes 0 (hardware verified) + val = (val > UOp.const(0.0, val.dtype)).where(val.minimum(UOp.const(1.0, val.dtype)), UOp.const(0.0, val.dtype)) if val.dtype in (dtypes.uint64, dtypes.int64, dtypes.float64): lo, hi = _split64(val) - raw_stores.extend([('vgpr', self.wvgpr_dyn(vdst_reg, lane, lo, exec_mask)), - ('vgpr', self.wvgpr_dyn(vdst_reg + _c(1), lane, hi, exec_mask))]) + lane_stores.extend([self.wvgpr_dyn(vdst_reg, lane, lo, exec_mask), self.wvgpr_dyn(vdst_reg + _c(1), lane, hi, exec_mask)]) elif val.dtype in (dtypes.half, dtypes.uint16, dtypes.int16): result, old_val = _val_to_u32(val), self.rvgpr_dyn(vdst_reg, lane) hi_result = (old_val & UOp.const(0xFFFF, dtypes.uint32)) | (result << UOp.const(16, dtypes.uint32)) @@ -751,34 +630,28 @@ class _Ctx: lo_result = (result & UOp.const(0xFFFF, dtypes.uint32)) if self.wave_size == 64 else \ (old_val & UOp.const(0xFFFF0000, dtypes.uint32)) | (result & UOp.const(0xFFFF, dtypes.uint32)) result = opsel_dst_hi.where(hi_result, lo_result) if isinstance(opsel_dst_hi, UOp) else hi_result if opsel_dst_hi else lo_result - raw_stores.append(('vgpr', self.wvgpr_dyn(vdst_reg, lane, result, exec_mask))) - else: raw_stores.append(('vgpr', self.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask))) + lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) + else: lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask)) elif dest.startswith('S0') and src0_off is not None: # Write back to src0 VGPR (e.g. v_swap_b32). src0_off is raw encoding (256+ = VGPR) - src0_vgpr = src0_off - _c(256) - raw_stores.append(('vgpr_s0', self.wvgpr_dyn(src0_vgpr, lane, _val_to_u32(val), exec_mask))) + lane_stores.append(self.wvgpr_dyn(src0_off - _c(256), lane, _val_to_u32(val), exec_mask)) elif dest.startswith('VCC'): vcc_val = val elif dest.startswith('EXEC'): exec_val = val - elif dest.startswith('SCC'): raw_stores.append(('scc', self.wsgpr_dyn(_c(SCC.offset), _to_u32(val)))) + elif dest.startswith('SCC'): scalar_stores.append(self.wsgpr_dyn(_c(SCC.offset), _to_u32(val))) - lane_stores = [s for t, s in raw_stores if t in ('vgpr', 'vgpr_s0', 'vgpr_direct')] - stores, scalar_stores = [], [s for t, s in raw_stores if t == 'scc'] - slice_stores = [s for t, s in raw_stores if t == 'vgpr_slice'] - if slice_stores: - result = self.rvgpr_dyn(vdst_reg, lane) - for lo_bit, width, val_bits in slice_stores: - mask = UOp.const(((1 << width) - 1) << lo_bit, dtypes.uint32) - result = (result & (mask ^ UOp.const(0xFFFFFFFF, dtypes.uint32))) | (val_bits << UOp.const(lo_bit, dtypes.uint32)) - lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) # VCC/EXEC mask writes must be computed BEFORE VGPR stores to avoid reading modified VGPRs. # When vdst overlaps with src operands (e.g. v_add_co_u32 v[0], vcc, s[8], v[0]), the carry # computation reads the original source values only if its range loop runs before the VGPR write loop. - mask_stores: list[UOp] = [] + stores: list[UOp] = [] for mask_val, reg in [(vcc_val, vcc_reg), (exec_val, EXEC_LO.offset)]: if mask_val is None: continue - def get_bit(l, v=mask_val): return (_to_u32(v.substitute({lane: l})) & _c(1)).cast(dtypes.uint32) - mask_stores.extend(self.wmask(_c(reg), self.unroll_lanes(get_bit, exec_mask, apply_exec=False))) - stores.extend(mask_stores) + # hardware zeroes the inactive lane bits of per-lane VCC writes (VCC = mask & EXEC), it never preserves them + stores.extend(self.wmask(_c(reg), self.unroll_lanes(lambda l, v=mask_val: (_to_u32(v.substitute({lane: l})) & _c(1)).cast(dtypes.uint32), + exec_mask, apply_exec=reg != EXEC_LO.offset))) + if slice_stores: # merge D0[hi:lo] slices into one read-modify-write of the destination VGPR + result = self.rvgpr_dyn(vdst_reg, lane) + for lo_bit, width, val_bits in slice_stores: result = _set_bits(result, val_bits, width, lo_bit) + lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) if lane_stores: stores.append(UOp.sink(*lane_stores).end(lane)) stores.extend(scalar_stores) return UOp.sink(*stores, *self.inc_pc()) @@ -793,9 +666,7 @@ def _compile_sopp(inst: ir3.SOPP | ir4.SOPP, ctx: _Ctx) -> UOp: return UOp.sink(ctx.wsgpr_dyn(_c(PC_LO_IDX), UOp.const(0xFFFFFFFF, dtypes.uint32)), ctx.wsgpr_dyn(_c(PC_HI_IDX), UOp.const(0xFFFFFFFF, dtypes.uint32))) # S_BARRIER: advance PC past the barrier instruction. The execution loop detects barriers before executing and handles synchronization. - barrier_ops = {ir3.SOPPOp.S_BARRIER, irc.SOPPOp.S_BARRIER} - if hasattr(ir4.SOPPOp, 'S_BARRIER_WAIT'): barrier_ops.add(ir4.SOPPOp.S_BARRIER_WAIT) - if inst.op in barrier_ops: return UOp.sink(*ctx.inc_pc()) + if inst.op in _BARRIER_OPS: return UOp.sink(*ctx.inc_pc()) # S_NOP and S_WAITCNT are no-ops in emulator (no pipeline/cache to wait on) if inst.op in (ir3.SOPPOp.S_NOP, ir4.SOPPOp.S_NOP, irc.SOPPOp.S_NOP, irc.SOPPOp.S_WAITCNT): return UOp.sink(*ctx.inc_pc()) # NOTE: we ignore SOPPs without PCODE @@ -845,7 +716,7 @@ def _compile_smem(inst: ir3.SMEM | ir4.SMEM, ctx: _Ctx) -> UOp: def _compile_sop(inst: ir3.SOP1|ir3.SOP2|ir3.SOPC|ir3.SOPK|ir4.SOP1|ir4.SOP2|ir4.SOPC|ir4.SOPK|irc.SOP1|irc.SOP2|irc.SOPC|irc.SOPK, ctx: _Ctx) -> UOp: bits = inst.canonical_op_bits - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') if isinstance(inst, (ir3.SOPK, ir4.SOPK, irc.SOPK)): sdst_off = ctx.inst_field(type(inst).sdst) @@ -963,11 +834,11 @@ def _dpp16_ctrl(lane: UOp, dpp: int, row_mask: int, bank_mask: int, wave_size: i def _load_dpp16_src0(ctx: _Ctx, inst, lane: UOp, fallback: UOp) -> UOp: """Load a DPP16-swizzled src0 value from vsrc0.""" - src_lane, enabled, valid = _dpp16_ctrl(lane, getattr(inst, 'dpp', 0) or 0, getattr(inst, 'row_mask', 0xf) or 0xf, - getattr(inst, 'bank_mask', 0xf) or 0xf, ctx.wave_size) + src_lane, enabled, valid = _dpp16_ctrl(lane, _iattr(inst, 'dpp'), _iattr(inst, 'row_mask', 0xf), + _iattr(inst, 'bank_mask', 0xf), ctx.wave_size) safe_src_lane = (enabled & valid).where(src_lane, _c(0, dtypes.int)) swizzled = ctx.rvgpr_dyn(ctx.inst_field(type(inst).vsrc0), safe_src_lane) - invalid = UOp.const(0, fallback.dtype) if getattr(inst, 'bc', 0) else fallback + invalid = UOp.const(0, fallback.dtype) if _iattr(inst, 'bc') else fallback return enabled.where(valid.where(swizzled, invalid), fallback) def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc.VOPC_SDWA_SDST, ctx: _Ctx) -> UOp: @@ -976,7 +847,7 @@ def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc exec_mask = ctx.rexec() # sd=1 means use sdst register, sd=0 means use VCC (for VOPC_SDWA_SDST and VOP2_SDWA_SDST) if isinstance(inst, (irc.VOP2_SDWA_SDST, irc.VOPC_SDWA_SDST)): - sdst_off = _c(inst.sdst.offset) if getattr(inst, 'sd', False) else _c(VCC_LO.offset) + sdst_off = _c(inst.sdst.offset) if _iattr(inst, 'sd') else _c(VCC_LO.offset) else: sdst_off = _c(VCC_LO.offset) # Read SDWA fields (these are dynamic but shared across lanes) @@ -1021,11 +892,7 @@ def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc if has_dst_sel: dst_sel = ctx.inst_field(type(inst).dst_sel) # type: ignore[union-attr] dst_unused = ctx.inst_field(type(inst).dst_unused) # type: ignore[union-attr] - srcs.update({'VCC': ctx.rmask(_c(VCC_LO.offset)), 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), - 'laneId': lane, 'VDST': vdst_reg, 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0), - 'ROUND_NEAREST_EVEN': _c(0), '_vgpr': ctx.vgpr, '_wave_size': ctx.wave_size, - 'SDWA_SRC0_SEL': _c(0), 'BYTE0': _c(0), 'BYTE1': _c(1), 'BYTE2': _c(2), 'BYTE3': _c(3), - 'WORD0': _c(0), 'WORD1': _c(1)}) + srcs.update(ctx.base_srcs(exec_mask, lane), VCC=ctx.rmask(_c(VCC_LO.offset)), VDST=vdst_reg) _, assigns = parse_pcode(pcode, srcs) stores = [] vcc_val = None @@ -1039,13 +906,11 @@ def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc result = _sdwa_write(old, result, dst_sel, dst_unused) stores.append(ctx.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) elif dest.startswith('VCC'): - old_vcc = ctx.rmask(_c(VCC_LO.offset)) - stores.extend(ctx.wmask(_c(VCC_LO.offset), _set_lane_bit(old_vcc, lane, val, exec_mask))) + stores.extend(ctx.wmask_lane_bit(_c(VCC_LO.offset), lane, val, exec_mask)) if vcc_val is not None: # Initialize sdst to 0 before lane loop (old value may be unrelated data), then set lane bits in loop init_stores = [ctx.wsgpr_dyn(sdst_off, _c(0)), ctx.wsgpr_dyn(sdst_off + _c(1), _c(0))] - old_sdst = ctx.rmask(sdst_off) - stores.extend(ctx.wmask(sdst_off, _set_lane_bit(old_sdst, lane, vcc_val, exec_mask))) + stores.extend(ctx.wmask_lane_bit(sdst_off, lane, vcc_val, exec_mask)) if stores: return UOp.sink(*init_stores, UOp.sink(*stores).end(lane), *ctx.inc_pc()) return UOp.sink(*init_stores, *ctx.inc_pc()) @@ -1053,6 +918,23 @@ def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc return UOp.sink(UOp.sink(*stores).end(lane), *ctx.inc_pc()) return UOp.sink(*ctx.inc_pc()) +def _load_vsrc0(ctx: _Ctx, inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | ir3.VOP2_DPP16 | + ir4.VOP1 | ir4.VOP1_SDST | ir4.VOP1_DPP16 | ir4.VOP2 | ir4.VOP2_DPP16 | + irc.VOP1 | irc.VOP1_DPP16 | irc.VOP2 | irc.VOP2_DPP16, + lane: UOp, bits: dict, literal: UOp | None, is_f64: bool, is_float: bool, fallback: UOp) -> tuple[UOp, UOp | None]: + """Load VOP src0: DPP16 swizzle (with abs/neg mods for floats), 16-bit VGPR hi-half (src0 >= 384), or plain operand. + Returns (value, src0_offset) - offset is None for DPP16.""" + if hasattr(type(inst), 'dpp') and hasattr(type(inst), 'vsrc0'): + s0 = _load_dpp16_src0(ctx, inst, lane, fallback) + if is_float: s0 = _apply_src_mods(s0, 0, 1 if _iattr(inst, 'src0_abs') else 0, 1 if _iattr(inst, 'src0_neg') else 0, bits['s0']) + return s0, None + src0_off = ctx.inst_field(type(inst).src0) + s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal, is_f64) + if bits['s0'] == 16: # src0 >= 384 means hi half of v[src0-384]. Guard index against OOB access. + src0_hi = src0_off >= _c(384) + s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_hi.where(src0_off - _c(384), _c(0)), lane)), s0) + return s0, src0_off + def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | ir3.VOP2_DPP16 | ir4.VOP1 | ir4.VOP1_SDST | ir4.VOP1_DPP16 | ir4.VOP2 | ir4.VOP2_DPP16 | irc.VOP1 | irc.VOP1_DPP16 | irc.VOP2 | irc.VOP2_DPP16, ctx: _Ctx) -> UOp: @@ -1066,7 +948,7 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | val = ctx.raccvgpr_dyn(acc_src0_off - _c(256), lane) return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane), *ctx.inc_pc()) lane, exec_mask, bits = ctx.range(), ctx.rexec(), inst.canonical_op_bits - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') is_f64 = 'F64' in op_name and 'B64' not in op_name is_float = any(x in op_name for x in ('F16', 'F32', 'F64')) is_dpp16 = hasattr(type(inst), 'dpp') and hasattr(type(inst), 'vsrc0') @@ -1074,50 +956,25 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | write_hi_half = bits['d'] == 16 and (vdst_reg >= _c(128)) if isinstance(write_hi_half, UOp): vdst_reg = write_hi_half.where(vdst_reg - _c(128), vdst_reg) elif write_hi_half: vdst_reg -= 128 - src0_off: UOp | None = None if isinstance(inst, (ir3.VOP1, ir4.VOP1, irc.VOP1)): - # Handle VOP1 hi-half source operand (src0 >= v[128] for 16-bit ops) d0 = _cond_hi16(write_hi_half, ctx.rvgpr_dyn(vdst_reg, lane)) - if is_dpp16: - s0 = _load_dpp16_src0(ctx, inst, lane, d0) - else: - src0_off = ctx.inst_field(type(inst).src0) - s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal, is_f64) - if bits['s0'] == 16 and not is_dpp16: - src0_hi = src0_off >= _c(384) - # Only compute hi-half when src0_off >= 384, use guarded index to prevent OOB access - src0_reg = src0_hi.where(src0_off - _c(384), _c(0)) - s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_reg, lane)), s0) - if is_dpp16 and is_float: - s0 = _apply_src_mods(s0, 0, 1 if getattr(inst, 'src0_abs', 0) else 0, 1 if getattr(inst, 'src0_neg', 0) else 0, bits['s0']) - srcs:dict[str, UOp | int] = {'S0': s0, 'D0': d0} + s0, src0_off = _load_vsrc0(ctx, inst, lane, bits, literal, is_f64, is_float, d0) + srcs: dict[str, UOp | int] = {'S0': s0, 'D0': d0} else: vsrc1_reg = ctx.inst_field(type(inst).vsrc1) vsrc1_hi = bits['s0'] == 16 and (vsrc1_reg >= _c(128)) - vsrc1_actual = _cond(vsrc1_hi, vsrc1_reg - _c(128), vsrc1_reg) if bits['s1'] == 64: s1 = _u64(ctx.rvgpr_dyn(vsrc1_reg, lane), ctx.rvgpr_dyn(vsrc1_reg + _c(1), lane)) d0 = _u64(ctx.rvgpr_dyn(vdst_reg, lane), ctx.rvgpr_dyn(vdst_reg + _c(1), lane)) else: - s1 = _cond_hi16(vsrc1_hi, ctx.rvgpr_dyn(vsrc1_actual, lane)) + s1 = _cond_hi16(vsrc1_hi, ctx.rvgpr_dyn(_cond(vsrc1_hi, vsrc1_reg - _c(128), vsrc1_reg), lane)) d0 = _cond_hi16(write_hi_half, ctx.rvgpr_dyn(vdst_reg, lane)) # FMAC/FMAMK hi-half dest needs hi-half accumulator - # Handle VOP2 hi-half src0 operand (src0 >= v[128] for 16-bit ops) - if is_dpp16: - s0 = _load_dpp16_src0(ctx, inst, lane, d0) - else: - src0_off = ctx.inst_field(type(inst).src0) - s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal, is_f64) - if bits['s0'] == 16 and not is_dpp16: - src0_hi = src0_off >= _c(384) - # Only compute hi-half when src0_off >= 384, use guarded index to prevent OOB access - src0_reg = src0_hi.where(src0_off - _c(384), _c(0)) - s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_reg, lane)), s0) + s0, src0_off = _load_vsrc0(ctx, inst, lane, bits, literal, is_f64, is_float, d0) if is_dpp16 and is_float: - s0 = _apply_src_mods(s0, 0, 1 if getattr(inst, 'src0_abs', 0) else 0, 1 if getattr(inst, 'src0_neg', 0) else 0, bits['s0']) - s1 = _apply_src_mods(s1, 0, 1 if getattr(inst, 'src1_abs', 0) else 0, 1 if getattr(inst, 'src1_neg', 0) else 0, bits['s1']) + s1 = _apply_src_mods(s1, 0, 1 if _iattr(inst, 'src1_abs') else 0, 1 if _iattr(inst, 'src1_neg') else 0, bits['s1']) srcs = {'S0': s0, 'S1': s1, 'D0': d0} # FMAAK_(DTYPE)_E32 series - if 'V_FMAA' in _op_name(inst) or 'V_FMAM' in _op_name(inst): + if 'V_FMAA' in op_name or 'V_FMAM' in op_name: assert literal is not None srcs['SIMM32'] = literal return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=write_hi_half, src0_off=src0_off) @@ -1144,7 +1001,7 @@ def _compile_vopc(inst: ir3.VOPC|ir3.VOPC_DPP16|ir3.VOP3|ir4.VOPC|ir4.VOPC_DPP16 src1_off = ctx.inst_field(type(inst).src1) # type: ignore[union-attr] dst_off = ctx.inst_field(type(inst).vdst) # type: ignore[union-attr] vsrc1_hi = False - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') is_float, is_f64, pcode = any(x in op_name for x in ('_F32', '_F64', '_F16')), '_F64' in op_name, get_pcode(inst.op) def get_cmp_bit(lane) -> UOp: @@ -1158,8 +1015,11 @@ def _compile_vopc(inst: ir3.VOPC|ir3.VOPC_DPP16|ir3.VOP3|ir4.VOPC|ir4.VOPC_DPP16 if bits['s0'] == 16 and opsel: s0, s1 = _apply_opsel(s0, 0, opsel), _apply_opsel(s1, 1, opsel) if is_float: if is_dpp16: - s0 = _apply_src_mods(s0, 0, 1 if getattr(inst, 'src0_abs', 0) else 0, 1 if getattr(inst, 'src0_neg', 0) else 0, bits['s0']) - s1 = _apply_src_mods(s1, 0, 1 if getattr(inst, 'src1_abs', 0) else 0, 1 if getattr(inst, 'src1_neg', 0) else 0, bits['s1']) + s0 = _apply_src_mods(s0, 0, 1 if _iattr(inst, 'src0_abs') else 0, 1 if _iattr(inst, 'src0_neg') else 0, bits['s0']) + s1 = _apply_src_mods(s1, 0, 1 if _iattr(inst, 'src1_abs') else 0, 1 if _iattr(inst, 'src1_neg') else 0, bits['s1']) + s0 = _apply_src_mods(s0, 0, abs_bits, neg_bits, bits['s0']) + s1 = _apply_src_mods(s1, 1, abs_bits, neg_bits, bits['s1']) + elif abs_bits or neg_bits: # int compares also honor abs/neg, as bit-level sign clear/flip (not integer abs/negate) s0 = _apply_src_mods(s0, 0, abs_bits, neg_bits, bits['s0']) s1 = _apply_src_mods(s1, 1, abs_bits, neg_bits, bits['s1']) for dest, val in parse_pcode(pcode, {'S0': s0, 'S1': s1, 'laneId': lc, 'D0': UOp.const(0, dtypes.uint64)})[1]: @@ -1170,12 +1030,9 @@ def _compile_vopc(inst: ir3.VOPC|ir3.VOPC_DPP16|ir3.VOP3|ir4.VOPC|ir4.VOPC_DPP16 # Both VOPC and VOP3 clear inactive lane bits (hardware verified) new_result = new_bits & exec_mask - # CMPX e32: writes EXEC only; CMPX e64: writes both EXEC and SDST; non-CMPX: writes dst only - if is_cmpx: - stores = ctx.wmask(_c(EXEC_LO.offset), new_result) - if not is_vopc: stores.extend(ctx.wmask(dst_off, new_result)) - else: - stores = ctx.wmask(dst_off, new_result) if not is_vopc else ctx.wmask(_c(VCC_LO.offset), new_result) + # CMPX writes EXEC only (hardware verified: e64 CMPX does not write SDST); non-CMPX writes SDST/VCC + if is_cmpx: stores = ctx.wmask(_c(EXEC_LO.offset), new_result) + else: stores = ctx.wmask(dst_off, new_result) if not is_vopc else ctx.wmask(_c(VCC_LO.offset), new_result) return UOp.sink(*stores, *ctx.inc_pc()) @@ -1188,7 +1045,7 @@ def _compile_bitop3(inst, ctx: _Ctx, exec_mask: UOp, bits: dict, op_name: str) - src1 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src1), lane, bits['s1'], None, 's1' in ops and ops['s1'][0] == Fmt.FMT_NUM_F64) src2 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src2), lane, bits['s2'], None, 's2' in ops and ops['s2'][0] == Fmt.FMT_NUM_F64) # Truth table: TTBL = { omod[1:0], abs[2:0], neg[2:0] } = 8-bit LUT - ttbl = ((getattr(inst, 'omod', 0) or 0) << 6) | ((getattr(inst, 'abs', 0) or 0) << 3) | (getattr(inst, 'neg', 0) or 0) + ttbl = ((_iattr(inst, 'omod')) << 6) | ((_iattr(inst, 'abs')) << 3) | (_iattr(inst, 'neg')) is_16 = 'B16' in op_name dt, mask = (dtypes.uint16, 0xFFFF) if is_16 else (dtypes.uint32, 0xFFFFFFFF) s0, s1, s2 = src0.cast(dt), src1.cast(dt), src2.cast(dt) @@ -1202,7 +1059,7 @@ def _compile_bitop3(inst, ctx: _Ctx, exec_mask: UOp, bits: dict, op_name: str) - def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: exec_mask = ctx.rexec() bits = inst.canonical_op_bits - opsel, op_name = getattr(inst, 'opsel', 0) or 0, _op_name(inst) + opsel, op_name = _iattr(inst, 'opsel'), _op_name(inst) # Lane operations if op_name in ('V_READLANE_B32', 'V_READFIRSTLANE_B32', 'V_READFIRSTLANE_B32_E64', 'V_WRITELANE_B32'): @@ -1214,7 +1071,7 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: # VOP3 VOPC (v_cmp_*_e64) - delegate to unified VOPC handler if 'V_CMP' in op_name or 'V_CMPX' in op_name: - return _compile_vopc(inst, ctx, opsel=opsel, abs_bits=getattr(inst, 'abs', 0) or 0, neg_bits=getattr(inst, 'neg', 0) or 0) + return _compile_vopc(inst, ctx, opsel=opsel, abs_bits=_iattr(inst, 'abs'), neg_bits=_iattr(inst, 'neg')) # BITOP3: abs/neg/omod encode truth table, not source modifiers if 'BITOP3' in op_name: @@ -1222,14 +1079,13 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: # VOP3 specific fields vdst_reg = ctx.inst_field(type(inst).vdst) - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] - abs_bits, neg_bits = getattr(inst, 'abs', 0) or 0, getattr(inst, 'neg', 0) or 0 + literal = ctx.optional_field(inst, 'literal') + abs_bits, neg_bits = _iattr(inst, 'abs'), _iattr(inst, 'neg') # VOP3_SDST: v_s_* instructions goes to SGPR if 'V_S_' in op_name: src0 = _apply_src_mods(ctx.rsrc_dyn(ctx.inst_field(type(inst).src0), _c(0, dtypes.int), bits['s0'], literal), 0, abs_bits, neg_bits, bits['s0']) - srcs = {'S0': src0, 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'laneId': _c(0, dtypes.int), - 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0)} + srcs = {**ctx.base_srcs(exec_mask, _c(0, dtypes.int)), 'S0': src0} _, assigns = parse_pcode(get_pcode(inst.op), srcs) stores = [ctx.wsgpr_dyn(vdst_reg, _val_to_u32(val)) for dest, val in assigns if dest.startswith('D0')] return UOp.sink(*stores, *ctx.inc_pc()) @@ -1252,7 +1108,7 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: # FMAC instructions need D0 (accumulator) from destination register if 'FMAC' in op_name: srcs['D0'] = ctx.rvgpr_dyn(vdst_reg, lane) opsel_dst_hi = bool(opsel & 0b1000) and bits['d'] == 16 - return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=opsel_dst_hi, clmp=getattr(inst, 'clmp', 0)) + return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=opsel_dst_hi, clmp=_iattr(inst, 'clmp')) def _compile_vinterp(inst: ir3.VINTERP | ir4.VINTERP, ctx: _Ctx) -> UOp: lane, exec_mask = ctx.range(), ctx.rexec() @@ -1274,13 +1130,13 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD | irc.VOP3SD, ctx: _Ctx) -> UO # Read operands dynamically from instruction encoding vdst_reg, sdst_off = ctx.inst_field(type(inst).vdst), ctx.inst_field(type(inst).sdst) src0_off, src1_off, src2_off = ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1), ctx.inst_field(type(inst).src2) - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') has_carry_in = 's2' in ops and ops['s2'][2] == OpType.OPR_SREG vcc_in_off = src2_off if has_carry_in else sdst_off def load_srcs(lane_uop): - ret = {'VCC': ctx.rmask(vcc_in_off), 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane_uop} + ret = {**ctx.base_srcs(exec_mask, lane_uop), 'VCC': ctx.rmask(vcc_in_off)} ret['S0'] = ctx.rsrc_dyn(src0_off, lane_uop, bits['s0'], literal, ops['s0'][0] == Fmt.FMT_NUM_F64) ret['S1'] = ctx.rsrc_dyn(src1_off, lane_uop, bits['s1'], literal, ops['s1'][0] == Fmt.FMT_NUM_F64) if 's2' in ops: ret['S2'] = ctx.rsrc_dyn(src2_off, lane_uop, bits['s2'], literal, ops['s2'][0] == Fmt.FMT_NUM_F64) @@ -1291,7 +1147,7 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD | irc.VOP3SD, ctx: _Ctx) -> UO _, assigns = parse_pcode(pcode, srcs) has_per_lane_vcc = any('[laneId]' in dest for dest, _ in assigns if dest.startswith('VCC') or dest.startswith('D0.u64')) - clmp = getattr(inst, 'clmp', 0) + clmp = _iattr(inst, 'clmp') if has_per_lane_vcc: # VCC computation: RANGE+REDUCE gets axis ID first (lower ID = runs first) # This ensures VCC reads source values BEFORE VGPR stores modify them @@ -1326,195 +1182,107 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD | irc.VOP3SD, ctx: _Ctx) -> UO else: return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, sdst_reg=inst.sdst.offset) +# MFMA shape -> (lanes per group, lane groups, output regs per lane) +_MFMA_SHAPES = {(16, 16): (16, 4, 4), (32, 32): (32, 2, 16), (4, 4): (4, 16, 4)} + def _compile_mfma(inst: irc.VOP3P|irc.VOP3PX2, ctx: _Ctx) -> UOp: - """CDNA MFMA matrix multiply-accumulate emulation. - - Uses local temp arrays to cache inputs, avoiding aliasing issues when vdst overlaps src0/src1. - Phase 1: Read all input f32 values from VGPRs into temp arrays (range loop over 64 lanes). - Phase 2: Compute 256 output values using temp arrays and write to VGPRs (range loop over 64 lanes) - - Register layout (wave64): - - 16x16: 4 groups of 16 lanes. Each lane in group holds k_per_grp elements. 4 output ACCVGPRs per lane. - - 32x32: 2 groups of 32 lanes. lanes%16 = M/N index within block, lanes//16 selects block. 16 output ACCVGPRs per lane. - - 4x4: 16 groups of 4 lanes. 4 output ACCVGPRs per lane. - """ - import re as _re - op_name = _op_name(inst) - exec_mask = ctx.rexec() + """CDNA MFMA matrix multiply-accumulate. Inputs are unpacked/converted into a local temp array (uint32 bit patterns, + avoiding aliasing when vdst overlaps src0/src1), then phase 2 computes the dot products and writes outputs. + wave64 register layout: 16x16 = 4 groups of 16 lanes (K split across groups), 32x32 = 2 groups of 32 lanes, + 4x4 = 16 independent groups of 4 lanes (K not split).""" + op_name, exec_mask = _op_name(inst), ctx.rexec() vdst_reg = ctx.inst_field(type(inst).vdst) - src0_off = ctx.inst_field(type(inst).src0) - src1_off = ctx.inst_field(type(inst).src1) - src0_r = src0_off - _c(256) # VGPR-relative index (only valid when src is VGPR) - src1_r = src1_off - _c(256) - src2_off = ctx.inst_field(type(inst).src2) - use_acc = bool(getattr(inst, 'acc_cd', 0)) - # Check if sources are VGPRs (offset >= 256) vs inline constants/SGPRs - src0_is_vgpr = src0_off >= _c(256) - src1_is_vgpr = src1_off >= _c(256) + src0_off, src1_off, src2_off = ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1), ctx.inst_field(type(inst).src2) + use_acc = bool(_iattr(inst, 'acc_cd')) scaled = isinstance(inst, irc.VOP3PX2) if scaled: assert isinstance(inst, irc.VOP3PX2) # F8F6F4 input formats: 0=FP8(E4M3), 1=BF8(E5M2). FP6/FP4 (2-4) not emulated. - src0_fmt, src1_fmt = int(inst.cbsz), int(inst.blgp) - if src0_fmt > 1 or src1_fmt > 1: raise RuntimeError(f"unsupported scaled MFMA formats cbsz={src0_fmt} blgp={src1_fmt}") + if int(inst.cbsz) > 1 or int(inst.blgp) > 1: raise RuntimeError(f"unsupported scaled MFMA formats cbsz={inst.cbsz} blgp={inst.blgp}") # scale_src0/scale_src1 are source operands pointing at 32-bit registers holding 4 packed E8M0 scale exponents. # The 2-bit opsel/opsel_hi select which byte applies to A/B for this instruction. - scale0_off = ctx.inst_field(type(inst).scale_src0) - scale1_off = ctx.inst_field(type(inst).scale_src1) - sel0, sel1 = int(inst.opsel) & 3, int(inst.opsel_hi) & 3 - def _scale_exp(off: UOp, sel: int, lane: UOp) -> UOp: - sv = ctx.rsrc_dyn(off, lane, 32) - byte = (sv >> UOp.const(sel * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) + scale0 = ctx.inst_field(type(inst).scale_src0), _iattr(inst, 'opsel') & 3 + scale1 = ctx.inst_field(type(inst).scale_src1), _iattr(inst, 'opsel_hi') & 3 + def _scale_exp(off_sel: tuple[UOp, int], lane: UOp) -> UOp: + byte = (ctx.rsrc_dyn(off_sel[0], lane, 32) >> UOp.const(off_sel[1] * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) return byte.cast(dtypes.int32) - UOp.const(127, dtypes.int32) - # combined A*B scale for this lane: 2^(ea-127) * 2^(eb-127) - def scale_factor(lane: UOp) -> UOp: - return UOp.exp2((_scale_exp(scale0_off, sel0, lane) + _scale_exp(scale1_off, sel1, lane)).cast(dtypes.float32)) + def scale_factor(lane: UOp) -> UOp: # combined A*B scale for this lane: 2^(ea-127) * 2^(eb-127) + return UOp.exp2((_scale_exp(scale0, lane) + _scale_exp(scale1, lane)).cast(dtypes.float32)) - m = _re.search(r'(\d+)X(\d+)X(\d+)', op_name) - if m is None: raise ValueError(f"could not parse MFMA dimensions from {op_name}") + if (m := re.search(r'(\d+)X(\d+)X(\d+)', op_name)) is None: raise ValueError(f"could not parse MFMA dimensions from {op_name}") M, N, K = int(m.group(1)), int(m.group(2)), int(m.group(3)) + if (M, N) not in _MFMA_SHAPES: raise RuntimeError(f"unsupported MFMA shape {M}x{N}x{K}") + grp_size, n_grps, out_per_lane = _MFMA_SHAPES[(M, N)] # lanes per group, lane groups, output regs per lane - is_bf16 = 'BF16' in op_name - is_fp8 = 'FP8' in op_name or 'F8' in op_name - is_i8 = 'I8' in op_name # Source type is the LAST type in the name: V_MFMA_F32_16X16X32_**F16** -> source is F16, not F32 - src_type = op_name.rsplit('_', 1)[-1] # e.g. "F16", "BF16", "F32", "I8" - is_f32_src = src_type == 'F32' + src_type = op_name.rsplit('_', 1)[-1] + is_bf16, is_fp8 = 'BF16' in op_name, 'FP8' in op_name or 'F8' in op_name + is_i8, is_f32_src = 'I8' in op_name, src_type == 'F32' is_int_out = 'I32' in op_name.split('_')[2] # V_MFMA_I32_... - - # Determine elements per VGPR and conversion function - if is_i8: vpg = 4 - elif is_f32_src: vpg = 1 - elif is_fp8: vpg = 4 - else: vpg = 2 - - # For 16x16: grp_size=16, n_grps=4, out_per_lane=4 - # For 32x32: grp_size=32, n_grps=2, out_per_lane=16 - # For 4x4: grp_size=4, n_grps=16, out_per_lane=4 - if M == 16 and N == 16: - grp_size, n_grps, out_per_lane = 16, 4, 4 - elif M == 32 and N == 32: - grp_size, n_grps, out_per_lane = 32, 2, 16 - elif M == 4 and N == 4: - grp_size, n_grps, out_per_lane = 4, 16, 4 - else: - raise RuntimeError(f"unsupported MFMA shape {M}x{N}x{K}") - - # For 4x4: each group independently computes a 4x4 block. K is NOT split across groups. - # For 16x16/32x32: K IS split across groups (each group has K/n_grps elements). - k_per_grp = K if M == 4 else K // n_grps - # Temp array size: for 4x4, store all 16 independent blocks; for others, store shared MxK/NxK - n_a_elems = n_grps * M * K if M == 4 else M * K - n_b_elems = n_grps * N * K if M == 4 else N * K - - # src2 can be VGPR (>=256) or inline constant/SGPR (<256) - src2_is_vgpr = src2_off >= _c(256) - src2_r = src2_off - _c(256) - if is_int_out: - acc_scalar = ctx.rsgpr_dyn(src2_off, src2_is_vgpr.ne(True)).cast(dtypes.int32) - else: - acc_scalar = ctx.rsgpr_dyn(src2_off, src2_is_vgpr.ne(True)).bitcast(dtypes.float32) - - # Phase 1: Read all A and B values from VGPRs into temp arrays. - # Layout: tmp[0..n_a_elems-1] = A[m][k], tmp[n_a_elems..n_a_elems+n_b_elems-1] = B[n][k] - # Within each group of lanes, lane%grp_sub gives M/N index, lane//grp_sub gives sub-block - grp_sub = min(M, 16) # lanes within group mapped to M/N dimension - b_off = UOp.const(n_a_elems, dtypes.int) + vpg = 4 if is_i8 else 1 if is_f32_src else 4 if is_fp8 else 2 # elements per VGPR acc_dt = dtypes.int32 if is_int_out else dtypes.float32 - # Use uint32 temp array to prevent optimizer from eliminating f16→f32 bitcast chains. - # The optimizer folds bitcast(uint32→float32) stores to float32 arrays, losing the conversion. - tmp = UOp.placeholder((n_a_elems + n_b_elems,), dtypes.uint32, slot=0, addrspace=AddrSpace.LOCAL) # Per-operand fp8 format ("fp8"=E4M3, "bf8"=E5M2) for A and B if 'F8F6F4' in op_name: assert isinstance(inst, (irc.VOP3P_MFMA, irc.VOP3PX2)) - _fmts = {0: "fp8", 1: "bf8"} - a_fmt, b_fmt = _fmts.get(int(inst.cbsz), "fp8"), _fmts.get(int(inst.blgp), "fp8") - elif is_fp8: - # A/B formats from name suffix, e.g. V_MFMA_F32_16X16X32_BF8_FP8 - suffixes = op_name.rsplit('_', 2)[-2:] - a_fmt, b_fmt = ("bf8" if sfx == "BF8" else "fp8" for sfx in suffixes) + a_fmt, b_fmt = {0: "fp8", 1: "bf8"}.get(int(inst.cbsz), "fp8"), {0: "fp8", 1: "bf8"}.get(int(inst.blgp), "fp8") + elif is_fp8: # A/B formats from name suffix, e.g. V_MFMA_F32_16X16X32_BF8_FP8 + a_fmt, b_fmt = ("bf8" if sfx == "BF8" else "fp8" for sfx in op_name.rsplit('_', 2)[-2:]) else: a_fmt = b_fmt = "fp8" - def cvt_elem(raw: UOp, sub_idx: int, fp8_fmt: str = "fp8") -> UOp: - if is_i8: - # Extract i8, sign-extend to i32 - byte_val = (raw >> UOp.const(sub_idx * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) - return (byte_val.cast(dtypes.int32) ^ UOp.const(0x80, dtypes.int32)) - UOp.const(0x80, dtypes.int32) - elif is_f32_src: - return raw # already uint32 (f32 bit pattern) - elif is_fp8: - return _FUNCS[f"{fp8_fmt}_to_f32"](raw >> UOp.const(sub_idx * 8, dtypes.uint32)).bitcast(dtypes.uint32) - elif is_bf16: - # bf16→f32 bits: just shift left by 16 (bf16 is upper 16 bits of f32) - return ((raw >> UOp.const(sub_idx * 16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32)) << UOp.const(16, dtypes.uint32) - else: - # f16→f32 conversion using float arithmetic to avoid UOp optimizer eliminating the conversion. - # The optimizer folds bitcast(uint32→float32) chains, so we compute the float value directly. - h = (raw >> UOp.const(sub_idx * 16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32) - sign = (h >> UOp.const(15, dtypes.uint32)) & UOp.const(1, dtypes.uint32) - exp = (h >> UOp.const(10, dtypes.uint32)) & UOp.const(0x1F, dtypes.uint32) - mant = h & UOp.const(0x3FF, dtypes.uint32) - # Use bf16 path: shift left by 16 to create bf16 bits, then shift mantissa and adjust exponent in float domain - # bf16 bits = (sign << 15) | (exp_bf16 << 7) | mant_bf16 -- but f16 and bf16 have different formats - # Instead: construct f32 bits properly, use a local uint32 array to force materialization - f32_bits = (sign << UOp.const(31, dtypes.uint32)) | \ - ((exp + UOp.const(112, dtypes.uint32)) << UOp.const(23, dtypes.uint32)) | \ - (mant << UOp.const(13, dtypes.uint32)) - is_zero = exp.eq(UOp.const(0, dtypes.uint32)) - # Return uint32 (f32 bit pattern) — stored directly to uint32 temp array, bitcast to float on read - return is_zero.where(UOp.const(0, dtypes.uint32), f32_bits) + # Phase 1: read all A and B values into tmp. Layout: tmp[0:n_a] = A[m][k], tmp[n_a:] = B[n][k]. + # For 4x4 each group is an independent block (K not split), for 16x16/32x32 groups share MxK/NxK and split K. + k_per_grp = K if M == 4 else K // n_grps + n_a_elems = n_grps * M * K if M == 4 else M * K + n_b_elems = n_grps * N * K if M == 4 else N * K + # Use a uint32 temp array: the optimizer folds bitcast(uint32->f32) chains on float arrays, losing conversions. + tmp = UOp.placeholder((n_a_elems + n_b_elems,), dtypes.uint32, slot=0, addrspace=AddrSpace.LOCAL) + + def cvt_elem(raw: UOp, sub_idx: int, fp8_fmt: str) -> UOp: + if is_i8: # extract i8, sign-extend to i32 + byte = (raw >> UOp.const(sub_idx * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) + return (byte.cast(dtypes.int32) ^ UOp.const(0x80, dtypes.int32)) - UOp.const(0x80, dtypes.int32) + if is_f32_src: return raw # already uint32 (f32 bit pattern) + if is_fp8: return _FUNCS[f"{fp8_fmt}_to_f32"](raw >> UOp.const(sub_idx * 8, dtypes.uint32)).bitcast(dtypes.uint32) + h = (raw >> UOp.const(sub_idx * 16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32) + if is_bf16: return h << UOp.const(16, dtypes.uint32) # bf16 is the upper 16 bits of f32 + # f16 -> f32 bit pattern, done in integer domain so the optimizer can't fold away the conversion + sign, exp, mant = (h >> _c(15)) & _c(1), (h >> _c(10)) & _c(0x1F), h & _c(0x3FF) + f32_bits = (sign << _c(31)) | ((exp + _c(112)) << _c(23)) | (mant << _c(13)) + return exp.eq(_c(0)).where(_c(0), f32_bits) + + def mn_idx(lane: UOp) -> UOp: # M/N matrix index held by a lane + if M == 32: # (lane%32)/16 selects the 16-wide block, (lane%32)%16 the index within it + return ((lane // UOp.const(16, dtypes.int)) % UOp.const(2, dtypes.int)) * UOp.const(16, dtypes.int) + lane % UOp.const(16, dtypes.int) + return lane % UOp.const(grp_size, dtypes.int) + + def grp_idx(lane: UOp) -> UOp: return lane // UOp.const(grp_size, dtypes.int) read_lane = ctx.range() - # For 32x32: lane%16 = M/N index within 16-wide block, lane//16 = which of 4 quarter-waves - # Groups: lanes 0-31 = group 0, lanes 32-63 = group 1 - # Within group: (lane%32)%16 = M/N[0-15], (lane%32)//16 selects M/N[0-15] or [16-31] - lane_in_grp = read_lane % UOp.const(grp_size, dtypes.int) - grp_idx = read_lane // UOp.const(grp_size, dtypes.int) - - if M == 32: - # 32x32: lane_in_grp%16 = sub-row/col (0-15), lane_in_grp//16 = block (0=rows 0-15, 1=rows 16-31) - sub_mn = lane_in_grp % UOp.const(16, dtypes.int) - block_mn = lane_in_grp // UOp.const(16, dtypes.int) - mn_idx = block_mn * UOp.const(16, dtypes.int) + sub_mn # actual M/N index (0-31) - else: - mn_idx = lane_in_grp # for 16x16 and 4x4 + mn, grp = mn_idx(read_lane), grp_idx(read_lane) + def mat_idx(base: int, dim: int, kl: int) -> UOp: # tmp index of element (mn, kl) of this lane's group + if M == 4: return UOp.const(base + kl, dtypes.int) + grp * UOp.const(dim * K, dtypes.int) + mn * UOp.const(K, dtypes.int) + return UOp.const(base + kl, dtypes.int) + mn * UOp.const(K, dtypes.int) + grp * UOp.const(k_per_grp, dtypes.int) read_stores = [] for kl in range(k_per_grp): reg_idx, sub_idx = kl // vpg, kl % vpg - # Read A/B sources. Use rsrc_dyn for inline constants/SGPRs (src_off < 256), rvgpr_dyn for VGPRs (src_off >= 256). - a_raw = src0_is_vgpr.where(ctx.rvgpr_dyn(src0_r + _c(reg_idx), read_lane), - ctx.rsrc_dyn(src0_off, _c(0, dtypes.int), 32)) - a_val = cvt_elem(a_raw, sub_idx, a_fmt) - if M == 4: - a_idx = grp_idx * UOp.const(M * K, dtypes.int) + mn_idx * UOp.const(K, dtypes.int) + UOp.const(kl, dtypes.int) - else: - a_idx = mn_idx * UOp.const(K, dtypes.int) + grp_idx * UOp.const(k_per_grp, dtypes.int) + UOp.const(kl, dtypes.int) - read_stores.append(tmp.index(a_idx).store(a_val)) - - b_raw = src1_is_vgpr.where(ctx.rvgpr_dyn(src1_r + _c(reg_idx), read_lane), - ctx.rsrc_dyn(src1_off, _c(0, dtypes.int), 32)) - b_val = cvt_elem(b_raw, sub_idx, b_fmt) - if M == 4: - b_idx = b_off + grp_idx * UOp.const(N * K, dtypes.int) + mn_idx * UOp.const(K, dtypes.int) + UOp.const(kl, dtypes.int) - else: - b_idx = b_off + mn_idx * UOp.const(K, dtypes.int) + grp_idx * UOp.const(k_per_grp, dtypes.int) + UOp.const(kl, dtypes.int) - read_stores.append(tmp.index(b_idx).store(b_val)) - + # src_off >= 256 means VGPR operand, otherwise inline constant/SGPR + for off, is_vgpr, fmt, base, dim in ((src0_off, src0_off >= _c(256), a_fmt, 0, M), (src1_off, src1_off >= _c(256), b_fmt, n_a_elems, N)): + raw = is_vgpr.where(ctx.rvgpr_dyn(off - _c(256 - reg_idx), read_lane), ctx.rsrc_dyn(off, _c(0, dtypes.int), 32)) + read_stores.append(tmp.index(mat_idx(base, dim, kl)).store(cvt_elem(raw, sub_idx, fmt))) read_phase = UOp.group(*read_stores).end(read_lane) - # Phase 2: Compute dot products and write outputs. - # For 16x16: each lane computes 4 outputs. n_idx = lane%16, grp selects which 4 rows. - # For 32x32: each lane computes 16 outputs. Layout: lane%16 selects n within block, lane//16 selects column block. - # Output mapping: out_reg r at lane l -> D[m][n] where - # n = (l%32)%16 + ((l%32)//16)*16, m = (l//32)*4 + r (for r in 0..3), with 4 groups of 4 rows -> 16 outputs total - # Actually: 16 ACCVGPRs per lane, organized as 4 groups (l//32 gives half, each half has 2 sub-groups) of 4 rows + # Phase 2: dot products and accumulate. acc reads src2 (VGPR, or scalar inline constant/SGPR broadcast). + src2_is_vgpr = src2_off >= _c(256) + acc_scalar = ctx.rsgpr_dyn(src2_off, src2_is_vgpr.ne(True)) + acc_scalar = acc_scalar.cast(dtypes.int32) if is_int_out else acc_scalar.bitcast(dtypes.float32) + acc_reader, acc_writer = (ctx.raccvgpr_dyn, ctx.waccvgpr_dyn) if use_acc else (ctx.rvgpr_dyn, ctx.wvgpr_dyn) tmp2 = tmp.after(read_phase) def _dot_accum(acc: UOp, a_row: UOp, b_row: UOp, lane: UOp) -> UOp: - """acc += sum_k A[a_row+k] * B[b_row+k]. For scaled MFMA, only the dot product is scaled: D = dot*scale + C.""" + """acc += sum_k A[a_row+k] * B[b_row+k] in order (FP-associativity matters). For scaled MFMA only the dot is scaled: D = dot*scale + C.""" def prod(k: int) -> UOp: return tmp2.index(a_row + UOp.const(k, dtypes.int)).bitcast(acc_dt) * tmp2.index(b_row + UOp.const(k, dtypes.int)).bitcast(acc_dt) if not scaled: @@ -1525,135 +1293,89 @@ def _compile_mfma(inst: irc.VOP3P|irc.VOP3PX2, ctx: _Ctx) -> UOp: return acc + dot * scale_factor(lane) compute_lane = ctx.range() + c_mn, c_grp = mn_idx(compute_lane), grp_idx(compute_lane) + b_off = UOp.const(n_a_elems, dtypes.int) + def out_ab(out_reg: int) -> tuple[UOp, UOp]: # A/B tmp base indices for one output element of this lane + if M == 32: # 16 outputs per lane: rows (lane//32)*16 + blocks of 4 within the wave + m_base = c_grp * UOp.const(16, dtypes.int) + UOp.const((out_reg // 4) * 4 + out_reg % 4, dtypes.int) + return m_base * UOp.const(K, dtypes.int), b_off + c_mn * UOp.const(K, dtypes.int) + if M == 4: # each group of 4 lanes computes an independent 4x4 block + a_base = c_grp * UOp.const(M * K, dtypes.int) + UOp.const(out_reg * K, dtypes.int) + return a_base, b_off + (c_grp * UOp.const(N, dtypes.int) + c_mn) * UOp.const(K, dtypes.int) + m_base = c_grp * UOp.const(out_per_lane, dtypes.int) + UOp.const(out_reg, dtypes.int) # 16x16: 4 outputs, one row each + return m_base * UOp.const(K, dtypes.int), b_off + c_mn * UOp.const(K, dtypes.int) + compute_stores = [] - - if M == 32 and N == 32: - # 32x32: each lane has 16 output ACCVGPRs - # Lane mapping: n = (lane%32)%16 + ((lane%32)//16)*16, gives column 0-31 - # Row groups: 4 groups of 4, covering rows 0-31. Group g (0-3): rows g*4 .. g*4+3 - # group assignment: lane//16 gives quarter (0-3), each quarter maps to 4 rows - c_lane_in_32 = compute_lane % UOp.const(32, dtypes.int) - c_sub = c_lane_in_32 % UOp.const(16, dtypes.int) - c_block = c_lane_in_32 // UOp.const(16, dtypes.int) - n_idx = c_block * UOp.const(16, dtypes.int) + c_sub - c_half = compute_lane // UOp.const(32, dtypes.int) # 0 or 1 - - for out_reg in range(16): - # Each half covers 8 rows. out_reg 0-3: rows 0-3 (half0) or 16-19 (half1) - # out_reg 4-7: rows 4-7 (half0) or 20-23 (half1), etc. - # Actually: for 32x32, the output layout per lane is: - # acc[0:3] -> rows 0-3 (half 0) or rows 0-3 (half 1)? - # Let me use the ISA doc: for 32x32, D has 16 dwords per lane. The mapping is: - # acc[r] at lane l -> D[m][n] where n = (l%32)%16 + ((l%32)//16)*16 - # m = (l//32)*16 + (r//4)*4 + (r%4) ... giving rows in blocks of 4 - # So: m_base = half * 16 + (out_reg // 4) * 4 + (out_reg % 4) - m_base = c_half * UOp.const(16, dtypes.int) + UOp.const((out_reg // 4) * 4 + (out_reg % 4), dtypes.int) - - acc_v = (ctx.raccvgpr_dyn if use_acc else ctx.rvgpr_dyn)(src2_r + _c(out_reg), compute_lane, src2_is_vgpr) - if is_int_out: acc_v = acc_v.cast(dtypes.int32) - else: acc_v = acc_v.bitcast(dtypes.float32) - acc = src2_is_vgpr.where(acc_v, acc_scalar) - - acc = _dot_accum(acc, m_base * UOp.const(K, dtypes.int), b_off + n_idx * UOp.const(K, dtypes.int), compute_lane) - - if is_int_out: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.cast(dtypes.uint32), exec_mask)) - else: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.bitcast(dtypes.uint32), exec_mask)) - else: - # 16x16 and 4x4: each lane computes out_per_lane outputs - n_idx = compute_lane % UOp.const(grp_sub, dtypes.int) - c_grp = compute_lane // UOp.const(grp_sub, dtypes.int) - - for out_reg in range(out_per_lane): - acc_v = (ctx.raccvgpr_dyn if use_acc else ctx.rvgpr_dyn)(src2_r + _c(out_reg), compute_lane, src2_is_vgpr) - if is_int_out: acc_v = acc_v.cast(dtypes.int32) - else: acc_v = acc_v.bitcast(dtypes.float32) - acc = src2_is_vgpr.where(acc_v, acc_scalar) - - if M == 4: - # 4x4: each group is independent. A/B indexed per-group. - m_base = c_grp * UOp.const(M * K, dtypes.int) + UOp.const(out_reg * K, dtypes.int) - b_base = b_off + c_grp * UOp.const(N * K, dtypes.int) + n_idx * UOp.const(K, dtypes.int) - else: - # 16x16: K is split across groups. Shared MxK/NxK arrays. - m_base = c_grp * UOp.const(out_per_lane, dtypes.int) + UOp.const(out_reg, dtypes.int) - b_base = b_off + n_idx * UOp.const(K, dtypes.int) - - acc = _dot_accum(acc, m_base if M == 4 else m_base * UOp.const(K, dtypes.int), b_base, compute_lane) - - if is_int_out: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.cast(dtypes.uint32), exec_mask)) - else: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.bitcast(dtypes.uint32), exec_mask)) - + for out_reg in range(out_per_lane): + acc_v = acc_reader(src2_off - _c(256 - out_reg), compute_lane, src2_is_vgpr) + acc_v = acc_v.cast(dtypes.int32) if is_int_out else acc_v.bitcast(dtypes.float32) + a_base, b_base = out_ab(out_reg) + acc = _dot_accum(src2_is_vgpr.where(acc_v, acc_scalar), a_base, b_base, compute_lane) + out_bits = acc.cast(dtypes.uint32) if is_int_out else acc.bitcast(dtypes.uint32) + compute_stores.append(acc_writer(vdst_reg + _c(out_reg), compute_lane, out_bits, exec_mask)) compute_phase = UOp.group(*compute_stores).end(compute_lane) return UOp.sink(read_phase, compute_phase, *ctx.inc_pc()) def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P, ctx: _Ctx) -> UOp: - op_name = _op_name(inst) - exec_mask = ctx.rexec() + """RDNA3/4 WMMA: D = A@B + C on 16x16 tiles. A/B are unpacked to flat f32/i32 arrays, then all 256 outputs are + computed directly with scalar ops (no lane loop - the wave32 lane structure is baked into the index maps).""" + op_name, exec_mask = _op_name(inst), ctx.rexec() vdst_reg = ctx.inst_field(type(inst).vdst) - src0_r = ctx.inst_field(type(inst).src0) - _c(256) - src1_r = ctx.inst_field(type(inst).src1) - _c(256) + src0_r, src1_r = ctx.inst_field(type(inst).src0) - _c(256), ctx.inst_field(type(inst).src1) - _c(256) src2_r = ctx.inst_field(type(inst).src2) src2_r = (src2_r >= 256).where(src2_r - _c(256), src2_r) output_type = op_name.split("WMMA_", 1)[1].split("_", 1)[0] - is_bf16 = 'BF16' in op_name - cvt = _FUNCS['bf16_to_f32'] if is_bf16 else _FUNCS['f16_to_f32'] - is_rdna4 = isinstance(inst, ir4.VOP3P) - sz = 8 if "8" in op_name else 16 - # read matrix from VGPRs → flat f32/i32 array[row*16+k] + is_bf16, is_rdna4 = 'BF16' in op_name, isinstance(inst, ir4.VOP3P) + cvt = _FUNCS['bf16_to_f32' if is_bf16 else 'f16_to_f32'] + sz = 8 if any(t in op_name for t in ('IU8', 'FP8', 'BF8')) else 16 # input element size + + # read a source element from VGPRs: (src, lane, vgpr, element-in-vgpr) -> f32/i32 def gval(src, lane, vgpr, ridx): v = ctx.rvgpr_dyn(src + _c(vgpr), UOp.const(lane, dtypes.int)) pkd = v >> UOp.const(ridx * sz, dtypes.uint32) if ridx > 0 else v pkd = pkd & UOp.const((1 << sz) - 1, dtypes.uint32) if "F" in output_type: return cvt(pkd) - return (pkd << _c(24, dtypes.uint)).bitcast(dtypes.int32) >> _c(24, dtypes.int32) # sign extend + return (pkd << _c(24, dtypes.uint)).bitcast(dtypes.int32) >> _c(24, dtypes.int32) # sign extend - # RDNA3 f16/bf16: 16 lanes × 8 VGPRs × 2 halves, k maps linearly - # RDNA3 iu8: 16 lanes × 4 VGPRs × 4 quarters, k maps linearly - # RDNA4: 32 lanes x 4 VGPRS x 2 halves, k bits are scrambled (k[2] goes to lane bit 4) + # RDNA3 f16/bf16: 16 lanes x 8 VGPRs x 2 halves, k maps linearly + # RDNA3 iu8: 16 lanes x 4 VGPRs x 4 quarters, k maps linearly + # RDNA4: 32 lanes x 4 VGPRs x 2 halves, k bits are scrambled (k[2] goes to lane bit 4) def read_mat(src): - n = 32 // sz # values per vgpr - # (row, k) → (lane, vgpr, row index) - def ab_map(i, k): + n = 32 // sz # values per vgpr + def ab_map(i, k): # (row, k) -> (lane, vgpr, element-in-vgpr) elem, lane = ((k & 3) | ((k >> 1) & 4), i + ((k >> 2) & 1) * 16) if is_rdna4 else (k, i) return lane, elem // n, elem % n return [gval(src, *ab_map(row, k)) for row in range(16) for k in range(16)] mat_a, mat_b = read_mat(src0_r), read_mat(src1_r) - # (row, col) -> (lane, vgpr) - def d_map(m, n): + def d_map(m, n): # output (row, col) -> (lane, vgpr) lane_bit, vgpr = (m >> 3, m & 7) if is_rdna4 else (m & 1, m >> 1) return n + lane_bit * 16, vgpr - if output_type in ["F16", "BF16"]: - # read accumulator C with f16 layout: for RDNA4, pairs of f32 vgprs pack into one f16 vgpr - # for RDNA3, same layout as f32 but only lo 16 bits used + + # Accumulator C. RDNA4 f16/bf16 packs two f32 accumulator VGPRs into one f16 VGPR; RDNA3 uses the lo half of each. + if output_type in ("F16", "BF16"): mat_c = [gval(src2_r, *((lane, vgpr // 2, vgpr % 2) if is_rdna4 else (lane, vgpr, 0))) for m in range(16) for n in range(16) for lane, vgpr in [d_map(m, n)]] - mat_d = [sum(mat_a[r*16+k] * mat_b[c*16+k] for k in range(16)) + mat_c[r*16+c] for r in range(16) for c in range(16)] - def f32_to_f16_bits(v: UOp) -> UOp: return v.cast(dtypes.half).bitcast(dtypes.uint16).cast(dtypes.uint32) - def f32_to_bf16_bits(v: UOp) -> UOp: return (v.bitcast(dtypes.uint32) >> UOp.const(16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32) - out_cvt = f32_to_bf16_bits if is_bf16 else f32_to_f16_bits - if is_rdna4: # pack 2 f16 per VGPR: adjacent m values share (lane, vgpr) since vgpr=m&7, half=m&1 - stores = [ctx.wvgpr_dyn(vdst_reg + _c(d_map(m, n)[1] // 2), UOp.const(d_map(m, n)[0], dtypes.int), - out_cvt(mat_d[m*16+n]) | (out_cvt(mat_d[(m+1)*16+n]) << UOp.const(16, dtypes.uint32)), exec_mask) - for n in range(16) for m in range(0, 16, 2)] - else: # (rdna3) 1 f16 per VGPR (lo half only) - stores = [ctx.wvgpr_dyn(vdst_reg + _c(d_map(m, n)[1]), UOp.const(d_map(m, n)[0], dtypes.int), out_cvt(mat_d[m*16+n]), exec_mask) - for m in range(16) for n in range(16)] - else: # f32/i32 + else: out_dt = dtypes.float32 if output_type == "F32" else dtypes.int32 - mat_c = [ctx.rvgpr_dyn(src2_r + _c(d_map(m, n)[1]), UOp.const(d_map(m, n)[0], dtypes.int)).bitcast(out_dt) - for m in range(16) for n in range(16)] - mat_d = [sum(mat_a[r*16+k] * mat_b[c*16+k] for k in range(16)) + mat_c[r*16+c] for r in range(16) for c in range(16)] - stores = [ctx.wvgpr_dyn(vdst_reg + _c(d_map(m, n)[1]), UOp.const(d_map(m, n)[0], dtypes.int), mat_d[m*16+n].bitcast(dtypes.uint32), exec_mask) - for m in range(16) for n in range(16)] + mat_c = [ctx.rvgpr_dyn(src2_r + _c(vgpr), UOp.const(lane, dtypes.int)).bitcast(out_dt) + for m in range(16) for n in range(16) for lane, vgpr in [d_map(m, n)]] + mat_d = [sum(mat_a[r*16+k] * mat_b[c*16+k] for k in range(16)) + mat_c[r*16+c] for r in range(16) for c in range(16)] + + def w_store(m: int, n: int, val: UOp, vgpr_off: int) -> UOp: # store one output element to its (lane, vgpr) slot + lane_i, _ = d_map(m, n) + return ctx.wvgpr_dyn(vdst_reg + _c(vgpr_off), UOp.const(lane_i, dtypes.int), val, exec_mask) + if output_type in ("F16", "BF16"): + def to_bits(v: UOp) -> UOp: # f32 result -> 16 output bits + return ((v.bitcast(dtypes.uint32) >> UOp.const(16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32)) if is_bf16 \ + else v.cast(dtypes.half).bitcast(dtypes.uint16).cast(dtypes.uint32) + if is_rdna4: # pack 2 outputs per VGPR (adjacent m values share a VGPR) + stores = [w_store(m, n, to_bits(mat_d[m*16+n]) | (to_bits(mat_d[(m+1)*16+n]) << UOp.const(16, dtypes.uint32)), d_map(m, n)[1] // 2) + for n in range(16) for m in range(0, 16, 2)] + else: # one output per VGPR (lo half) + stores = [w_store(m, n, to_bits(mat_d[m*16+n]), d_map(m, n)[1]) for m in range(16) for n in range(16)] + else: # f32/i32 + stores = [w_store(m, n, mat_d[m*16+n].bitcast(dtypes.uint32), d_map(m, n)[1]) for m in range(16) for n in range(16)] return UOp.sink(*stores, *ctx.inc_pc()) def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P | irc.VOP3PX2, ctx: _Ctx) -> UOp: @@ -1693,56 +1415,40 @@ def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P | irc.VOP3PX2, ctx: _ is_pk_f32 = 'PK' in op_name and 'F32' in op_name and 'MOV' not in op_name # CDNA packed F32 ops is_pk_mov_b32 = 'PK_MOV_B32' in op_name # CDNA packed MOV needs special handling do_cast = any(x in op_name for x in ('F16', 'F32', 'BF16')) and 'IU' not in op_name and not is_pk_f32 - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') src0 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src0), lane, 16, literal=literal, do_cast=do_cast) src1 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src1), lane, 16, literal=literal, do_cast=do_cast) src2 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src2), lane, 16, literal=literal, do_cast=do_cast) - opsel, opsel_hi = getattr(inst, 'opsel', 0) or 0, getattr(inst, 'opsel_hi', 3) if getattr(inst, 'opsel_hi', 3) is not None else 3 - opsel_hi2 = getattr(inst, 'opsel_hi2', 1) if getattr(inst, 'opsel_hi2', 1) is not None else 1 - neg, neg_hi = getattr(inst, 'neg', 0) or 0, getattr(inst, 'neg_hi', 0) or 0 + opsel, opsel_hi = _iattr(inst, 'opsel'), _iattr(inst, 'opsel_hi', 3) + opsel_hi2 = _iattr(inst, 'opsel_hi2', 1) + neg, neg_hi = _iattr(inst, 'neg'), _iattr(inst, 'neg_hi') + + def _pk_sel(src_lo: UOp, src_off: UOp, sel: int) -> UOp: + """Lo (sel=0) / hi (sel!=0) half of a packed source: VGPR/SGPR pairs select from the two registers of the pair + (off>=256 is a VGPR pair, off<128 an SGPR pair), inline constants (128<=off<256) broadcast to both halves.""" + if not sel: return (src_off >= _c(256)).where(ctx.rvgpr_dyn(src_off - _c(256), lane), src_lo) + is_sgpr_pair = src_off < _c(128) + return (src_off >= _c(256)).where(ctx.rvgpr_dyn(src_off - _c(256) + _c(1), lane), + is_sgpr_pair.where(ctx.rsgpr_dyn(src_off + _c(1), is_sgpr_pair), src_lo)) if is_pk_mov_b32: # v_pk_mov_b32: D[lo] = src0[opsel_bit0 ? hi : lo], D[hi] = src1[opsel_bit1 ? hi : lo] - src_offs = [ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1)] - def _pk_mov_sel(src_lo: UOp, src_off: UOp, sel_bit: int) -> UOp: - is_vgpr = src_off >= _c(256) - vgpr_lo = ctx.rvgpr_dyn(src_off - _c(256), lane) if lane is not None else _c(0) - vgpr_hi = ctx.rvgpr_dyn(src_off - _c(256) + _c(1), lane) if lane is not None else _c(0) - is_sgpr_pair = src_off < _c(128) - sgpr_hi = ctx.rsgpr_dyn(src_off + _c(1), is_sgpr_pair) - scalar_sel = is_sgpr_pair.where(sgpr_hi, src_lo) if sel_bit else src_lo - return is_vgpr.where(vgpr_hi if sel_bit else vgpr_lo, scalar_sel) - lo_val = _pk_mov_sel(src0, src_offs[0], opsel & 1) - hi_val = _pk_mov_sel(src1, src_offs[1], opsel & 2) - result = _u64(lo_val, hi_val) - lo_out, hi_out = _split64(result) + lo_val = _pk_sel(src0, ctx.inst_field(type(inst).src0), opsel & 1) + hi_val = _pk_sel(src1, ctx.inst_field(type(inst).src1), opsel & 2) + lo_out, hi_out = _split64(_u64(lo_val, hi_val)) stores = [ctx.wvgpr_dyn(vdst_reg, lane, lo_out, exec_mask), ctx.wvgpr_dyn(vdst_reg + _c(1), lane, hi_out, exec_mask)] return UOp.sink(UOp.group(*stores).end(lane), *ctx.inc_pc()) srcs: dict[str, UOp | int] = {} if is_pk_f32: - # CDNA packed F32: read 32-bit sources, build 64-bit packed values using opsel. - # For VGPRs: opsel selects between v[reg] (0) and v[reg+1] (1) for each half. - # For SGPR pairs (off < 128): s[N] = lo float32, s[N+1] = hi float32. - # For inline constants (128 <= off < 256): broadcast same value to both halves. + # CDNA packed F32: read 32-bit sources, build 64-bit packed values per opsel, with per-half negation src_offs = [ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1), ctx.inst_field(type(inst).src2)] - def build_pk_f32(src_lo: UOp, src_off: UOp, opsel_lo: int, opsel_hi_bit: int, neg_lo: int, neg_hi_bit: int) -> UOp: - is_vgpr = src_off >= _c(256) - vgpr_lo = ctx.rvgpr_dyn(src_off - _c(256), lane) if lane is not None else _c(0) - vgpr_hi = ctx.rvgpr_dyn(src_off - _c(256) + _c(1), lane) if lane is not None else _c(0) - # For SGPR pairs, opsel selects between s[N] (0) and s[N+1] (1); inline constants always broadcast. - is_sgpr_pair = src_off < _c(128) - sgpr_hi = ctx.rsgpr_dyn(src_off + _c(1), is_sgpr_pair) - scalar_lo_sel = src_lo if not opsel_lo else is_sgpr_pair.where(sgpr_hi, src_lo) - scalar_hi_sel = src_lo if not opsel_hi_bit else is_sgpr_pair.where(sgpr_hi, src_lo) - lo = is_vgpr.where(vgpr_hi if opsel_lo else vgpr_lo, scalar_lo_sel) - hi = is_vgpr.where(vgpr_hi if opsel_hi_bit else vgpr_lo, scalar_hi_sel) - if neg_lo: lo = lo ^ UOp.const(0x80000000, dtypes.uint32) - if neg_hi_bit: hi = hi ^ UOp.const(0x80000000, dtypes.uint32) - return _u64(lo, hi) - srcs = {'S0': build_pk_f32(src0, src_offs[0], opsel & 1, opsel_hi & 1, neg & 1, neg_hi & 1), - 'S1': build_pk_f32(src1, src_offs[1], opsel & 2, opsel_hi & 2, neg & 2, neg_hi & 2), - 'S2': build_pk_f32(src2, src_offs[2], opsel & 4, 1 if opsel_hi2 else 0, neg & 4, neg_hi & 4)} + hi_bits = (opsel_hi & 1, opsel_hi & 2, 1 if opsel_hi2 else 0) + for i, (bit, s0) in enumerate(zip((1, 2, 4), (src0, src1, src2))): + lo, hi = _pk_sel(s0, src_offs[i], opsel & bit), _pk_sel(s0, src_offs[i], hi_bits[i]) + if neg & bit: lo = lo ^ UOp.const(0x80000000, dtypes.uint32) + if neg_hi & bit: hi = hi ^ UOp.const(0x80000000, dtypes.uint32) + srcs[f'S{i}'] = _u64(lo, hi) elif 'FMA_MIX' in op_name or 'MAD_MIX' in op_name: combined_opsel_hi = (opsel_hi & 0x3) | ((opsel_hi2 & 0x1) << 2) # For FMA_MIX: neg_hi is ABS (not neg!), neg is actual negation @@ -1805,12 +1511,12 @@ def _compile_vopd(inst: ir3.VOPD | ir4.VOPD, ctx: _Ctx) -> UOp: if label == 'Y': srcs = {'S0': srcy0, 'S1': srcy1, 'D0': ctx.rvgpr_dyn(vdst_reg, lane)} else: srcs = {'S0': ctx.rsrc_dyn(src0_off, lane, literal=literal), 'S1': ctx.rvgpr_dyn(vsrc1_reg, lane), 'D0': ctx.rvgpr_dyn(vdst_reg, lane)} # VOP2_FMAAK/FMAMK_(DTYPE)_E32 - if vop in (ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOP2Op.V_FMAMK_F32_E32, ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOP2Op.V_FMAMK_F32_E32): + if vop in (ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOP2Op.V_FMAMK_F32_E32): assert literal is not None srcs['SIMM32'] = literal if op in (ir3.VOPDOp.V_DUAL_CNDMASK_B32, ir4.VOPDOp.V_DUAL_CNDMASK_B32): srcs['VCC'] = ctx.rmask(_c(VCC_LO.offset)) pcode = get_pcode(vop) - srcs.update({'VCC': ctx.rmask(_c(VCC_LO.offset)), 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane}) + srcs.update(ctx.base_srcs(exec_mask, lane), VCC=ctx.rmask(_c(VCC_LO.offset))) for dest, val in parse_pcode(pcode, srcs)[1]: if dest.startswith('D0'): all_stores.append(ctx.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask, after=srcy1)) return UOp.sink(UOp.group(*all_stores).end(lane), *ctx.inc_pc()) @@ -1828,37 +1534,26 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA is_lds = isinstance(inst, (ir3.DS, ir4.DS, irc.DS)) is_scratch = isinstance(inst, (ir3.SCRATCH, ir4.VSCRATCH, irc.SCRATCH)) # CDNA acc bit: when set, VGPR operands (vdst/vdata) target ACCVGPR file instead of VGPR - use_acc = bool(getattr(inst, 'acc', 0)) + use_acc = bool(_iattr(inst, 'acc')) mem = ctx.lds if is_lds else ctx.scratch if is_scratch else ctx.vmem addr_shift = UOp.const(2, dtypes.uint32 if is_lds else dtypes.uint64) - # Extract register info - all dynamic for deduplication + # Field names differ per format: DS (addr/data0/offset0+offset1), RDNA4 V* (vaddr/vsrc/ioffset), RDNA3+CDNA (addr/data/offset) + if is_lds: addr_field, data_field = 'addr', 'data0' + elif isinstance(inst, (ir4.VGLOBAL, ir4.VSCRATCH, ir4.VFLAT)): addr_field, data_field = 'vaddr', 'vsrc' + else: addr_field, data_field = 'addr', 'data' + addr_reg = ctx.inst_field(getattr(type(inst), addr_field)) + vdata_reg, vdst_reg = ctx.inst_field(getattr(type(inst), data_field)), ctx.inst_field(type(inst).vdst) if is_lds: - addr_reg = ctx.inst_field(type(inst).addr) # type: ignore[union-attr] - vdata_reg = ctx.inst_field(type(inst).data0) # type: ignore[union-attr] - vdst_reg = ctx.inst_field(type(inst).vdst) - offset0 = ctx.inst_field(type(inst).offset0) # type: ignore[union-attr] - offset1 = ctx.inst_field(type(inst).offset1) # type: ignore[union-attr] - offset = (offset1 << _c(8)) | offset0 # DS offset is 16-bit: (offset1 << 8) | offset0 - saddr_reg = None - elif isinstance(inst, (ir4.VGLOBAL, ir4.VSCRATCH, ir4.VFLAT)): # RDNA4: vaddr, vsrc, ioffset - addr_reg = ctx.inst_field(type(inst).vaddr) - vdata_reg = ctx.inst_field(type(inst).vsrc) - vdst_reg = ctx.inst_field(type(inst).vdst) - offset = ctx.inst_field_signed(type(inst).ioffset) - offset0, offset1 = _c(0), _c(0) - saddr_reg = ctx.inst_field(type(inst).saddr) if hasattr(type(inst), 'saddr') else None - else: # RDNA3: addr, data, offset - addr_reg = ctx.inst_field(type(inst).addr) - vdata_reg = ctx.inst_field(type(inst).data) # type: ignore[union-attr] - vdst_reg = ctx.inst_field(type(inst).vdst) - offset = ctx.inst_field_signed(type(inst).offset) # type: ignore[union-attr] - offset0, offset1 = _c(0), _c(0) - saddr_reg = ctx.inst_field(type(inst).saddr) if hasattr(type(inst), 'saddr') else None # type: ignore[union-attr] + offset0, offset1 = ctx.inst_field(type(inst).offset0), ctx.inst_field(type(inst).offset1) # type: ignore[union-attr] + offset, saddr_reg = (offset1 << _c(8)) | offset0, None # DS offset is 16-bit: (offset1 << 8) | offset0 + else: + offset0, offset1, saddr_reg = _c(0), _c(0), ctx.optional_field(inst, 'saddr') + offset = ctx.inst_field_signed(getattr(type(inst), 'ioffset' if hasattr(type(inst), 'ioffset') else 'offset')) # Data width from canonical_op_bits (32/64/96/128), default to 32 for untyped ops data_bits_mem = inst.canonical_op_bits.get('data', 32) - is_atomic, glc = 'ATOMIC' in op_name, getattr(inst, 'glc', 0) + is_atomic, glc = 'ATOMIC' in op_name, _iattr(inst, 'glc') has_data1 = is_lds and hasattr(inst, 'data1') and inst.data1 is not None data1_reg = ctx.inst_field(type(inst).data1) if is_lds else _c(0) # type: ignore[union-attr] @@ -1885,7 +1580,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA scratch_stride = ctx.rsgpr_dyn(_c(SCRATCH_STRIDE_IDX)).cast(dtypes.uint64) base = lane.cast(dtypes.uint64) * scratch_stride # SVE (Scratch VGPR Enable): when SVE=1, VADDR is used as offset; when SVE=0, VADDR is ignored - sve = getattr(inst, 'sve', 0) + sve = _iattr(inst, 'sve') vaddr = ctx.rvgpr_dyn(addr_reg, lane).cast(dtypes.uint64) addr_offset = vaddr if sve == 1 else UOp.const(0, dtypes.uint64) # Add saddr value only if use_saddr is true (saddr < 124) @@ -1916,17 +1611,13 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA def make_srcs(lane: UOp) -> dict: addr = make_addr(lane) if is_lds: - if data_bits_mem == 128: - data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), 'DATA1': ctx.rvgpr_dyn(vdata_reg + _c(1), lane), - 'DATA2': ctx.rvgpr_dyn(vdata_reg + _c(2), lane), 'DATA3': ctx.rvgpr_dyn(vdata_reg + _c(3), lane)} - elif data_bits_mem == 96: - data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), 'DATA1': ctx.rvgpr_dyn(vdata_reg + _c(1), lane), - 'DATA2': ctx.rvgpr_dyn(vdata_reg + _c(2), lane)} - elif data_bits_mem <= 32: + if data_bits_mem <= 32: data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), 'DATA2': ctx.rvgpr_dyn(data1_reg, lane) if has_data1 else UOp.const(0, dtypes.uint32)} - else: + elif data_bits_mem == 64: # DATA/DATA2 are the 64-bit input registers, formed from VGPR pairs data = {'DATA': _u64(ctx.rvgpr_dyn(vdata_reg, lane), ctx.rvgpr_dyn(vdata_reg + _c(1), lane)), 'DATA2': _u64(ctx.rvgpr_dyn(data1_reg, lane), ctx.rvgpr_dyn(data1_reg + _c(1), lane)) if has_data1 else UOp.const(0, dtypes.uint64)} + else: # 96/128-bit: one VGPR per dword + data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), **{f'DATA{i}': ctx.rvgpr_dyn(vdata_reg + _c(i), lane) for i in range(1, data_bits_mem // 32)}} # RDNA3 uses ADDR/OFFSET, RDNA4 uses vgpr_a/offset (lowercase) + CalcDsAddr function return {'ADDR': addr, 'ADDR_BASE': addr, 'OFFSET': offset, 'OFFSET0': offset0, 'OFFSET1': offset1, '_lds': mem, 'laneId': lane, 'vgpr_a': ctx.rvgpr_dyn(addr_reg, lane), 'offset': offset, 'offset0': offset0, 'offset1': offset1, **data} @@ -1969,7 +1660,8 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA if is_scratch: return _mem_store_bytes(mem, val[0], val[1], active, data_bits) return _mem_store(mem, val[0], val[1], active, 64, data_bits) if dest.startswith('RETURN_DATA') and writes_return_data: - _wdata = (lambda r, v, l, e: ctx.waccvgpr_dyn(r, l, v, e)) if use_acc else (lambda r, v, l, e: ctx.wvgpr_dyn(r, l, v, e)) + write_gpr = ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn + _wdata = lambda r, v, l, e: write_gpr(r, l, v, e) # noqa: E731 (arg order: reg, val, lane, exec) if (m := re.match(r'RETURN_DATA\[(\d+)\s*:\s*(\d+)\]', dest)): bit_width, dword_idx = int(m.group(1)) - int(m.group(2)) + 1, int(m.group(2)) // 32 return _write_val(bit_width, val, _wdata, vdst_reg + _c(dword_idx), lane, exec_mask) @@ -2023,7 +1715,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA def _compile_mubuf(inst: irc.MUBUF, ctx: _Ctx) -> UOp: """CDNA MUBUF: linear buffer address = base + soffset + (stride * index) + vgpr_offset + inst_offset""" exec_mask, op_name = ctx.rexec(), _op_name(inst) - use_acc, is_store, is_lds = bool(getattr(inst, 'acc', 0)), 'STORE' in op_name, bool(getattr(inst, 'lds', 0)) + use_acc, is_store, is_lds = bool(_iattr(inst, 'acc')), 'STORE' in op_name, bool(_iattr(inst, 'lds')) n_dwords = 4 if 'X4' in op_name else 2 if 'X2' in op_name else 1 # instruction fields @@ -2076,28 +1768,26 @@ def _compile_mubuf(inst: irc.MUBUF, ctx: _Ctx) -> UOp: stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)(vdata + _c(i), lane, val, exec_mask)) return UOp.sink(UOp.group(*stores).end(lane), *ctx.inc_pc()) -# Dispatch table: instruction type -> handler function -_INST_HANDLERS: dict[type, Callable[..., UOp]] = { - ir3.SOPP: _compile_sopp, ir3.SMEM: _compile_smem, ir3.SOP1: _compile_sop, ir3.SOP2: _compile_sop, ir3.SOPC: _compile_sop, ir3.SOPK: _compile_sop, - ir3.VOP1: _compile_vop12, ir3.VOP1_SDST: _compile_vop12, ir3.VOP1_DPP16: _compile_vop12, ir3.VOP2: _compile_vop12, ir3.VOP2_DPP16: _compile_vop12, - ir3.VOPC: _compile_vopc, ir3.VOPC_DPP16: _compile_vopc, ir3.VOP3: _compile_vop3, ir3.VINTERP: _compile_vinterp, - ir3.VOP3_SDST: _compile_vop3, ir3.VOP3SD: _compile_vop3sd, ir3.VOP3P: _compile_vop3p, ir3.VOPD: _compile_vopd, - ir3.DS: _compile_mem_op, ir3.FLAT: _compile_mem_op, ir3.GLOBAL: _compile_mem_op, ir3.SCRATCH: _compile_mem_op, - # RDNA4 instruction classes - ir4.SOPP: _compile_sopp, ir4.SMEM: _compile_smem, ir4.SOP1: _compile_sop, ir4.SOP2: _compile_sop, ir4.SOPC: _compile_sop, ir4.SOPK: _compile_sop, - ir4.VOP1: _compile_vop12, ir4.VOP1_SDST: _compile_vop12, ir4.VOP1_DPP16: _compile_vop12, ir4.VOP2: _compile_vop12, ir4.VOP2_DPP16: _compile_vop12, - ir4.VOPC: _compile_vopc, ir4.VOPC_DPP16: _compile_vopc, ir4.VOP3: _compile_vop3, ir4.VINTERP: _compile_vinterp, - ir4.VOP3_SDST: _compile_vop3, ir4.VOP3SD: _compile_vop3sd, ir4.VOP3P: _compile_vop3p, ir4.VOPD: _compile_vopd, - ir4.DS: _compile_mem_op, ir4.VFLAT: _compile_mem_op, ir4.VGLOBAL: _compile_mem_op, ir4.VSCRATCH: _compile_mem_op, - # CDNA instruction classes - irc.SOPP: _compile_sopp, irc.SMEM: _compile_smem, irc.SOP1: _compile_sop, irc.SOP2: _compile_sop, irc.SOPC: _compile_sop, irc.SOPK: _compile_sop, - irc.VOP1: _compile_vop12, irc.VOP1_DPP16: _compile_vop12, irc.VOP2: _compile_vop12, irc.VOP2_DPP16: _compile_vop12, - irc.VOPC: _compile_vopc, irc.VOP3: _compile_vop3, - irc.VOP3_SDST: _compile_vop3, irc.VOP3SD: _compile_vop3sd, irc.VOP3P: _compile_vop3p, irc.VOP3PX2: _compile_vop3p, - irc.VOP1_SDWA: _compile_sdwa, irc.VOP2_SDWA: _compile_sdwa, irc.VOP2_SDWA_SDST: _compile_sdwa, irc.VOPC_SDWA_SDST: _compile_sdwa, - irc.DS: _compile_mem_op, irc.FLAT: _compile_mem_op, irc.GLOBAL: _compile_mem_op, irc.SCRATCH: _compile_mem_op, - irc.MUBUF: _compile_mubuf, -} +# Dispatch table: instruction type -> handler function. Classes are looked up by name across all three archs. +def _inst_kinds(*names: str) -> tuple[type, ...]: + return tuple(getattr(m, n) for m in (ir3, ir4, irc) for n in names if hasattr(m, n)) + +_COMMON_HANDLERS: list[tuple[Callable[..., UOp], tuple[str, ...]]] = [ + (_compile_sopp, ('SOPP',)), + (_compile_smem, ('SMEM',)), + (_compile_sop, ('SOP1', 'SOP2', 'SOPC', 'SOPK')), + (_compile_vop12, ('VOP1', 'VOP1_SDST', 'VOP1_DPP16', 'VOP2', 'VOP2_DPP16')), + (_compile_vopc, ('VOPC', 'VOPC_DPP16')), + (_compile_vop3, ('VOP3', 'VOP3_SDST')), + (_compile_vinterp, ('VINTERP',)), + (_compile_vop3sd, ('VOP3SD',)), + (_compile_vop3p, ('VOP3P', 'VOP3PX2')), + (_compile_vopd, ('VOPD',)), + (_compile_sdwa, ('VOP1_SDWA', 'VOP2_SDWA', 'VOP2_SDWA_SDST', 'VOPC_SDWA_SDST')), + (_compile_mem_op, ('DS', 'FLAT', 'GLOBAL', 'SCRATCH', 'VFLAT', 'VGLOBAL', 'VSCRATCH')), +] +_INST_HANDLERS: dict[type, Callable[..., UOp]] = {t: h for h, names in _COMMON_HANDLERS for t in _inst_kinds(*names)} +_INST_HANDLERS[irc.MUBUF] = _compile_mubuf # CDNA only (rdna3 also has a MUBUF class, intentionally unhandled) # ═══════════════════════════════════════════════════════════════════════════════ # PROGRAM DECODE AND COMPILATION @@ -2249,7 +1939,6 @@ def _init_wave(lib: int, wave_start: int, total_threads: int, lx: int, ly: int, def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int, rsrc2: int = 0x19c, scratch_size: int = 0, arch: str = "rdna3", user_data: list[int]|None = None) -> int: """Execute AMD assembly program. scratch_size is private_segment_fixed_size from kernel descriptor (per-lane).""" - from tinygrad.renderer.amd.dsl import Inst program: dict[int, tuple[Callable, list[int], bool, Inst]] = {} # pc -> (fxn, globals, is_barrier, inst) lds_size = ((rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE_SHIFT) * 512 total_threads = lx * ly * lz @@ -2262,7 +1951,7 @@ def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, # Initialize SQTT encoder — emits packets inline as instructions execute (only when profiling) if PROFILE: - sqtt_emit, sqtt_finish, sqtt_finalize = _init_sqtt_encoder() + sqtt_emit, sqtt_finish, sqtt_finalize = _make_sqtt_encoder() def _ensure_compiled(pc: int) -> tuple[Callable, list[int], bool, Inst]: if pc not in program: @@ -2276,53 +1965,47 @@ def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, print(colored(msg, 'green') if len(_canonical_runner_cache) > prev_len else msg) return program[pc] + def _run_workgroup(gidx: int, gidy: int, gidz: int, tracing: bool): + """Initialize all wavefronts for one workgroup and execute them with barrier synchronization. + Each wave runs until it hits s_barrier or s_endpgm. When all waves have stopped, release barrier waves.""" + waves: list[tuple[WaveState, list]] = [] + for wave_start in range(0, total_threads, wave_size): + st = _init_wave(lib, wave_start, total_threads, lx, ly, lz, args_ptr, rsrc2, scratch_size, arch, gidx, gidy, gidz, user_data, wave_size) + waves.append((st, [ctypes.c_uint64(st.sgpr_buf._buf.va_addr), ctypes.c_uint64(st.vgpr_buf._buf.va_addr), + ctypes.c_uint64(vmem_buf._buf.va_addr), ctypes.c_uint64(lds_buf._buf.va_addr), + ctypes.c_uint64(scratch_buf._buf.va_addr if scratch_buf else 0), + ctypes.c_uint64(st.accvgpr_buf._buf.va_addr)])) + done = [False] * len(waves) + for _ in range(10_000_000): + if all(done): return + for wi, (st, c_bufs) in enumerate(waves): + if done[wi]: continue + # Run this wave until barrier or endpgm + for _ in range(1_000_000): + pc = st.pc + if pc == ENDPGM_PC: + done[wi] = True + if tracing: sqtt_finish(wi) + break + fxn, globals_list, is_barrier, inst = _ensure_compiled(pc) + if DEBUG >= 5: print(f" exec gid=({gidx},{gidy},{gidz}) w={wi} PC={pc - lib}: {inst!r}", flush=True) + fxn(*[c_bufs[g] for g in globals_list]) + if tracing: + inst_op = inst.op.value if hasattr(inst, 'op') else 0 + sqtt_emit(wi, inst, (st.pc != ENDPGM_PC and st.pc != pc + inst.size()) if inst_op in _BRANCH_OPS else None) + if is_barrier: break # s_barrier hit: PC already advanced past it, pause this wave + else: raise RuntimeError("exceeded 1M instructions in single wave, likely infinite loop") + # All waves have either hit barrier or endpgm — release barrier waves for next round + raise RuntimeError("exceeded 10M total scheduling rounds") + # Set DAZ+FTZ during emulator execution, restore afterward to avoid breaking hypothesis tests # Only trace the first workgroup (like real HW traces one CU/SIMD), subsequent workgroups run but don't add to trace tracing = bool(PROFILE) - with _MXCSRContext(): - for gidz in range(gz): - for gidy in range(gy): - for gidx in range(gx): - # Initialize all wavefronts for this workgroup - waves: list[tuple[WaveState, list]] = [] - for wave_start in range(0, total_threads, wave_size): - st = _init_wave(lib, wave_start, total_threads, lx, ly, lz, args_ptr, rsrc2, scratch_size, arch, gidx, gidy, gidz, user_data, - wave_size) - c_bufs = [ctypes.c_uint64(st.sgpr_buf._buf.va_addr), ctypes.c_uint64(st.vgpr_buf._buf.va_addr), - ctypes.c_uint64(vmem_buf._buf.va_addr), ctypes.c_uint64(lds_buf._buf.va_addr), - ctypes.c_uint64(scratch_buf._buf.va_addr if scratch_buf else 0), - ctypes.c_uint64(st.accvgpr_buf._buf.va_addr)] - waves.append((st, c_bufs)) - - # Execute wavefronts with barrier synchronization - # Each wave runs until it hits s_barrier or s_endpgm. When all waves have stopped, release barrier waves. - done = [False] * len(waves) - for total_inst in range(10_000_000): - if all(done): break - for wi, (st, c_bufs) in enumerate(waves): - if done[wi]: continue - # Run this wave until barrier or endpgm - for _ in range(1_000_000): - pc = st.pc - if pc == ENDPGM_PC: - done[wi] = True - if tracing: sqtt_finish(wi) - break - fxn, globals_list, is_barrier, inst = _ensure_compiled(pc) - if DEBUG >= 5: print(f" exec gid=({gidx},{gidy},{gidz}) w={wi} PC={pc - lib}: {inst!r}", flush=True) - fxn(*[c_bufs[g] for g in globals_list]) - if tracing: - inst_op = inst.op.value if hasattr(inst, 'op') else 0 - sqtt_emit(wi, inst, (st.pc != ENDPGM_PC and st.pc != pc + inst.size()) if inst_op in _BRANCH_OPS else None) - if is_barrier: break # s_barrier hit: PC already advanced past it, pause this wave - else: raise RuntimeError("exceeded 1M instructions in single wave, likely infinite loop") - # All waves have either hit barrier or endpgm — release barrier waves for next round - else: raise RuntimeError("exceeded 10M total scheduling rounds") - tracing = False # only trace the first workgroup - - # Reset LDS for next workgroup - if lds_size > 0: ctypes.memset(lds_buf._buf.va_addr, 0, max(lds_size, 4)) + for gidz, gidy, gidx in itertools.product(range(gz), range(gy), range(gx)): + _run_workgroup(gidx, gidy, gidz, tracing) + tracing = False # only trace the first workgroup + if lds_size > 0: ctypes.memset(lds_buf._buf.va_addr, 0, max(lds_size, 4)) # reset LDS for next workgroup if PROFILE: sqtt_traces.append(sqtt_finalize()) return 0 diff --git a/tinygrad_repo/test/mockgpu/amd/pcode.py b/tinygrad_repo/test/mockgpu/amd/pcode.py index bb66ffe03..8d49b6b42 100644 --- a/tinygrad_repo/test/mockgpu/amd/pcode.py +++ b/tinygrad_repo/test/mockgpu/amd/pcode.py @@ -1,5 +1,20 @@ # Tokenizer-based expression parser for AMD pcode +import ast, itertools, operator, re from typing import Any, Callable + +_BINOPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, ast.LShift: operator.lshift, ast.RShift: operator.rshift, + ast.BitAnd: operator.and_, ast.BitOr: operator.or_, ast.BitXor: operator.xor} +def _const_int(expr: str) -> int: + """Evaluate a compile-time integer expression (integer literals and basic arithmetic only).""" + def ev(node: ast.AST) -> int: + if isinstance(node, ast.Expression): return ev(node.body) + if isinstance(node, ast.Constant) and isinstance(node.value, int): return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + return (-1 if isinstance(node.op, ast.USub) else 1) * ev(node.operand) + if isinstance(node, ast.BinOp) and type(node.op) in _BINOPS: return _BINOPS[type(node.op)](ev(node.left), ev(node.right)) + raise ValueError(f"not a constant integer expression: {expr!r}") + return ev(ast.parse(expr.strip(), mode='eval')) from tinygrad.dtype import dtypes from tinygrad.uop.ops import Ops, UOp from tinygrad.codegen.decomp.dtype import f2f @@ -8,6 +23,7 @@ from tinygrad.codegen.decomp.dtype import f2f VarVal = UOp | tuple[str, list[str], str] def _const(dt, v): return UOp.const(v, dt) +def _single_value(v: UOp): return v.vmin if v.vmin == v.vmax else None def _u32(v): return _const(dtypes.uint32, v) def _u64(v): return _const(dtypes.uint64, v) def _to_u32(v): return v if v.dtype == dtypes.uint32 else v.bitcast(dtypes.uint32) if v.dtype.itemsize == 4 else v.cast(dtypes.uint32) @@ -55,8 +71,8 @@ def _expr_bits(v: UOp) -> int: if v.op in (Ops.AND, Ops.XOR): widths: list[int] = [] for src in v.src: - if src.op == Ops.CONST and isinstance(src.val, int) and src.val > 0 and (src.val & (src.val + 1)) == 0: - widths.append(src.val.bit_length()) + if isinstance(sv:=_single_value(src), int) and sv > 0 and (sv & (sv + 1)) == 0: + widths.append(sv.bit_length()) if widths: return max(widths) return v.dtype.bitsize @@ -144,9 +160,9 @@ def _minmax_reduce(is_max: bool, dt, *args: UOp) -> UOp: def _find_two_pi_mul(x): if x.op != Ops.MUL or len(x.src) != 2: return None for i, s in enumerate(x.src): - if s.op == Ops.CONST and abs(s.val - 6.283185307179586) < 1e-5: return (x.src[1-i], 6.283185307179586) + if (sv:=_single_value(s)) is not None and abs(sv - 6.283185307179586) < 1e-5: return (x.src[1-i], 6.283185307179586) if s.op == Ops.MUL and len(s.src) == 2: - vals = [ss.val for ss in s.src if ss.op == Ops.CONST] + [ss.src[0].val for ss in s.src if ss.op == Ops.CAST and ss.src[0].op == Ops.CONST] + vals = [sv for ss in s.src if (sv:=_single_value(ss)) is not None] if len(vals) == 2 and abs(vals[0] * vals[1] - 6.283185307179586) < 1e-5: return (x.src[1-i], vals[0] * vals[1]) return None @@ -163,7 +179,7 @@ def _trig_reduce(x, phase=0.0): def _signext(val: UOp) -> UOp: for bits, mask, ext in [(4, 0xF, 0xFFFFFFF0), (8, 0xFF, 0xFFFFFF00), (16, 0xFFFF, 0xFFFF0000)]: - if (val.op == Ops.AND and len(val.src) == 2 and val.src[1].op == Ops.CONST and val.src[1].val == mask) or val.dtype.itemsize == bits // 8: + if (val.op == Ops.AND and len(val.src) == 2 and _single_value(val.src[1]) == mask) or val.dtype.itemsize == bits // 8: v32 = val.cast(dtypes.uint32) if val.dtype != dtypes.uint32 else val sb = (v32 >> _u32(bits - 1)) & _u32(1) return sb.ne(_u32(0)).where(v32 | _u32(ext), v32).cast(dtypes.int) @@ -185,7 +201,20 @@ def _abs(val: UOp) -> UOp: def _f_to_u(f, dt): clamped = (f < _const(f.dtype, 0.0)).where(_const(f.dtype, 0.0), f) truncated = UOp(Ops.TRUNC, src=(clamped,)) - return (truncated >= _const(f.dtype, 2**(dt.itemsize*8))).where(_const(dt, dt.max), truncated.cast(dt)) + res = (truncated >= _const(f.dtype, 2**(dt.itemsize*8))).where(_const(dt, dt.max), truncated.cast(dt)) + return _isnan(f).where(_const(dt, 0), res) # float->uint conversion of NaN is 0 on hardware + +def _f_to_i32(a: UOp) -> UOp: + """v_cvt_i32_f32/f64: truncate toward zero, saturate to [INT_MIN, INT_MAX], NaN -> 0. + (x86 cvttss2si returns 0x80000000 for all of these, which matches hardware only for negative overflow.)""" + res = (a >= _const(a.dtype, 2147483648.0)).where(_const(dtypes.int, 0x7FFFFFFF), UOp(Ops.TRUNC, src=(a,)).cast(dtypes.int)) + return _isnan(a).where(_const(dtypes.int, 0), res) + +def _ftz_f32(v: UOp) -> UOp: + """Flush f32 denormals to signed zero (RDNA default float mode flushes denormal f32 inputs on select-style ops).""" + bits = v.bitcast(dtypes.uint32) if v.dtype == dtypes.float32 else v + return ((bits & _u32(0x7FFFFFFF)) < _u32(0x00800000)).where((bits & _u32(0x80000000)).bitcast(dtypes.float32), + v if v.dtype == dtypes.float32 else v.bitcast(dtypes.float32)) def _cvt_quiet(val: UOp) -> UOp: bits, _, _, qb, _ = _float_info(val) @@ -230,18 +259,51 @@ def _ldexp(val: UOp, exp: UOp) -> UOp: if val.dtype == dtypes.uint32: val = val.bitcast(dtypes.float32) elif val.dtype == dtypes.uint64: val = val.bitcast(dtypes.float64) if exp.dtype in (dtypes.uint32, dtypes.uint64): exp = exp.cast(dtypes.int if exp.dtype == dtypes.uint32 else dtypes.int64) - return val * UOp(Ops.EXP2, src=(exp.cast(val.dtype),)) + bits = val.bitcast(dtypes.uint32) if val.dtype == dtypes.float32 else val.bitcast(dtypes.uint64) + abs_max = _const(bits.dtype, 0x7F800000 if val.dtype == dtypes.float32 else 0x7FF0000000000000) + sign_mask = _const(bits.dtype, 0x80000000 if val.dtype == dtypes.float32 else 0x8000000000000000) + # hardware flushes denormal inputs to signed zero + magn_mask = _const(bits.dtype, 0x7FFFFFFF if val.dtype == dtypes.float32 else 0x7FFFFFFFFFFFFFFF) + is_denorm = ((bits & abs_max).eq(_const(bits.dtype, 0))) & ((bits & magn_mask).ne(_const(bits.dtype, 0))) + val = is_denorm.where((bits & sign_mask).bitcast(val.dtype), val) + # hardware propagates 0/+-inf/NaN unchanged (avoids 0*inf = NaN on the host) + res = val * UOp(Ops.EXP2, src=(exp.cast(val.dtype),)) + is_special = (bits & abs_max).eq(_const(bits.dtype, 0)) | ((bits & abs_max) >= abs_max) + return is_special.where(val, res) def _frexp_mant(val: UOp) -> UOp: val = val.bitcast(dtypes.float32) if val.dtype == dtypes.uint32 else val.bitcast(dtypes.float64) if val.dtype == dtypes.uint64 else val - if val.dtype == dtypes.float32: return ((val.bitcast(dtypes.uint32) & _u32(0x807FFFFF)) | _u32(0x3f000000)).bitcast(dtypes.float32) - return ((val.bitcast(dtypes.uint64) & _const(dtypes.uint64, 0x800FFFFFFFFFFFFF)) | - _const(dtypes.uint64, 0x3fe0000000000000)).bitcast(dtypes.float64) + if val.dtype == dtypes.float32: + bits = val.bitcast(dtypes.uint32) + # denormal/zero inputs (exponent field == 0) return signed zero on hardware + return ((bits & _u32(0x7F800000)).ne(_u32(0))).where(((bits & _u32(0x807FFFFF)) | _u32(0x3F000000)).bitcast(dtypes.float32), + (bits & _u32(0x80000000)).bitcast(dtypes.float32)) + bits = val.bitcast(dtypes.uint64) + return ((bits & _const(dtypes.uint64, 0x7FF0000000000000)).ne(_const(dtypes.uint64, 0))).where( + ((bits & _const(dtypes.uint64, 0x800FFFFFFFFFFFFF)) | _const(dtypes.uint64, 0x3fe0000000000000)).bitcast(dtypes.float64), + (bits & _const(dtypes.uint64, 0x8000000000000000)).bitcast(dtypes.float64)) + +def _msb(val: UOp, bits: int) -> UOp: + """Index of the highest set bit, or -1 if val == 0.""" + dt = dtypes.uint64 if bits > 32 else dtypes.uint32 + val = val.cast(dt) if val.dtype != dt else val + result = _const(dtypes.int, -1) + for i in range(bits - 1, -1, -1): + cond = ((val >> _const(dt, i)) & _const(dt, 1)).ne(_const(dt, 0)) & result.eq(_const(dtypes.int, -1)) + result = cond.where(_const(dtypes.int, i), result) + return result def _frexp_exp(val: UOp) -> UOp: val = val.bitcast(dtypes.float32) if val.dtype == dtypes.uint32 else val.bitcast(dtypes.float64) if val.dtype == dtypes.uint64 else val - if val.dtype == dtypes.float32: return ((val.bitcast(dtypes.uint32) >> _u32(23)) & _u32(0xFF)).cast(dtypes.int) - _const(dtypes.int, 126) - return ((val.bitcast(dtypes.uint64) >> _const(dtypes.uint64, 52)) & _const(dtypes.uint64, 0x7FF)).cast(dtypes.int) - _const(dtypes.int, 1022) + if val.dtype == dtypes.float32: + e = (val.bitcast(dtypes.uint32) >> _u32(23)) & _u32(0xFF) + return e.ne(_u32(0)).where(e.cast(dtypes.int) - _const(dtypes.int, 126), _const(dtypes.int, 0)) # f32 denormals -> 0 (hardware verified) + bits = val.bitcast(dtypes.uint64) + e = (bits >> _const(dtypes.uint64, 52)) & _const(dtypes.uint64, 0x7FF) + mant = bits & _const(dtypes.uint64, 0xFFFFFFFFFFFFF) + # f64 denormals: normalized exponent = highest set mantissa bit - 1073, zero -> 0 (hardware verified) + denorm = mant.ne(_const(dtypes.uint64, 0)).where(_msb(mant, 52) - _const(dtypes.int, 1073), _const(dtypes.int, 0)) + return e.ne(_const(dtypes.uint64, 0)).where(e.cast(dtypes.int) - _const(dtypes.int, 1022), denorm) TWO_OVER_PI = int( "0145f306dc9c882a53f84eafa3ea69bb81b6c52b3278872083fca2c757bd778ac36e48dc74849ba5c00c925dd413a32439fc3bd" @@ -299,9 +361,9 @@ _FUNCS: dict[str, Callable[..., UOp]] = { 'fma': lambda a, b, c: a * b + c, 'i32_to_f32': lambda a: a.cast(dtypes.int).cast(dtypes.float32), 'u32_to_f32': lambda a: a.cast(dtypes.uint32).cast(dtypes.float32), - 'f32_to_i32': lambda a: UOp(Ops.TRUNC, src=(a.bitcast(dtypes.float32),)).cast(dtypes.int), + 'f32_to_i32': lambda a: _f_to_i32(a.bitcast(dtypes.float32)), 'f32_to_u32': lambda a: _f_to_u(a.bitcast(dtypes.float32), dtypes.uint32), - 'f64_to_i32': lambda a: UOp(Ops.TRUNC, src=(a.bitcast(dtypes.float64),)).cast(dtypes.int), + 'f64_to_i32': lambda a: _f_to_i32(a.bitcast(dtypes.float64)), 'f64_to_u32': lambda a: _f_to_u(a.bitcast(dtypes.float64), dtypes.uint32), 'f16_to_f32': lambda a: _f16_extract(a).cast(dtypes.float32), 'f32_to_f16': lambda a: a.cast(dtypes.half), @@ -360,22 +422,13 @@ _FUNCS: dict[str, Callable[..., UOp]] = { 'fp8_to_f32': _fp8_to_f32, 'bf8_to_f32': _bf8_to_f32, 'f32_to_fp8': _f32_to_fp8, 'f32_to_bf8': _f32_to_bf8, 'f32_to_bf16': _f32_to_bf16, 'f32_to_bf16_SR': _f32_to_bf16_sr, 'f32_to_bf16_sr': _f32_to_bf16_sr, } -for is_max, name in [(False, 'min'), (True, 'max')]: - for dt, sfx in [(dtypes.float32, 'f32'), (dtypes.int, 'i32'), (dtypes.uint32, 'u32'), (dtypes.int16, 'i16'), (dtypes.uint16, 'u16')]: - _FUNCS[f'v_{name}_{sfx}'] = lambda *a, im=is_max, d=dt: _minmax_reduce(im, d, *a) - _FUNCS[f'v_{name}3_{sfx}'] = lambda *a, im=is_max, d=dt: _minmax_reduce(im, d, *a) -# f16 min/max/min3/max3/med3 -for is_max, name in [(False, 'min'), (True, 'max')]: - _FUNCS[f'v_{name}_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}3_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}_num_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}_num_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) - _FUNCS[f'v_{name}3_num_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}3_num_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) - _FUNCS[f'v_{name}imum_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}imum_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) - _FUNCS[f'v_{name}imum3_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}imum3_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) +# min/max family: min/max + 3-input (x3), IEEE num variants (f16/f32 only), and long names minimum/maximum (f16/f32 only) +for is_max, name, full in [(False, 'min', 'minimum'), (True, 'max', 'maximum')]: + for dt, sfx, pre in [(dtypes.float32, 'f32', None), (dtypes.int, 'i32', None), (dtypes.uint32, 'u32', None), + (dtypes.int16, 'i16', None), (dtypes.uint16, 'u16', None), (dtypes.half, 'f16', _f16_extract)]: + def mm(*a, im=is_max, d=dt, p=pre): return _minmax_reduce(im, d, *(a if p is None else [p(x) for x in a])) + extra = (f'v_{name}_num_{sfx}', f'v_{name}3_num_{sfx}', f'v_{full}_{sfx}', f'v_{full}3_{sfx}') if dt in (dtypes.float32, dtypes.half) else () + for fn in (f'v_{name}_{sfx}', f'v_{name}3_{sfx}', *extra): _FUNCS[fn] = mm # ═══════════════════════════════════════════════════════════════════════════════ # TOKENIZER/PARSER @@ -497,7 +550,7 @@ class Parser: if not dtypes.is_int(right.dtype): right = right.cast(dtypes.uint32) return (left >> right) if op == '>>' else (left << right) case '+' | '-': - if op == '-' and left.op == Ops.CONST and right.op == Ops.CONST: return _const(left.dtype, left.val - right.val) + if op == '-' and (lv:=_single_value(left)) is not None and (rv:=_single_value(right)) is not None: return _const(left.dtype, lv - rv) return (left + right) if op == '+' else (left - right) case '*' | '/': # Integer promotion: promote 16-bit integers to 32-bit before multiply to avoid overflow @@ -507,7 +560,7 @@ class Parser: left, right = left.cast(pdt), right.cast(pdt) if op == '*': return left * right return (left // right) if dtypes.is_int(left.dtype) else (left / right) - case '**': return UOp(Ops.EXP2, src=(right.cast(left.dtype),)) if left.op == Ops.CONST and left.val == 2.0 else left + case '**': return UOp(Ops.EXP2, src=(right.cast(left.dtype),)) if _single_value(left) == 2.0 else left _PREC = [('||',), ('&&',), ('|',), ('^',), ('&',), ('==', '!=', '<>'), ('>=', '<=', '>', '<'), ('>>', '<<'), ('+', '-'), ('*', '/'), ('**',)] @@ -529,8 +582,8 @@ class Parser: return inner.eq(_const(inner.dtype, 0)) if self.try_eat_val('-', 'OP'): inner = self.unary() - if inner.op == Ops.CONST: - return _const(dtypes.int if inner.dtype == dtypes.uint32 else inner.dtype, -inner.val) + if (v:=_single_value(inner)) is not None: + return _const(dtypes.int if inner.dtype == dtypes.uint32 else inner.dtype, -v) return inner.neg() if self.try_eat_val('+', 'OP'): return self.unary() return self.postfix() @@ -669,15 +722,13 @@ class Parser: self.eat('OP') width = self.parse() self.eat('RBRACKET') - if width.op == Ops.CONST: - w = int(width.val) + if isinstance(w:=_single_value(width), int): return (base >> _to_u32(first)) & _const(base.dtype, (1 << w) - 1) return base if self.try_eat('COLON'): second = self.parse() self.eat('RBRACKET') - if first.op == Ops.CONST and second.op == Ops.CONST: - a, b = int(first.val), int(second.val) + if isinstance(a:=_single_value(first), int) and isinstance(b:=_single_value(second), int): if a < b: return _bitreverse(base, b - a + 1) hi, lo = a, b if lo >= base.dtype.itemsize * 8: @@ -698,8 +749,7 @@ class Parser: dt_suffix = DTYPES.get(self.eat('IDENT').val, dtypes.uint32) if var_name is None: var_name = self._find_var_name(base) - if first.op == Ops.CONST: - idx = int(first.val) + if isinstance(idx:=_single_value(first), int): # Check for array element (var@idx) if var_name and f'{var_name}@{idx}' in self.vars: v = self.vars[f'{var_name}@{idx}'] @@ -872,7 +922,7 @@ class Parser: def _coerce_cmp(self, l: UOp, r: UOp) -> tuple[UOp, UOp]: if l.dtype != r.dtype: - if r.dtype == dtypes.int and r.op == Ops.CONST and r.val < 0: l = l.cast(dtypes.int) + if r.dtype == dtypes.int and isinstance(rv:=_single_value(r), int) and rv < 0: l = l.cast(dtypes.int) else: r = r.cast(l.dtype) return l, r @@ -890,6 +940,8 @@ class Parser: return result & _isnan(l).logical_not() & _isnan(r).logical_not() return result +_break_var_ids = itertools.count() # unique names for per-loop break-tracking variables + def _match_bracket(toks: list[Token], start: int) -> tuple[int, list[Token]]: """Match brackets from start, return (end_idx, inner_tokens).""" j, depth = start + 1, 1 @@ -968,9 +1020,7 @@ def parse_block(lines: list[str], start: int, env: dict[str, VarVal], funcs: dic p.eat('NUM') p.eat('QUOTE') if p.at('NUM'): return int(p.eat('NUM').val.rstrip('UuLl')) - expr = p.parse().simplify() - assert expr.op == Ops.CONST, f"loop bound must be constant, got {expr}" - return int(expr.val) + return int(p.parse()) start_val = parse_bound() p.eat('COLON') end_val = parse_bound() @@ -987,7 +1037,7 @@ def parse_block(lines: list[str], start: int, env: dict[str, VarVal], funcs: dic i += 1 # Execute loop with break support has_break = any('break' in bl.lower() for bl in body_lines) - found_var = f'_found_{id(body_lines)}' if has_break else None + found_var = f'_found_{next(_break_var_ids)}' if has_break else None if found_var: env[found_var] = block_assigns[found_var] = _const(dtypes.bool, False) for loop_i in range(start_val, end_val + 1): subst_lines = [_subst_loop_var(bl, loop_var, loop_i) for bl in body_lines if not (has_break and bl.strip().lower() == 'break')] @@ -1087,7 +1137,7 @@ def parse_block(lines: list[str], start: int, env: dict[str, VarVal], funcs: dic j, slice_toks = _match_bracket(toks, j) slice_str = _tok_str(slice_toks) hi_str, lo_str = slice_str.split(':') - hi_val, lo_val = int(eval(hi_str.strip())), int(eval(lo_str.strip())) + hi_val, lo_val = _const_int(hi_str), _const_int(lo_str) if j < len(toks) and toks[j].type == 'DOT': j += 2 # skip .type suffix if j < len(toks) and toks[j].type == 'EQUALS': j += 1 ln = parse_tokens(lane_toks, env, funcs) @@ -1145,7 +1195,7 @@ def parse_block(lines: list[str], start: int, env: dict[str, VarVal], funcs: dic hi_str = ' '.join(t.val for t in toks[bracket_start:colon_pos] if t.type != 'EOF') lo_str = ' '.join(t.val for t in toks[colon_pos+1:j] if t.type != 'EOF') try: - hi_val, lo_val = int(eval(hi_str)), int(eval(lo_str)) + hi_val, lo_val = _const_int(hi_str), _const_int(lo_str) hi, lo = max(hi_val, lo_val), min(hi_val, lo_val) j += 1 if j < len(toks) and toks[j].type == 'DOT': j += 2 @@ -1159,7 +1209,7 @@ def parse_block(lines: list[str], start: int, env: dict[str, VarVal], funcs: dic block_assigns[var] = env[var] = _set_bits(old, _val_to_bits(val), hi - lo + 1, lo) i += 1 continue - except Exception: pass + except (ValueError, SyntaxError): pass # non-constant slice bounds - fall through to other statement forms elif toks[1].type == 'LBRACKET': # bit index: var[expr] (only for var[...], not var.type[...]) existing = block_assigns.get(var, env.get(var)) if existing is not None and isinstance(existing, UOp) and \ @@ -1360,3 +1410,25 @@ def parse_block(lines: list[str], start: int, env: dict[str, VarVal], funcs: dic def parse_expr(expr: str, env: dict[str, VarVal], funcs: dict | None = None) -> UOp: return parse_tokens(tokenize(expr.strip().rstrip(';')), env, funcs) + +def parse_pcode(pcode: str, srcs: dict[str, UOp | int] | None = None) -> tuple[dict, list]: + env: dict = srcs.copy() if srcs else {} + assigns: list[tuple[str, UOp]] = [] + raw_lines = [l.strip().rstrip(';') for l in pcode.split('\n') if l.strip() and not l.strip().startswith('//')] + # TODO: pcode.py should tokenize full pcode string instead of line-by-line, then this hack can be removed + lines: list[str] = [] + for l in raw_lines: + if lines and re.search(r'(&&|\|\||[&|+\-*/^])\s*$', lines[-1]): lines[-1] = lines[-1] + ' ' + l + else: lines.append(l) + _, final, _ = parse_block(lines, 0, env, assigns=assigns) + sliced = set(d.split('[')[0] for d, _ in assigns if '[' in d) + for var, val in final.items(): + if var in ['D0', 'S0', 'SCC', 'VCC', 'EXEC', 'PC', 'RETURN_DATA', 'VDATA'] and isinstance(val, UOp): + if var in sliced and not any(re.match(rf'{var}\.\w+\s*=', l) for l in lines): continue + for l in lines: + if (m := re.match(rf'{var}\.(\w+(?:\[\w+\])?)', l)): + assigns.append((f'{var}.{m.group(1)}', val)) + break + else: assigns.append((var, val)) + return env, assigns + diff --git a/tinygrad_repo/test/mockgpu/amd/sqtt_enc.py b/tinygrad_repo/test/mockgpu/amd/sqtt_enc.py new file mode 100644 index 000000000..924d7ac29 --- /dev/null +++ b/tinygrad_repo/test/mockgpu/amd/sqtt_enc.py @@ -0,0 +1,100 @@ +# SQTT trace encoder for the emulator (the decoder lives in tinygrad/renderer/amd/sqtt.py). +# run_asm emits packets inline as instructions execute; finished traces end up in emu.sqtt_traces. +from __future__ import annotations +from tinygrad.renderer.amd.dsl import Inst +from tinygrad.renderer.amd.sqtt import (_build_decode_tables, PACKET_TYPES_RDNA3, PacketType, InstOp, + LAYOUT_HEADER, WAVESTART, WAVEEND, INST, IMMEDIATE, VALUINST) + +_NIB_COUNTS = {cls: nc for _, (cls, nc, *_) in _build_decode_tables(PACKET_TYPES_RDNA3)[0].items()} + +def _emit_nibbles(nibbles: list[int], pkt_cls: type[PacketType], **kwargs): + raw = pkt_cls.encoding.default + for k, v in kwargs.items(): raw = pkt_cls.__dict__[k].set(raw, v) + nibbles.extend((raw >> (i * 4)) & 0xF for i in range(_NIB_COUNTS[pkt_cls])) + +def make_encoder(): + """Build an SQTT trace encoder for the emulator. Returns (emit, finish, finalize).""" + from tinygrad.runtime.autogen.amd.rdna3.enum import SOPPOp as SOPPOp3 + from tinygrad.runtime.autogen.amd.rdna4.enum import SOPPOp as SOPPOp4 + from tinygrad.runtime.autogen.amd.rdna3 import ins as ir3 + from tinygrad.runtime.autogen.amd.rdna4 import ins as ir4 + from tinygrad.runtime.autogen.amd.cdna import ins as irc + import re + + def _kinds(*names: str) -> tuple[type[Inst], ...]: + return tuple(getattr(m, n) for m in (ir3, ir4, irc) for n in names if hasattr(m, n)) + _SOPP, _SMEM, _DS = _kinds('SOPP'), _kinds('SMEM'), _kinds('DS') + _GLOBAL, _FLAT, _SCRATCH = _kinds('GLOBAL', 'VGLOBAL'), _kinds('FLAT', 'VFLAT'), _kinds('SCRATCH', 'VSCRATCH') + _VALU = _kinds('VOP1', 'VOP2', 'VOP3', 'VOP3P', 'VOP3PX2', 'VOPC', 'VOPD', 'VOP3SD', 'VOP3_SDST', 'VOP1_SDST') + + # SOPP classification sets + _SOPP_SKIP = {SOPPOp3.S_ENDPGM.value, SOPPOp3.S_ENDPGM_SAVED.value, SOPPOp3.S_ENDPGM_ORDERED_PS_DONE.value, SOPPOp3.S_DELAY_ALU.value} + _SOPP_IMMEDIATE = {SOPPOp3.S_NOP.value, SOPPOp3.S_CLAUSE.value, SOPPOp3.S_WAITCNT.value, SOPPOp3.S_WAITCNT_DEPCTR.value, + SOPPOp3.S_WAIT_IDLE.value, SOPPOp3.S_WAIT_EVENT.value, SOPPOp3.S_SLEEP.value, SOPPOp3.S_SET_INST_PREFETCH_DISTANCE.value} + for _op in (SOPPOp4.S_WAIT_ALU, SOPPOp4.S_WAIT_LOADCNT, SOPPOp4.S_WAIT_STORECNT, SOPPOp4.S_WAIT_SAMPLECNT, + SOPPOp4.S_WAIT_BVHCNT, SOPPOp4.S_WAIT_EXPCNT, SOPPOp4.S_WAIT_DSCNT, SOPPOp4.S_WAIT_KMCNT, + SOPPOp4.S_WAIT_LOADCNT_DSCNT, SOPPOp4.S_WAIT_STORECNT_DSCNT): + _SOPP_IMMEDIATE.add(_op.value) + _SOPP_BARRIER = {SOPPOp3.S_BARRIER.value} + if hasattr(SOPPOp4, 'S_BARRIER_WAIT'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_WAIT.value) + if hasattr(SOPPOp4, 'S_BARRIER_LEAVE'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_LEAVE.value) + _SOPP_BRANCH = {SOPPOp3.S_BRANCH.value, SOPPOp3.S_CBRANCH_SCC0.value, SOPPOp3.S_CBRANCH_SCC1.value, + SOPPOp3.S_CBRANCH_VCCZ.value, SOPPOp3.S_CBRANCH_VCCNZ.value, + SOPPOp3.S_CBRANCH_EXECZ.value, SOPPOp3.S_CBRANCH_EXECNZ.value} + + # VALU sub-classification patterns + _VALUT_4_RE = re.compile(r'V_(EXP|LOG|RCP|RSQ|SQRT|SIN|COS|CEIL|FLOOR|TRUNC|RNDNE|FRACT|FREXP)_') + _VALUB_2_RE = re.compile(r'V_(LSHLREV|LSHRREV|ASHRREV)_(B|I)64') + _VALUB_4_RE = re.compile(r'V_MAD_(U|I)64') + _VALUB_16_RE = re.compile(r'V_\w+_F64') + + def _valu_op(op_name: str) -> InstOp|None: + if 'CMPX' in op_name: return InstOp.VALU1_WR_EXEC + if _VALUB_2_RE.search(op_name): return InstOp.VALUB_2 + if _VALUB_4_RE.search(op_name): return InstOp.VALUB_4 + if _VALUB_16_RE.search(op_name): return InstOp.VALUB_16 + if _VALUT_4_RE.search(op_name): return InstOp.VALUT_4 + return None + + def _mem_op(t: type[Inst], op_name: str) -> InstOp: + is_store = "STORE" in op_name + if issubclass(t, _DS): return InstOp.LDS_WR_2 if is_store else InstOp.LDS_RD + if issubclass(t, _GLOBAL): return InstOp.SGMEM_WR_2 if is_store else InstOp.SGMEM_RD_1 + if issubclass(t, _FLAT) or issubclass(t, _SCRATCH): return InstOp.FLAT_WR_3 if is_store else InstOp.FLAT_RD_2 + return InstOp.SALU + + nibbles: list[int] = [] + started: set[int] = set() + _emit_nibbles(nibbles, LAYOUT_HEADER, layout=3, sel_a=6) + + def emit(wave_id: int, inst: Inst, branch_taken: bool|None): + """Emit an SQTT packet for one executed instruction.""" + w = wave_id & 0x1F + if wave_id not in started: + _emit_nibbles(nibbles, WAVESTART, delta=1, simd=0, wgp=0, wave=w, id7=wave_id) + started.add(wave_id) + inst_type, inst_op, op_name = type(inst), inst.op.value if hasattr(inst, 'op') else 0, inst.op.name if hasattr(inst, 'op') else "" + if issubclass(inst_type, _SOPP): + if inst_op in _SOPP_SKIP: return + if inst_op in _SOPP_IMMEDIATE: _emit_nibbles(nibbles, IMMEDIATE, delta=1, wave=w) + elif inst_op in _SOPP_BARRIER: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.BARRIER) + elif inst_op in _SOPP_BRANCH: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.JUMP if branch_taken else InstOp.JUMP_NO) + else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SALU) + elif issubclass(inst_type, _VALU): + if (op := _valu_op(op_name)) is None: _emit_nibbles(nibbles, VALUINST, delta=1, wave=w) + else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=op) + elif issubclass(inst_type, _SMEM): _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SMEM_RD) + else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=_mem_op(inst_type, op_name)) + + def finish(wave_id: int): + """Emit WAVEEND for a completed wave.""" + if wave_id in started: _emit_nibbles(nibbles, WAVEEND, delta=1, simd=0, wgp=0, wave=wave_id & 0x1F) + + def finalize() -> bytes: + """Pad and return the encoded SQTT blob.""" + while len(nibbles) % 2 != 0: nibbles.append(0) + nibbles.extend([0] * 32) + while len(nibbles) % 64 != 0: nibbles.append(0) + return bytes(nibbles[i] | ((nibbles[i + 1] if i + 1 < len(nibbles) else 0) << 4) for i in range(0, len(nibbles), 2)) + + return emit, finish, finalize diff --git a/tinygrad_repo/test/mockgpu/usb.py b/tinygrad_repo/test/mockgpu/usb.py index 746c951d5..f18b8cfdb 100644 --- a/tinygrad_repo/test/mockgpu/usb.py +++ b/tinygrad_repo/test/mockgpu/usb.py @@ -160,7 +160,7 @@ class MockUSB3: elif request == 0xE5: self.state._xram_write_byte(value, index) elif request == 0xF2: - op = ("sram_read" if value & 0x8000 else "sram_write", 0xF000, (value & 0x7FFF) * 512) + op = ("sram_read" if value & 0x8000 else "sram_write", 0xF000 + (index & 0xFF) * 0x4000, (value & 0x7FFF) * 512) if value & 0x8000: self._bulk_read_op = op else: self._bulk_write_op = op elif request == 0xF0: @@ -193,19 +193,33 @@ class MockUSB3: op, address, size = self._bulk_write_op assert len(data) == size if op == "sram_write": - host_addr, region_size = self.state._dma_regions[address] - ctypes.memmove(host_addr, data, min(len(data), region_size)) + ctrl, (host_addr, region_size) = next((ca, r) for ca, r in self.state._dma_regions.items() if ca <= address < ca + r[1]) + ctypes.memmove(host_addr + (address - ctrl), data, min(len(data), region_size - (address - ctrl))) + self.state.driver._emulate_execute() # landed data may un-stall a ring polling on it (e.g. copyin sentinels) elif op == "pcie_write": self.state._pcie_write(address, data) else: raise RuntimeError(f"cannot bulk write for {op}") self._bulk_write_op = None + def bulk_write_async(self, payload:memoryview, timeout:int=10000) -> int: # the mock completes transfers synchronously + self.bulk_write(bytes(payload), timeout) + return 0 + + def control_write_async(self, request:int, value:int=0, index:int=0, data:bytes=b"", timeout:int=1000) -> int: + self.control_write(request, value, index, data, timeout) + return 0 + + def control_read_async(self, request:int, length:int, value:int=0, index:int=0, timeout:int=1000) -> tuple[int, memoryview]: + return 0, self.control_read(request, length, value, index, timeout) + + def bulk_wait(self, tag:int): pass + def bulk_read(self, length:int, timeout:int=1000) -> memoryview: assert self._bulk_read_op is not None op, address, size = self._bulk_read_op assert length == size if op == "sram_read": - host_addr, region_size = self.state._dma_regions[address] - data = bytes((ctypes.c_ubyte * min(length, region_size)).from_address(host_addr)) + ctrl, (host_addr, region_size) = next((ca, r) for ca, r in self.state._dma_regions.items() if ca <= address < ca + r[1]) + data = bytes((ctypes.c_ubyte * min(length, region_size - (address - ctrl))).from_address(host_addr + (address - ctrl))) elif op == "pcie_read": data = self.state._pcie_read(address, length) else: raise RuntimeError(f"cannot bulk read for {op}") self._bulk_read_op = None diff --git a/tinygrad_repo/test/models/test_whisper.py b/tinygrad_repo/test/models/test_whisper.py index e6e3e507c..3c0fa2c18 100644 --- a/tinygrad_repo/test/models/test_whisper.py +++ b/tinygrad_repo/test/models/test_whisper.py @@ -74,6 +74,7 @@ class TestWhisper(unittest.TestCase): err ) + @slow def test_transcribe_file1(self): self.assertEqual(transcribe_file(self.model, self.enc, TEST_FILE_1), TRANSCRIPTION_1) @@ -89,6 +90,7 @@ class TestWhisper(unittest.TestCase): self.assertEqual(TRANSCRIPTION_1, transcriptions[0]) self.assertEqual(TRANSCRIPTION_2, transcriptions[1]) + @slow def test_transcribe_batch21(self): waveforms = [load_file_waveform(TEST_FILE_2), load_file_waveform(TEST_FILE_1)] transcriptions = transcribe_waveform(self.model, self.enc, waveforms) diff --git a/tinygrad_repo/test/null/test_const_folding.py b/tinygrad_repo/test/null/test_const_folding.py index 5e365c423..e7072dae0 100644 --- a/tinygrad_repo/test/null/test_const_folding.py +++ b/tinygrad_repo/test/null/test_const_folding.py @@ -1,38 +1,8 @@ import unittest, itertools, math -from tinygrad import Tensor, dtypes, Context -from tinygrad.dtype import DType, ConstType, truncate +from tinygrad import dtypes, Context +from tinygrad.dtype import DType, ConstType from tinygrad.uop.ops import Ops, UOp from test.helpers import full_rewrite -import numpy as np - -def _check_ast_count(desired_count:int, t:Tensor): - # NOTE: this has side effect because everything can be scheduled only once - linear = t.schedule_linear() - asts = [s for s in linear.src if s.src[0].op is Ops.SINK] - len(asts) - # NOT SUPPORTED ANYMORE - #assert len(asts) == desired_count, f"{len(asts)} != {desired_count}" - -class TestUnaryOpsConstFolding(unittest.TestCase): - def test_all_consts_ops(self): - _check_ast_count(0, Tensor.ones(4).exp()) - _check_ast_count(0, Tensor.ones(4).sqrt()) - _check_ast_count(0, Tensor.ones(4) + Tensor.ones(4)) - _check_ast_count(0, Tensor.ones(4) / Tensor.ones(4)) - - def test_cast(self): - _check_ast_count(0, Tensor.ones(4).cast(dtypes.int16)) - _check_ast_count(0, Tensor.full(4, fill_value=-1).cast(dtypes.uint16)) - - def test_neg_folding(self): - _check_ast_count(0, Tensor([1, 2, 3]).mul(-1).neg()) - _check_ast_count(0, Tensor([1, 2, 3]).neg().mul(-1)) - _check_ast_count(0, Tensor([1, 2, 3]).neg().neg()) - - def test_neg_realized_no_fold(self): - x = Tensor.randn(32, 32) - x = x.clip(0, 1).realize() - _check_ast_count(1, x.neg()) class TestWeakConstFolding(unittest.TestCase): def test_weakint_math(self): @@ -51,90 +21,19 @@ class TestWeakConstFolding(unittest.TestCase): def test_invalid_poison(self): self.assertTrue(UOp.invalid().alu(Ops.CDIV, UOp.const(0)).simplify().is_invalid) - def test_cast_commits_to_dtype_grid(self): - # committing a weak const to a stated width puts the value on that width's grid, same as storage packing and native compilers - v = 1/123008 # not representable in float16 - out = UOp.const(v).cast(dtypes.half).simplify() - self.assertEqual((out.op, out.dtype, out.val), (Ops.CONST, dtypes.half, truncate[dtypes.half](v))) - self.assertNotEqual(out.val, v) - # the grid commit preserves the sign of zero - self.assertEqual(math.copysign(1, UOp.const(-0.0).cast(dtypes.half).simplify().val), -1) - # observable at tensor level: the const-folded comparison agrees with the committed value - self.assertTrue((Tensor(-3.2).cast(dtypes.float32) <= truncate[dtypes.float32](-3.2)).item()) - -class TestBinaryOpsConstFolding(unittest.TestCase): - def test_add_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + 0) - def test_add_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + Tensor.zeros(4)) - def test_literal_zero_add(self): - _check_ast_count(0, 0 + Tensor([1.0, 2, 3, 4])) - def test_tensor_zero_add(self): - _check_ast_count(0, Tensor.zeros(4) + Tensor([1.0, 2, 3, 4])) - - def test_sub_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) - 0) - def test_sub_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) - Tensor.zeros(4)) - - def test_mul_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * 0) - def test_mul_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * Tensor.zeros(4)) - def test_literal_zero_mul(self): - _check_ast_count(0, 0 * Tensor([1.0, 2, 3, 4]) * 0) - def test_tensor_zero_mul(self): - _check_ast_count(0, Tensor.zeros(4) * Tensor([1.0, 2, 3, 4])) - - def test_mul_literal_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * 1) - def test_mul_tensor_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * Tensor.ones(4)) - def test_literal_one_mul(self): - _check_ast_count(0, 1 * Tensor([1.0, 2, 3, 4])) - def test_tensor_one_mul(self): - _check_ast_count(0, Tensor.ones(4) * Tensor([1.0, 2, 3, 4])) - - def test_bool_tensor_mul_bool(self): - _check_ast_count(0, Tensor([True, False]) * True) - _check_ast_count(0, Tensor([True, False]) * False) - def test_bool_mul_bool_tensor(self): - _check_ast_count(0, True * Tensor([True, False])) - _check_ast_count(0, False * Tensor([True, False])) - - def test_div_literal_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) / 1) - def test_div_tensor_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) / Tensor.ones(4)) - - def test_floordiv_literal_one(self): - _check_ast_count(0, Tensor([1, 2, 3, 4]) // 1) - def test_floordiv_tensor_one(self): - _check_ast_count(0, Tensor([1, 2, 3, 4]) // Tensor.ones(4, dtype=dtypes.int32)) - - def test_pow_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** 0) - def test_pow_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** Tensor.zeros(4)) - - def test_pow_literal_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** 1) - def test_pow_tensor_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** Tensor.ones(4)) - def test_literal_one_pow(self): - _check_ast_count(0, 1 ** Tensor([1.0, 2, 3, 4])) - def test_tensor_one_pow(self): - _check_ast_count(0, Tensor.ones(4) ** Tensor([1.0, 2, 3, 4])) - class TestBitcastConstFolding(unittest.TestCase): + def test_out_of_range_source_value(self): + for val, src_dt, dst_dt, bits in ((3000000000, dtypes.int32, dtypes.uint32, 3000000000), + (70000, dtypes.int16, dtypes.uint16, 4464), + (-5, dtypes.uint32, dtypes.int32, -5)): + self.assertIs(UOp.const(val, src_dt).bitcast(dst_dt).simplify(), UOp.const(bits, dst_dt)) + def test_scalar_bitcast(self): def t(cases: dict[DType, ConstType]): for (from_dt, from_v), (to_dt, to_v) in itertools.product(cases.items(), cases.items()): if not math.isnan(from_v): - r = full_rewrite(UOp.const(from_v, from_dt).bitcast(to_dt).sink()).src[0] - self.assertEqual(r.op, Ops.CONST, msg:=f"{from_dt} -> {to_dt} ({from_v} -> {to_v})") - self.assertEqual(r.dtype, to_dt, msg) - np.testing.assert_equal(r.val, to_v, msg) + r = UOp.const(from_v, from_dt).bitcast(to_dt).simplify() + self.assertIs(r, UOp.const(to_v, to_dt), f"{from_dt} -> {to_dt} ({from_v} -> {to_v})") t({dtypes.int8: 0, dtypes.uint8: 0, dtypes.bool: False}) t({dtypes.int8: 1, dtypes.uint8: 1, dtypes.bool: True}) @@ -155,24 +54,9 @@ class TestBitcastConstFolding(unittest.TestCase): def test_vec_bitcast(self): with Context(SPEC=0): - srcs = full_rewrite(UOp.const((-1, -2**31, 75), dtypes.int32).bitcast(dtypes.uint32).sink()).src - self.assertTrue(all(r.op is Ops.CONST and r.dtype == dtypes.uint32 for r in srcs)) - self.assertEqual(tuple(x.val for x in srcs), (2**32-1, 2**31, 75)) - -# folds advance indexing into basic indexing -class TestIndexingConstFolding(unittest.TestCase): - def test_scalar_index(self): - t = Tensor.arange(16).float().reshape(1,1,4,4).clone().realize() - _check_ast_count(1, t[:,:,Tensor(1),:]) - _check_ast_count(1, t[:,:,Tensor(1)+2,:]) - _check_ast_count(1, t[:,:,Tensor(1),Tensor(0)]) - - def test_const_tensor_index(self): - # TODO: these can be 0, implement const tensor folded indexing - t = Tensor.arange(16).float().reshape(1,1,4,4).clone().realize() - _check_ast_count(1, t[:,:,Tensor.ones(2,1,dtype=dtypes.int),:]) - _check_ast_count(1, t[:,:,Tensor.ones(1,2,dtype=dtypes.int)+2,:]) - _check_ast_count(1, t[:,:,Tensor.ones(1,1,dtype=dtypes.int),Tensor.zeros(2,1,2,dtype=dtypes.int)]) + result = full_rewrite(UOp.const((-1, -2**31, 75), dtypes.int32).bitcast(dtypes.uint32).sink()) + expected = full_rewrite(UOp.const((2**32-1, 2**31, 75), dtypes.uint32).sink()) + self.assertEqual(result.src, expected.src) if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/null/test_dtype_spec.py b/tinygrad_repo/test/null/test_dtype_spec.py index 2aaebefe0..363ae5b75 100644 --- a/tinygrad_repo/test/null/test_dtype_spec.py +++ b/tinygrad_repo/test/null/test_dtype_spec.py @@ -51,10 +51,6 @@ class TestHelpers(unittest.TestCase): assert dtypes.is_float(dtypes.fp8e4m3) assert dtypes.is_float(dtypes.fp8e5m2) - @given(strat.sampled_from([d for d in DTYPES_DICT.values() if dtypes.is_float(d) or dtypes.is_int(d)])) - def test_scalar(self, dtype): - assert dtype.scalar() == dtype - def test_from_py(self): assert dtypes.from_py(True) == dtypes.bool assert dtypes.from_py(Invalid) == dtypes.bool @@ -110,7 +106,8 @@ class TestHelpers(unittest.TestCase): def test_float_to_bf16(self): max_bf16 = torch.finfo(torch.bfloat16).max - for a in [1, 1.1, 1234, 23456, -777.777, max_bf16, max_bf16 * 1.00001, -max_bf16, -max_bf16 * 1.00001, math.inf, -math.inf]: + for a in [1, 1.1, 1234, 23456, -777.777, max_bf16, max_bf16 * 1.00001, -max_bf16, -max_bf16 * 1.00001, + max_bf16 * 2, -max_bf16 * 2, math.inf, -math.inf]: self.assertEqual(float_to_bf16(a), torch.tensor([a], dtype=torch.bfloat16).item()) self.assertTrue(math.isnan(float_to_bf16(math.nan))) @@ -422,7 +419,7 @@ class TestAutoCastType(unittest.TestCase): self.check_where_alternate_input_other(3, True, dtypes.weakint) def test_where_non_bool_cond_raises(self): - with self.assertRaises(RuntimeError): Tensor([1, 0, 2]).where(1, 0) + with self.assertRaises(RuntimeError): Tensor([1, 0, 2]).where(1, 0).dtype self.check_where_alternate_input_other(False, True, dtypes.bool) @given(strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes)) diff --git a/tinygrad_repo/test/null/test_elf.py b/tinygrad_repo/test/null/test_elf.py index 0f9a3d48f..ac8bf6c41 100644 --- a/tinygrad_repo/test/null/test_elf.py +++ b/tinygrad_repo/test/null/test_elf.py @@ -1,6 +1,7 @@ import unittest, subprocess, platform from tinygrad.runtime.support.compiler_cpu import ClangCompiler from tinygrad.runtime.support.elf import elf_loader +from tinygrad.runtime.support.c import DLL class TestElfLoader(unittest.TestCase): def test_load_clang_jit_strtab(self): @@ -23,7 +24,7 @@ class TestElfLoader(unittest.TestCase): } ''' with self.assertRaisesRegex(RuntimeError, 'evil_external_function'): - ClangCompiler([{'AMD64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine(), m), "native"]).compile(src) + elf_loader(ClangCompiler([{'AMD64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine(), m), "native"]).compile(src)) def test_link(self): src = ''' float powf(float, float); // from libm @@ -32,7 +33,7 @@ class TestElfLoader(unittest.TestCase): args = ('-x', 'c', '-c', '-target', f'{platform.machine()}-none-unknown-elf', '-march=native', '-fPIC', '-O2', '-ffreestanding', '-nostdlib') obj = subprocess.check_output(('clang',) + args + ('-', '-o', '-'), input=src.encode()) with self.assertRaisesRegex(RuntimeError, 'powf'): elf_loader(obj) - elf_loader(obj, link_libs=['m']) + elf_loader(obj, link_libs=[DLL('m', 'm')]) if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/null/test_gpudims.py b/tinygrad_repo/test/null/test_gpudims.py index 8d73d907b..cde27aadc 100644 --- a/tinygrad_repo/test/null/test_gpudims.py +++ b/tinygrad_repo/test/null/test_gpudims.py @@ -107,7 +107,7 @@ class TestGroupedDims(unittest.TestCase): def test_global_prod_max(self): g, l = UOp.range(256, 0, AxisType.GLOBAL), UOp.range(256, 1, AxisType.LOCAL) - sink = UOp.param(0, dtypes.float, (512,)).index(g + l).store(UOp.const(1.0)).end(g, l).sink(arg=KernelInfo()) + sink = UOp.param(0, dtypes.float, 512).index(g + l).store(UOp.const(1.0)).end(g, l).sink(arg=KernelInfo()) class R(Renderer): global_max, local_max, global_prod_max = (256, 256, 256), (128, 128, 128), (128, 128, 128) specials = [u for u in add_gpudims(R(Target()), sink).toposort() if u.op is Ops.SPECIAL] self.assertGreater(len([s for s in specials if "lidx" in s.arg]), 1) diff --git a/tinygrad_repo/test/null/test_graph_rewrite.py b/tinygrad_repo/test/null/test_graph_rewrite.py index 9194cc01b..a1f874517 100644 --- a/tinygrad_repo/test/null/test_graph_rewrite.py +++ b/tinygrad_repo/test/null/test_graph_rewrite.py @@ -1,8 +1,7 @@ import unittest, math from tinygrad import dtypes -from tinygrad.dtype import AddrSpace from tinygrad.helpers import all_same, Context -from tinygrad.uop.ops import GroupOp, UOp, Ops, exec_alu, PatternMatcher, TrackedPatternMatcher, UPat +from tinygrad.uop.ops import GroupOp, UOp, Ops, PatternMatcher, TrackedPatternMatcher, UPat from test.helpers import full_rewrite from hypothesis import given, strategies as strat @@ -11,125 +10,14 @@ from hypothesis import given, strategies as strat def apply_rewrite(expr): return full_rewrite(expr.sink()).src[0] -@Context(SPEC=0) -def apply_rewrite_values(expr): - srcs = full_rewrite(expr.sink()).src - if len(srcs) == 1: - if srcs[0].op is Ops.CONST: return (srcs[0].val,) - if srcs[0].op is Ops.STACK: return tuple(s.val for s in srcs[0].src) - return tuple(s.val for s in srcs) - -def evaluate_uop(uop, variables): - if uop.op == Ops.CONST: - return uop.val - elif uop.op == Ops.PARAM and uop.arg.addrspace is AddrSpace.ALU: - return variables[uop.expr] - elif uop.op in GroupOp.ALU: - src_values = [evaluate_uop(src, variables) for src in uop.src] - return exec_alu(uop.op, uop.dtype, src_values) - else: - raise NotImplementedError(f"Unsupported UOp {uop.op}") - -class TestArithmeticSimplifications(unittest.TestCase): - def test_full_graph_rewrite_division_by_zero(self): - optimized_div_uop = apply_rewrite(UOp.const(10.0) / UOp.const(0.0)) - self.assertEqual(optimized_div_uop.op, Ops.CONST) - self.assertTrue(math.isinf(optimized_div_uop.val) or math.isnan(optimized_div_uop.val)) - - def test_full_graph_rewrite_redundant_operations(self): - optimized_uop = apply_rewrite((UOp.const(10.0) + UOp.const(0.0)) * UOp.const(1.0)) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, 10.0) - - def test_full_graph_rewrite_large_graph(self): - prev_uop = UOp.const(0) - for i in range(1, 101): - prev_uop += UOp.const(i) - optimized_uop = apply_rewrite(prev_uop) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, sum(range(1, 101))) - - def test_full_graph_rewrite_division_by_one(self): - optimized_uop = apply_rewrite(UOp.const(42.0) / UOp.const(1.0)) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, 42.0) - - def test_full_graph_rewrite_modulo_by_one(self): - optimized_uop = apply_rewrite(UOp.const(42) % UOp.const(1)) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, 0) - - -class TestFoldingAndReduction(unittest.TestCase): - @unittest.skip("reduce is removed now") - def test_full_graph_rewrite_constant_reduction_folding(self): - const1 = UOp.const(5) - const2 = UOp.const(10) - const3 = UOp.const(20) - optimized_sink = apply_rewrite((const1 + const2 + const3).reduce(Ops.ADD)) - expected_sum = 5 + 10 + 20 - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("reduce is removed now") - def test_full_graph_rewrite_reduction_with_unused_range(self): - const1 = UOp.const(15) - const2 = UOp.const(25) - rng = UOp.range(10, idx=0) - optimized_sink = apply_rewrite((const1 + const2).reduce(Ops.ADD, rng)) - expected_sum = 10 * (15 + 25) - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("currently failing") - def test_full_graph_rewrite_range_reduction(self): - simple_range = UOp.range(5, idx=0) - optimized_sink = apply_rewrite(simple_range.reduce(Ops.ADD, simple_range)) - expected_sum = sum(range(5)) - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("currently failing") - def test_full_graph_rewrite_simple_reduction_folding(self): - simple_range = UOp.range(4, idx=0) - add_uop = simple_range + UOp.const(1) - optimized_sink = apply_rewrite(add_uop.reduce(Ops.ADD, simple_range)) - expected_sum = sum(i + 1 for i in range(4)) - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("currently failing") - def test_full_graph_rewrite_nested_loop_collapse(self): - outer_range = UOp.range(8, 0) - inner_range = UOp.range(4, 1) - expr = (outer_range * 10) + inner_range - optimized_reduce_uop = apply_rewrite(expr.reduce(Ops.ADD, outer_range, inner_range)) - self.assertEqual(optimized_reduce_uop.op, Ops.CONST) - self.assertEqual(optimized_reduce_uop.val, sum((i * 10) + j for i in range(8) for j in range(4))) - +def const_value(uop:UOp): + if uop.op is Ops.CAST: uop = uop.src[0] + assert uop.op is Ops.CONST + return uop.val class TestModuloAndDivisionFolding(unittest.TestCase): - def test_full_graph_rewrite_modulo_folding_with_define_var(self): - # index dtype because div-mod rules only work on index - x_var_uop = UOp.variable('x', 0, 100).cast(dtypes.weakint) - optimized_mod_uop = apply_rewrite(((x_var_uop * 4) + 2) % 4) - self.assertEqual(optimized_mod_uop.op, Ops.CONST) - self.assertEqual(optimized_mod_uop.val, 2) - - def test_full_graph_rewrite_division_folding_with_define_var(self): - # index dtype because div-mod rules only work on index - n_var_uop = UOp.variable('n', 1, 1000).cast(dtypes.weakint) - optimized_div_uop = apply_rewrite((n_var_uop * 6) // 3) - self.assertEqual(optimized_div_uop.op, Ops.MUL) - self.assertEqual(optimized_div_uop.src[1].val, 2) - - def test_full_graph_rewrite_complex_mod_div_folding(self): - # index dtype because div-mod rules only work on index - k_var_uop = UOp.variable('k', 0, 50).cast(dtypes.weakint) - optimized_div_uop = apply_rewrite(((k_var_uop * 12 + 8) % 6) // 2) - self.assertEqual(optimized_div_uop.op, Ops.CONST) - self.assertEqual(optimized_div_uop.val, 1) - def test_graph_rewrite_div_folding_bug(self): - lhs = UOp(Ops.ADD, src=( - UOp(Ops.STACK, arg=None, src=(UOp(Ops.SPECIAL, src=(UOp.const(32),), arg='lidx0'),)*4), - UOp.const((0, 256, 512, 768)))) + lhs = UOp.stack(*(UOp.special(32, 'lidx0'),)*4) + UOp.const((0, 256, 512, 768)) rhs = UOp.const((2,)*4) unopt = lhs 0, - f"Expected +inf for reciprocal(0.0), got {optimized_recip_zero.val}") - - @unittest.skip("broken") - def test_full_graph_rewrite_modulo_negative_dividend(self): - x_var_uop = UOp.variable('x', -5, -1) - optimized_sink = full_rewrite((x_var_uop % 3).sink()) - for x_value in range(-5, 0): - self.assertEqual(x_value % 3, evaluate_uop(optimized_sink.src[0], {'x': x_value})) - - @unittest.skip("broken") - def test_full_graph_rewrite_division_negative_divisor(self): - x_var_uop = UOp.variable('x', 1, 5) - optimized_sink = full_rewrite((x_var_uop // -2).sink()) - for x_value in range(1, 6): - self.assertEqual(x_value // -2, evaluate_uop(optimized_sink.src[0], {'x': x_value})) + log2_neg, recip_zero = const_value(optimized_log2_neg), const_value(optimized_recip_zero) + self.assertTrue(math.isnan(log2_neg), f"Expected NaN for log2(-1.0), got {log2_neg}") + self.assertTrue(math.isinf(recip_zero) and recip_zero > 0, f"Expected +inf for reciprocal(0.0), got {recip_zero}") class TestGEPAndVectorizeRewrite(unittest.TestCase): def test_gep_single_element_extraction(self): # GEP on a vector dtype to extract a single element base_vector = UOp.const((1.0, 2.0, 3.0, 4.0)) - self.assertEqual(apply_rewrite(base_vector.index(2)).val, 3.0) + self.assertIs(apply_rewrite(base_vector.index(2)), apply_rewrite(base_vector.src[2])) def test_gep_tuple_extraction(self): # GEP on a vector dtype to extract multiple elements as a vector base_vector = UOp.const((1.0, 2.0, 3.0, 4.0)) - self.assertEqual(list(apply_rewrite_values(UOp.stack(*[base_vector.index(i) for i in (2, 3)]))), [3.0, 4.0]) - - def test_gep_on_const_stack(self): - # GEP on a const STACK to extract a single element - const_stack = UOp.const((1.0, 2.0, 3.0, 4.0)) - self.assertEqual(apply_rewrite(const_stack.index(2)).val, 3.0) - - def test_gep_tuple_on_const_stack(self): - # GEP on a const STACK using a tuple to extract multiple elements - const_stack = UOp.const((7.0, 8.0, 9.0, 10.0)) - self.assertEqual(list(apply_rewrite_values(UOp.stack(*[const_stack.index(i) for i in (1, 3)]))), [8.0, 10.0]) + self.assertIs(apply_rewrite(UOp.stack(*[base_vector.index(i) for i in (2, 3)])), + apply_rewrite(UOp.stack(base_vector.src[2], base_vector.src[3]))) def test_vectorize_multiple_elements(self): # Vectorizing multiple elements using GEP base_vector = UOp.const((5.0, 10.0, 15.0, 20.0)) - vectorized_uop = UOp(Ops.STACK, src=tuple(base_vector.index(i) for i in range(4))) - self.assertEqual(list(apply_rewrite_values(vectorized_uop)), [5.0, 10.0, 15.0, 20.0]) + vectorized_uop = UOp.stack(*(base_vector.index(i) for i in range(4))) + self.assertIs(apply_rewrite(vectorized_uop), apply_rewrite(base_vector)) import inspect @@ -256,16 +101,6 @@ class TestSubstitute(unittest.TestCase): ret = substitute(ret, {a.sin():b}) self.assertIs(ret, b.sin()) - # broken due to infinite recursion - # NOTE: VIZ hangs and doesn't recover if you click this one - @unittest.skip("recursion error no longer raised") - def test_assert_inf_recurse(self): - a = UOp.variable('a', 0, 10) - n1 = a.sin() - ret = n1 - with self.assertRaises(RecursionError): - ret = substitute(ret, {n1:n1.sqrt()}) - def test_sin_to_sqrt(self): a = UOp.variable('a', 0, 10, dtype=dtypes.float) n1 = a.sin() diff --git a/tinygrad_repo/test/null/test_linearizer_failures.py b/tinygrad_repo/test/null/test_linearizer_failures.py index 620cd8d0f..5c89d4c3d 100644 --- a/tinygrad_repo/test/null/test_linearizer_failures.py +++ b/tinygrad_repo/test/null/test_linearizer_failures.py @@ -7,14 +7,14 @@ from tinygrad.codegen import to_program class TestLinearizerFailures(unittest.TestCase): def test_fail_1(self): - c0 = UOp.param(0, dtypes.float, (64,)) + c0 = UOp.param(0, dtypes.float, 64) c1 = UOp.range(UOp.const(2), 1, AxisType.WEAK) c2 = UOp.range(UOp.const(32), 2, AxisType.WEAK) c3 = ((c1*UOp.const(32))+c2) - c4 = UOp.param(1, dtypes.float, (163840,)) + c4 = UOp.param(1, dtypes.float, 163840) c5 = UOp.range(UOp.const(2560), 0, AxisType.REDUCE) c6 = c4.index(((((((c5//UOp.const(8))%UOp.const(8))*UOp.const(8))+(c5%UOp.const(8)))+(((c2*UOp.const(40))+(c5//UOp.const(64)))*UOp.const(64)))+(c1*UOp.const(81920)))) - c7 = UOp.param(2, dtypes.float, (64,)) + c7 = UOp.param(2, dtypes.float, 64) c8 = c7.index(c3) c9 = ((((c6+(c8*UOp.const(-1.0)))*(c6+(c8*UOp.const(-1.0)))).reduce(c5, arg=Ops.ADD)*UOp.const(0.000390625))+UOp.const(1e-05)).sqrt().reciprocal() c10 = c0.index(c3).store(c9).end(c1, c2) diff --git a/tinygrad_repo/test/null/test_linearizer_rewrite.py b/tinygrad_repo/test/null/test_linearizer_rewrite.py index 031c86bb5..12890cf6e 100644 --- a/tinygrad_repo/test/null/test_linearizer_rewrite.py +++ b/tinygrad_repo/test/null/test_linearizer_rewrite.py @@ -2,7 +2,7 @@ import unittest from tinygrad import Tensor, Context, Device from tinygrad.codegen import to_program from tinygrad.codegen.opt import Opt, OptOps -from tinygrad.uop.ops import KernelInfo +from tinygrad.uop.ops import KernelInfo, AxisType class TestLinearizerRewrite(unittest.TestCase): def test_reduction(self): @@ -11,8 +11,8 @@ class TestLinearizerRewrite(unittest.TestCase): with Context(SPLIT_REDUCEOP=0): si = out.schedule_linear().src[-1] opts_to_apply = [] - opts_to_apply.append(Opt(OptOps.UPCAST, 0, 4)) - opts_to_apply.append(Opt(OptOps.UNROLL, 0, 4)) + opts_to_apply.append(Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))) + opts_to_apply.append(Opt(OptOps.SPLIT, 2, (4, AxisType.UNROLL))) ast = si.src[0].replace(arg=KernelInfo(opts_to_apply=tuple(opts_to_apply))) prg = to_program(ast, Device["CPU"].renderer) print(prg.src[2].arg) @@ -22,7 +22,7 @@ class TestLinearizerRewrite(unittest.TestCase): with Context(SPLIT_REDUCEOP=0): si = out.schedule_linear().src[-1] opts_to_apply = [] - opts_to_apply.append(Opt(OptOps.UPCAST, 0, 4)) + opts_to_apply.append(Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))) ast = si.src[0].replace(arg=KernelInfo(opts_to_apply=tuple(opts_to_apply))) prg = to_program(ast, Device["CPU"].renderer) print(prg.src[2].arg) diff --git a/tinygrad_repo/test/null/test_llm_tokenizer.py b/tinygrad_repo/test/null/test_llm_tokenizer.py index 2675dae44..a526c81a3 100644 --- a/tinygrad_repo/test/null/test_llm_tokenizer.py +++ b/tinygrad_repo/test/null/test_llm_tokenizer.py @@ -95,6 +95,26 @@ class TestLLMTokenizer(unittest.TestCase): self.assertEqual(template.end_turn(), "[/INST]") self.assertEqual(template.role("assistant"), "") + def test_tekken_gpt4o_split(self): + split = {p: SimpleTokenizer({}, {}, p)._split_to_word.findall for p in ("tekken", "gpt-4o")} + shared = { + "HelloWorld": ["Hello", "World"], + " ÜNICODE": [" ÜNICODE"], # Ü: non-ascii upper joins the run + "é café": ["é", " café"], # first é is e + U+0301 combining acute (NFD) + "เพื่อน วิ": ["เพื่อน", " วิ"], # thai vowel marks stay in the word + "a/b\r\n x": ["a", "/b", "\r\n", " x"], # punct tail eats / + } + for s, want in shared.items(): + self.assertEqual(split["tekken"](s), want, f"tekken {s!r}") + self.assertEqual(split["gpt-4o"](s), want, f"gpt-4o {s!r}") + differ = [ + ("12345", list("12345"), ["123", "45"]), # digits: tekken single, o200k groups {1,3} + ("it's I'M don'T", ["it", "'s", " I", "'M", " don", "'T"], ["it's", " I'M", " don'T"]), # contraction: o200k inline suffix + ] + for s, tk, go in differ: + self.assertEqual(split["tekken"](s), tk, f"tekken {s!r}") + self.assertEqual(split["gpt-4o"](s), go, f"gpt-4o {s!r}") + def test_stream_decoder(self): """stream_decoder buffers incomplete UTF-8: token 25677 has 3/4 of emoji, token 138 completes it.""" bs = [*range(33, 127), *range(161, 173), *range(174, 256)] diff --git a/tinygrad_repo/test/null/test_memory_planner.py b/tinygrad_repo/test/null/test_memory_planner.py index 63be20332..c879630fa 100644 --- a/tinygrad_repo/test/null/test_memory_planner.py +++ b/tinygrad_repo/test/null/test_memory_planner.py @@ -27,10 +27,15 @@ def _make_linear(buffer_lists, copies=None): calls.append(UOp(Ops.CALL, src=(src0, *bufs))) return UOp(Ops.LINEAR, src=tuple(calls)) +def _get_planned_view(buf:UOp) -> tuple[UOp, int, int]|None: + view = buf.src[0] if buf.op is Ops.BITCAST else buf + if view.op is not Ops.SHRINK or view.src[0].op is not Ops.BUFFER: return None + return (arena:=view.src[0]), view.src[1].val * arena.dtype.itemsize, view.src[2].val * arena.dtype.itemsize + def _get_arena(buf, linear, result): for orig_si, new_si in zip(linear.src, result.src): for orig, new in zip(orig_si.src[1:], new_si.src[1:]): - if orig is buf and new.op is Ops.SLICE: return new.src[0] + if orig is buf and (planned:=_get_planned_view(new)) is not None: return planned[0] return None def check_assign(buffer_lists, copies=None): @@ -41,8 +46,8 @@ def check_assign(buffer_lists, copies=None): replace_map: dict[int, tuple[UOp, int, int]] = {} for orig_si, new_si in zip(linear.src, result.src): for orig, new in zip(orig_si.src[1:], new_si.src[1:]): - if new.op is Ops.SLICE and id(orig) not in replace_map: - replace_map[id(orig)] = (new.src[0], new.src[1].val * new.src[0].dtype.itemsize, new.arg * new.dtype.itemsize) + if (planned:=_get_planned_view(new)) is not None and id(orig) not in replace_map: + replace_map[id(orig)] = planned # verify pinned buffers are not planned for buf in held_bufs: diff --git a/tinygrad_repo/test/null/test_mnist_dataset.py b/tinygrad_repo/test/null/test_mnist_dataset.py index bc334d1ca..4f959b0be 100644 --- a/tinygrad_repo/test/null/test_mnist_dataset.py +++ b/tinygrad_repo/test/null/test_mnist_dataset.py @@ -1,6 +1,7 @@ import unittest from tinygrad.helpers import GlobalCounters from tinygrad.nn.datasets import mnist +from test.helpers import KernelCountException class TestDataset(unittest.TestCase): def test_dataset_is_realized(self): @@ -8,7 +9,7 @@ class TestDataset(unittest.TestCase): X_train[0].contiguous().realize() GlobalCounters.reset() X_train[0].contiguous().realize() - self.assertLessEqual(GlobalCounters.kernel_count, 1) # 0 if SLICE (zero-copy), 1 otherwise + if GlobalCounters.kernel_count > 1: raise KernelCountException(1, GlobalCounters.kernel_count) # 0 if SLICE (zero-copy), 1 otherwise if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/null/test_pattern_matcher.py b/tinygrad_repo/test/null/test_pattern_matcher.py index ca9b6bf0e..20dc78f9f 100644 --- a/tinygrad_repo/test/null/test_pattern_matcher.py +++ b/tinygrad_repo/test/null/test_pattern_matcher.py @@ -40,7 +40,7 @@ class TestPatternMatcher(unittest.TestCase): assert len(x.src) == 0 return UOp(Ops.CONST, src=(UOp(Ops.CONST),)) matcher = PatternMatcher([(UPat(Ops.CONST, src=(), name="x"), fxn)]) - c1 = UOp(Ops.CONST, dtypes.float, arg=1.0) + c1 = UOp(Ops.CONST, arg=1.0) # second rewrite shouldn't match anything c1 = matcher.rewrite(c1) c1 = matcher.rewrite(c1) diff --git a/tinygrad_repo/test/null/test_process_replay.py b/tinygrad_repo/test/null/test_process_replay.py index b75e4b09c..182c39b0a 100644 --- a/tinygrad_repo/test/null/test_process_replay.py +++ b/tinygrad_repo/test/null/test_process_replay.py @@ -2,6 +2,7 @@ import unittest from tinygrad import Tensor, Device, Context from tinygrad.codegen import do_to_program from tinygrad.codegen.opt import Opt, OptOps +from tinygrad.uop.ops import AxisType from test.external.process_replay.process_replay import replay_to_program from test.helpers import replace_opts @@ -27,7 +28,7 @@ class TestProcessReplay(unittest.TestCase): def test_replay_with_opt(self): # opts=[Opt(...)] means apply a specific opt - opts = [Opt(OptOps.UPCAST, 0, 4)] + opts = [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))] ast = replace_opts(self.ast, opts) p = do_to_program(ast, self.renderer) good, compare, _ = replay_to_program(p, ast, self.renderer) diff --git a/tinygrad_repo/test/null/test_simplify_valid_idx.py b/tinygrad_repo/test/null/test_simplify_valid_idx.py index 754d713fe..05f831fb8 100644 --- a/tinygrad_repo/test/null/test_simplify_valid_idx.py +++ b/tinygrad_repo/test/null/test_simplify_valid_idx.py @@ -3,7 +3,7 @@ import unittest, itertools from tinygrad.codegen.late.coalesce import indexing_simplify from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, Ops, graph_rewrite -from tinygrad.uop.weak import pm_lower_index_dtype +from tinygrad.uop.weak import pm_commit_weak from tinygrad.uop.symbolic import simplify_valid, sym, pm_move_where_on_load from tinygrad.helpers import Context from test.helpers import full_rewrite @@ -15,17 +15,13 @@ def simplify_valid_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move def simplify_image_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move_where_on_load+indexing_simplify, name="simplify_image_idx") def get_gated_load_uop(valid:UOp, idx:UOp): - return UOp(Ops.LOAD, src=( - UOp.param(0, dtypes.float, (1024,)).index(idx.valid(valid)), - )) + return UOp.param(0, dtypes.float, 1024).index(idx.valid(valid)).load() def get_load_image_uop(image_shape:tuple[int, ...], valid:UOp, idx:tuple[UOp, UOp]): - return UOp(Ops.LOAD, src=( - UOp.param(0, dtypes.float, image_shape).index(idx[1].valid(valid), idx[0].valid(valid)), - )) + return UOp.param(0, dtypes.float, image_shape).index(idx[1].valid(valid), idx[0].valid(valid)).load() -def Special(expr, nmax): return UOp(Ops.SPECIAL, src=(UOp.const(nmax),), arg=expr) -def Variable(expr, nmin, nmax): return UOp.variable(expr, nmin, nmax) +def Special(expr, nmax): return UOp.special(nmax, expr) +def Variable(expr, nmin, nmax): return UOp.variable(expr, nmin, nmax, param=True) def Range(n, nmax): return UOp.range(nmax, n) class TestValidIdxSimplification(unittest.TestCase): @@ -500,7 +496,7 @@ class TestImageSimplification(unittest.TestCase): idx_y = (f + UOp.const(1.0)).cast(dtypes.int) load = get_load_image_uop((10, 10, 4), (UOp.const(-1) < idx_y) & (idx_y < UOp.const(10)), (Special("gidx0", 10), idx_y)) - off = graph_rewrite(load.sink(), pm_lower_index_dtype+indexing_simplify, ctx={}).src[0].src[0] + off = graph_rewrite(load.sink(), pm_commit_weak+indexing_simplify).src[0].src[0] self.assertEqual(off.src[1].get_valid(), UOp.const(True)) class TestDropTrueGate(unittest.TestCase): @@ -509,10 +505,10 @@ class TestDropTrueGate(unittest.TestCase): from tinygrad.codegen.late.coalesce import indexing_simplify from tinygrad.uop.ops import graph_rewrite from tinygrad.uop.symbolic import sym - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) idx = UOp.const(0) true_gate = UOp.const(True) - index_with_gate = UOp(Ops.INDEX, src=(buf, idx.valid(true_gate))) + index_with_gate = buf.index(idx.valid(true_gate)) # apply the optimization result = graph_rewrite(index_with_gate, sym+indexing_simplify) # the True valid should be dropped (INDEX should only have 2 sources) @@ -524,13 +520,17 @@ class TestRangeShrink(unittest.TestCase): result = full_rewrite(sink) return [u for u in result.toposort() if u.op is Ops.RANGE] + def assert_range_end(self, ranges:list[UOp], end:int): + self.assertEqual(len(ranges), 1) + with Context(NOOPT=1, SPEC=0): expected = full_rewrite(UOp.const(end, dtypes.int).sink()).src[0] + self.assertIs(ranges[0].src[0], expected) + def test_range_shrink_single_guard(self): # range 0..203 guarded by r < 4 everywhere -> shrink to 0..3 r = Range(0, 204) load = get_gated_load_uop(r < UOp.const(4), r) ranges = self.get_ranges(load.sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 4) + self.assert_range_end(ranges, 4) def test_range_shrink_picks_max_guard(self): # two loads guard the same range with r < 4 and r < 8 -> shrink to max(4, 8) = 8 @@ -538,25 +538,22 @@ class TestRangeShrink(unittest.TestCase): load1 = get_gated_load_uop(r < UOp.const(4), r) load2 = get_gated_load_uop(r < UOp.const(8), r) ranges = self.get_ranges(UOp.sink(load1, load2)) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 8) + self.assert_range_end(ranges, 8) def test_range_no_shrink_guard_ge_max(self): # guard r < 300 with range max 204 -> no shrink (guard doesn't constrain) r = Range(0, 204) load = get_gated_load_uop(r < UOp.const(300), r) ranges = self.get_ranges(load.sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 204) + self.assert_range_end(ranges, 204) def test_range_no_shrink_when_unguarded_elsewhere(self): # one load guards r < 4, but another load uses r without a gate -> no shrink r = Range(0, 204) load1 = get_gated_load_uop(r < UOp.const(4), r) - load2 = UOp(Ops.LOAD, src=(UOp.param(1, dtypes.float, (204,)).index(r),)) + load2 = UOp.param(1, dtypes.float, 204).index(r).load() ranges = self.get_ranges(UOp.sink(load1, load2)) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 204) + self.assert_range_end(ranges, 204) def test_range_no_shrink_when_used_in_reduce(self): # range used in both a gated load AND directly in the reduce expression -> no shrink @@ -564,8 +561,7 @@ class TestRangeShrink(unittest.TestCase): gated_load = get_gated_load_uop(r < UOp.const(4), r) red = (r.cast(dtypes.float) + gated_load).reduce(r, arg=Ops.ADD) ranges = self.get_ranges(red.sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 204) + self.assert_range_end(ranges, 204) def test_range_shrink_to_single_iteration(self): # guard r < 1 shrinks range to 1 -> single iteration, range eliminated entirely @@ -579,18 +575,16 @@ class TestRangeShrink(unittest.TestCase): from tinygrad.dtype import Invalid r = Range(0, 204) x = (r < 4).where(UOp.const(1.0), Invalid) - ranges = self.get_ranges(UOp.param(0, dtypes.float, (204,)).index(r).store((r < 4).where(x, Invalid)).sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 4) + ranges = self.get_ranges(UOp.param(0, dtypes.float, 204).index(r).store((r < 4).where(x, Invalid)).sink()) + self.assert_range_end(ranges, 4) def test_range_shrink_store_where_invalid_flipped(self): # above, but flipped from tinygrad.dtype import Invalid r = Range(0, 204) x = (r < 4).where(UOp.const(1.0), Invalid) - ranges = self.get_ranges(UOp.param(0, dtypes.float, (204,)).index(r).store((r >= 4).where(Invalid, x)).sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 4) + ranges = self.get_ranges(UOp.param(0, dtypes.float, 204).index(r).store((r >= 4).where(Invalid, x)).sink()) + self.assert_range_end(ranges, 4) if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/null/test_tensor.py b/tinygrad_repo/test/null/test_tensor.py index 4156b3717..560ce3f5e 100644 --- a/tinygrad_repo/test/null/test_tensor.py +++ b/tinygrad_repo/test/null/test_tensor.py @@ -69,7 +69,7 @@ class TestIdxUpcast(unittest.TestCase): if not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, NIRRenderer)): assert idx.op is Ops.INDEX idx_val = idx.src[1] - self.assertFalse(idx_val.overflows(idx_val.dtype.scalar())) + self.assertFalse(idx_val.overflows(idx_val.dtype)) # use expand to generate kernel that uses large idx def do_op_then_assert(self, dtype: DType, dim1, dim2, dim3): @@ -171,6 +171,11 @@ class TestTensorConstLike(unittest.TestCase): t = Tensor.ones(8, 4).shard(("NULL:0", "NULL:1"), axis=0) with self.assertRaises(RuntimeError): t.full_like(5, device="NULL") +class TestTensorShape(unittest.TestCase): + def test_float_shape_raises(self): + for dim in (2.0, 2.5): + with self.subTest(dim=dim), self.assertRaisesRegex(RuntimeError, "shape must be int"): Tensor.ones(dim) + class TestTensorDevice(unittest.TestCase): def test_create_from_single_device_tuple(self): (Tensor([1.0], device=(Device.DEFAULT,)) + Tensor([2.0])).realize() diff --git a/tinygrad_repo/test/null/test_transcendental_helpers.py b/tinygrad_repo/test/null/test_transcendental_helpers.py index 1653b176c..760218424 100644 --- a/tinygrad_repo/test/null/test_transcendental_helpers.py +++ b/tinygrad_repo/test/null/test_transcendental_helpers.py @@ -9,7 +9,7 @@ class TestTranscendentalFunctions(unittest.TestCase): def test_payne_hanek_reduction(self): # TODO: Test constant input when constant folding is fixed (or maybe test both variants) # Load input value from a buffer to prevent constant folding - input_buf = UOp.param(1, dtypes.double, (1,)) + input_buf = UOp.param(1, dtypes.double, 1) loaded_value = input_buf.index(UOp.const(0)).load() def eval_payne_hanek_reduction(v:float) -> tuple[float, int]: return tuple(eval_uop(u, [(dtypes.float64, [v])]) for u in payne_hanek_reduction(loaded_value)) diff --git a/tinygrad_repo/test/null/test_uop_graph.py b/tinygrad_repo/test/null/test_uop_graph.py index b7ff3fbeb..577e39afe 100644 --- a/tinygrad_repo/test/null/test_uop_graph.py +++ b/tinygrad_repo/test/null/test_uop_graph.py @@ -1,10 +1,9 @@ import unittest, pytest from tinygrad import dtypes, Variable, Device from tinygrad.dtype import AddrSpace -from tinygrad.helpers import DEBUG, Context from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite, GroupOp, AxisType, broadcast_axes, KernelInfo from tinygrad.uop.symbolic import sym -from test.helpers import to_uops_list +from test.helpers import full_rewrite, to_uops_list from tinygrad.codegen import full_rewrite_to_sink simple_pm = PatternMatcher([ @@ -14,43 +13,27 @@ simple_pm = PatternMatcher([ ((UPat.var('x') + UPat.cvar('c1')) + UPat.cvar('c2'), lambda x,c1,c2: x + (c1.val+c2.val)), ]) -def const_values(u:UOp): - if u.op is Ops.CONST: return (u.val,) - if u.op is Ops.STACK: return tuple(x.val for x in u.src) - raise AssertionError(f"expected const-like UOp, got {u.op}") - class TestGraphRewriteConst(unittest.TestCase): def test_gep_const(self): v1 = UOp.const((0,1,2), dtypes.int) v2 = v1.index(1) ret = graph_rewrite(v2, sym) - self.assertEqual(ret.dtype, dtypes.int) - self.assertEqual(ret.val, 1) + self.assertIs(ret, UOp.const(1, dtypes.int)) def test_add_const(self): v1 = UOp.const((0,1,2)) v2 = UOp.const((5,6,7)) - ret = graph_rewrite(v1+v2, sym) - self.assertEqual(ret.op, Ops.STACK) - self.assertEqual(const_values(ret), (5,7,9)) - - def test_add_const_lose_v(self): - v1 = UOp.const((0,1,2)) - v2 = UOp.const((2,1,0)) - ret = graph_rewrite(v1+v2, sym) - self.assertEqual(ret.op, Ops.STACK) - self.assertEqual(const_values(ret), (2,2,2)) + self.assertIs(graph_rewrite(v1+v2, sym), UOp.const((5,7,9))) def xfail_broken_const_wraparound(fn): fn = pytest.mark.xfail(reason="const folding does not properly implement modular arithmetic")(fn) return unittest.expectedFailure(fn) class TestModularWraparound(unittest.TestCase): def _test(self, uop:UOp, expected:int): - results = to_uops_list([uop]) - self.assertEqual(len(results), 2) # +1 for SINK - self.assertEqual(results[0].op, Ops.CONST) - self.assertEqual(results[0].dtype, uop.dtype) - self.assertEqual(results[0].val, expected) + result = uop.simplify() + self.assertEqual(result.op, Ops.CONST) + self.assertEqual(result.dtype, uop.dtype) + self.assertEqual(result.val, expected) @xfail_broken_const_wraparound def test_cast(self): @@ -157,7 +140,7 @@ class TestGraphRewrite(unittest.TestCase): self.assertEqual(nout.val, 3.0) def test_depth_2_fold(self): - v = UOp.variable("v", 0, 1, dtypes.float) + v = UOp.variable("v", 0, 1, dtypes.float, param=True) c1 = UOp.const(1.0) c2 = UOp.const(2.0) nout = graph_rewrite(v+c1+c2, simple_pm) @@ -191,63 +174,25 @@ class TestGraphRewrite(unittest.TestCase): self.assertEqual(len([x for x in sink.toposort() if x.op is Ops.CONST]), 1) class TestUOpGraph(unittest.TestCase): - def test_add_constant_fold(self): - c1 = UOp.const(1.0, dtypes.float) - c2 = UOp.const(2.0, dtypes.float) - out = c1+c2 - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 3.0) - def test_where_same_fold(self): v = UOp.variable('tmp', 0, 1) c0 = UOp.const(0) vc = v != c0 c1 = UOp.const(1.0, dtypes.float) out = vc.where(c1, c1) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 1.0) + self.assertIs(out.simplify(), c1) def test_where_const_fold(self): bf = UOp.const(False) c1 = UOp.const(1.0, dtypes.float) c2 = UOp.const(2.0, dtypes.float) out = bf.where(c1, c2) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 2.0) + self.assertIs(out.simplify(), c2) def test_const_cast(self): bf = UOp.const(False) out = bf.cast(dtypes.int) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 0) - - def test_const_bitcast(self): - bf = UOp.const(1.0, dtypes.float) - out = bf.bitcast(dtypes.uint32) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 0x3F800000) - - @unittest.expectedFailure - def test_const_shape_change_bitcast(self): - bf = UOp.const(0x3F).cast(dtypes.uint8) - out = bf.bitcast(dtypes.half) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK + self.assertIs(full_rewrite(out.sink()).src[0], full_rewrite(UOp.const(0, dtypes.int).sink()).src[0]) def test_devectorize_derives_lane_dtype(self): from tinygrad.codegen import do_devectorize @@ -257,189 +202,137 @@ class TestUOpGraph(unittest.TestCase): invalid_lane_mul = next(u for u in out.src[0].toposort() if u.op is Ops.MUL) self.assertIs(invalid_lane_mul.dtype, dtypes.bool) - @unittest.skip("this test isn't valid uops") - def test_noop_vectorize_fold(self): - d0 = UOp.param(0, dtypes.float, (1,)) - idx = UOp.const(0) - ld = d0.load(idx, dtype=dtypes.float) - vec = UOp(Ops.STACK, dtypes.float, (ld,)) - x = vec.index(0) - alu = UOp(Ops.SQRT, src=(x, )) - out = UOp(Ops.STORE, src=(d0, idx, alu)) - uops = to_uops_list([out]) - self.assertEqual(len([x for x in uops if x.op is Ops.STACK]), 0) - - @unittest.skip("this test isn't valid uops") - def test_gep_vec_fold(self): - d0 = UOp.param(0, dtypes.float, (1,)) - d1 = UOp.param(1, dtypes.float, (1,)) - d2 = UOp.param(2, dtypes.float, (1,)) - idx = UOp.const(0) - def _test_vec(geps, count=4): - vec = UOp(Ops.STACK, dtypes.float, geps) - out = d0.index(idx).store(vec) - uops = to_uops_list([out]) - if DEBUG >= 4: - from tinygrad import Device - print(Device[Device.DEFAULT].renderer.render(uops)) - return uops[-2].src[-1] # -2 to skip SINK - - # possible - val = d1.index(idx).load(dtype=dtypes.float) - xyzw = tuple(val.index(i) for i in range(4)) - self.assertIs(_test_vec(xyzw).op, Ops.LOAD) - - # unaligned - val = d1.index(idx).load(dtype=dtypes.float) - wzyx = tuple(val.index(i) for i in reversed(range(4))) - self.assertIs(_test_vec(wzyx).op, Ops.STACK) - - # different_size - val = d1.index(idx).load(dtype=dtypes.float) - xy = tuple(val.index(i) for i in range(2)) - self.assertIs(_test_vec(xy+xy).op, Ops.STACK) - val = d1.index(idx).load(dtype=dtypes.float) - xy = tuple(val.index(i) for i in range(2)) - self.assertIs(_test_vec(xy, count=2).op, Ops.STACK) - - # different vals - val1 = d1.index(idx).load(dtype=dtypes.float) - val2 = d2.index(idx).load(dtype=dtypes.float) - xy1 = tuple(val1.index(i) for i in range(2)) - xy2 = tuple(val2.index(i) for i in range(2)) - self.assertIs(_test_vec(xy1+xy2).op, Ops.STACK) + def test_devectorize_zero_sized_scalar_expand(self): + from tinygrad.codegen import devectorizer2 + expanded = UOp.const(1.0).reshape(1, 1).expand(0, 3) + self.assertEqual(graph_rewrite(expanded, devectorizer2).shape, (0, 3)) def test_gep_vec_const_fold(self): for vec_size in [2, 4, 8]: consts = [UOp.const(float(i), dtypes.float) for i in range(vec_size)] - vec = UOp(Ops.STACK, src=tuple(consts)) - with Context(SPEC=0): - uops = to_uops_list([vec.index(i) for i in range(vec_size)]) - for uop, const in zip(uops, consts): - self.assertEqual(uop, const) + vec = UOp.stack(*consts) + for i, const in enumerate(consts): self.assertIs(vec.index(i), const) def test_cast_alu_fold(self): - d0 = UOp.param(0, dtypes.bool, (1,)) - d1 = UOp.param(1, dtypes.int, (1,)) + d0 = UOp.param(0, dtypes.bool, 1) + d1 = UOp.param(1, dtypes.int, 1) idx = UOp.const(0) ld = d1.index(idx) alu = (ld<1).cast(dtypes.bool) out = d0.index(idx).store(alu) uops = to_uops_list([out]) - self.assertEqual(len([x for x in uops if x.op is Ops.CAST]), 0) + self.assertEqual(len([x for x in uops if x.op is Ops.CAST and x.src[0].op is not Ops.CONST]), 0) def test_double_cast_fold(self): - d0 = UOp.param(0, dtypes.float, (1,)) - d1 = UOp.param(1, dtypes.int, (1,)) + d0 = UOp.param(0, dtypes.float, 1) + d1 = UOp.param(1, dtypes.int, 1) idx = UOp.const(0, dtypes.int) ld = d1.index(idx) alu = ld.cast(dtypes.float).cast(dtypes.float) out = d0.index(idx).store(alu) uops = to_uops_list([out]) - self.assertEqual(len([x for x in uops if x.op is Ops.CAST]), 1) + self.assertEqual(len([x for x in uops if x.op is Ops.CAST and x.src[0].op is not Ops.CONST]), 1) def test_depth_2_const_fold(self): - v = UOp.variable("tmp", 0, 1, dtypes.int) - c2 = UOp.const(2, dtypes.int) - c4 = UOp.const(4, dtypes.int) + v = UOp.variable("tmp", 0, 1, dtypes.int, param=True) + c2 = UOp.const(2) + c4 = UOp.const(4) vc = v+c2 out = vc+c4 - uops = to_uops_list([out]) - self.assertEqual(len(uops), 5) # +1 for SINK, +1 for the PARAM shape STACK - out = uops[-2] # -2 to skip SINK - self.assertEqual(out.op, Ops.ADD) - self.assertEqual(out.src[1].op, Ops.CONST) - self.assertEqual(out.src[1].val, 6) + self.assertIs(out.simplify(), (v+UOp.const(6)).simplify()) def test_bitcast_to_same_dtype_fold(self): for dt in dtypes.ints + dtypes.floats + (dtypes.bool,): - d0 = UOp.param(0, dt, (1,)) + d0 = UOp.param(0, dt, 1) v = d0.index(UOp.const(0)) uops = to_uops_list([v.bitcast(dt)]) self.assertEqual(len([x for x in uops if x.op is Ops.BITCAST and x.dtype is dt]), 0, f"dtype = {dt}") def test_sub_with_cast_folds(self): a = Variable("a", 0, 5) - uops = to_uops_list([a.cast(dtypes.int)+(-a).cast(dtypes.int)]) - assert uops[0] == UOp.const(0, dtypes.int) - assert uops[-1].op == Ops.SINK + out = a+(-a) + self.assertIs(full_rewrite(out.sink()).src[0], full_rewrite(UOp.const(0, dtypes.int).sink()).src[0]) def test_where_on_gated_load_fold(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.long, (100,)) + d0 = UOp.param(0, dtypes.long, 100) ld = d0.index(ridx0.valid(ridx0<50)) w = (ridx0<50).where(ld, 5) - out = UOp.param(1, dtypes.long, (100,)) + out = UOp.param(1, dtypes.long, 100) uops = to_uops_list([out.index(ridx0).store(w)]) + expected = full_rewrite(UOp.const(5, dtypes.long).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: assert u.src[1].val==5 + if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: self.assertIs(u.src[1], expected) def test_where_on_gated_load_folds_swapped_branches(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.long, (100,)) + d0 = UOp.param(0, dtypes.long, 100) ld = d0.index(ridx0.valid((ridx0<50).logical_not())) w = (ridx0<50).where(5, ld) uops = to_uops_list([w]) + expected = full_rewrite(UOp.const(5, dtypes.long).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.LOAD: assert u.src[1].val==5 + if u.op is Ops.LOAD: self.assertIs(u.src[1], expected) def test_where_on_gated_load_with_cast(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.int, (100,)) + d0 = UOp.param(0, dtypes.int, 100) gate_idx = ridx0.valid((ridx0<50)) ld = d0.index(gate_idx).cast(dtypes.float) w = (ridx0<50).where(ld, 5.0) - out = UOp.param(1, dtypes.float, (100,)) + out = UOp.param(1, dtypes.float, 100) uops = to_uops_list([out.index(ridx0).store(w)]) + expected = full_rewrite(UOp.const(5, dtypes.int).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: assert u.src[1].val == 5 + if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: self.assertIs(u.src[1], expected) def test_where_on_casted_gated_load_extra_cond(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.float, (100,)) + d0 = UOp.param(0, dtypes.float, 100) ld = d0.index(ridx0.valid(ridx0<50)) w = ((ridx0<50) & (ridx0>30)).where(ld, UOp.const(0.0)).cast(dtypes.half) - out = UOp.param(1, dtypes.half, (100,)) + out = UOp.param(1, dtypes.half, 100) uops = to_uops_list([out.index(ridx0).store(w)]) for u in uops: assert u.op is not Ops.WHERE def test_where_on_casted_gated_load_extra_cond_swapped(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.float, (100,)) + d0 = UOp.param(0, dtypes.float, 100) ld = d0.index(ridx0.valid(ridx0<50)) w = ((ridx0<50) & (ridx0>30)).where(UOp.const(0.0), ld).cast(dtypes.half) - out = UOp.param(1, dtypes.half, (100,)) + out = UOp.param(1, dtypes.half, 100) uops = to_uops_list([out.index(ridx0).store(w)]) for u in uops: assert u.op is not Ops.WHERE def test_where_in_store_becomes_gate(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.long, (100,)) + d0 = UOp.param(0, dtypes.long, 100) idx = d0.index(ridx0) ld = idx.load() val = (ridx0<50).where(5, ld) st = idx.store(val).end(ridx0) uops = to_uops_list([st]) + expected = full_rewrite(UOp.const(5, dtypes.long).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.STORE: assert u.src[1].val==5 + if u.op is Ops.STORE: self.assertIs(u.src[1], expected) def test_load_idx_becomes_int(self): # mnist indexing with split reduceop # Make sure we are not doign math on the loaded index, which would promote it to long - c0 = UOp.param(0, dtypes.uchar, (128000,)) + c0 = UOp.param(0, dtypes.uchar, 128000) c1 = UOp.range(UOp.const(512), 1, AxisType.WEAK) c2 = UOp.range(UOp.const(250), 2, AxisType.WEAK) - c3 = UOp.param(1, dtypes.int, (512,)) + c3 = UOp.param(1, dtypes.int, 512) c4 = c3.index(c1) c5 = UOp.range(UOp.const(240), 0, AxisType.REDUCE) c6 = ((c2*UOp.const(240))+c5) - c7 = UOp.param(2, dtypes.uchar, (60000,)) + c7 = UOp.param(2, dtypes.uchar, 60000) c8 = c7.index(c6) c9 = ((c4<0).where((c4+60000), c4)!=c6.cast(dtypes.int)).where(0, c8.cast(dtypes.uint).cast(dtypes.uchar)).reduce(c5, arg=Ops.ADD) c10 = c0.index(((c1*UOp.const(250))+c2)).store(c9).end(c1, c2) @@ -449,14 +342,14 @@ class TestUOpGraph(unittest.TestCase): def test_load_idx_no_math_on_loaded(self): # test the (x+y)> 0, 0, 8, "a") + def test_xor_self_inverse(self): self.helper_test_variable((Variable("a", 0, 8, dtypes.int) ^ 5) ^ 5, 0, 8, "a", test_z3=False) @@ -441,10 +448,20 @@ class TestSymbolic(unittest.TestCase): def test_and_remove(self): self.helper_test_variable(uand([uconst(1), Variable("a", 0, 1)]), 0, 1, "a") + def test_zero_div_zero_bottom_up(self): + # codegen runs symbolic_simple bottom_up, so the 0/0 is rewritten before its consts fold. + # without the guard the unsound x/x -> 1 below it claims this one. + z = UOp.const(0.0) + self.assertTrue(math.isnan(graph_rewrite(z/z, symbolic_simple, bottom_up=True).arg)) + def test_masked_shr_fold(self): - x = UOp.variable('x', 0, 255, dtype=dtypes.uint32) + x = UOp.variable('x', 0, 255, dtype=dtypes.uint32, param=True) self.helper_test_variable((x & -4) >> 2, 0, 63, "(x>>2)") + def test_masked_idiv_fold(self): + x = UOp.variable('x', 0, 255, dtype=dtypes.uint32, param=True) + self.helper_test_variable((x & -4) // 4, 0, 63, "(x//4)") + def test_bool_or_not_tautology(self): a = Variable("a", 0, 10) c = a<10 @@ -483,12 +500,12 @@ class TestSymbolic(unittest.TestCase): def test_div_drop_small_terms(self): # from openpilot, shouldnt simplify - gidx0 = UOp.variable("gidx0", 0, 10) - gidx1 = UOp.variable("gidx1", 0, 10) - lidx0 = UOp.variable("lidx0", 0, 1) - lidx1 = UOp.variable("lidx1", 0, 1) - ridx1005 = UOp.variable("ridx1005", 0, 2) - ridx1006 = UOp.variable("ridx1006", 0, 2) + gidx0 = UOp.variable("gidx0", 0, 10, param=True) + gidx1 = UOp.variable("gidx1", 0, 10, param=True) + lidx0 = UOp.variable("lidx0", 0, 1, param=True) + lidx1 = UOp.variable("lidx1", 0, 1, param=True) + ridx1005 = UOp.variable("ridx1005", 0, 2, param=True) + ridx1006 = UOp.variable("ridx1006", 0, 2, param=True) self.helper_test_variable((lidx1+((gidx1*18)+(ridx1005*18)+(lidx0*162))+(gidx0*2)+(ridx1006*2)+-40)//18, -3, 20, "(gidx1+ridx1005+lidx0*9+(gidx0+ridx1006+7)//9+-3)") @@ -948,6 +965,11 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable(cond.where(u0, u1), 0, 1, "((a<2)!=True)") self.helper_test_variable(cond.where(u0, u1).where(u0, u1), 0, 1, "(a<2)") + def test_equivalent_const_max(self): + x = Variable("x", -10, 10) + self.helper_test_variable((x < 0).where(0, x), 0, 10, "x.maximum(0)") + self.helper_test_variable((0 < x).where(x, 0), 0, 10, "x.maximum(0)") + def test_where_combine(self): cond = Variable("x", 0, 3) < 2 a = Variable("a", 0, 3) @@ -992,7 +1014,7 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable(cond.ne(False), 0, 1, "(x<2)") def test_bitcast_chain(self): - a = UOp.variable("a", 0, 3, dtype=dtypes.int32) + a = UOp.variable("a", 0, 3, dtype=dtypes.int32, param=True) self.assertIs(graph_rewrite(a.bitcast(dtypes.float32).bitcast(a.dtype), sym), a) def test_negation_in_where(self): @@ -1008,20 +1030,19 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable(-a<-b, False, True, "(b NOOP rule. This rule matches patterns that EMERGE during simplification.""" def test_store_load_folding(self): # store(idx, load(idx)) -> NOOP, including emergent patterns like store(idx, load(idx) + 0) - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) index = buf.index(UOp.const(0)) # Direct: store(idx, load(idx)) -> NOOP self.assertEqual(graph_rewrite(index.store(index.load()), sym).op, Ops.NOOP) @@ -1386,7 +1417,7 @@ class TestStoreLoadFolding(unittest.TestCase): class TestMoveWhereOnLoad(unittest.TestCase): def test_bool_index_preserves_dtype(self): - buf = UOp.param(0, dtypes.bool, (8,)) + buf = UOp.param(0, dtypes.bool, 8) a = Variable("a", 0, 7) r = UOp.range(8, 0) # cond has a range that the rewrite can move into the valid: gate (a<4) goes into load valid @@ -1434,12 +1465,17 @@ class TestGatedUopGivenValid(unittest.TestCase): self.assertEqual(idx, (r0 < 3).where(expected_vec, UOp.invalid())) class TestRangeSplitting(unittest.TestCase): + def test_end_preserves_constant_backedge(self): + loop, backedge = UOp.loop(0), UOp.const(False) + end = graph_rewrite(UOp(Ops.NOOP).end(loop, backedge), sym) + self.assertEqual(end.src, (UOp(Ops.NOOP), loop, backedge)) + def test_range_split_on_mod(self): # test that mark_range_mod splits RANGE(8) into RANGE(4)*2 + RANGE(2) when used with %2 from tinygrad.codegen.simplify import pm_split_ranges, pm_flatten_range r0 = UOp.range(uconst(8), 0) # create a simple expression using the range with mod: store range%2 to a buffer - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) val = (r0 % uconst(2)).cast(dtypes.int) store = UOp(Ops.STORE, src=(buf.index(uconst(0)), val)) sink = UOp(Ops.SINK, src=(UOp(Ops.END, src=(store, r0)),)) diff --git a/tinygrad_repo/test/null/test_uop_vmin_vmax.py b/tinygrad_repo/test/null/test_uop_vmin_vmax.py index 2dc3f27c3..fcc783408 100644 --- a/tinygrad_repo/test/null/test_uop_vmin_vmax.py +++ b/tinygrad_repo/test/null/test_uop_vmin_vmax.py @@ -82,7 +82,7 @@ class TestVminVmaxProperties(unittest.TestCase): def test_vmin_vmax_multiplication_0_inf(self): # vmin and vmax for multiplication with a variable x = UOp.const(0.0) - y = UOp.load(UOp.param(0, dtypes.float, (1,)), UOp.const(0), dtype=dtypes.float) + y = UOp.load(UOp.param(0, dtypes.float, 1), UOp.const(0)) uop = x * y # TODO: these should be 0, but definitely should not be nan self.assertEqual(uop.vmin, -math.inf) @@ -332,7 +332,7 @@ class TestVminVmaxVConst(unittest.TestCase): def test_vmin_vmax_vector_with_gep(self): # vmin and vmax for a vector constant of bool values - d1 = UOp.param(1, dtypes.int, (1,)) + d1 = UOp.param(1, dtypes.int, 1) idx = UOp.const(0) val = UOp(Ops.LOAD, src=(d1.index(idx),)) uop = (val // 32) diff --git a/tinygrad_repo/test/null/test_uops.py b/tinygrad_repo/test/null/test_uops.py index c1d6931fd..2138b3070 100644 --- a/tinygrad_repo/test/null/test_uops.py +++ b/tinygrad_repo/test/null/test_uops.py @@ -5,8 +5,8 @@ from tinygrad.tensor import Tensor from tinygrad.helpers import Timing, Context, cdiv from tinygrad.dtype import dtypes, AddrSpace, ConstFloat, Invalid # noqa: F401 from tinygrad.device import Device -from tinygrad.uop.ops import Ops, ParamArg, PatternMatcher, UOp, UPat, dtype_from_uop, exec_alu, graph_rewrite # noqa: F401 # ParamArg used by eval(str(uop)) roundtrip tests -from tinygrad.uop.weak import pm_lower_index_dtype +from tinygrad.uop.ops import Ops, AxisType, ParamArg, PatternMatcher, UOp, UPat, dtype_from_uop, exec_alu, graph_rewrite # noqa: F401 # ParamArg used by eval(str(uop)) roundtrip tests +from tinygrad.uop.weak import pm_lower_weak from tinygrad.uop.spec import spec_program, spec_shared, type_verify from tinygrad.uop.symbolic import sym, pm_remove_invalid from test.helpers import eval_uop, to_uops_list @@ -38,20 +38,9 @@ class TestDTypeFromUOp(unittest.TestCase): self.assertEqual(UOp(Ops.CONST, arg=ConstFloat(3.0)).dtype, dtypes.weakfloat) self.assertEqual(UOp(Ops.CONST, arg=True).dtype, dtypes.bool) self.assertEqual(UOp(Ops.CONST, arg=Invalid).dtype, dtypes.bool) - # an explicit (strong) const dtype is legal until the field is removed + # UOp.const at a strong dtype builds the CAST that carries it self.assertEqual(UOp.const(3, dtypes.int32).dtype, dtypes.int32) - def test_weak_dtype_rejected_by_program_spec(self): - for weak, concrete, value in ((dtypes.weakint, dtypes.int32, 1), (dtypes.weakfloat, dtypes.float32, 1.0)): - with self.assertRaises(RuntimeError): type_verify(UOp.const(value, weak).sink(), spec_program) - type_verify(UOp.const(value, concrete).sink(), spec_program) - - def test_invalid_stated_dtype(self): - # UOp.const normalizes a stated dtype away (const_like/full pass their position's); the core constructor does not, - # and the spec is what rejects a non-bool Invalid - self.assertIs(UOp.const(Invalid, dtypes.float32), UOp.invalid()) - with self.assertRaises(RuntimeError): type_verify(UOp(Ops.CONST, dtypes.float32, arg=Invalid), spec_shared) - def test_invalid_dtype_and_consumers(self): invalid = UOp.invalid() self.assertIs(invalid.dtype, dtypes.bool) @@ -61,15 +50,15 @@ class TestDTypeFromUOp(unittest.TestCase): if u.is_invalid)), (dtypes.float32, dtypes.float32, dtypes.bool)) invalid, value = UOp.invalid(), UOp.const(1, dtypes.float32) for u in (UOp.param(0, dtypes.bool, ()).where(value, invalid), value+invalid, UOp.stack(value, invalid)): self.assertIs(u.src[-1], invalid) - for u in (UOp(Ops.STACK, dtypes.float32, src=(value, invalid)), UOp(Ops.ADD, dtypes.float32, src=(value, invalid)), + for u in (UOp(Ops.STACK, src=(value, invalid)), UOp(Ops.ADD, src=(value, invalid)), UOp.const(True).where(value, invalid), UOp(Ops.CMPLT, src=(invalid, value)), UOp(Ops.CMPLT, src=(value, invalid)), - UOp.param(0, dtypes.float32, (4,)).index(invalid)): type_verify(u, spec_shared) + UOp.param(0, dtypes.float32, 4).index(invalid)): type_verify(u, spec_shared) gate, value = UOp.param(0, dtypes.bool, ()), UOp.param(1, dtypes.float, ()) self.assertIs((out:=graph_rewrite(gate.where(value, UOp.invalid()), pm_remove_invalid)).src[2], UOp.const(0, dtypes.float)) type_verify(out.sink(), spec_program) def test_remove_invalid_stack_lanes(self): - stack = UOp(Ops.STACK, dtypes.half, (UOp.const(1, dtypes.half), UOp.invalid())) + stack = UOp(Ops.STACK, src=(UOp.const(1, dtypes.half), UOp.invalid())) out = graph_rewrite(stack, pm_remove_invalid) self.assertEqual(out.src, (UOp.const(1, dtypes.half), UOp.const(0, dtypes.half))) type_verify(out.sink(), spec_program) @@ -78,20 +67,21 @@ class TestLowerIndexDtype(unittest.TestCase): def test_gated_shrink_lowers_to_selected_width(self): # coalesce builds gated SHRINKs for masked vectorized loads; lowering must resolve them at the # width the offset bounds select (this one needs long) - buf = UOp.param(0, dtypes.float, (2**31+64,)) + buf = UOp.param(0, dtypes.float, 2**31+64) i = UOp.variable("i", 0, 2**28) shrink = UOp(Ops.SHRINK, src=(buf, (i*24).valid(i < 2**28), UOp.const(4))) - lowered = graph_rewrite(shrink.sink(), pm_lower_index_dtype) - self.assertTrue(all(u.dtype != dtypes.weakint for u in lowered.backward_slice_with_self), "lowering must resolve all weakint") + lowered = graph_rewrite(shrink.sink(), pm_lower_weak) + self.assertTrue(all(u.op is Ops.CONST for u in lowered.backward_slice_with_self if u.dtype in dtypes.weaks), + "lowering must resolve every weak width, except a typed literal's value half") sh = next(u for u in lowered.backward_slice_with_self if u.op is Ops.SHRINK) self.assertEqual(sh.src[1].dtype, dtypes.long) def test_reg_buffer_size_lowers(self): reg = UOp.placeholder((4,), dtypes.float, 0, addrspace=AddrSpace.REG) - self.assertEqual(reg.src[0].dtype, dtypes.weakint) - lowered = graph_rewrite(reg.sink(), pm_lower_index_dtype) - self.assertTrue(all(u.dtype != dtypes.weakint for u in lowered.backward_slice_with_self), "lowering must resolve all weakint") - self.assertEqual(next(u for u in lowered.backward_slice_with_self if u.op is Ops.BUFFER).src[0].dtype, dtypes.int) + self.assertEqual(reg.arg.size, 4) + lowered = graph_rewrite(reg.sink(), pm_lower_weak) + self.assertTrue(all(u.op is Ops.CONST for u in lowered.backward_slice_with_self if u.dtype in dtypes.weaks), + "lowering must resolve every weak width, except a typed literal's value half") class TestSafeCast(unittest.TestCase): def test_cast_folds(self): @@ -134,7 +124,7 @@ class TestConstFloatEq(unittest.TestCase): self.assertFalse(Invalid != HoldsInvalid()) def test_matchers_agree_on_nan(self): - n = UOp.const(math.nan, dtypes.float32) + n = UOp.const(math.nan) for compiled in (False, True): pm = PatternMatcher([(UPat(Ops.CONST, arg=math.nan), lambda: True)], compiled=compiled) self.assertTrue(pm.rewrite(n), f"{compiled=}") @@ -224,7 +214,7 @@ class TestExecALU(unittest.TestCase): class TestGatedStoreRewrite(unittest.TestCase): def test_tiny_gate_store(self): - gmem = UOp.param(0, dtypes.float, (8,)) + gmem = UOp.param(0, dtypes.float, 8) gidx0 = UOp.special(4, 'gidx0') gate = gidx0>6)*18725)>>17) instead of (int)((((long)(ridx0)*1198373)>>29)) - self.assertNotIn(Ops.CAST, ops) + self.assertNotIn(dtypes.long, [x.dtype for x in uops]) @unittest.expectedFailure def test_fast_idiv_overflow(self): # This will be possible with a slightly different method for fast_idiv - g = UOp.param(0, dtypes.uint32, (8,)) + g = UOp.param(0, dtypes.uint32, 8) c = UOp.const(7).cast(dtypes.uint) l = UOp(Ops.LOAD, src=(g.index(c),)) a = UOp(Ops.CDIV, src=(l, c)) @@ -367,8 +357,8 @@ class TestFastIdiv(unittest.TestCase): self.assertNotIn(Ops.CDIV, ops) def test_disable_fast_idiv(self): - g = UOp.param(0, dtypes.uint32, (4,)) - c = UOp.const(3).cast(dtypes.uint) + g = UOp.param(0, dtypes.uint32, 4) + c = UOp.const(3) l = g.index(c) a = UOp(Ops.CDIV, src=(l, c)) with Context(DISABLE_FAST_IDIV=1): @@ -410,8 +400,8 @@ class TestUOpMethod(unittest.TestCase): self.assertIs((x != x).simplify(), x.const_like(False, dtypes.bool)) def test_replace(self): - x = UOp.param(0, dtypes.int, (1,)) - self.assertEqual(x.replace(arg=UOp.param(1, dtypes.int, (1,)).arg).arg.slot, 1) + x = UOp.param(0, dtypes.int, 1) + self.assertEqual(x.replace(arg=UOp.param(1, dtypes.int, 1).arg).arg.slot, 1) with self.assertRaises(AssertionError): x.replace(field="a") def test_const_zero_neg_zero_different(self): @@ -463,11 +453,19 @@ class TestUopsObject(unittest.TestCase): self.assertEqual(a.device, Device.DEFAULT) class TestUOpRender(unittest.TestCase): + def test_render_ssimplified_marg_outside_toposort(self): + r = UOp.range(UOp.const(16, dtypes.int), 2, AxisType.WEAK, dtype=dtypes.int) + offset = (r * 2) + (r * 2) + shrink = UOp(Ops.SHRINK, src=(UOp.param(0, dtypes.uint, 32), offset, UOp.const(2, dtypes.int))) + self.assertIsNot(shrink.src[1], shrink.marg[0][0]) + self.assertEqual(shrink.render(simplify=False), "p0.shrink((((r2*4), 2),))") + self.assertEqual(UOp.range(1, 0, src=(shrink,), dtype=dtypes.int).render(simplify=False), "r0") + def test_render_vectorize_empty(self): - u = UOp(Ops.STACK, dtype=dtypes.void, src=()) + u = UOp(Ops.STACK, src=()) self.assertEqual(u.render(simplify=False), "{}") def test_render_vectorize_empty_simplified(self): - u = UOp(Ops.STACK, dtype=dtypes.void, src=()) + u = UOp(Ops.STACK, src=()) self.assertEqual(u.render(), "{}") def test_render_vectorize_same(self): u = UOp(Ops.STACK, src=(UOp.const(0),)*3) diff --git a/tinygrad_repo/test/null/test_uops_stats.py b/tinygrad_repo/test/null/test_uops_stats.py index edab94807..831ed67db 100644 --- a/tinygrad_repo/test/null/test_uops_stats.py +++ b/tinygrad_repo/test/null/test_uops_stats.py @@ -4,7 +4,7 @@ from tinygrad.helpers import GlobalCounters from tinygrad.engine.realize import compile_linear, estimate_uop from tinygrad.codegen import to_program from tinygrad.renderer import Estimates -from tinygrad.uop.ops import Ops, UOp +from tinygrad.uop.ops import Ops, UOp, AxisType from tinygrad.dtype import dtypes from tinygrad.codegen.opt import Opt, OptOps, KernelOptError from tinygrad.device import Device @@ -146,7 +146,7 @@ class TestUOpsStats(unittest.TestCase): #MULACC should have the same stats as MUL + ADD def test_mulacc(self): - globl = UOp.param(0, dtypes.int, (3,)) + globl = UOp.param(0, dtypes.int, 3) o1 = UOp.const(1, dtypes.int) o2 = UOp.const(2, dtypes.int) u1 = globl.index(o1) @@ -156,7 +156,7 @@ class TestUOpsStats(unittest.TestCase): u5 = UOp(Ops.ADD, src=(u4,u3)) uops = tuple(u5.toposort()) - globl = UOp.param(0, dtypes.int, (3,)) + globl = UOp.param(0, dtypes.int, 3) o1 = UOp.const(1, dtypes.int) o2 = UOp.const(2, dtypes.int) u1 = globl.index(o1) @@ -190,7 +190,7 @@ class TestStatsOptimized(unittest.TestCase): @unittest.skip("fails locally on AMD") def test_gemm_tc_unroll_half(self): try: - p = to_program(replace_opts(self.ast_gemm_half, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UNROLL, 0, 2)]), + p = to_program(replace_opts(self.ast_gemm_half, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL))]), renderer=Device[Device.DEFAULT].renderer) except KernelOptError: raise unittest.SkipTest("no tensor cores") @@ -199,7 +199,7 @@ class TestStatsOptimized(unittest.TestCase): def test_gemm_tc_unroll(self): try: - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UNROLL, 0, 2)]), + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL))]), renderer=Device[Device.DEFAULT].renderer) except KernelOptError: raise unittest.SkipTest("no tensor cores") @@ -209,20 +209,22 @@ class TestStatsOptimized(unittest.TestCase): # this is a good lesson about why UPCASTing is a good idea def test_gemm_one_upcasted(self): - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.UPCAST, 0, 4)]), renderer=Device[Device.DEFAULT].renderer) + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))]), renderer=Device[Device.DEFAULT].renderer) self.check_gemm(p) self.assertEqual(p.src[0].arg.estimates.lds, N*N*N*4 + N*N*N*4//4 + 4*N*N) def test_gemm_upcasted(self): - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4)]), + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), + Opt(OptOps.SPLIT, 4, (4, AxisType.UNROLL))]), renderer=Device[Device.DEFAULT].renderer) self.check_gemm(p) self.assertEqual(p.src[0].arg.estimates.lds, 2*N*N*N*4//4 + 4*N*N) def test_gemm_upcasted_locals(self): try: - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.LOCAL, 0, 4), - Opt(OptOps.LOCAL, 1, 4)]), renderer=Device[Device.DEFAULT].renderer) + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), + Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (4, AxisType.LOCAL))]), + renderer=Device[Device.DEFAULT].renderer) except KernelOptError: raise unittest.SkipTest("no locals") self.check_gemm(p) @@ -230,7 +232,7 @@ class TestStatsOptimized(unittest.TestCase): def test_gemm_group(self): try: - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.GROUP, 0, 4)]), renderer=Device[Device.DEFAULT].renderer) + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.SPLIT, 2, (4, AxisType.GROUP_REDUCE))]), renderer=Device[Device.DEFAULT].renderer) except KernelOptError: raise unittest.SkipTest("no locals") SZ = N*N*4 @@ -245,13 +247,5 @@ class TestStatsOptimized(unittest.TestCase): self.assertEqual(est.ops, N*N) self.assertEqual(est.mem, N*N*4 + 4) - def test_reduce_group(self): - try: - p = to_program(replace_opts(self.ast_reduce, [Opt(OptOps.GROUP, 0, 50)]), renderer=Device[Device.DEFAULT].renderer) - except KernelOptError: - raise unittest.SkipTest("no locals") - est = p.src[0].arg.estimates - print(p.arg.name, est.ops, est.mem, est.lds) - if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/tinygrad_repo/test/null/test_validate_oob.py b/tinygrad_repo/test/null/test_validate_oob.py index 2d52f8e70..00295880a 100644 --- a/tinygrad_repo/test/null/test_validate_oob.py +++ b/tinygrad_repo/test/null/test_validate_oob.py @@ -1,52 +1,54 @@ import unittest -from tinygrad import dtypes, Variable +from tinygrad import dtypes from tinygrad.dtype import AddrSpace from tinygrad.helpers import Context from tinygrad.uop.ops import Ops, UOp, AxisType from test.helpers import to_uops_list +def Variable(name, nmin, nmax): return UOp.variable(name, nmin, nmax, param=True) + class TestValidateOOB(unittest.TestCase): """Test z3 validation of index bounds for different ALU ops and patterns.""" # basic index patterns def test_const_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(UOp.const(0)).load(dtype=dtypes.int)]) # valid - to_uops_list([buf.index(UOp.const(15)).load(dtype=dtypes.int)]) # valid (last element) + buf = UOp.param(0, dtypes.int, 16) + to_uops_list([buf.index(UOp.const(0)).load()]) # valid + to_uops_list([buf.index(UOp.const(15)).load()]) # valid (last element) with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.const(16)).load(dtype=dtypes.int)]) # off by one + to_uops_list([buf.index(UOp.const(16)).load()]) # off by one with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.const(42)).load(dtype=dtypes.int)]) # way out + to_uops_list([buf.index(UOp.const(42)).load()]) # way out def test_variable_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(Variable("i", 0, 15)).load(dtype=dtypes.int)]) # valid + buf = UOp.param(0, dtypes.int, 16) + to_uops_list([buf.index(Variable("i", 0, 15)).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(Variable("i", 0, 20)).load(dtype=dtypes.int)]) # oob + to_uops_list([buf.index(Variable("i", 0, 20)).load()]) # oob with self.assertRaises(RuntimeError): - to_uops_list([buf.index(Variable("i", -5, 10)).load(dtype=dtypes.int)]) # negative + to_uops_list([buf.index(Variable("i", -5, 10)).load()]) # negative def test_range_with_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(42, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r.valid(r < 16)).load(dtype=dtypes.int)]) # valid + to_uops_list([buf.index(r.valid(r < 16)).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r.valid(r < 17)).load(dtype=dtypes.int)]) # oob + to_uops_list([buf.index(r.valid(r < 17)).load()]) # oob def test_variable_with_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) v = Variable("v", -5, 80) - to_uops_list([buf.index(v.valid((v >= 0) & (v < 16))).load(dtype=dtypes.int)]) # valid + to_uops_list([buf.index(v.valid((v >= 0) & (v < 16))).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(v.valid(v < 20)).load(dtype=dtypes.int)]) # negative not masked + to_uops_list([buf.index(v.valid(v < 20)).load()]) # negative not masked def test_gated_store(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) v = Variable("v", 0, 20) to_uops_list([buf.index(v.valid(v < 16)).store(0)]) # valid with self.assertRaises(RuntimeError): @@ -55,101 +57,101 @@ class TestValidateOOB(unittest.TestCase): # ALU ops in index def test_floordiv(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(UOp.range(32, 0, AxisType.GLOBAL) // 2).load(dtype=dtypes.int)]) # 0..15 valid + buf = UOp.param(0, dtypes.int, 16) + to_uops_list([buf.index(UOp.range(32, 0, AxisType.GLOBAL) // 2).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.range(34, 0, AxisType.GLOBAL) // 2).load(dtype=dtypes.int)]) # 0..16 oob + to_uops_list([buf.index(UOp.range(34, 0, AxisType.GLOBAL) // 2).load()]) # 0..16 oob def test_mod(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(100, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r % 16).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(r % 16).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r % 20).load(dtype=dtypes.int)]) # 0..19 oob + to_uops_list([buf.index(r % 20).load()]) # 0..19 oob def test_shr(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(UOp.range(64, 0, AxisType.GLOBAL) >> 2).load(dtype=dtypes.int)]) # 0..15 valid + buf = UOp.param(0, dtypes.int, 16) + to_uops_list([buf.index(UOp.range(64, 0, AxisType.GLOBAL) >> 2).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.range(128, 0, AxisType.GLOBAL) >> 2).load(dtype=dtypes.int)]) # 0..31 oob + to_uops_list([buf.index(UOp.range(128, 0, AxisType.GLOBAL) >> 2).load()]) # 0..31 oob def test_shl(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (64,)) + buf = UOp.param(0, dtypes.int, 64) r = UOp.range(8, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r << 2).load(dtype=dtypes.int)]) # 0..28 valid + to_uops_list([buf.index(r << 2).load()]) # 0..28 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r << 4).load(dtype=dtypes.int)]) # 0..112 oob + to_uops_list([buf.index(r << 4).load()]) # 0..112 oob def test_and(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(100, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r & 15).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(r & 15).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r & 31).load(dtype=dtypes.int)]) # 0..31 oob + to_uops_list([buf.index(r & 31).load()]) # 0..31 oob # align masks round down to a multiple of 2^k - to_uops_list([buf.index((r & -4).valid(r < 16)).load(dtype=dtypes.int)]) # 0..12 valid + to_uops_list([buf.index((r & -4).valid(r < 16)).load()]) # 0..12 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r & -2).load(dtype=dtypes.int)]) # 0..100 oob + to_uops_list([buf.index(r & -2).load()]) # 0..100 oob # other masks can't be modeled as mod with self.assertRaisesRegex(RuntimeError, "z3 int AND only supports"): - to_uops_list([buf.index(r & 21).load(dtype=dtypes.int)]) + to_uops_list([buf.index(r & 21).load()]) def test_max(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(Variable("v", -10, 15).maximum(0)).load(dtype=dtypes.int)]) # 0..15 valid + buf = UOp.param(0, dtypes.int, 16) + to_uops_list([buf.index(Variable("v", -10, 15).maximum(0)).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(Variable("v2", -10, 20).maximum(0)).load(dtype=dtypes.int)]) # 0..20 oob + to_uops_list([buf.index(Variable("v2", -10, 20).maximum(0)).load()]) # 0..20 oob def test_xor_in_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(32, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r.valid((r < 8) ^ ((r >= 8) & (r < 16)))).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(r.valid((r < 8) ^ ((r >= 8) & (r < 16)))).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r.valid((r < 10) ^ (r >= 20))).load(dtype=dtypes.int)]) # 0..9,20..31 oob + to_uops_list([buf.index(r.valid((r < 10) ^ (r >= 20))).load()]) # 0..9,20..31 oob # cast patterns def test_float_cast_in_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(20, 0) i = (r.cast(dtypes.float) * 0.68).trunc().cast(dtypes.int) - to_uops_list([buf.index(i.valid((i >= 0) & (i < 16))).load(dtype=dtypes.int)]) + to_uops_list([buf.index(i.valid((i >= 0) & (i < 16))).load()]) def test_bool_cast_in_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) r = UOp.range(20, 0) - to_uops_list([buf.index(r.valid(r.cast(dtypes.bool).logical_not())).load(dtype=dtypes.int)]) # only r=0 valid + to_uops_list([buf.index(r.valid(r.cast(dtypes.bool).logical_not())).load()]) # only r=0 valid # load result as index/mask def test_load_as_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf0 = UOp.param(0, dtypes.int, (16,)) - buf1 = UOp.param(1, dtypes.int, (64,)) + buf0 = UOp.param(0, dtypes.int, 16) + buf1 = UOp.param(1, dtypes.int, 64) r = UOp.range(42, 0, AxisType.GLOBAL) - ld0 = buf0.index(r.valid(r < 8)).load(dtype=dtypes.int).cast(dtypes.weakint) - to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 32))).load(dtype=dtypes.int)]) # valid + ld0 = buf0.index(r.valid(r < 8)).load().cast(dtypes.weakint) + to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 32))).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 64))).load(dtype=dtypes.int)]) # oob + to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 64))).load()]) # oob def test_load_from_shrink_as_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf0 = UOp.param(0, dtypes.int, (16,)) - buf1 = UOp.param(1, dtypes.int, (64,)) + buf0 = UOp.param(0, dtypes.int, 16) + buf1 = UOp.param(1, dtypes.int, 64) shrink = UOp(Ops.SHRINK, src=(buf0, UOp.const(0, dtypes.int), UOp.const(4))) - ld0 = shrink.load(dtype=dtypes.int).index(0) - to_uops_list([buf1.index(ld0.valid((ld0 >= 0) & (ld0 < 64))).load(dtype=dtypes.int)]) + ld0 = shrink.load().index(0) + to_uops_list([buf1.index(ld0.valid((ld0 >= 0) & (ld0 < 64))).load()]) def test_load_bool_as_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf_bool = UOp.param(0, dtypes.bool, (16,)) - buf_int = UOp.param(1, dtypes.int, (8,)) + buf_bool = UOp.param(0, dtypes.bool, 16) + buf_int = UOp.param(1, dtypes.int, 8) gidx = UOp(Ops.SPECIAL, src=(UOp.const(16),), arg="gidx0") ld_bool = buf_bool.index(gidx).load() with self.assertRaises(RuntimeError): @@ -160,7 +162,7 @@ class TestValidateOOB(unittest.TestCase): def test_in_bounds_access_gated_local(self): with Context(CHECK_OOB=1): # Define buffers - gbuf = UOp.param(0, dtypes.uint, (400,)) + gbuf = UOp.param(0, dtypes.uint, 400) sbuf = UOp.placeholder((8,), dtypes.uint, slot=0, addrspace=AddrSpace.LOCAL) # Define indices, valids and barrier @@ -184,8 +186,8 @@ class TestValidateOOB(unittest.TestCase): @unittest.skip("Bool load is not supported yet") def test_load_mask(self): with Context(CHECK_OOB=1): - glbl0 = UOp.param(0, dtypes.int, (16,)) - mask = UOp.param(0, dtypes.bool, (16,)) + glbl0 = UOp.param(0, dtypes.int, 16) + mask = UOp.param(0, dtypes.bool, 16) ridx = UOp.range(20, 0) ld0 = UOp(Ops.LOAD, src=(glbl0.index(UOp.const(ridx<16&mask, ridx)))) to_uops_list([ld0]) diff --git a/tinygrad_repo/test/null/test_viz.py b/tinygrad_repo/test/null/test_viz.py index c6cdb6213..fae51d311 100644 --- a/tinygrad_repo/test/null/test_viz.py +++ b/tinygrad_repo/test/null/test_viz.py @@ -1,5 +1,5 @@ -import unittest, decimal, sys, json, contextlib, tempfile, pickle, io, math -from pathlib import Path +import unittest +import decimal, sys, json, contextlib, tempfile, pickle, io, math, pathlib from dataclasses import dataclass from typing import Generator @@ -43,7 +43,7 @@ def save_viz(): Buffer.profile_events.clear() cpu_events.clear() viz = VizTrace() - with Context(VIZ=-1, TRACK_MATCH_STATS=2, PROFILE=1): + with Context(VIZ=-1, TRACK_MATCH_STATS=2, PROFILE=1, PARALLEL=0): yield viz viz.set_data() @@ -185,18 +185,18 @@ class TestViz(unittest.TestCase): @dataclass(frozen=True) class TestStruct: colored_field: str - a = UOp(Ops.CUSTOM, arg=TestStruct(colored("xyz", "magenta")+colored("12345", "blue"))) + a = UOp(Ops.PYLITERAL, arg=TestStruct(colored("xyz", "magenta")+colored("12345", "blue"))) a2 = uop_to_json(VizData(), a)[id(a)] - self.assertEqual(ansistrip(a2["label"]), f"CUSTOM\n{TestStruct.__qualname__}(colored_field='xyz12345')") + self.assertEqual(ansistrip(a2["label"]), f"PYLITERAL\n{TestStruct.__qualname__}(colored_field='xyz12345')") def test_colored_label_multiline(self): with save_viz() as viz: arg = colored("x", "green")+"\n"+colored("y", "red")+colored("z", "yellow")+colored("ww\nw", "magenta") src = [Tensor.empty(1).uop for _ in range(10)] - a = UOp(Ops.CUSTOM, src=tuple(src), arg=arg) + a = UOp(Ops.PYLITERAL, src=tuple(src), arg=arg) exec_rewrite(a, [PatternMatcher([])]) a2 = next(viz.get_details(0, 0))["graph"][id(a)] - self.assertEqual(ansistrip(a2["label"]), "CUSTOM\nx\nyzww\nw") + self.assertEqual(ansistrip(a2["label"]), "PYLITERAL\nx\nyzww\nw") def test_inf_loop(self): a = UOp.const(3) @@ -228,28 +228,27 @@ class TestViz(unittest.TestCase): with save_viz() as viz: inner = UOp.const(3) call = UOp(Ops.CALL, src=(UOp(Ops.SINK, src=(inner,)),)) - func = UOp(Ops.FUNCTION, src=(UOp(Ops.TUPLE, src=(call,)),)) - graph_rewrite(func, TrackedPatternMatcher(pm.patterns), enter_calls=True) + graph_rewrite(call, TrackedPatternMatcher(pm.patterns), enter_calls=True) details = list(viz.get_details(0, 0)) self.assertTrue(details[-1]["change"], "viz replay should detect change inside CALL") def test_const_node_visibility(self): with save_viz() as viz: a = UOp.variable("a", 0, 10, dtype=dtypes.int) - z = UOp.const(0, a.dtype) - y = UOp.const(math.pi, dtypes.float) + z = UOp.const(0) + y = UOp.const(math.pi) alu = a*z ret = exec_rewrite(sink:=UOp.sink(alu, y), [sym]) lst = viz.list_items() self.assertEqual(len(lst), 1) graphs = [x["graph"] for x in viz.get_details(0, 0)] # const is always in the graph, client side hides exclude=True nodes by default - self.assertEqual(list(graphs[0]), [id(a.src[0]), id(a), id(z), id(alu), id(y), id(sink)]) + self.assertEqual(list(graphs[0]), [id(a), id(z), id(alu), id(y), id(sink)]) self.assertTrue(graphs[0][id(z)]["exclude"]) self.assertTrue(graphs[0][id(y)]["exclude"]) self.assertFalse(graphs[0][id(alu)]["exclude"]) self.assertEqual(graphs[0][id(y)]["label"].split("\n")[:2], ["CONST", "3.14159"]) - self.assertEqual(list(graphs[1]), [id(z), id(y), id(ret)]) + self.assertEqual(list(graphs[1]), [id(u) for u in ret.toposort()]) # rewrite graph keys follow the rewritten sink's toposort def test_const_reshape_expand_folded(self): # CONST->EXPAND should be folded into the ALU node, not shown as separate EXPAND nodes @@ -305,10 +304,10 @@ class TestVizTree(unittest.TestCase): def test_tree_view(self): with save_viz() as viz: - a = UOp.variable("a",0,10) - b = UOp.variable("b",0,10) - c = UOp.variable("c",0,10) - d = UOp.variable("d",0,10) + a = UOp.variable("a",0,10,param=True) + b = UOp.variable("b",0,10,param=True) + c = UOp.variable("c",0,10,param=True) + d = UOp.variable("d",0,10,param=True) sink = UOp.sink(a+b, c+d) def tree_rewrite(): return graph_rewrite(sink, root, name="root") tree_rewrite() @@ -347,7 +346,7 @@ class TestVizGC(unittest.TestCase): init = bufs_allocated() a = UOp.new_buffer("NULL", 10, dtypes.char) a.buffer.allocate() - exec_rewrite(UOp(Ops.CUSTOM, src=(a,), arg=a), [PatternMatcher([])]) + exec_rewrite(UOp(Ops.PYLITERAL, src=(a,), arg=a), [PatternMatcher([])]) del a self.assertEqual(bufs_allocated()-init, 0) lst = viz.list_items() @@ -474,7 +473,7 @@ class TestVizIntegration(unittest.TestCase): def custom_fn(X:UOp): X = X.flatten() i = UOp.range(X.numel(), 0) - custom_op = UOp(Ops.CUSTOMI, src=(X[i],), arg="{} + undeclared_name") + custom_op = UOp(Ops.CUSTOMI, src=(X[i],), arg=("{} + undeclared_name", X.dtype)) return X[i].store(custom_op).end(i).sink(arg=KernelInfo(name=f"custom_fn_{X.numel()}")) x = Tensor.custom_kernel(Tensor.empty(1, device="CPU"), fxn=custom_fn)[0] with save_viz() as viz: @@ -516,6 +515,22 @@ class TestVizIntegration(unittest.TestCase): src_render = get_render(viz.data, steps[src_idx]["query"])["src"] self.assertEqual(src, src_render) + def test_profiler_duplicate_name(self): + kernel_name = "duplicate_name" + def one(A:UOp): return A[0].store(UOp.const(1.0, dtypes.float)).sink(arg=KernelInfo(kernel_name)) + def zero(A:UOp): return A[0].store(UOp.const(0.0, dtypes.float)).sink(arg=KernelInfo(kernel_name)) + with save_viz() as viz: + @TinyJit + def f(a:Tensor, b:Tensor): return Tensor.custom_kernel(a, fxn=one)[0], Tensor.custom_kernel(b, fxn=zero)[0] + a, b = Tensor.empty(4, device="NULL"), Tensor.empty(4, device="NULL") + # warmup + for _ in range(2): Tensor.realize(*f(a, b)) + Tensor.realize(*f(a, b)) + kernels = {i for i,c in enumerate(viz.list_items()) if c["name"] == kernel_name} + profile = decode_profile(unwrap(get_profile(viz.data, cpu_events))) + events = [e for e in profile["layout"]["NULL"]["events"] if e["name"] == kernel_name] + self.assertEqual({e["ref"] for e in events}, kernels) + from tinygrad.device import ProfileDeviceEvent, ProfileGraphEvent, ProfileGraphEntry from tinygrad.viz.serve import get_profile from tinygrad.viz.cli import decode_profile @@ -819,17 +834,15 @@ from extra.gemm.amd_asm_matmul import Kernel @needs_tracked_pm class TestCfg(unittest.TestCase): - def setUp(self): self.arch = "gfx1100" - def get_cfg(self, name:str, k:Kernel): insts = k.finalize() def fxn(out:UOp) -> UOp: lidx = UOp.special(1, "lidx0") gidx = UOp.special(1, "gidx0") sink = UOp.sink(out.base, lidx, gidx, arg=KernelInfo(name=name)) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) with save_viz() as viz: - with Context(DEV=f"NULL::{self.arch}"): + with Context(DEV="NULL::gfx1100"): out = Tensor.custom_kernel(Tensor.empty(1), fxn=fxn)[0] _ = do_to_program(out.schedule_linear().src[-1].src[0], Device[out.device].renderer) codegen_rewrites = next(s for s in viz.list_items() if s["name"] == name) @@ -1011,8 +1024,8 @@ def run_cli(*cli_args) -> list[dict]: @contextlib.contextmanager def write_files(viz) -> list[str]: with tempfile.TemporaryDirectory() as tmpdir: - (r:=Path(tmpdir)/"rewrites.pkl").write_bytes(pickle.dumps(viz.data.trace)) - (p:=Path(tmpdir)/"profile.pkl").write_bytes(pickle.dumps(cpu_events)) + (r:=pathlib.Path(tmpdir)/"rewrites.pkl").write_bytes(pickle.dumps(viz.data.trace)) + (p:=pathlib.Path(tmpdir)/"profile.pkl").write_bytes(pickle.dumps(cpu_events)) yield ["--rewrites-path", str(r), "--profile-path", str(p)] class TestCLI(unittest.TestCase): diff --git a/tinygrad_repo/test/opt/test_gen_float4.py b/tinygrad_repo/test/opt/test_gen_float4.py index b946cb6bd..6306f5ef7 100644 --- a/tinygrad_repo/test/opt/test_gen_float4.py +++ b/tinygrad_repo/test/opt/test_gen_float4.py @@ -1,6 +1,6 @@ import unittest from tinygrad import Device, Tensor, Variable, dtypes -from tinygrad.uop.ops import UOp, Ops +from tinygrad.uop.ops import UOp, Ops, AxisType from tinygrad.codegen import to_program from tinygrad.codegen.opt import Opt, OptOps @@ -10,12 +10,12 @@ from test.helpers import replace_opts class TestFloat4(unittest.TestCase): @staticmethod def count_float4(uops: list[UOp], n=4): - return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype.scalar() == dtypes.float and uop.shape == (4,)]), - len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype.scalar() == dtypes.float and uop.shape == (4,)])) + return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype == dtypes.float and uop.shape == (4,)]), + len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype == dtypes.float and uop.shape == (4,)])) @staticmethod def count_half4(uops: list[UOp]): - return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype.scalar() == dtypes.half and uop.shape == (4,)]), - len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype.scalar() == dtypes.half and uop.shape == (4,)])) + return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype == dtypes.half and uop.shape == (4,)]), + len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype == dtypes.half and uop.shape == (4,)])) def test_float4_basic(self): a = Tensor.empty(2, 8).realize() @@ -24,7 +24,7 @@ class TestFloat4(unittest.TestCase): s = c.schedule_linear().src[0] realized_ast = s.src[0] - opts_to_apply = [Opt(op=OptOps.UPCAST, axis=0, arg=4)] + opts_to_apply = [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))] program = to_program(replace_opts(realized_ast, opts_to_apply), renderer=Device[Device.DEFAULT].renderer) assert TestFloat4.count_float4(tuple(program.src[1].src)) == (2, 1) @@ -35,7 +35,8 @@ class TestFloat4(unittest.TestCase): c = a + b s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=0, arg=2)]), + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=0, arg=(2, AxisType.UPCAST))]), renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (4, 2) @@ -46,7 +47,7 @@ class TestFloat4(unittest.TestCase): s = c.schedule_linear().src[0] realized_ast = s.src[0] - opts_to_apply = [Opt(op=OptOps.UPCAST, axis=0, arg=4)] + opts_to_apply = [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))] program = to_program(replace_opts(realized_ast, opts_to_apply), renderer=Device[Device.DEFAULT].renderer) assert TestFloat4.count_float4(tuple(program.src[1].src)) == (0, 1) @@ -57,7 +58,8 @@ class TestFloat4(unittest.TestCase): c = a + b s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=1, arg=4), Opt(op=OptOps.UPCAST, axis=1, arg=2)]), + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=1, arg=(4, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=1, arg=(2, AxisType.UPCAST))]), renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (0, 2) @@ -70,7 +72,8 @@ class TestFloat4(unittest.TestCase): # float4 should be emitted (the reduce axis of size 4 is the float4 axis here) s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UNROLL, axis=0, arg=4)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=1, arg=(4, AxisType.UNROLL))]), + renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (0, 0) @@ -84,7 +87,8 @@ class TestFloat4(unittest.TestCase): # UPDATE: now we do this fusion s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.UNROLL, axis=0, arg=0)]), + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(0, AxisType.UPCAST)), + Opt(op=OptOps.SPLIT, axis=1, arg=(0, AxisType.UNROLL))]), renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) in {(0,1), (1,1)} @@ -98,7 +102,8 @@ class TestFloat4(unittest.TestCase): # since the top axis is not contiguous. s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=4)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))]), + renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (0, 1) @@ -110,7 +115,8 @@ class TestFloat4(unittest.TestCase): # should float4 b but not a s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=4)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))]), + renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (1, 1) @@ -123,7 +129,8 @@ class TestFloat4(unittest.TestCase): # should float4 both s = c.linear_with_vars()[0].src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=4)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))]), + renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (2, 1) @@ -136,7 +143,8 @@ class TestFloat4(unittest.TestCase): # should float4 a but not b s = c.linear_with_vars()[0].src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=4)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.SPLIT, axis=0, arg=(4, AxisType.UPCAST))]), + renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (1, 1) diff --git a/tinygrad_repo/test/opt/test_kernel_opts.py b/tinygrad_repo/test/opt/test_kernel_opts.py index af90b6932..41c88bbb3 100644 --- a/tinygrad_repo/test/opt/test_kernel_opts.py +++ b/tinygrad_repo/test/opt/test_kernel_opts.py @@ -1,6 +1,8 @@ import unittest from tinygrad import Device, Tensor, dtypes +from tinygrad.helpers import Context from tinygrad.codegen.opt import Opt, OptOps, KernelOptError +from tinygrad.uop.ops import AxisType # TODO: write a clean version of this from test.backend.test_linearizer import helper_linearizer_opt @@ -15,23 +17,28 @@ class TestKernelOpts(unittest.TestCase): b = Tensor.rand(4, 4, N) r = (b.sqrt() + ((a+1).sum(axis=3).exp())) helper_linearizer_opt(r, [ - [Opt(OptOps.LOCAL, 0, 2)], - [Opt(OptOps.LOCAL, 0, 8)], - [Opt(OptOps.LOCAL, 0, 16)], # Checking how it works with locals - [Opt(OptOps.GROUPTOP, 0, 2)], - [Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.GROUPTOP, 0, 64)], # Checking how it works with grouped reduce - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 2)], - [Opt(OptOps.LOCAL, 0, 16), Opt(OptOps.GROUPTOP, 0, 16)], - [Opt(OptOps.LOCAL, 0, 32), Opt(OptOps.GROUPTOP, 0, 2)], + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL))], + [Opt(OptOps.SPLIT, 0, (8, AxisType.LOCAL))], + [Opt(OptOps.SPLIT, 0, (16, AxisType.LOCAL))], # Checking how it works with locals + [Opt(OptOps.SPLIT, 1, (2, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 1, (32, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 1, (64, AxisType.GROUP_REDUCE, True))], # Checking how it works with grouped reduce + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 0, (16, AxisType.LOCAL)), Opt(OptOps.SPLIT, 2, (16, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 0, (32, AxisType.LOCAL)), Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE, True))], # Checking how it works with locals + grouped reduce - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 64)], + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 2, (64, AxisType.GROUP_REDUCE, True))], # Checking how it works with locals + grouped reduce + upcasts - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.UPCAST, 0, 8), Opt(OptOps.UNROLL, 1, 4)], + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE, True)), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST)), + Opt(OptOps.SPLIT, 4, (4, AxisType.UNROLL))], # many local + many group - [Opt(OptOps.GROUP, 0, 2)] * 4, - [Opt(OptOps.LOCAL, 0, 2)] * 4, - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUP, 0, 2)] * 4, + [Opt(OptOps.SPLIT, 1, (2, AxisType.GROUP_REDUCE)), Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE)), + Opt(OptOps.SPLIT, 3, (2, AxisType.GROUP_REDUCE)), Opt(OptOps.SPLIT, 4, (2, AxisType.GROUP_REDUCE))], + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL))] * 4, + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE)), + Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (2, AxisType.GROUP_REDUCE)), + Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 6, (2, AxisType.GROUP_REDUCE)), + Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 8, (2, AxisType.GROUP_REDUCE))], ]) def test_upcasts(self): @@ -41,9 +48,9 @@ class TestKernelOpts(unittest.TestCase): b = Tensor.rand(N, N) r = (a+b).sqrt() * ((a+1).exp()) helper_linearizer_opt(r, [ - [Opt(OptOps.UPCAST, 0, 2)], - [Opt(OptOps.UPCAST, 0, 4)], - [Opt(OptOps.UPCAST, 0, 8)], # Checking how it works with upcasts + [Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST))], # Checking how it works with upcasts ]) def test_full_upcast(self): @@ -52,7 +59,7 @@ class TestKernelOpts(unittest.TestCase): b = Tensor.rand(4) r = (a+b).sqrt() * ((a+1).exp()) helper_linearizer_opt(r, [ - [Opt(OptOps.UPCAST, 0, 4)], # Checking how it works with upcasts + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))], # Checking how it works with upcasts ]) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") @@ -64,24 +71,28 @@ class TestKernelOpts(unittest.TestCase): b = Tensor.rand(N, N) r = a@b helper_linearizer_opt(r, [ - [Opt(OptOps.UPCAST, 0, 2)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4)], # Checking how it works with upcasts - [Opt(OptOps.LOCAL, 0, 2)], - [Opt(OptOps.LOCAL, 1, 32)], - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4)], - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 32)], - [Opt(OptOps.LOCAL, 0, 16), Opt(OptOps.LOCAL, 1, 8)], # Checking how it works with locals - [Opt(OptOps.GROUPTOP, 0, 2)], - [Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.GROUPTOP, 0, 32), Opt(OptOps.UNROLL, 0, 4)], # Checking how it works with grouped_reduce - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 0, 4)], # Checking how it works with local+grouped_reduce + [Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST))], # Checking how it works with upcasts + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL))], + [Opt(OptOps.SPLIT, 1, (32, AxisType.LOCAL))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (4, AxisType.LOCAL))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (32, AxisType.LOCAL))], + [Opt(OptOps.SPLIT, 0, (16, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (8, AxisType.LOCAL))], # Checking how it works with locals + [Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 2, (32, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 2, (32, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 2, (4, AxisType.UNROLL))], # Checking how it works with grouped_reduce + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (32, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 0, (8, AxisType.LOCAL)), Opt(OptOps.SPLIT, 3, (32, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 0, (8, AxisType.LOCAL)), + Opt(OptOps.SPLIT, 4, (4, AxisType.GROUP_REDUCE, True))], # Checking how it works with local+grouped_reduce # Checking all together - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 4), - Opt(OptOps.UPCAST, 1, 2)], + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (8, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 4, (4, AxisType.UNROLL)), Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), + Opt(OptOps.SPLIT, 1, (2, AxisType.UPCAST))], # Full global upcast + local - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 8)], + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (8, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 4, (4, AxisType.UNROLL)), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST))], ]) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") @@ -93,21 +104,34 @@ class TestKernelOpts(unittest.TestCase): r = a.sum(axis=(1,3)) helper_linearizer_opt(r, [ # openCL / DEV=CL is 256 max threads - [Opt(OptOps.GROUPTOP, 0, 2)], [Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.GROUPTOP, 1, 2)], [Opt(OptOps.GROUPTOP, 1, 32)], # Checking how it works with 1 grouped_reduce. - [Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 2)], - [Opt(OptOps.GROUPTOP, 0, 16), Opt(OptOps.GROUPTOP, 1, 2)], - [Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 64)], # Checking how it works with 2 grouped_reduces. - [Opt(OptOps.GROUPTOP, 0, 16), Opt(OptOps.GROUPTOP, 1, 2), Opt(OptOps.UNROLL, 0, 4)], - [Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UNROLL, 2, 4)], # Checking how it works with 2 grouped_reduces + upcasts. - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 4)], + [Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE, True))], [Opt(OptOps.SPLIT, 2, (32, AxisType.GROUP_REDUCE, True))], + # Checking how it works with 1 grouped_reduce. + [Opt(OptOps.SPLIT, 3, (2, AxisType.GROUP_REDUCE, True))], [Opt(OptOps.SPLIT, 3, (32, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE, True)), Opt(OptOps.SPLIT, 4, (2, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 2, (16, AxisType.GROUP_REDUCE, True)), Opt(OptOps.SPLIT, 4, (2, AxisType.GROUP_REDUCE, True))], + [Opt(OptOps.SPLIT, 2, (4, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 4, (64, AxisType.GROUP_REDUCE, True))], # Checking how it works with 2 grouped_reduces. + [Opt(OptOps.SPLIT, 2, (16, AxisType.GROUP_REDUCE, True)), Opt(OptOps.SPLIT, 4, (2, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 2, (4, AxisType.UNROLL))], + # Checking how it works with 2 grouped_reduces + upcasts. + [Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE, True)), Opt(OptOps.SPLIT, 4, (32, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 4, (4, AxisType.UNROLL))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (4, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 6, (4, AxisType.GROUP_REDUCE, True))], # Checking how it works with 2 grouped_reduces + upcasts + locals. - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UNROLL, 1, 4)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2), - Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4)], # Checking how it works with 2 grouped_reduces + upcasts + locals. - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2), - Opt(OptOps.UPCAST, 0, 2)], # No globals + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (2, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 6, (32, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 5, (4, AxisType.UNROLL))], + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (8, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 6, (4, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (8, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 6, (4, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST)), Opt(OptOps.SPLIT, 4, (4, AxisType.UNROLL)), + Opt(OptOps.SPLIT, 5, (4, AxisType.UNROLL))], # Checking how it works with 2 grouped_reduces + upcasts + locals. + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (4, AxisType.LOCAL)), Opt(OptOps.SPLIT, 4, (4, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 6, (4, AxisType.GROUP_REDUCE, True)), + Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST)), Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST))], # No globals ]) @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") @@ -121,17 +145,18 @@ class TestKernelOpts(unittest.TestCase): atol, rtol = 0.25, 0.01 helper_linearizer_opt(r, [ [], - [Opt(OptOps.UPCAST, 0, 4)], - [Opt(OptOps.UPCAST, 1, 4)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4)], # check upcasts - [Opt(OptOps.UNROLL, 0, 2)], # check unroll - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 2)], # check combo of unroll and local - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 2)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4)], - [Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 0, 4)], # check permutations - [Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 0, 4)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 1, 4)], - [Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 4)], + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST))], # check upcasts + [Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL))], # check unroll + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 5, (2, AxisType.UNROLL))], # check combo of unroll and upcast + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 6, (2, AxisType.UNROLL))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 6, (4, AxisType.UNROLL))], + [Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))], # check permutations + [Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL)), Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 5, (2, AxisType.UNROLL)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), + Opt(OptOps.SPLIT, 6, (4, AxisType.UNROLL))], ], apply_tc=True, atol=atol, rtol=rtol) @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") @@ -145,34 +170,12 @@ class TestKernelOpts(unittest.TestCase): r = a.matmul(b, dtype=dtypes.half) atol, rtol = 0.25, 0.01 helper_linearizer_opt(r, [ - [Opt(OptOps.UNROLL, 0, 0)], # check full unroll of reduce with locals - [Opt(OptOps.LOCAL, 0, 4)], # check local - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.LOCAL, 0, 2)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 0, 4)], - ], apply_tc=True, atol=atol, rtol=rtol) - - @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_shared, "test requires shared memory") - @unittest.skipUnless(any(tc.dtype_in == tc.dtype_out == dtypes.half for tc in Device[Device.DEFAULT].renderer.tensor_cores), - "test requires tensor cores with accumulation in half") # testing with half suffices. - # NOTE: the METAL test is broken, likely due to a compiler bug. passes on CI with -O0 and with default opt level locally on M3 - @unittest.skipIf(Device.DEFAULT == "METAL", "broken for METAL") - @unittest.skip("feature was removed") - def test_tensor_core_opts_group(self): - N = 128 - Tensor.manual_seed(1552) - a, b = Tensor.rand(N, N, dtype=dtypes.half), Tensor.rand(N, N, dtype=dtypes.half) - r = a.matmul(b, dtype=dtypes.half) - atol, rtol = 0.25, 0.01 - helper_linearizer_opt(r, [ - [Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.GROUPTOP, 0, 4)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 1, 2)], + [Opt(OptOps.SPLIT, 4, (0, AxisType.UNROLL))], # check full unroll of reduce with locals + [Opt(OptOps.SPLIT, 0, (4, AxisType.LOCAL))], # check local + [Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 6, (4, AxisType.UNROLL)), + Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL))], + [Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)), Opt(OptOps.SPLIT, 1, (4, AxisType.UPCAST)), Opt(OptOps.SPLIT, 6, (2, AxisType.UNROLL)), + Opt(OptOps.SPLIT, 0, (4, AxisType.UPCAST))], ], apply_tc=True, atol=atol, rtol=rtol) def test_padto_matmul(self): @@ -187,7 +190,7 @@ class TestKernelOpts(unittest.TestCase): [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.PADTO, 1, 32)], [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.PADTO, 1, 32), Opt(OptOps.PADTO, 2, 32)], # can optimize further post PADTO - [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.PADTO, 1, 32), Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.UPCAST, 1, 2),], + [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.PADTO, 1, 32), Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST)), Opt(OptOps.SPLIT, 1, (2, AxisType.UPCAST)),], ]) def test_padto_upcasted_not_ok(self): @@ -195,19 +198,19 @@ class TestKernelOpts(unittest.TestCase): a = Tensor.rand(N, N) b = Tensor.rand(N, N) helper_linearizer_opt(a@b, [ - [Opt(OptOps.UPCAST, 0, 0)], - [Opt(OptOps.UPCAST, 1, 0)], - [Opt(OptOps.UNROLL, 0, 0)], + [Opt(OptOps.SPLIT, 0, (0, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 1, (0, AxisType.UPCAST))], + [Opt(OptOps.SPLIT, 2, (0, AxisType.UNROLL))], [Opt(OptOps.PADTO, 0, 8)], [Opt(OptOps.PADTO, 1, 8)], [Opt(OptOps.PADTO, 2, 8)], ]) with self.assertRaises(KernelOptError): - helper_linearizer_opt(a@b, [[Opt(OptOps.UPCAST, 0, 0), Opt(OptOps.PADTO, 1, 8)]]) + helper_linearizer_opt(a@b, [[Opt(OptOps.SPLIT, 0, (0, AxisType.UPCAST)), Opt(OptOps.PADTO, 1, 8)]]) with self.assertRaises(KernelOptError): - helper_linearizer_opt(a@b, [[Opt(OptOps.UPCAST, 1, 0), Opt(OptOps.PADTO, 1, 8)]]) + helper_linearizer_opt(a@b, [[Opt(OptOps.SPLIT, 1, (0, AxisType.UPCAST)), Opt(OptOps.PADTO, 1, 8)]]) with self.assertRaises(KernelOptError): - helper_linearizer_opt(a@b, [[Opt(OptOps.UNROLL, 0, 0), Opt(OptOps.PADTO, 2, 8)]]) + helper_linearizer_opt(a@b, [[Opt(OptOps.SPLIT, 2, (0, AxisType.UNROLL)), Opt(OptOps.PADTO, 2, 8)]]) def test_padto_sum_ok(self): N = 18 @@ -217,11 +220,11 @@ class TestKernelOpts(unittest.TestCase): helper_linearizer_opt(a.sum(0), [ [Opt(OptOps.PADTO, 0, 32)], - [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.UPCAST, 0, 8),], + [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST)),], ]) helper_linearizer_opt(a.sum(1), [ [Opt(OptOps.PADTO, 0, 32)], - [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.UPCAST, 0, 8),], + [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST)),], ]) for axis in (0, 1): @@ -239,6 +242,72 @@ class TestKernelOpts(unittest.TestCase): helper_linearizer_opt(a.sum().exp(), [[Opt(OptOps.PADTO, 0, 32)],]) helper_linearizer_opt(a.sum(0).exp(), [[Opt(OptOps.PADTO, 1, 32)],]) + @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") + @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_shared, "test requires shared") + def test_padto_group_full_unroll_sum(self): + a = Tensor.ones(2, 28, 4096).realize() + out = ((a * 0.5).float().square()).sum(axis=(0, 2)) + opts_to_apply = [Opt(OptOps.SPLIT, 2, (256, AxisType.GROUP_REDUCE, True)), Opt(OptOps.PADTO, 3, 32), Opt(OptOps.SPLIT, 3, (0, AxisType.UNROLL)), + Opt(OptOps.SPLIT, 0, (7, AxisType.UPCAST))] + helper_linearizer_opt(out, [opts_to_apply], check_default_opt=False) + + def test_padto_unrolled_sum(self): + a = Tensor.arange(4*17, dtype=dtypes.float).reshape(4, 17).clone().realize() + for amt in (4, 0): + helper_linearizer_opt(a.sum(1), [[Opt(OptOps.PADTO, 1, 32), Opt(OptOps.SPLIT, 1, (amt, AxisType.UNROLL))]]) + + def test_padto_unrolled_max(self): + a = (Tensor.arange(4*17, dtype=dtypes.float).reshape(4, 17) - 100).clone().realize() + for amt in (4, 0): + helper_linearizer_opt(a.max(1), [[Opt(OptOps.PADTO, 1, 32), Opt(OptOps.SPLIT, 1, (amt, AxisType.UNROLL))]]) + + def test_padto_unrolled_upcast(self): + a = Tensor.arange(4*17, dtype=dtypes.float).reshape(4, 17).clone().realize() + helper_linearizer_opt(a.sum(1), [[Opt(OptOps.PADTO, 1, 32), Opt(OptOps.SPLIT, 1, (0, AxisType.UNROLL)), + Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST))]]) + + @unittest.skipUnless(any(tc.dtype_in in (dtypes.half, dtypes.float) for tc in Device[Device.DEFAULT].renderer.tensor_cores), + "test requires half or float tensor cores") + def test_tc_shape_padded(self): + tc = next(tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in in (dtypes.half, dtypes.float)) + Tensor.manual_seed(3) + a, b = Tensor.rand(17, 23, dtype=tc.dtype_in).realize(), Tensor.rand(23, 29, dtype=tc.dtype_in).realize() + with Context(ALLOW_TF32=1): + helper_linearizer_opt(a.matmul(b, dtype=tc.dtype_out), [[Opt(OptOps.TC, 0, (-1, 2, 2))]], check_default_opt=False, atol=3e-2, rtol=1e-3) + + @unittest.skipUnless(any(tc.dtype_in in (dtypes.half, dtypes.float) for tc in Device[Device.DEFAULT].renderer.tensor_cores), + "test requires half or float tensor cores") + def test_tc_padto_full_upcast(self): + # a fully upcast pad lane makes a WMMA operand entirely Invalid + tc = next(tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in in (dtypes.half, dtypes.float)) + Tensor.manual_seed(3) + a, b = Tensor.rand(17, 23, dtype=tc.dtype_in).realize(), Tensor.rand(23, 29, dtype=tc.dtype_in).realize() + with Context(ALLOW_TF32=1): + helper_linearizer_opt(a.matmul(b, dtype=tc.dtype_out), + [[Opt(OptOps.TC, 0, (-1, 2, 1)), Opt(OptOps.PADTO, 0, 4), Opt(OptOps.SPLIT, 0, (0, AxisType.UPCAST))]], + check_default_opt=False, atol=3e-2, rtol=1e-3) + + def test_padto_nested_reduce(self): + a = (Tensor.arange(2*3, dtype=dtypes.float).reshape(2, 3) + 1).clone().realize() # [[1, 2, 3], [4, 5, 6]] + # the pad gate has the outer reduce's range, the inner reduce must not resolve it with its own identity + pad_outer = [[Opt(OptOps.PADTO, 1, 4)]] + helper_linearizer_opt(a.max(1).sum(0), pad_outer, wanna_output=[[3+6]]) + helper_linearizer_opt((-a).sum(1).max(0), pad_outer, wanna_output=[[-6]]) + helper_linearizer_opt(a.prod(1).sum(0), pad_outer, wanna_output=[[6+120]]) + # both reduce axes padded: the outer clause lifts out, the inner clause is the inner reduce's identity + helper_linearizer_opt(a.max(1).sum(0), [[Opt(OptOps.PADTO, 0, 4), Opt(OptOps.PADTO, 1, 4)]], wanna_output=[[3+6]]) + + def test_padto_unrolled_prod(self): + a = (Tensor.arange(4*17, dtype=dtypes.float).reshape(4, 17) / 100 + 1).clone().realize() + helper_linearizer_opt(a.prod(1), [[Opt(OptOps.PADTO, 1, 32), Opt(OptOps.SPLIT, 1, (0, AxisType.UNROLL)), + Opt(OptOps.SPLIT, 0, (2, AxisType.UPCAST))]]) + + def test_padto_arg(self): + a = Tensor.arange(4*17, dtype=dtypes.float).reshape(4, 17).clone().realize() + for arg in (-4, 0, 1, True): + with self.assertRaises(KernelOptError): + helper_linearizer_opt(a.sum(1), [[Opt(OptOps.PADTO, 1, arg)]]) + def test_padto_sum(self): N = 18 # NOTE: this setup prevents 17 * 17 contiguous merged into one dimension @@ -257,11 +326,11 @@ class TestKernelOpts(unittest.TestCase): helper_linearizer_opt(a.max(0), [ [Opt(OptOps.PADTO, 0, 32)], - [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.UPCAST, 0, 8),], + [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST)),], ]) helper_linearizer_opt(a.max(1), [ [Opt(OptOps.PADTO, 0, 32)], - [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.UPCAST, 0, 8),], + [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST)),], ]) helper_linearizer_opt(a.max(), [[Opt(OptOps.PADTO, 0, 32)],]) @@ -273,7 +342,7 @@ class TestKernelOpts(unittest.TestCase): a = (Tensor.randn(N, N).realize().max(axis=0, keepdim=True) > 1).where(1, 0).int() helper_linearizer_opt(a.max(0), [ [Opt(OptOps.PADTO, 0, 32)], - [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.UPCAST, 0, 8),], + [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST)),], ]) def test_padto_where_multioutput(self): @@ -284,7 +353,7 @@ class TestKernelOpts(unittest.TestCase): a1 = r.where(2, 0).int() helper_linearizer_opt([a0.max(0), a1.max(0)], [ [Opt(OptOps.PADTO, 0, 32)], - [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.UPCAST, 0, 8),], + [Opt(OptOps.PADTO, 0, 32), Opt(OptOps.SPLIT, 0, (8, AxisType.UPCAST)),], ]) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") @@ -296,16 +365,21 @@ class TestKernelOpts(unittest.TestCase): b = Tensor.rand(N, N) r = a@b opts_shapes = [ - ([Opt(OptOps.LOCAL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("red",32)]), - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 2)], [("blue",16),("blue",32),("cyan",2),("green",2),("red",16)]), + ([Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL))], [("blue",16),("blue",32),("cyan",2),("red",32)]), + ([Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)),Opt(OptOps.SPLIT, 3, (2, AxisType.GROUP_REDUCE))], + [("blue",16),("blue",32),("cyan",2),("green",2),("red",16)]), # check to ensure local_dims are stable for full UNROLL of the first reduce - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UNROLL, 0, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.UNROLL, 0, 0),Opt(OptOps.LOCAL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)),Opt(OptOps.SPLIT, 3, (0, AxisType.UNROLL))], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.SPLIT, 2, (0, AxisType.UNROLL)),Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL))], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), # check behavior for full UNROLL on an existing GROUP - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UNROLL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("green",16),("magenta",2)]), - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UNROLL, 0, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.GROUP, 0, 0),Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UNROLL, 0, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.GROUP, 0, 2),Opt(OptOps.UNROLL, 0, 0)], [("blue",32),("blue",32),("red",16),("magenta",2)]), + ([Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)),Opt(OptOps.SPLIT, 3, (0, AxisType.GROUP_REDUCE)),Opt(OptOps.SPLIT, 3, (2, AxisType.UNROLL))], + [("blue",16),("blue",32),("cyan",2),("green",16),("magenta",2)]), + ([Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)),Opt(OptOps.SPLIT, 3, (0, AxisType.GROUP_REDUCE)),Opt(OptOps.SPLIT, 3, (0, AxisType.UNROLL))], + [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.SPLIT, 2, (0, AxisType.GROUP_REDUCE)),Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL)),Opt(OptOps.SPLIT, 2, (0, AxisType.UNROLL))], + [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.SPLIT, 2, (2, AxisType.GROUP_REDUCE)),Opt(OptOps.SPLIT, 2, (0, AxisType.UNROLL))], + [("blue",32),("blue",32),("red",16),("magenta",2)]), ] helper_linearizer_opt(r, [x[0] for x in opts_shapes], color_sizes=[x[1] for x in opts_shapes]) @@ -316,39 +390,21 @@ class TestKernelOpts(unittest.TestCase): a = Tensor.arange(128).clone() # NOTE: arange no longer has reduce ops available for opt helper_linearizer_opt(a, [ - #[Opt(OptOps.GROUP, 0, 32)], - #[Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(op=OptOps.LOCAL, axis=0, arg=8)], - [Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0)], - #[Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.GROUP, axis=0, arg=8)], - #[Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.GROUP, axis=0, arg=8), Opt(op=OptOps.UNROLL, axis=1, arg=4)], # noqa: E501 + [Opt(op=OptOps.SPLIT, axis=0, arg=(8, AxisType.LOCAL))], + [Opt(op=OptOps.SPLIT, axis=0, arg=(8, AxisType.LOCAL)), Opt(op=OptOps.SPLIT, axis=0, arg=(0, AxisType.UPCAST))], ]) - @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_threads, "test requires threads") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.global_max is not None and - Device[Device.DEFAULT].renderer.global_max[0] > 1, "test requires multicore") - def test_thread_opts(self): - a = Tensor.rand(4, 4, 4, 4) - b = Tensor.rand(4, 4, 4) - r = (b.sqrt() + ((a+1).sum(axis=3).exp())) - helper_linearizer_opt(r, [ - [Opt(OptOps.THREAD, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.THREAD, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.THREAD, 0, 2), Opt(OptOps.UNROLL, 0, 2)], - ] + [[Opt(OptOps.THREAD, 0, 4)] if Device[Device.DEFAULT].renderer.global_max[0] >= 4 else []] - + [[Opt(OptOps.THREAD, 0, 8)] if Device[Device.DEFAULT].renderer.global_max[0] >= 8 else []]) - def test_double_sum_group(self): a = Tensor.rand(4, 4, 4) r = a.sum((1, 2)).sum() with self.assertRaises(KernelOptError): - helper_linearizer_opt(r, [[Opt(OptOps.GROUPTOP, 0, 16)],]) + helper_linearizer_opt(r, [[Opt(OptOps.SPLIT, 0, (16, AxisType.GROUP_REDUCE, True))],]) r = a.sum((1, 2)).sum() with self.assertRaises(KernelOptError): - helper_linearizer_opt(r, [[Opt(OptOps.UNROLL, 1, 4), Opt(OptOps.GROUPTOP, 0, 16)],]) + helper_linearizer_opt(r, [[Opt(OptOps.SPLIT, 1, (4, AxisType.UNROLL)), Opt(OptOps.SPLIT, 0, (16, AxisType.GROUP_REDUCE, True))],]) r = a.sum((1, 2)).sum() with self.assertRaises(KernelOptError): - helper_linearizer_opt(r, [[Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.GROUPTOP, 0, 16)],]) + helper_linearizer_opt(r, [[Opt(OptOps.SPLIT, 1, (4, AxisType.GROUP_REDUCE, True)), Opt(OptOps.SPLIT, 1, (16, AxisType.GROUP_REDUCE, True))],]) if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/opt/test_tensor_cores.py b/tinygrad_repo/test/opt/test_tensor_cores.py index c25eec469..ed0a57cab 100644 --- a/tinygrad_repo/test/opt/test_tensor_cores.py +++ b/tinygrad_repo/test/opt/test_tensor_cores.py @@ -3,7 +3,7 @@ import unittest from tinygrad import Device, Tensor, dtypes from tinygrad.tensor import _to_np_dtype -from tinygrad.uop.ops import Ops, UOp, buffers +from tinygrad.uop.ops import Ops, UOp, AxisType, buffers from tinygrad.dtype import DType from tinygrad.device import Buffer from tinygrad.helpers import DEV, Context @@ -11,7 +11,8 @@ from test.helpers import slow, replace_opts from tinygrad.engine.realize import run_linear from tinygrad.codegen import to_program from tinygrad.codegen.opt import Opt, OptOps, KernelOptError -from tinygrad.codegen.opt.tc import amd_cdna_1616128 +from tinygrad.codegen.opt.postrange import Scheduler +from tinygrad.renderer.tc import amd_cdna_1616128 # TODO: write a clean version of this from test.backend.test_linearizer import helper_realized_ast, helper_linearizer_opt @@ -52,14 +53,15 @@ def helper_tc_ensure_uops_and_opts_count(N: int, M:int, K:int, dtype_in:DType, d assert False, "OptOps.TC triggered, expected KernelOptError" except KernelOptError: pass -def helper_tc_allclose(N:int, M:int, K:int, dtype_in:DType, dtype_out:DType, axis:int=0, tc_select:int=-1, tc_opt:int=0, use_tensor_cores:int=1): +def helper_tc_allclose(N:int, M:int, K:int, dtype_in:DType, dtype_out:DType, axis:int=0, tc_select:int=-1, tc_opt:int=0, use_tensor_cores:int=1, + extra_opts:list[Opt]=[]): _skip_unsupported_tc_dtypes(dtype_in, dtype_out) a, b = _tc_rand(M, K, dtype=dtype_in), _tc_rand(K, N, dtype=dtype_in) np_a, np_b = a.numpy(), b.numpy() r = a.matmul(b, dtype=dtype_out) if dtype_in == dtypes.bfloat16: r = r.float() realized_ast, bufs = helper_realized_ast(r) - opts = [Opt(op=OptOps.TC, axis=axis, arg=(tc_select, tc_opt, use_tensor_cores))] + opts = [Opt(op=OptOps.TC, axis=axis, arg=(tc_select, tc_opt, use_tensor_cores))] + extra_opts ast = replace_opts(realized_ast, opts) pu = to_program(ast, Device[Device.DEFAULT].renderer) if use_tensor_cores == 1: assert len([uop for uop in pu.src[1].src if uop.op is Ops.WMMA]) > 0, "wmma not triggered" @@ -79,7 +81,34 @@ class TestTensorCores(unittest.TestCase): @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") def test_tensor_cores(self): for tc in Device[Device.DEFAULT].renderer.tensor_cores: - helper_tc_allclose(tc.dims[0], tc.dims[1], tc.dims[2], tc.dtype_in, tc.dtype_out, axis=0, tc_opt=0) + with self.subTest(tc=tc): + helper_tc_allclose(tc.dims[0], tc.dims[1], tc.dims[2], tc.dtype_in, tc.dtype_out, axis=0, tc_opt=0) + + @Context(ALLOW_TF32=1) + @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") + def test_tensor_cores_extra_locals(self): + # LOCAL splits after the TC opt: the WARP must keep a whole hardware local dim, its lanes are consecutive threads + for tc in Device[Device.DEFAULT].renderer.tensor_cores: + with self.subTest(tc=tc): + helper_tc_allclose(tc.dims[0]*8, tc.dims[1]*8, tc.dims[2], tc.dtype_in, tc.dtype_out, + extra_opts=[Opt(OptOps.SPLIT, 0, (2, AxisType.LOCAL))]*3) + + @Context(ALLOW_TF32=1) + @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") + def test_tensor_cores_group_reduce(self): + tc = next(tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in not in dtypes.fp8s) + sche = Scheduler(Tensor.empty(16, 64, dtype=tc.dtype_in).matmul(Tensor.empty(64, 16, dtype=tc.dtype_in), dtype=tc.dtype_out) + .schedule_linear().src[-1].src[0], Device[Device.DEFAULT].renderer) + sche.apply_opt(Opt(OptOps.TC, 0, (-1, 0, 1))) + axis = sche.axis_types.index(AxisType.REDUCE) + if AxisType.UNROLL in sche.axis_types: + # this tc keeps an unrolled reduce outside the WMMA, grouping inside it must be rejected + with self.assertRaises(KernelOptError): sche.apply_opt(Opt(OptOps.SPLIT, axis, (2, AxisType.GROUP_REDUCE))) + else: + x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) + helper_linearizer_opt(x.matmul(y, dtype=tc.dtype_out), + [[Opt(OptOps.SPLIT, axis, (amt, AxisType.GROUP_REDUCE, top))] for amt in (2, 4) for top in (False, True)], + apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") def test_tensor_cores_nested_reduce(self): @@ -115,16 +144,6 @@ class TestTensorCores(unittest.TestCase): for tc in Device[Device.DEFAULT].renderer.tensor_cores: helper_tc_allclose(tc.dims[0]+(pad:=1), tc.dims[1]+pad, tc.dims[2]+pad, tc.dtype_in, tc.dtype_out, tc_opt=2) - # AMD compiler bug: AMD miscompiles non-zero padded tc kernels with -O3, producing wrong results, nans or hang (see #9606) - # Internal bug: zero-stride dimensions combined with a mask may produce wrong index/valid for pad == 1 on AMD - @unittest.skipUnless((Device.DEFAULT == "AMD") or (Device.DEFAULT == "PYTHON" and Device.default.renderer.target.device == "AMD"), - "test for AMD's tc") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") - @unittest.skip("warp elements not duplicated properly across lanes") - def test_tensor_cores_padded_amd(self): - for tc in Device[Device.DEFAULT].renderer.tensor_cores: - helper_tc_allclose(tc.dims[0]+(pad:=1), tc.dims[1]+pad, tc.dims[2]+pad, tc.dtype_in, tc.dtype_out, tc_opt=2) - @Context(ALLOW_TF32=1) @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") def test_tensor_cores_padded_uops(self): @@ -185,13 +204,13 @@ class TestTensorCores(unittest.TestCase): # skip fp8 tcs: the unoptimized ALU baseline quantizes products to fp8 (JAX promotion), which legitimately # differs from the MFMA path (f32 accumulation), so the baseline-vs-TC numerical gate can't hold for fp8. tc = next(tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in not in dtypes.fp8s) - x, y = Tensor.rand(64, 64, dtype=tc.dtype_in), Tensor.rand(64, 64, dtype=tc.dtype_in) + x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) - opts = [Opt(OptOps.UNROLL, 0, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3) - for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): - if u.op is Ops.WMMA: - assert u.src[-1].src[0].op != Ops.STORE + opts = [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL))] + ast = helper_linearizer_opt(r, [opts[1:]], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) + wmmas = [u for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src) if u.op is Ops.WMMA] + self.assertGreater(len(wmmas), 0) + for u in wmmas: assert u.src[-1].src[0].op != Ops.STORE @Context(ALLOW_TF32=1) @unittest.skipIf(Device.DEFAULT == "PYTHON", "slow on EMULATED device") @@ -199,14 +218,13 @@ class TestTensorCores(unittest.TestCase): @unittest.skipIf(Device.DEFAULT in {"CPU"}, "CPU does not support using a different type for accumulation") def test_tensor_cores_unroll_casted_phi(self): tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] - x, y = Tensor.rand(64, 64, dtype=tc.dtype_in), Tensor.rand(64, 64, dtype=tc.dtype_in) + x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) - opts = [Opt(OptOps.UNROLL, 0, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3) - for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): - if u.op is Ops.WMMA: - #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) - assert u.src[-1].src[0].op != Ops.STORE + opts = [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL))] + ast = helper_linearizer_opt(r, [opts[1:]], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) + wmmas = [u for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src) if u.op is Ops.WMMA] + self.assertGreater(len(wmmas), 0) + for u in wmmas: assert u.src[-1].src[0].op != Ops.STORE @Context(ALLOW_TF32=1) @unittest.skipIf(Device.DEFAULT == "PYTHON", "slow on EMULATED device") @@ -215,14 +233,13 @@ class TestTensorCores(unittest.TestCase): def test_tensor_cores_unroll_casted_phi_with_children(self): # all STORE children are outside the loop tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] - x, y = Tensor.rand(64, 64, dtype=tc.dtype_in), Tensor.rand(64, 64, dtype=tc.dtype_in) + x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out).relu() - opts = [Opt(OptOps.UNROLL, 0, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3) - for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): - if u.op is Ops.WMMA: - #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) - assert u.src[-1].src[0].op != Ops.STORE + opts = [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.SPLIT, 4, (2, AxisType.UNROLL))] + ast = helper_linearizer_opt(r, [opts[1:]], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) + wmmas = [u for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src) if u.op is Ops.WMMA] + self.assertGreater(len(wmmas), 0) + for u in wmmas: assert u.src[-1].src[0].op != Ops.STORE if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/testextra/test_bench_log.py b/tinygrad_repo/test/testextra/test_bench_log.py index 975bef2fe..4067985a0 100644 --- a/tinygrad_repo/test/testextra/test_bench_log.py +++ b/tinygrad_repo/test/testextra/test_bench_log.py @@ -21,7 +21,7 @@ class TestBenchLog(unittest.TestCase): # check event list for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 1) - self.assertGreater(_events[event]["wall"][0], 0) + self.assertGreater(_events[event]["wall"][0][0], 0) def test_log_double_wall_time(self): for event in BenchEvent: @@ -35,8 +35,8 @@ class TestBenchLog(unittest.TestCase): # check event list for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 2) - self.assertGreater(_events[event]["wall"][0], 0) - self.assertGreater(_events[event]["wall"][1], 0) + self.assertGreater(_events[event]["wall"][0][0], 0) + self.assertGreater(_events[event]["wall"][1][0], 0) @skipIf(_SKIP_KERNEL_TIMING, "ci timing is not accurate") def test_log_single_kernel_time(self): @@ -52,8 +52,8 @@ class TestBenchLog(unittest.TestCase): # check event list for event in BenchEvent: self.assertEqual(len(_events[event]["kernel"]), 1) - self.assertLess(_events[event]["kernel"][0], wall_times[0]) - self.assertGreater(_events[event]["kernel"][0], 0) + self.assertLess(_events[event]["kernel"][0][0], wall_times[0]) + self.assertGreater(_events[event]["kernel"][0][0], 0) @skipIf(_SKIP_KERNEL_TIMING, "ci cuda timing is not accurate") def test_interleaved_wall_kernel_time(self): @@ -74,8 +74,8 @@ class TestBenchLog(unittest.TestCase): for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 1) self.assertEqual(len(_events[event]["kernel"]), 1) - self.assertLess(_events[event]["kernel"][0], wall_times[0]) - self.assertGreater(_events[event]["kernel"][0], 0) + self.assertLess(_events[event]["kernel"][0][0], wall_times[0]) + self.assertGreater(_events[event]["kernel"][0][0], 0) @skipIf(_SKIP_KERNEL_TIMING, "ci cuda timing is not accurate") def test_stacked_wall_kernel_time(self): @@ -93,10 +93,10 @@ class TestBenchLog(unittest.TestCase): for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 2) self.assertEqual(len(_events[event]["kernel"]), 2) - self.assertLess(_events[event]["kernel"][0], _events[event]["wall"][0]) - self.assertGreater(_events[event]["kernel"][0], 0) - self.assertLess(_events[event]["kernel"][1], _events[event]["wall"][1]) - self.assertGreater(_events[event]["kernel"][1], 0) + self.assertLess(_events[event]["kernel"][0][0], _events[event]["wall"][0][0]) + self.assertGreater(_events[event]["kernel"][0][0], 0) + self.assertLess(_events[event]["kernel"][1][0], _events[event]["wall"][1][0]) + self.assertGreater(_events[event]["kernel"][1][0], 0) def test_log_instant_event(self): for event in InstantBenchEvent: @@ -105,7 +105,7 @@ class TestBenchLog(unittest.TestCase): # check event list for event in InstantBenchEvent: self.assertEqual(len(_events[event]), 1) - self.assertEqual(_events[event][0], 1000) + self.assertEqual(_events[event][0][0], 1000) if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/unit/test_allreduce.py b/tinygrad_repo/test/unit/test_allreduce.py index 0997e5597..602f5aa57 100644 --- a/tinygrad_repo/test/unit/test_allreduce.py +++ b/tinygrad_repo/test/unit/test_allreduce.py @@ -2,6 +2,8 @@ import unittest from tinygrad import Tensor, UOp, dtypes from tinygrad.helpers import Context from tinygrad.uop.ops import Ops +from test.helpers import KernelCountException +from tinygrad.engine.realize import run_linear class TestRingAllReduce(unittest.TestCase): def test_schedule_ring(self): @@ -13,20 +15,33 @@ class TestRingAllReduce(unittest.TestCase): copies = [si for si in linear.src if si.src[0].op is Ops.COPY] pairs = [(c.src[1].buffer.device, c.src[2].buffer.device) for c in copies] # N*(N-1) scatter reduce, and N*(N-1) allgather - self.assertEqual(len(pairs), N*(N-1)*2) + if len(pairs) != N*(N-1)*2: raise KernelCountException(N*(N-1)*2, len(pairs)) # copy topology forms a ring self.assertEqual(len(set(pairs)), N) def test_schedule_all2all(self): with Context(ALL2ALL=2): N = 4 + M = N*100 ds = tuple(f"CPU:{i}" for i in range(N)) - t = Tensor.empty(N, N*100).shard(ds, axis=0).realize() - linear = t.sum(0).mul(2.0).contiguous().linear_with_vars()[0] + x = Tensor.arange(N*M, dtype=dtypes.float).reshape(N, M) + t = (x*x).clone().shard(ds, axis=0).realize() + out = t.sum(0).mul(2.).contiguous() + linear, var_vals = out.linear_with_vars() copies = [si for si in linear.src if si.src[0].op is Ops.COPY] sinks = [si for si in linear.src if si.src[0].op is Ops.SINK] - self.assertEqual(len(copies), 24) - self.assertEqual(len(sinks), 26) + # N*(N-1) copies for input and output + copy_count = N*(N-1)*2 + if len(copies) != copy_count: raise KernelCountException(copy_count, len(copies)) + # N*(N-1) shrinks from other devices becoming contigs, N ALU, N extra contig, reassembly (cat), and mul + sink_count = (N*(N-1))+(N)+(N)+(1)+(1) + if len(sinks) != sink_count: raise KernelCountException(sink_count, len(sinks)) + # correctness + run_linear(linear, var_vals) + expected = [2*sum((d*M+i)**2 for d in range(N)) for i in range(M)] + dev_nums = Tensor.arange(1, N+1, dtype=dtypes.float).reshape(N, 1).expand(N, M).shard(ds, axis=0) + shards = out.reshape(1, M).expand(N, M)+dev_nums + self.assertListEqual(shards.tolist(), [[x+d+1 for x in expected] for d in range(N)]) @Context(RING=0, ALL2ALL=0) def test_schedule_naive(self): @@ -39,8 +54,8 @@ class TestRingAllReduce(unittest.TestCase): sinks = [si for si in linear.src if si.src[0].op is Ops.SINK] pairs = [(c.src[1].buffer.device, c.src[2].buffer.device) for c in copies] - self.assertEqual(len(pairs), N*(N-1)) - self.assertEqual(len(sinks), 2) + if len(pairs) != N*(N-1): raise KernelCountException(N*(N-1), len(pairs)) + if len(sinks) != 2: raise KernelCountException(2, len(sinks)) self.assertTrue(all(dst != src for dst, src in pairs)) def test_symbolic_shape(self): @@ -64,7 +79,7 @@ class TestAllreduceCast(unittest.TestCase): with Context(ALLREDUCE_CAST=allreduce_cast, RING=0, SCACHE=0): t = Tensor.empty(4, 4, dtype=dtype).shard(ds, axis=0) linear = t.sum(0).linear_with_vars()[0] - return {si.src[1].buffer.dtype.scalar() for si in linear.src if si.src[0].op is Ops.COPY} + return {si.src[1].buffer.dtype for si in linear.src if si.src[0].op is Ops.COPY} def test_allreduce_cast_bf16(self): # with ALLREDUCE_CAST, allreduce copies stay in bfloat16 instead of promoting to float32 diff --git a/tinygrad_repo/test/unit/test_attention.py b/tinygrad_repo/test/unit/test_attention.py index 622712cc6..7a3947378 100644 --- a/tinygrad_repo/test/unit/test_attention.py +++ b/tinygrad_repo/test/unit/test_attention.py @@ -1,10 +1,12 @@ import unittest import numpy as np -from tinygrad import Tensor, dtypes +from tinygrad import Tensor, dtypes, nn from tinygrad.llm.model import ( GatedDeltaNetBlock, SSMConfig, TransformerBlock, TransformerConfig, apply_rope as apply_rope_new, precompute_freqs_cis, pairwise_topk, ) +from tinygrad.llm.kernels.amd import Linear, gated_delta_prefill, amd_custom_kernels_supported +from tinygrad.llm.gguf import ggml_data_to_tensor def apply_rope(x:Tensor, start_pos:int): B, H, T, Hd = x.shape @@ -12,6 +14,15 @@ def apply_rope(x:Tensor, start_pos:int): freqs_cis = precompute_freqs_cis(Hd, start_pos+T)[start_pos:start_pos+T] return apply_rope_new(x, freqs_cis) +class TestLinear(unittest.TestCase): + def test_recovers_packed_ggml_weight(self): + for ggml_type,packed_size,words in ((13, 176, 44), (14, 210, 53), (23, 136, 34)): + packed = Tensor.empty(packed_size+4, dtype=dtypes.uint8, device="CPU")[4:] + decoded = ggml_data_to_tensor(packed, 256, ggml_type).reshape(1, 256) + linear = Linear(256, 1, bias=False) + linear.set_quantized(decoded) + self.assertEqual((linear.ggml_type, linear.weight.numel()), (ggml_type, words)) + class TestAttention(unittest.TestCase): def test_apply_rope(self): x = Tensor.randn(1, 2, 4, 8, dtype=dtypes.float32) @@ -41,14 +52,31 @@ class TestAttention(unittest.TestCase): np.testing.assert_allclose(block.cache_kv[0, :, :, :seqlen, :].numpy(), expected.numpy(), rtol=1e-5, atol=1e-5) class TestGatedDeltaNetBlock(unittest.TestCase): + def test_gated_delta_rectangular_state_and_row_decay(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + q, k = (rng.normal(size=(1, 1, 3, 32)).astype(np.float32) for _ in range(2)) + v, beta = rng.normal(size=(1, 1, 3, 4)).astype(np.float32), rng.uniform(size=(1, 1, 3)).astype(np.float32) + alpha, initial = rng.uniform(0.8, 1, size=(1, 1, 3, 4)).astype(np.float32), rng.normal(size=(1, 1, 4, 32)).astype(np.float32) + expected_state, expected_out = initial.copy(), np.empty_like(v) + for t in range(3): + previous, av = expected_state.copy(), alpha[:, :, t, :, None] + delta = (v[:, :, t] - (previous*k[:, :, t, None]).sum(-1)*alpha[:, :, t]) * beta[:, :, t, None] + expected_state = previous*av + delta[..., None]*k[:, :, t, None, :] + expected_out[:, :, t] = (previous*q[:, :, t, None]).sum(-1)*alpha[:, :, t] + delta*(q[:, :, t]*k[:, :, t]).sum(-1) + state = Tensor(initial).contiguous().realize() + out = gated_delta_prefill(Tensor(q), Tensor(k), Tensor(v), Tensor(beta), Tensor(alpha), state).realize() + np.testing.assert_allclose(out.numpy(), expected_out, rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(state.numpy(), expected_state, rtol=1e-4, atol=1e-4) + def _tensor_linspace(self, start:float, stop:float, shape:tuple[int, ...]) -> Tensor: return Tensor.linspace(start, stop, int(np.prod(shape)), dtype=dtypes.float32).reshape(*shape) def _make_config(self, **kwargs): - return TransformerConfig(**({"num_blocks":1, "dim":4, "hidden_dim":8, "n_heads":1, "n_kv_heads":1, - "norm_eps":1e-5, "vocab_size":32, "head_dim":4, "rope_theta":10000.0, - "rope_dim":4, "v_head_dim":4, "max_context":4, "ssm_layers":(True,), - "ssm":SSMConfig(conv_kernel=2, state_size=2, group_count=1, time_step_rank=1, inner_size=2)} | kwargs)) + return TransformerConfig(**({"num_blocks":1, "dim":8, "hidden_dim":16, "n_heads":1, "n_kv_heads":1, + "norm_eps":1e-5, "vocab_size":32, "head_dim":8, "rope_theta":10000.0, + "rope_dim":8, "v_head_dim":8, "max_context":4, "ssm_layers":(True,), + "ssm":SSMConfig(conv_kernel=2, state_size=4, group_count=1, time_step_rank=1, inner_size=4)} | kwargs)) def _make_block(self, config:TransformerConfig) -> GatedDeltaNetBlock: block = GatedDeltaNetBlock(config, config.ssm) @@ -79,6 +107,10 @@ class TestGatedDeltaNetBlock(unittest.TestCase): recurrent_state = cache[:, conv_flat:].reshape(cache.shape[0], block.num_v_heads, block.head_v_dim, block.head_v_dim) return conv_state, recurrent_state + def _reset_state(self, block:GatedDeltaNetBlock): + Tensor.realize(block.conv_state.assign(block.conv_state.const_like(0)), + block.recurrent_state.assign(block.recurrent_state.const_like(0))) + def _linear_np(self, x:np.ndarray, weight:np.ndarray) -> np.ndarray: return x.astype(np.float32) @ weight.T.astype(np.float32) @@ -86,7 +118,7 @@ class TestGatedDeltaNetBlock(unittest.TestCase): x_float = x.astype(np.float32) return (x_float / np.sqrt((x_float * x_float).mean(axis=-1, keepdims=True) + eps)) * weight.astype(np.float32) - def _normalize_np(self, x:np.ndarray, eps:float=1e-12) -> np.ndarray: + def _normalize_np(self, x:np.ndarray, eps:float=1e-6) -> np.ndarray: return x / np.maximum(np.sqrt((x * x).sum(axis=-1, keepdims=True)), eps) def _softplus_np(self, x:np.ndarray) -> np.ndarray: @@ -148,6 +180,12 @@ class TestGatedDeltaNetBlock(unittest.TestCase): x = Tensor.linspace(-1.0, 1.0, 3 * config.dim, dtype=dtypes.float32).reshape(1, 3, config.dim) expected_outs, expected_conv, expected_recurrent = self._naive_attention(block, x) + out = self._run_attention(block, x, 0) + conv_state, recurrent_state = self._cache_views(block) + np.testing.assert_allclose(out, np.concatenate(expected_outs, axis=1), rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(conv_state, expected_conv[-1], rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(recurrent_state, expected_recurrent[-1], rtol=1e-3, atol=1e-3) + self._reset_state(block) for step in range(x.shape[1]): out = self._run_attention(block, x[:, step:step+1], step) @@ -163,7 +201,7 @@ class TestGatedDeltaNetBlock(unittest.TestCase): prompt = Tensor.linspace(0.75, -0.75, 2 * config.dim, dtype=dtypes.float32).reshape(1, 2, config.dim) for i in range(warmup.shape[1]): self._run_attention(block, warmup[:, i:i+1], i) - Tensor.realize(*block._state_reset_ops()) + self._reset_state(block) expected_outs, expected_conv, expected_recurrent = self._naive_attention(block, prompt) for step in range(prompt.shape[1]): @@ -177,18 +215,65 @@ class TestGatedDeltaNetBlock(unittest.TestCase): err_msg=f"GatedDeltaNet reset recurrent cache mismatch at step {step}") def test_kda_channel_decay(self): - config = self._make_config(n_heads=2, ssm=SSMConfig(conv_kernel=2, state_size=2, group_count=2, time_step_rank=2, inner_size=4, kda=True)) - block, x = GatedDeltaNetBlock(config, config.ssm), Tensor([[[1., 2., 0., 0.]]]) - # f_b(f_a(x)) = [1, 2, 3, 4] + config = self._make_config(dim=4, hidden_dim=8, n_heads=2, head_dim=4, rope_dim=4, v_head_dim=4, + ssm=SSMConfig(conv_kernel=2, state_size=2, group_count=2, time_step_rank=2, inner_size=4, kda=True)) + block, x = GatedDeltaNetBlock(config, config.ssm), Tensor([[[1., 2., 0., 0.], [2., 1., 0., 0.]]]) block.ssm_f_a.weight = Tensor([[1., 0., 0., 0.], [0., 1., 0., 0.]]) block.ssm_f_b.weight = Tensor([[1., 0.], [0., 1.], [1., 1.], [2., 1.]]) block._init_state(x) initial_state = Tensor.arange(8, dtype=dtypes.float32).reshape(1, 2, 2, 2) block.recurrent_state.assign(initial_state).realize() block.ssm_a = Tensor([[-1.], [-1.]]) - block._attention(x, 0).realize() - alpha = np.exp(-self._softplus_np(np.arange(1, 5)).reshape(1, 2, 1, 2)) - np.testing.assert_allclose(block.recurrent_state.numpy(), initial_state.numpy() * alpha, rtol=1e-5, atol=1e-5) + block._attention(x, x.shape[1]).realize() + alpha = np.exp(-self._softplus_np(np.array([[1, 2, 3, 4], [2, 1, 3, 5]])).reshape(2, 2, 2)).prod(0) + np.testing.assert_allclose(block.recurrent_state.numpy(), initial_state.numpy() * alpha[..., None], rtol=1e-5, atol=1e-5) + + def test_kda_prefill_matches_decode(self): + config = self._make_config(ssm=SSMConfig(conv_kernel=2, state_size=4, group_count=1, time_step_rank=1, inner_size=4, kda=True)) + block = GatedDeltaNetBlock(config, config.ssm) + for p in nn.state.get_parameters(block): + p.replace(self._tensor_linspace(-0.05, 0.05, p.shape) if len(p.shape) > 1 else self._tensor_linspace(0.05, 0.1, p.shape)) + x = self._tensor_linspace(-0.5, 0.5, (1, 3, config.dim)) + prefill = self._run_attention(block, x, 0) + prefill_conv, prefill_recurrent = self._cache_views(block) + self._reset_state(block) + decode = np.concatenate([self._run_attention(block, x[:, i:i+1], i) for i in range(3)], axis=1) + decode_conv, decode_recurrent = self._cache_views(block) + np.testing.assert_allclose(prefill, decode, rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(prefill_conv, decode_conv, rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(prefill_recurrent, decode_recurrent, rtol=1e-3, atol=1e-3) + + def test_varied_chunk_sizes_match_decode(self): + for kda in (False, True): + ssm = SSMConfig(conv_kernel=2, state_size=4, group_count=1, time_step_rank=1, inner_size=4, kda=kda) + config = self._make_config(ssm=ssm) + if kda: + block = GatedDeltaNetBlock(config, config.ssm) + for p in nn.state.get_parameters(block): + p.replace(self._tensor_linspace(-0.05, 0.05, p.shape) if len(p.shape) > 1 else self._tensor_linspace(0.05, 0.1, p.shape)) + else: block = self._make_block(config) + x = self._tensor_linspace(-0.5, 0.5, (1, 4, config.dim)) + decode = np.concatenate([self._run_attention(block, x[:, i:i+1], i) for i in range(4)], axis=1) + decode_conv, decode_recurrent = self._cache_views(block) + for chunking in ([4], [2, 2], [1, 3], [3, 1], [2, 1, 1]): + self._reset_state(block) + outs, start = [], 0 + for size in chunking: + outs.append(self._run_attention(block, x[:, start:start+size], start)) + start += size + chunked_conv, chunked_recurrent = self._cache_views(block) + np.testing.assert_allclose(np.concatenate(outs, axis=1), decode, rtol=1e-3, atol=1e-3, err_msg=f"{kda=} {chunking=}") + np.testing.assert_allclose(chunked_conv, decode_conv, rtol=1e-3, atol=1e-3, err_msg=f"{kda=} {chunking=}") + np.testing.assert_allclose(chunked_recurrent, decode_recurrent, rtol=1e-3, atol=1e-3, err_msg=f"{kda=} {chunking=}") + + def test_start_zero_resets_realized_state(self): + config = self._make_config(max_context=3) + x = self._tensor_linspace(-1, 1, (1, 3, config.dim)) + block = self._make_block(config) + self._run_attention(block, x, 0) + restarted = self._run_attention(block, x[:, :2], 0) + fresh = self._run_attention(self._make_block(config), x[:, :2], 0) + np.testing.assert_allclose(restarted, fresh, rtol=1e-3, atol=1e-3) class TestPairwiseTopk(unittest.TestCase): def test_basic_topk(self): diff --git a/tinygrad_repo/test/unit/test_bnxt.py b/tinygrad_repo/test/unit/test_bnxt.py new file mode 100644 index 000000000..0716da3da --- /dev/null +++ b/tinygrad_repo/test/unit/test_bnxt.py @@ -0,0 +1,115 @@ +import struct, unittest +from types import SimpleNamespace +from unittest.mock import patch + +from tinygrad.runtime.autogen import bnxt +from extra.bnxt_driver.bnxtdev import BNXT_BACKING_STORE, BNXTDev, BNXTQP, _queue, _qwrite, ipv4_to_gid + +class FakePCI: + def __init__(self): self.next_addr, self.allocations = 0x100000, [] + def alloc_sysmem(self, size, contiguous=False): + pages = [self.next_addr+i*0x1000 for i in range((size+0xfff)//0x1000)] + self.next_addr += len(pages)*0x1000 + self.allocations.append(mem := bytearray(size)) + return mem, pages + +class FakeDev: + def __init__(self): self.pci_dev, self.calls = FakePCI(), [] + def hwrm(self, name, **fields): + self.calls.append((name, fields)) + typ = fields.get("type", 0) + return SimpleNamespace(ctx_init_value=0x5a, ctx_init_offset=4, entry_size=16 if typ == 0 else 4, + subtype_valid_cnt=typ == 0, split_entry_0=2, instance_bit_map=5 if typ == 0 else 1, min_num_entries=0) + +class FakeRCFW: + def __init__(self): self.calls, self.doorbells = [], [] + def exec(self, name, **fields): + self.calls.append((name, fields)) + return SimpleNamespace(xid={"create_cq":77, "create_qp":88, "register_mr":0x5678}.get(name, 0)) + def doorbell(self, *args, **kwargs): self.doorbells.append((args, kwargs)) + +class FakeQPDev: + def __init__(self): self.pci_dev, self.fw, self.gid_id, self.nq_id = FakePCI(), FakeRCFW(), 9, 41 + def rcfw(self, *args, **kwargs): return self.fw.exec(*args, **kwargs) + def doorbell(self, *args, **kwargs): self.fw.doorbell(*args, **kwargs) + +class TestMemory(unittest.TestCase): + def test_cmdq_and_sq_aux(self): + dev = FakeDev() + cmdq, sq = _queue(dev), _queue(dev, aux=True) + self.assertEqual((cmdq["level"], cmdq["base"]), (0, 0x100000)) + _qwrite(sq, 3, b"ABCDEFGH", aux=True) + self.assertEqual(bytes(sq["mem"][0x1018:0x1020]), b"ABCDEFGH") + + def test_f320_backing_layout_and_final_marker(self): + self.assertEqual(len(BNXT_BACKING_STORE), 9) + dev = FakeDev() + small = ((0, 6), (15, 0)) + with patch("extra.bnxt_driver.bnxtdev.BNXT_BACKING_STORE", small): BNXTDev.setup_backing_store(dev) + cfg = [fields for name, fields in dev.calls if name == "func_backing_store_cfg_v2"] + self.assertEqual([(x["type"], x["instance"]) for x in cfg], [(0, 0), (0, 2), (15, 0)]) + self.assertTrue(all(not x["flags"] for x in cfg[:-1])) + self.assertEqual(cfg[-1]["flags"], bnxt.FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE) + self.assertEqual((dev.pci_dev.allocations[0][4], dev.pci_dev.allocations[0][20]), (0x5a, 0x5a)) + +class TestRCFW(unittest.TestCase): + def setUp(self): + patch("extra.bnxt_driver.bnxtdev.System.memory_barrier").start() + self.addCleanup(patch.stopall) + + def test_doorbell_encodes_xid_type_and_index(self): + dev = BNXTDev.__new__(BNXTDev) + dev.db, dev.db_off = [0]*1024, 0x1000 + dev.doorbell(0x123456, bnxt.DBC_DBC_TYPE_CQ_ARMALL, 0x456, epoch=1) + key = dev.db[0x1000//8] + self.assertEqual(key >> 32, + 0x123456 & bnxt.DBC_DBC_XID_MASK | bnxt.DBC_DBC_PATH_ROCE | bnxt.DBC_DBC_TYPE_CQ_ARMALL | bnxt.BNXT_QPLIB_DBR_VALID) + self.assertEqual(key & 0xffffffff, 0x456 | 1< Tensor: + # update a cache at a symbolic offset, like an attention KV cache update + return Tensor(cache.uop.after(cache[sp:sp+x.shape[0]].uop.store(x.uop)))[:sp+x.shape[0]].sum() + sp_v, nt_v = UOp.variable("sp", 0, 8), UOp.variable("nt", 1, 8) + t = Tensor.arange(16).float().realize() + sp, nt = sp_v.bind(0), nt_v.bind(8) + store(t[sp:sp+nt].clone().realize(), sp).realize() + np.testing.assert_equal(cache.numpy()[:8], t[:8].numpy()) + np.testing.assert_equal(cache.numpy()[8:], np.zeros(8)) + def test_precompile_schedule_cache_hit(self): """two instances of the same @function should produce identical function body keys (schedule cache hit)""" @function(precompile=True) @@ -231,9 +251,9 @@ class TestCallSchedule(unittest.TestCase): a = Tensor.empty(4, 8) b = Tensor.empty(4, 8) r0, r1 = f(a), f(b) - # find the FUNCTION nodes - c0 = next(u for u in r0.uop.toposort() if u.op is Ops.FUNCTION) - c1 = next(u for u in r1.uop.toposort() if u.op is Ops.FUNCTION) + # find the call nodes + c0 = next(u for u in r0.uop.toposort() if u.op is Ops.CALL and u.num_returned) + c1 = next(u for u in r1.uop.toposort() if u.op is Ops.CALL and u.num_returned) # the function bodies (src[0]) should have identical keys self.assertEqual(c0.src[0].key, c1.src[0].key) @@ -256,6 +276,54 @@ class TestCallSchedule(unittest.TestCase): out = f(a) + 2 np.testing.assert_allclose(out.numpy(), np.arange(8, dtype=np.float32).reshape(4, 2) + 3) +class TestArgOrder(unittest.TestCase): + """RETURNED placeholders can appear anywhere in a call's srcs: slots are src positions, nothing reorders""" + def make_intersperse_call(self, x, precompile=False): + # call with sources (body, returned(slot=0), input(slot=1)): the input is the input, the output binds the RETURNED + dev = x.device if isinstance(x.device, str) else (x.device or (Device.DEFAULT,))[0] + r0 = UOp.returned(0, x.dtype, x.shape, device=dev) + o0 = UOp.param(0, x.dtype, x.shape, dev) + p1 = UOp.param(1, x.dtype, x.shape, dev) + from tinygrad.uop.ops import CallInfo + return UOp(Ops.CALL, src=(UOp.sink(o0.store(p1.reshape(x.shape) * 2)), r0, x.uop), + arg=CallInfo(None, 't', precompile, False, None)) + + def test_intersperse_returned(self): + x = Tensor.arange(3, dtype=dtypes.int).realize() + call = self.make_intersperse_call(x) + out = Tensor(call.returned_outputs[0], device=x.device) + 1 + np.testing.assert_equal(out.numpy(), [1, 3, 5]) + + def test_intersperse_returned_precompile(self): + x = Tensor.arange(3, dtype=dtypes.int).realize() + call = self.make_intersperse_call(x, precompile=True) + # the transform must preserve the RETURNED's src position: its placeholder is at src 1, the input stays at src 2 + from tinygrad.tensor import transform_precompiled_call + new = transform_precompiled_call(call) + new_call = new.src[0].src[1].src[1] + # the out buffer takes the RETURNED's position (src 1), the input value keeps its position (src 2) + self.assertEqual(new_call.src[1].op, Ops.BUFFER) + self.assertEqual(new_call.src[1].arg.size, 3) + self.assertEqual(new_call.src[2].op, Ops.ADD) + # the body binds positionally: store dest at slot 0 (the RETURNED's position), input param at slot 1 + store = [u for u in new_call.src[0].toposort(enter_calls=False) if u.op is Ops.STORE][0] + self.assertEqual(store.src[0].arg.slot, 0) + self.assertEqual([u.arg.slot for u in store.src[1].toposort(enter_calls=False) if u.op is Ops.PARAM], [1]) + + def test_intersperse_returned_gradient(self): + x = Tensor([1.0, 2.0, 3.0]).realize() + x.requires_grad = True + dev = x.device if isinstance(x.device, str) else (x.device or (Device.DEFAULT,))[0] + r0 = UOp.returned(0, dtypes.float, x.shape, device=dev) + o0 = UOp.param(0, dtypes.float, x.shape, dev) + p1 = UOp.param(1, dtypes.float, x.shape, dev) + from tinygrad.uop.ops import CallInfo + body = UOp.sink(o0.store(p1.reshape(x.shape) * p1.reshape(x.shape))) + call = UOp(Ops.CALL, src=(body, r0, x.uop), arg=CallInfo(None, 't', False, False, None)) + y = Tensor(call.returned_outputs[0], device=x.device) + y.sum().backward() + np.testing.assert_equal(x.grad.numpy(), [2, 4, 6]) + class TestCallMultiSharded(unittest.TestCase): # TODO: multi-output + sharded needs per-device CALL execution, which requires reworking how MULTI propagates through TUPLE bodies def test_tuple_sharded(self): diff --git a/tinygrad_repo/test/unit/test_dtype_weak.py b/tinygrad_repo/test/unit/test_dtype_weak.py index a4e9740eb..b8b022829 100644 --- a/tinygrad_repo/test/unit/test_dtype_weak.py +++ b/tinygrad_repo/test/unit/test_dtype_weak.py @@ -3,11 +3,12 @@ import tempfile, unittest, math from tinygrad import Tensor, dtypes, TinyJit from tinygrad.helpers import Context from tinygrad.dtype import least_upper_float -from tinygrad.uop.ops import UOp, Ops, dtype_from_uop, graph_rewrite -from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak +from tinygrad.uop.ops import UOp, Ops, GroupOp, dtype_from_uop, graph_rewrite +from tinygrad.uop.weak import pm_commit_weak from tinygrad.uop.symbolic import symbolic_simple from tinygrad.uop.spec import spec_shared, type_verify from tinygrad.engine.jit import JitError +from test.helpers import full_rewrite class TestWeakPromotion(unittest.TestCase): @@ -53,6 +54,13 @@ class TestWeakPromotion(unittest.TestCase): self.assertEqual((r.dtype, r.tolist()), (dt, [1])) self.assertNotIn(Ops.CAST, [u.op for u in r._uop.toposort()]) + def test_promote_keeps_shape_args(self): + # the shape arg is the same CONST as the value, only the value lifts + self.assertEqual((Tensor(5).expand(5) + 1.5).tolist(), [6.5]*5) + self.assertEqual((Tensor(2).reshape(1,1).expand(2,2).pad(((0,2),(0,0))) + 0.5).tolist(), [[2.5,2.5],[2.5,2.5],[0.5,0.5],[0.5,0.5]]) + x, _ = Tensor(5).reshape(1).pad((1,1))._broadcasted(0.5) + self.assertEqual((x._uop.op, x._uop.base.dtype, x._uop.src[1].dtype), (Ops.PAD, dtypes.weakfloat, dtypes.weakint)) + def test_broadcasted_keeps_const_weak(self): # a python scalar stays a bare weak CONST through _broadcasted, lifted only to the KIND of the lub x, y = Tensor([1], dtype=dtypes.int8)._broadcasted(3) @@ -73,13 +81,18 @@ class TestWeakPromotion(unittest.TestCase): recips = [u for u in (x / y)._uop.toposort() if u.op is Ops.RECIPROCAL] self.assertEqual([(u.dtype, u.src[0].dtype) for u in recips], [(dtypes.float32, dtypes.float32)]) with Context(DEFAULT_FLOAT=dtypes.float16): - committed = graph_rewrite((UOp.const(1).cast(dtypes.int32) + UOp.const(1.0)).cast(dtypes.float32), pm_lower_index_dtype, ctx={}) + committed = graph_rewrite((UOp.const(1).cast(dtypes.int32) + UOp.const(1.0)).cast(dtypes.float32), pm_commit_weak) self.assertEqual([u.dtype for u in committed.toposort() if u.op is Ops.ADD], [dtypes.float32]) + def test_div_sub_operand_kept_weak(self): + a = Tensor.empty(4, dtype=dtypes.float32) + for t in (a / 1, a - 0): + self.assertEqual(t.uop.src[1].dtype, dtypes.weakfloat) + def test_cast_weak_expression_commits_at_cast_floor(self): # the floor never narrows: a cast BELOW the default does not pull the compute width down with it with Context(DEFAULT_FLOAT=dtypes.float32): - narrowed = graph_rewrite((UOp.const(1.0) + UOp.const(2.0)).cast(dtypes.float16), pm_lower_index_dtype, ctx={}) + narrowed = graph_rewrite((UOp.const(1.0) + UOp.const(2.0)).cast(dtypes.float16), pm_commit_weak) self.assertEqual((narrowed.dtype, narrowed.src[0].dtype), (dtypes.float16, dtypes.float32)) def test_cast_weak_expression_value_uses_cast_floor(self): @@ -88,6 +101,13 @@ class TestWeakPromotion(unittest.TestCase): out = Tensor(1.0, dtype=dtypes.float32, device="CPU") / denom self.assertAlmostEqual(out.item(), 1 / (70000 + 1e-5), places=10) + def test_stacked_weak_casts_convert_each_kind(self): + # each weak cast is a kind conversion: weakint truncates before weakfloat re-lifts (neither is only a marker) + x = Tensor([2.5, -3.7], dtype=dtypes.float32, device="CPU") + stacked = x.cast(dtypes.weakint).cast(dtypes.weakfloat) + self.assertIs(stacked.dtype, dtypes.weakfloat) + self.assertEqual(stacked.tolist(), [2.0, -3.0]) + def test_uop_scalar_const_lifts_kind(self): for dtype, value, out_dtype, const_dtype in ((dtypes.weakint, 1, dtypes.weakint, dtypes.weakint), (dtypes.int32, 1, dtypes.int32, dtypes.weakint), @@ -101,27 +121,35 @@ class TestWeakPromotion(unittest.TestCase): self.assertIsInstance((x + 2).src[1].val, float) self.assertIs(x + UOp.const(2), x + 2) - def test_index_dtype_ignores_weakness(self): - with Context(SPEC=2): - idx = UOp.const(0).cast(dtypes.int32) - weak = UOp.const(1.0).expand((1,)) - self.assertEqual(UOp(Ops.INDEX, dtypes.float32, (weak, idx)).dtype, dtypes.float32) - with self.assertRaisesRegex(RuntimeError, "bad dtype"): UOp(Ops.INDEX, dtypes.int32, (weak, idx)) - def test_store_weak_value_uses_destination_dtype(self): with Context(DEFAULT_FLOAT=dtypes.float16): - dst = UOp.param(0, dtypes.bfloat16, (1,)).index(UOp.const(0).cast(dtypes.int32)) + dst = UOp.param(0, dtypes.bfloat16, 1).index(UOp.const(0).cast(dtypes.int32)) gate = UOp.const(True) - out = graph_rewrite(dst.store(UOp.const(5.0), gate), pm_lower_index_dtype, ctx={}) + out = graph_rewrite(dst.store(UOp.const(5.0), gate), pm_commit_weak) # a bare weak CONST commits directly: the pass runs without symbolic, so a CAST here would survive it self.assertEqual((out.src[1], out.src[2]), (UOp.const(5.0, dtypes.bfloat16), gate)) def test_weak_srcs_commit_only_at_a_concrete_lub(self): weak_lub = UOp(Ops.ADD, src=(UOp.const(1), UOp.const(1.0))) - self.assertIs(graph_rewrite(weak_lub, pm_lower_index_dtype, ctx={}), weak_lub) + self.assertIs(graph_rewrite(weak_lub, pm_commit_weak), weak_lub) concrete = UOp.const(2.0).cast(dtypes.float16) - where = graph_rewrite(UOp(Ops.WHERE, src=(UOp.const(True), concrete, UOp.const(1.0))), pm_lower_index_dtype, ctx={}) - self.assertEqual(tuple(x.dtype for x in where.src), (dtypes.bool, dtypes.float16, dtypes.float16)) + # the weak arm stays bare: its sibling states the width, so the WHERE already derives float16 for it + where = graph_rewrite(UOp(Ops.WHERE, src=(UOp.const(True), concrete, UOp.const(1.0))), pm_commit_weak) + self.assertEqual((where.dtype, tuple(x.dtype for x in where.src)), (dtypes.float16, (dtypes.bool, dtypes.float16, dtypes.weakfloat))) + + def test_derivable_const_rounds_at_the_derived_width(self): + # re-rounds a derivable const in place (still bare) so value-keyed folds (x*1 -> x, x*-1 -> NEG) still fire + x = UOp.param(0, dtypes.float32, 1).index(UOp.const(0).cast(dtypes.int32)).load() + mul = graph_rewrite(x * UOp.const(-0.9999999893980771), symbolic_simple+pm_commit_weak) + self.assertIs(mul.src[1], UOp.const(-1.0)) + self.assertIs(graph_rewrite(x * UOp.const(1.0000000106), symbolic_simple+pm_commit_weak), x) + + def test_committed_const_conversion_folds_for_native_format(self): + folded = graph_rewrite(UOp.const(16256, dtypes.ushort).cast(dtypes.uint), symbolic_simple) + self.assertIs(folded, UOp.const(16256, dtypes.uint)) + # fmt-less targets are lowered by renderer rewrites, where collapsing this pair would cycle with float-intermediate insertion. + emulated = UOp.const(1.0, dtypes.float).cast(dtypes.bfloat16) + self.assertIs(graph_rewrite(emulated, symbolic_simple), emulated) def test_weak_shift_lhs_commits_the_node(self): # a shift derives its lhs's dtype, so committing the lhs restates the root (WGSL's packed store writes `mask << shift_am`) @@ -162,15 +190,15 @@ class TestWeakPromotion(unittest.TestCase): self.assertEqual(dtype_from_uop(Ops.SHL, (UOp.const(1, dtypes.int8), UOp.const(1, dtypes.uint32)), None), dtypes.int8) self.assertEqual(UOp.const(1).alu(Ops.SHL, UOp.const(1, dtypes.uint)).dtype, dtypes.weakint) self.assertEqual((v & 3).dtype, dtypes.weakint) - with self.assertRaises(RuntimeError): Tensor.const(1.0) << Tensor.const(1.0) - with self.assertRaises(RuntimeError): UOp.const(1, dtypes.int32).alu(Ops.SHL, UOp.const(1, dtypes.float64)) + with self.assertRaises(RuntimeError): (Tensor.const(1.0) << Tensor.const(1.0)).dtype + with self.assertRaises(RuntimeError): UOp.const(1, dtypes.int32).alu(Ops.SHL, UOp.const(1, dtypes.float64)).dtype for op in (Ops.SHL, Ops.SHR): with self.assertRaises(RuntimeError): - UOp.const(1, dtypes.float32).alu(op, UOp.const(1, dtypes.int32)) + UOp.const(1, dtypes.float32).alu(op, UOp.const(1, dtypes.int32)).dtype # float bitwise builds, the spec rejects it with Context(SPEC=1): f32, wf = UOp.const(1.0, dtypes.float32), UOp.const(1.0) - for bad in (f32.alu(Ops.AND, f32), UOp(Ops.AND, dtypes.float32, (f32, f32)), UOp(Ops.AND, dtypes.int32, (wf, wf))): + for bad in (f32.alu(Ops.AND, f32), UOp(Ops.AND, (f32, f32)), UOp(Ops.AND, (wf, wf))): with self.assertRaises(RuntimeError): type_verify([bad], spec_shared) def test_integer_values(self): @@ -205,6 +233,22 @@ class TestWeakPromotion(unittest.TestCase): self.assertNotIn(out.uop.buffer.dtype, dtypes.weaks) +class TestWeakBounds(unittest.TestCase): + def test_bounds_survive_movement(self): + moved = Tensor(5).reshape(1).expand(2).pad((1, 1)).detach().contiguous_backward() + self.assertEqual((moved.uop.vmin, moved.uop.vmax, moved.uop.bufferize().vmax), (0, 5, 5)) + self.assertEqual(moved.numpy().dtype, Tensor(5).numpy().dtype) # a moved weak int reads at the same dtype as the bare one + + def test_wide_src_keeps_its_width(self): + # the node's result fits int32, its variable does not: the shift runs at long, only the result narrows + v = UOp.variable("v", 0, 2**40).bind(2**35+7) + for t in (Tensor(v) // 2**31, (Tensor(v) - 1) // 2**31, Tensor(v).reshape(1) // 2**31): self.assertEqual(t.item(), 16) + + def test_padded_weak_const_keeps_its_zeros(self): + self.assertEqual(Tensor(1).expand(1).cat(Tensor(2).expand(2), Tensor(3).expand(3)).tolist(), [1, 2, 2, 3, 3, 3]) + self.assertEqual((Tensor(5).reshape(1).pad((1, 1)) == 5).tolist(), [False, True, False]) + self.assertEqual((Tensor(5).reshape(1,1).expand(1,2).pad(((0,2),(0,0))) + Tensor([[1],[2],[3]])).tolist(), [[6,6],[2,2],[3,3]]) + class TestWeakStorageBoundary(unittest.TestCase): # weak has no storage: a weak assignment source casts when it defers to the destination, everything else raises def test_weak_source(self): @@ -219,6 +263,25 @@ class TestWeakStorageBoundary(unittest.TestCase): ddst = Tensor.empty(2, dtype=dtypes.int32, device=f"DISK:{td}/t") with self.assertRaises(RuntimeError): ddst.assign(w05.expand(2)) + def test_weak_commits_by_bounds(self): + big = Tensor(2**40) + edges = (big.clone(), big.sum(), big.reshape(1).max(), big.reshape(1).mean(), Tensor.stack(big, Tensor(1)).sum() - 1, + Tensor([2**40]), big.full_like(2**40)) + for t in edges: self.assertEqual(t.item(), 2**40) + self.assertEqual(Tensor(UOp.variable("b", 0, 2**40).bind(2**35+3)).clone().item(), 2**35+3) + self.assertEqual(Tensor([10, 20, 30])[[2**32+1]].tolist(), [0]) # a wide list index is out of range, not wrapped + with Context(DEFAULT_INT=dtypes.int64): self.assertEqual(Tensor(2).clone().dtype, dtypes.int64) + + def test_literal_beyond_any_int_raises(self): + for make in (lambda: Tensor(2**64).item(), lambda: Tensor([2**64]), lambda: Tensor.full((2,), -2**63-1)): + with self.assertRaises(OverflowError): make() + + def test_weak_sentinels_commit_first(self): + # max_pool2d, scatter_reduce and cummax pad with the dtype's min/max, which a weak dtype does not have + self.assertEqual(Tensor(-5).expand(1, 1, 2, 2).max_pool2d(2, padding=1).dtype, Tensor(-5).clone().dtype) + self.assertEqual(Tensor(-5).expand(2).scatter_reduce(0, Tensor([0]), Tensor(-5).expand(1), "amax", include_self=False).tolist(), [-5, -5]) + self.assertEqual(Tensor(2**40).expand(3).cummax(0)[0].tolist(), [2**40]*3) + def test_weak_has_no_storage(self): import numpy as np with self.assertRaises(RuntimeError): Tensor(np.ones(2, dtype=np.float32), dtype=dtypes.weakfloat) @@ -276,5 +339,18 @@ class TestSignedUint64Weakfloat(unittest.TestCase): self.assertAlmostEqual((i64 + u64).sin().item(), math.sin(2), places=5) # Unary lowers before transcendental +class TestNoRedundantWide(unittest.TestCase): + def wide_alu(self, t:Tensor) -> int: + return sum(sum(1 for u in full_rewrite(call.src[0]).toposort() if u.op in GroupOp.ALU and u.dtype in {dtypes.long, dtypes.ulong}) + for call in t.schedule_linear().src if call.src[0].op is Ops.SINK) + + def test_unbounded_long_stays_long(self): + self.assertGreater(self.wide_alu(Tensor.empty(16, dtype=dtypes.long)*3 + 1), 0) + + def test_fancy_index_has_no_wide_alu(self): + j, o = Tensor([0, 1, 2]).reshape(3, 1), Tensor([0, 1]).reshape(1, 2) + self.assertEqual(self.wide_alu(Tensor.empty(8, 9, 10, 11, 12)[1, j, 2, o, 2]), 0) + + if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/unit/test_function.py b/tinygrad_repo/test/unit/test_function.py index 2bc6577e8..0ebcaef77 100644 --- a/tinygrad_repo/test/unit/test_function.py +++ b/tinygrad_repo/test/unit/test_function.py @@ -4,7 +4,7 @@ from tinygrad.function import function from tinygrad import Tensor, GlobalCounters, Device from tinygrad.dtype import Invalid from tinygrad.uop.ops import UOp, Ops, KernelInfo, ProgramInfo -from test.helpers import assert_kernel_count +from test.helpers import assert_kernel_count, KernelCountException class TestFunction(unittest.TestCase): def test_simple(self): @@ -15,6 +15,16 @@ class TestFunction(unittest.TestCase): b = Tensor([4,5,6]) np.testing.assert_equal(f(a,b).numpy(), [5,7,9]) + def test_two_return(self, precompile=False): + @function(precompile=precompile) + def f(a:Tensor, b:Tensor) -> tuple[Tensor, Tensor]: + return (a+b, (a+b)*2) + a = Tensor([1,2,3]) + b = Tensor([4,5,6]) + c = f(a,b) + np.testing.assert_equal((c[0]+c[1]).numpy(), [5*3,7*3,9*3]) + def test_two_return_precompiled(self): self.test_two_return(True) + def test_simple_same(self): @function def f(a:Tensor, b:Tensor) -> Tensor: return a+b @@ -174,13 +184,13 @@ class TestFunction(unittest.TestCase): def test_name(self): @function def f(a:Tensor) -> Tensor: return a + 1 - assert f(Tensor([1])).uop.src[0].arg.name.endswith("f") + assert f(Tensor([1])).uop.src[1].arg.name.endswith("f") def test_method_name(self): class Foo: @function def __call__(self, x:Tensor) -> Tensor: return x + 1 - assert Foo()(Tensor([1])).uop.src[0].arg.name.endswith("Foo.__call__") + assert Foo()(Tensor([1])).uop.src[1].arg.name.endswith("Foo.__call__") def test_callable_instance(self): class Foo: @@ -189,7 +199,7 @@ class TestFunction(unittest.TestCase): foo = Foo() f = function(foo, allow_implicit=True) np.testing.assert_equal(f(Tensor([1,2,3])).numpy(), [11,22,33]) - assert f(Tensor([1,2,3])).uop.src[0].arg.name.endswith("Foo") + assert f(Tensor([1,2,3])).uop.src[1].arg.name.endswith("Foo") def test_iadd(self): @function @@ -425,6 +435,15 @@ class TestFunctionTuple(unittest.TestCase): np.testing.assert_allclose(x.grad.numpy(), [1., 1., 1.]) np.testing.assert_allclose(y.grad.numpy(), [1., 1., 1.]) + def test_grad_fxn_more_outputs_than_inputs(self): + def grad_fxn(grad:UOp, call:UOp): return (grad,) + + x = Tensor([2.]).contiguous() + @function(grad_fxn=grad_fxn) + def f(x:Tensor): return (x+1, x+2) + _, y = f(x) + self.assertEqual(y.sum().gradient(x)[0].item(), 1.0) + def test_grad_unused_tuple_output_recursive(self): # only one output is used @function(precompile=True, precompile_backward=True) @@ -516,7 +535,7 @@ class TestFunctionTuple(unittest.TestCase): Tensor.realize(a) c = f(a) - self.assertEqual(count_kernels(c), 1) + if count_kernels(c) != 1: raise KernelCountException(1, count_kernels(c)) c.sum().backward() Tensor.realize(a.grad) diff --git a/tinygrad_repo/test/unit/test_gguf.py b/tinygrad_repo/test/unit/test_gguf.py index 8b74d88aa..7fcdda8c1 100644 --- a/tinygrad_repo/test/unit/test_gguf.py +++ b/tinygrad_repo/test/unit/test_gguf.py @@ -5,17 +5,33 @@ from tinygrad.llm.gguf import _ggml_iq_grid, ggml_data_to_tensor, gguf_load from tinygrad.runtime.autogen import ggml_common as _ggml import numpy as np from gguf import GGUFReader, GGUFValueType, GGMLQuantizationType, GGML_QUANT_SIZES, dequantize, quantize -from gguf.quants import IQ2_S, IQ3_S, IQ3_XXS +from gguf.quants import IQ1_S, IQ2_S, IQ2_XS, IQ2_XXS, IQ3_S, IQ3_XXS ggml_test_block_count = 4 supported_dtypes = Device[Device.DEFAULT].renderer.supported_dtypes() class TestGGUFTables(unittest.TestCase): + def test_iq2_xxs_grid_matches_gguf_py(self): + IQ2_XXS.init_grid() + grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq2xxs_grid, (256, 8)).numpy() + np.testing.assert_equal(grid, IQ2_XXS.grid.reshape(256, 8)) + + def test_iq2_xs_grid_matches_gguf_py(self): + IQ2_XS.init_grid() + grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq2xs_grid, (512, 8)).numpy() + np.testing.assert_equal(grid, IQ2_XS.grid.reshape(512, 8)) + def test_iq2_s_grid_matches_gguf_py(self): IQ2_S.init_grid() grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq2s_grid, (1024, 8)).numpy() np.testing.assert_equal(grid, IQ2_S.grid.reshape(1024, 8)) + def test_iq1_s_grid_matches_gguf_py(self): + IQ1_S.init_grid() + grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq1s_grid, (2048, 8)).numpy() + grid = np.where(grid > 127, grid - 256, grid) + np.testing.assert_equal(grid, IQ1_S.grid.reshape(2048, 8)) + def test_iq3_xxs_grid_matches_gguf_py(self): IQ3_XXS.init_grid() grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq3xxs_grid, (256, 4)).numpy() @@ -39,6 +55,56 @@ class TestGGUF(unittest.TestCase): expected = np.arange(1, 33, dtype=np.float32) * 2.0 np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 32, GGMLQuantizationType.Q8_0.value).numpy().flatten(), expected) + def test_dequantization_q2_k_hardcoded(self): + # Q2_K: scales[16] + qs[64] + d(fp16) + dmin(fp16). 16 sub-blocks of 16, x = d*(scale&0xF)*q - dmin*(scale>>4) + scales, qs = bytes([0x11]*16), bytes([0x55]*64) # scale=1, min=1; qs=0x55 -> 2-bit quants of 1 + d, dmin = np.float16(1.0).tobytes(), np.float16(0.0).tobytes() + block = np.frombuffer(scales + qs + d + dmin, dtype=np.uint8).copy() + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 10).numpy().flatten(), np.ones(256, dtype=np.float32)) + + def test_dequantization_q3_k_hardcoded(self): + # Q3_K: hmask[32] + qs[64] + scales[12] + d(fp16). 16 sub-blocks of 16, x = d * (scale-32) * (q - (hbit?0:4)) + # 6-bit scales 32..47 so (scale-32) = 0..15; qs=0x55 -> 2-bit quants of 1; d=1.0 + scales = bytes([0x80, 0x91, 0xA2, 0xB3, 0xC4, 0xD5, 0xE6, 0xF7, 0xAA, 0xAA, 0xAA, 0xAA]) + d = np.float16(1.0).tobytes() + qs, ones = bytes([0x55]*64), np.ones(16, dtype=np.float32) + # hmask all-ones: high bit set, q=1; hmask zeros: subtract 4, q=-3 + for hmask, q in ((bytes([0xFF]*32), 1.0), (bytes([0x00]*32), -3.0)): + block = np.frombuffer(hmask + qs + scales + d, dtype=np.uint8).copy() + expected = np.concatenate([q * s * ones for s in range(16)]) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 11).numpy().flatten(), expected) + + def test_dequantization_iq2_xxs_hardcoded(self): + # IQ2_XXS: d + 8 groups of (4 grid bytes + uint32 signs/scale). grid[0]=all 0x08, scale=0, signs=0 + # db = 1.0 * (0.5 + 0) * 0.25 = 0.125; 0.125 * 8 = 1.0 + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(64), dtype=np.uint8).copy() + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 16).numpy().flatten(), np.ones(256, dtype=np.float32)) + + def test_dequantization_iq2_xs_hardcoded(self): + # IQ2_XS: d + 32 uint16 qs + 8 scale bytes. qs=0 -> grid[0]=all 0x08, signs=0; scales=0 + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(64) + bytes(8), dtype=np.uint8).copy() + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 17).numpy().flatten(), np.ones(256, dtype=np.float32)) + + def test_dequantization_iq1_s_hardcoded(self): + # IQ1_S: d + qs[32] + qh[16]. qs=qh=0 -> grid[0]=all -1, scale=1, delta=+0.125 -> -0.875 + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(48), dtype=np.uint8).copy() + expected = np.full(256, -0.875, dtype=np.float32) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 19).numpy().flatten(), expected) + + def test_dequantization_iq1_m_hardcoded(self): + # IQ1_M: qs[32] + qh[16] + scales[8]. f16 1.0=0x3C00 packed in high nibbles; qs=qh=0 -> -0.875 + scales = bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x00, 0x30]) + block = np.frombuffer(bytes(48) + scales, dtype=np.uint8).copy() + expected = np.full(256, -0.875, dtype=np.float32) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 29).numpy().flatten(), expected) + + def test_dequantization_iq4_nl_hardcoded(self): + # IQ4_NL: 2-byte fp16 scale + 16 packed bytes. low nibbles first, then high + lut = list(_ggml.kvalues_iq4nl) + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(range(16)), dtype=np.uint8).copy() + expected = np.array(lut + [lut[0]]*16, dtype=np.float32) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 32, 20).numpy().flatten(), expected) + def test_dequantization_mxfp4_hardcoded(self): # MXFP4: 1 byte shared exponent E + 16 packed bytes (32 x 4-bit values) # nibble: bit3=sign, bit2:1=exp, bit0=mant; E=128 gives scale=1.0 @@ -52,13 +118,20 @@ class TestGGUF(unittest.TestCase): def test_dequantization_q5_0(self): self._test_dequantization(GGMLQuantizationType.Q5_0) def test_dequantization_q5_1(self): self._test_dequantization(GGMLQuantizationType.Q5_1) def test_dequantization_q8_0(self): self._test_dequantization(GGMLQuantizationType.Q8_0) + def test_dequantization_q2_k(self): self._test_dequantization(GGMLQuantizationType.Q2_K) + def test_dequantization_q3_k(self): self._test_dequantization(GGMLQuantizationType.Q3_K) def test_dequantization_q4_k(self): self._test_dequantization(GGMLQuantizationType.Q4_K) def test_dequantization_q5_k(self): self._test_dequantization(GGMLQuantizationType.Q5_K) def test_dequantization_q6_k(self): self._test_dequantization(GGMLQuantizationType.Q6_K) + def test_dequantization_iq2_xxs(self): self._test_dequantization(GGMLQuantizationType.IQ2_XXS) + def test_dequantization_iq2_xs(self): self._test_dequantization(GGMLQuantizationType.IQ2_XS) def test_dequantization_iq3_xxs(self): self._test_dequantization(GGMLQuantizationType.IQ3_XXS) + def test_dequantization_iq1_s(self): self._test_dequantization(GGMLQuantizationType.IQ1_S) + def test_dequantization_iq4_nl(self): self._test_dequantization(GGMLQuantizationType.IQ4_NL) def test_dequantization_iq3_s(self): self._test_dequantization(GGMLQuantizationType.IQ3_S) def test_dequantization_iq2_s(self): self._test_dequantization(GGMLQuantizationType.IQ2_S) def test_dequantization_iq4_xs(self): self._test_dequantization(GGMLQuantizationType.IQ4_XS) + def test_dequantization_iq1_m(self): self._test_dequantization(GGMLQuantizationType.IQ1_M) def test_dequantization_mxfp4(self): self._test_dequantization(GGMLQuantizationType.MXFP4) @unittest.skipUnless(dtypes.bfloat16 in supported_dtypes, "Backend must support bfloat16") def test_dequantization_bf16(self): self._test_dequantization(GGMLQuantizationType.BF16) @@ -203,12 +276,23 @@ class TestGGUFGEMV(unittest.TestCase): q_data = rng.integers(0, 256, size=n_blocks * type_size, dtype=np.uint8).reshape(n_blocks, type_size) scales = np.float16(rng.standard_normal(n_blocks * 4)).view(np.uint8).reshape(n_blocks, -1) if qtype in (GGMLQuantizationType.Q5_0, GGMLQuantizationType.Q8_0, - GGMLQuantizationType.IQ3_XXS, - GGMLQuantizationType.IQ2_S, + GGMLQuantizationType.IQ2_XXS, GGMLQuantizationType.IQ2_XS, + GGMLQuantizationType.IQ3_XXS, GGMLQuantizationType.IQ4_NL, + GGMLQuantizationType.IQ1_S, GGMLQuantizationType.IQ2_S, GGMLQuantizationType.IQ3_S, GGMLQuantizationType.IQ4_XS): q_data[:, :2] = scales[:, :2] # d at offset 0 elif qtype in (GGMLQuantizationType.Q5_1, GGMLQuantizationType.Q4_K, GGMLQuantizationType.Q5_K): q_data[:, :4] = scales[:, :4] # d, m/dmin at offset 0 - elif qtype == GGMLQuantizationType.Q6_K: q_data[:, -2:] = scales[:, :2] # d at end + elif qtype == GGMLQuantizationType.Q2_K: q_data[:, -4:] = scales[:, :4] # d, dmin at end + elif qtype in (GGMLQuantizationType.Q6_K, GGMLQuantizationType.Q3_K): q_data[:, -2:] = scales[:, :2] # d at end + elif qtype == GGMLQuantizationType.IQ1_M: + s = np.float16(rng.standard_normal(n_blocks)).view(np.uint16) + sc = q_data[:, -8:].copy().view(np.uint16).reshape(n_blocks, 4) + sc &= np.uint16(0x0FFF) + sc[:, 0] |= (s & np.uint16(0x000F)) << 12 + sc[:, 1] |= (s & np.uint16(0x00F0)) << 8 + sc[:, 2] |= (s & np.uint16(0x0F00)) << 4 + sc[:, 3] |= (s & np.uint16(0xF000)) + q_data[:, -8:] = sc.reshape(n_blocks, -1).view(np.uint8) elif qtype == GGMLQuantizationType.MXFP4: q_data[:, 0] = rng.integers(120, 136, size=n_blocks, dtype=np.uint8) # constrain byte0 q_data = q_data.flatten() ref = dequantize(q_data, qtype).reshape(rows, cols) @@ -235,13 +319,20 @@ class TestGGUFGEMV(unittest.TestCase): def test_gguf_gemv_q8_0(self): self._test_gguf_gemv(GGMLQuantizationType.Q8_0) def test_gguf_gemv_q5_0(self): self._test_gguf_gemv(GGMLQuantizationType.Q5_0) def test_gguf_gemv_q5_1(self): self._test_gguf_gemv(GGMLQuantizationType.Q5_1) + def test_gguf_gemv_q2_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q2_K) + def test_gguf_gemv_q3_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q3_K) def test_gguf_gemv_q4_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q4_K) def test_gguf_gemv_q5_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q5_K) def test_gguf_gemv_q6_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q6_K) + def test_gguf_gemv_iq2_xxs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ2_XXS) + def test_gguf_gemv_iq2_xs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ2_XS) def test_gguf_gemv_iq3_xxs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ3_XXS) + def test_gguf_gemv_iq1_s(self): self._test_gguf_gemv(GGMLQuantizationType.IQ1_S) + def test_gguf_gemv_iq4_nl(self): self._test_gguf_gemv(GGMLQuantizationType.IQ4_NL) def test_gguf_gemv_iq3_s(self): self._test_gguf_gemv(GGMLQuantizationType.IQ3_S) def test_gguf_gemv_iq2_s(self): self._test_gguf_gemv(GGMLQuantizationType.IQ2_S) def test_gguf_gemv_iq4_xs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ4_XS) + def test_gguf_gemv_iq1_m(self): self._test_gguf_gemv(GGMLQuantizationType.IQ1_M) def test_gguf_gemv_mxfp4(self): self._test_gguf_gemv(GGMLQuantizationType.MXFP4) @unittest.skipUnless(dtypes.bfloat16 in supported_dtypes, "Backend must support bfloat16") def test_gguf_gemv_bf16(self): self._test_gguf_gemv(GGMLQuantizationType.BF16) diff --git a/tinygrad_repo/test/unit/test_gradient.py b/tinygrad_repo/test/unit/test_gradient.py index 8a1f91c16..5f704aeb3 100644 --- a/tinygrad_repo/test/unit/test_gradient.py +++ b/tinygrad_repo/test/unit/test_gradient.py @@ -51,6 +51,10 @@ class TestTensorGradient(unittest.TestCase): with self.assertRaises(RuntimeError): x.sum().gradient(x) with self.assertRaises(RuntimeError): x.float().sum().gradient(x) + def test_const_target_raise(self): + t = Tensor(2.0) + with self.assertRaises(RuntimeError): (t * 2.0).gradient(t) + def test_copy_to_device_gradient(self): t = Tensor([1.0, 2, 3]).realize() t.to("CPU:1").square().sum().backward() @@ -100,7 +104,7 @@ class TestTensorGradient(unittest.TestCase): def test_implicit_broadcast_where_gradient(self): # WHERE with a bare ()-shape branch: the scalar's gradient counts the positions where it is selected - cond, x, w = Tensor([True, False, True]), Tensor([1.0, 2.0, 3.0]), Tensor(4.0) + cond, x, w = Tensor([True, False, True]), Tensor([1.0, 2.0, 3.0]), Tensor(4.0, dtype=dtypes.float32) dw = Tensor(cond.uop.alu(Ops.WHERE, x.uop, w.uop)).sum().gradient(w)[0] self.assertEqual(dw.shape, ()) self.assertEqual(dw.item(), 1.0) @@ -109,7 +113,7 @@ class TestTensorGradient(unittest.TestCase): def test_implicit_broadcast_alu_gradient(self): # MUL with a bare ()-shape src, no EXPAND in the graph - x, w = Tensor([1.0, 2.0, 3.0]), Tensor(2.0) + x, w = Tensor([1.0, 2.0, 3.0]), Tensor(2.0, dtype=dtypes.float32) m = x.uop.alu(Ops.MUL, w.uop) self.assertIs(m.src[1], w.uop) dw = Tensor(m).sum().gradient(w)[0] @@ -118,7 +122,7 @@ class TestTensorGradient(unittest.TestCase): def test_implicit_broadcast_intermediate_accumulation(self): # s is used directly and through an implicit broadcast edge, each edge's gradient reduces to s's shape before they sum - x, p = Tensor([1.0, 2.0, 3.0]), Tensor(0.5) + x, p = Tensor([1.0, 2.0, 3.0]), Tensor(0.5, dtype=dtypes.float32) s = p.sin() z = Tensor(x.uop.alu(Ops.MUL, s.uop)).sum() + s dp = z.gradient(p)[0] diff --git a/tinygrad_repo/test/unit/test_hcq_graph.py b/tinygrad_repo/test/unit/test_hcq_graph.py index 2ca6fd7ef..fd3737576 100644 --- a/tinygrad_repo/test/unit/test_hcq_graph.py +++ b/tinygrad_repo/test/unit/test_hcq_graph.py @@ -25,10 +25,10 @@ class TestHCQUnit(unittest.TestCase): cpu_call = UOp(Ops.PROGRAM, src=(UOp.sink(),)).call(UOp.new_buffer("CPU", 1, dtypes.float)) gpu_devs = [d0] - # local MMIO: GPU works alone and with CPU in batch (cpu_support=True) + # CPU uses HCQ2 and is no longer batched into legacy HCQ graphs. assert HCQGraph.supports_uop(gpu_devs, gpu_call) is True - assert HCQGraph.supports_uop(gpu_devs, cpu_call) is True - assert HCQGraph.supports_uop(gpu_devs + [cpu_dev], gpu_call) is True + assert HCQGraph.supports_uop(gpu_devs, cpu_call) is False + assert HCQGraph.supports_uop(gpu_devs + [cpu_dev], gpu_call) is False # USB MMIO: GPU-only still works, but CPU batching must be rejected (cpu_support=False) orig_view = d0.timeline_signal.base_buf.view diff --git a/tinygrad_repo/test/unit/test_invalid_tensor.py b/tinygrad_repo/test/unit/test_invalid_tensor.py index 1c8630af3..1277ce6b8 100644 --- a/tinygrad_repo/test/unit/test_invalid_tensor.py +++ b/tinygrad_repo/test/unit/test_invalid_tensor.py @@ -23,6 +23,12 @@ class TestInvalidTensor(unittest.TestCase): out = mask.where(Tensor([1.0, 2.0, 3.0, 4.0]), Invalid) self._invalid_test_helper(out, [1.0, 2.0, None, None]) + def test_where_padded_invalid_cast(self): + # a padded Invalid is not Invalid: its zeros take the cast + a, b = Tensor.full((1,), Invalid).pad((1,1)), Tensor.full((1,), Invalid).pad((2,0)) + out = Tensor([True, False, True]).where(a, b).cast(dtypes.float) + Tensor([1., 2., 3.]) + self.assertEqual((out.dtype, out.tolist()), (dtypes.float, [1.0, 2.0, 3.0])) + def test_where_invalid_x(self): mask = Tensor.arange(4) < 2 out = mask.where(Invalid, Tensor([1.0, 2.0, 3.0, 4.0])) diff --git a/tinygrad_repo/test/unit/test_llm_amd.py b/tinygrad_repo/test/unit/test_llm_amd.py new file mode 100644 index 000000000..718a44599 --- /dev/null +++ b/tinygrad_repo/test/unit/test_llm_amd.py @@ -0,0 +1,122 @@ +import unittest +import numpy as np +from tinygrad import Tensor, UOp, dtypes, nn, function +from tinygrad.llm.kernels.amd import Linear, amd_custom_kernels_supported, q8_quantize, flash_attention +from tinygrad.llm.gguf import ggml_data_to_tensor + +class TestQ8Quantize(unittest.TestCase): + def test_word_quant_weights_use_typed_buffer_view(self): + for ggml_type, type_size in ((13, 176), (23, 136)): + with self.subTest(ggml_type=ggml_type): + raw = Tensor(np.zeros(type_size + 4, dtype=np.uint8), device="CPU").contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 256, ggml_type).reshape(1, 256) + linear = Linear(256, 1, bias=False) + linear.set_quantized(decoded) + self.assertEqual(linear.ggml_type, ggml_type) + self.assertEqual(linear.weight.dtype, dtypes.uint32) + self.assertEqual(linear.weight.nbytes(), type_size) + self.assertEqual(linear.weight.uop.buf_uop.buffer.offset, 4) + + def test_values_and_scales(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + x = np.linspace(-3.1, 2.7, 64, dtype=np.float32).reshape(2, 32) + quant, scale, gsum = q8_quantize(Tensor(x), 2, 32) + scale_np = np.maximum(np.max(np.abs(x), axis=-1, keepdims=True) / 127, 1e-8) + expected = np.clip(np.rint(x / scale_np), -127, 127).astype(np.int8) + np.testing.assert_array_equal(quant.bitcast(dtypes.int8).reshape(2, 32).numpy(), expected) + np.testing.assert_allclose(scale.numpy(), scale_np, rtol=1e-6) + # xsum holds the two per-16 sums per 32-wide group + np.testing.assert_array_equal(gsum.numpy().reshape(2, 2), expected.reshape(2, 2, 16).sum(-1).astype(np.float32)) + + def test_q6_linear_compiles_in_function(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + packed = rng.integers(0, 256, 210, dtype=np.uint8) + packed[-2:] = np.array([0.01], dtype=np.float16).view(np.uint8) + raw = Tensor(np.pad(packed, (4, 0))).contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 256, 14).reshape(1, 256) + linear = Linear(256, 1, bias=False) + nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) + @function(allow_implicit=True) + def run(x:Tensor): return linear(x) + self.assertTrue(np.isfinite(run(Tensor.randn(1, 256)).realize().item())) + # the Q6 weight is repacked: 210-byte blocks padded to 212 (one block = 53 words) + self.assertEqual(linear.weight.uop.buf_uop.buffer.nbytes, 53*4) + self.assertEqual(linear.weight.dtype, dtypes.uint32) + + def test_q4_k_linear(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + in_features, blocks = 2048, 16*2048//256 + packed = rng.integers(0, 256, blocks*144, dtype=np.uint8) + for i in range(blocks): packed[i*144:i*144+4] = np.array([0.01, 0.002], dtype=np.float16).view(np.uint8) + raw = Tensor(np.pad(packed, (4, 0))).contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 16*in_features, 12).reshape(16, in_features) + weight = decoded.numpy() + linear = Linear(in_features, 16, bias=False) + nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) + x = rng.normal(size=(3, in_features)).astype(np.float32) + scale = np.maximum(np.abs(x).reshape(3, in_features//32, 32).max(-1, keepdims=True) / 127, 1e-8) + xq = np.clip(np.rint(x.reshape(3, in_features//32, 32) / scale), -127, 127) * scale + np.testing.assert_allclose(linear(Tensor(x)).numpy(), xq.reshape(3, in_features) @ weight.T, rtol=2e-3, atol=2e-2) + self.assertEqual(linear.ggml_type, 12) + + def test_q6_linear_multiple_tokens(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + in_features, blocks = 2048, 16*2048//256 + packed = rng.integers(0, 256, blocks*210, dtype=np.uint8) + for i in range(blocks): packed[i*210+208:i*210+210] = np.array([0.01], dtype=np.float16).view(np.uint8) + raw = Tensor(np.pad(packed, (4, 0))).contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 16*in_features, 14).reshape(16, in_features) + weight = decoded.numpy() + linear = Linear(in_features, 16, bias=False) + nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) + x = rng.normal(size=(3, in_features)).astype(np.float32) + scale = np.maximum(np.abs(x).reshape(3, in_features//32, 32).max(-1, keepdims=True) / 127, 1e-8) + xq = np.clip(np.rint(x.reshape(3, in_features//32, 32) / scale), -127, 127) * scale + np.testing.assert_allclose(linear(Tensor(x)).numpy(), xq.reshape(3, in_features) @ weight.T, rtol=2e-3, atol=2e-2) + self.assertEqual(linear.ggml_type, 14) + + # symbolic token counts take the padded kernel path and give the same results + generic = Linear(in_features, 16, bias=False) + nn.state.load_state_dict(generic, {"weight":decoded}, verbose=False, realize=False) + sym = Tensor(np.concatenate([x, np.zeros((1, in_features), np.float32)])).contiguous()[:UOp.variable("tokens", 1, 4).bind(3)] + np.testing.assert_allclose(generic(sym)[:3].numpy(), xq.reshape(3, in_features) @ weight.T, rtol=2e-3, atol=2e-2) + self.assertTrue(generic.use_custom_quant) + self.assertEqual(generic.ggml_type, 14) + + def test_attention_uses_physical_cache_length(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + q, k, v = Tensor.zeros(1, 2, 1, 32), Tensor.randn(1, 1, 1, 32), Tensor.randn(1, 1, 1, 32) + cache = Tensor.empty(2, 1, 1, 256, 32, dtype=dtypes.half).contiguous() + assigned = Tensor(cache.uop.after(cache[:, :, :, 0:1, :].uop.store(Tensor.stack(k, v).cast(dtypes.half).uop))) + out = flash_attention(q, assigned, 1).realize() + np.testing.assert_allclose(out.numpy(), v.expand(1, 2, 1, 32).numpy(), rtol=2e-2, atol=2e-2) + + def test_flash_attention_decode_gqa_output_layout(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + Tensor.manual_seed(42) + q = Tensor.randn(1, 4, 1, 128, dtype=dtypes.half).realize() + cache = Tensor.randn(2, 1, 1, 256, 128, dtype=dtypes.half).realize() + out = flash_attention(q, cache, 3).realize() + expected = q.scaled_dot_product_attention(cache[0, :, :, :3], cache[1, :, :, :3], enable_gqa=True) + np.testing.assert_allclose(out.numpy(), expected.numpy(), rtol=2e-3, atol=2e-3) + + def test_prefill_attention_unaligned_start(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + start_pos = 1718 + q = Tensor.zeros(1, 8, 32, 128) + old_kv = rng.normal(size=(2, 1, 1, start_pos, 128)).astype(np.float32) + new_kv = rng.normal(size=(2, 1, 1, 32, 128)).astype(np.float32) + cache = Tensor.zeros(2, 1, 1, 2048, 128, dtype=dtypes.half).contiguous() + Tensor.realize(cache[:, :, :, :start_pos].assign(Tensor(old_kv).cast(dtypes.half))) + sp = UOp.variable("start_pos", 0, 2047).bind(start_pos) + assigned = Tensor(cache.uop.after(cache[:, :, :, sp:sp+32, :].uop.store(Tensor(new_kv).cast(dtypes.half).uop))) + out = flash_attention(q, assigned, sp+32).realize() + values = np.concatenate([old_kv[1, 0, 0], new_kv[1, 0, 0]]).astype(np.float16).astype(np.float32) + expected = np.stack([values[:start_pos+i+1].mean(0) for i in range(32)])[None, None].repeat(8, axis=1) + np.testing.assert_allclose(out.numpy(), expected, rtol=2e-3, atol=2e-3) + +if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/unit/test_llm_moe.py b/tinygrad_repo/test/unit/test_llm_moe.py index 56a486ab5..b5d6aafcb 100644 --- a/tinygrad_repo/test/unit/test_llm_moe.py +++ b/tinygrad_repo/test/unit/test_llm_moe.py @@ -2,7 +2,7 @@ import unittest import numpy as np from dataclasses import replace from tinygrad import Tensor -from tinygrad.llm.model import TransformerBlock, TransformerConfig +from tinygrad.llm.model import ExpertGating, TransformerBlock, TransformerConfig def _moe_config(dim=8, hidden=16, n_heads=2, num_experts=4, num_experts_per_tok=2): return TransformerConfig( @@ -96,5 +96,32 @@ class TestMoEFeedForward(unittest.TestCase): expected = moe_expected + shared_expected np.testing.assert_allclose(out.numpy(), expected, rtol=1e-2) + def test_moe_feed_forward_gating_funcs(self): + dim, hidden, n_heads = 8, 16, 2 + num_experts, k = 4, 2 + logits = np.array([4.0, 3.0, 0.0, -1.0], dtype=np.float32) + def softmax(x): + probs = np.exp(x - x.max()) + return probs / probs.sum() + for gating_func in ExpertGating: + for norm_topk_prob in (False, True): + block = TransformerBlock(replace(_moe_config(dim, hidden, n_heads, num_experts, k), + expert_gating_func=gating_func, norm_topk_prob=norm_topk_prob)) + block.ffn_gate_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) for _ in range(num_experts)]) + block.ffn_up_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) * (i + 1) for i in range(num_experts)]) + block.ffn_down_exps.weight = Tensor.stack(*[Tensor.eye(dim, hidden) for _ in range(num_experts)]) + block.ffn_gate_inp.weight = Tensor((logits / dim)[None, :].repeat(dim, 0).T) + out = block._feed_forward(Tensor.ones(1, 1, dim)).numpy()[0, 0, 0] + + if gating_func == ExpertGating.SOFTMAX: selection_scores = softmax(logits) + elif gating_func == ExpertGating.SIGMOID: selection_scores = 1 / (1 + np.exp(-logits)) + elif gating_func == ExpertGating.SOFTMAX_WEIGHT: selection_scores = logits + else: selection_scores = np.sqrt(np.logaddexp(0, logits)) + sel = np.argsort(selection_scores)[-k:] + weights = softmax(logits[sel]) if gating_func == ExpertGating.SOFTMAX_WEIGHT else selection_scores[sel] + if norm_topk_prob: weights /= weights.sum() + expected = (weights * (sel + 1)).sum() / (1 + np.exp(-1)) + np.testing.assert_allclose(out, expected, rtol=1e-3) + if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/unit/test_llm_server.py b/tinygrad_repo/test/unit/test_llm_server.py index 4eee0e522..e8ade873f 100644 --- a/tinygrad_repo/test/unit/test_llm_server.py +++ b/tinygrad_repo/test/unit/test_llm_server.py @@ -1,6 +1,8 @@ import unittest +import numpy as np from unittest.mock import patch from tinygrad import Tensor, UOp +from tinygrad.nn.state import get_state_dict from tinygrad.schedule import schedule_cache from tinygrad.llm.model import Transformer, TransformerConfig from tinygrad.llm.serve import StreamRouter @@ -13,12 +15,18 @@ V_TOKS = UOp.variable("toks", 1, 32) # 32 is the default chunk_size in generate class TestTransformerGenerate(unittest.TestCase): def test_warmup(self): model, calls = Transformer(TEST_CONFIG), [] - def generate(tokens): + def generate(tokens, **kwargs): calls.append(tokens) yield from (1, 2) with patch.object(model, "generate", generate): model.warmup() self.assertEqual(calls, [[0], [0]]) + def test_warmup_then_generate_with_default_chunk(self): + # warmup must not capture JIT graphs that generate()'s default chunk_size then rejects + model = Transformer(TEST_CONFIG) + model.warmup() + self.assertIsInstance(next(model.generate([5, 6, 7, 8])), int) + def test_first_recurrent_generate_before_state_init(self): model = Transformer(TEST_CONFIG) model.has_recurrent_block = True @@ -36,7 +44,19 @@ class TestTransformerGenerate(unittest.TestCase): return Tensor([[42]]) with patch.object(Transformer, '__call__', mock_call): next(model.generate([1, 2, 3, 4, 5, 42, 10])) - self.assertEqual(calls, [((1, 1), V_START_POS.bind(5)), ((1, 1), V_START_POS.bind(6))]) + # resumes from the reused state at position 5 and consumes the 2 new tokens (one chunk or two decode steps) + self.assertEqual(calls[0][1], V_START_POS.bind(5)) + def ntok(shape): return shape[1] if isinstance(shape[1], int) else shape[1].unbind()[1] + self.assertEqual(sum(ntok(c[0]) for c in calls), 2) + + def test_recurrent_divergent_prompt_restarts(self): + model, calls = Transformer(TEST_CONFIG), [] + model.has_recurrent_block, model._cached_tokens = True, [1, 2, 9] + def mock_call(self, tokens, start_pos, temperature): + calls.append(start_pos) + return Tensor([[42]]) + with patch.object(Transformer, '__call__', mock_call): next(model.generate([1, 2, 10, 11])) + self.assertEqual(calls[0], V_START_POS.bind(0)) def test_template_starts_reasoning(self): router = StreamRouter(reasoning=True) @@ -137,6 +157,22 @@ class TestTransformerGenerate(unittest.TestCase): # 4 tokens, chunk_size=4 -> 1 prefill chunk self.assertEqual(get_prefill_flags(list(range(4)), 4), [True, False, False]) + def test_chunked_prefill_kv_cache_matches_single_chunk(self): + config = TransformerConfig(num_blocks=1, dim=8, hidden_dim=16, n_heads=1, n_kv_heads=1, norm_eps=1e-5, + vocab_size=32, head_dim=4, rope_theta=1000000, rope_dim=4, qk_norm=4, v_head_dim=4, max_context=16) + def model(): + m = Transformer(config) + rng = np.random.RandomState(1234) + for t in get_state_dict(m).values(): + t.assign(Tensor(rng.uniform(-1, 1, t.shape).astype(np.float32))).realize() + return m + def prefill(m, chunk_size): + gen = m.generate(list(range(1, 9)), chunk_size=chunk_size, temperature=0.0) + next(gen) + return [b.cache_kv.numpy() for b in m.blk] + for g, r in zip(prefill(model(), 4), prefill(model(), 8)): + np.testing.assert_allclose(g[:, :, :, :8, :], r[:, :, :, :8, :], atol=1e-5) + def test_kv_cache_resume_matches_fresh(self): model = Transformer(TEST_CONFIG) @@ -178,6 +214,12 @@ class TestTransformerGenerate(unittest.TestCase): # with temperature=2.0, we should see at least 2 distinct outputs across 5 runs self.assertGreater(len(runs), 1, "high temperature should produce varied outputs") + def test_recurrent_temperature_high_produces_variety(self): + model = Transformer(TEST_CONFIG) + model.has_recurrent_block = True + outputs = {model.forward(Tensor([[1]]), 0, Tensor([2.0])).item() for _ in range(5)} + self.assertGreater(len(outputs), 1) + def test_temperature_passed_to_forward(self): """Temperature from generate should be passed through to __call__.""" model = Transformer(TEST_CONFIG) diff --git a/tinygrad_repo/test/unit/test_metal_graph.py b/tinygrad_repo/test/unit/test_metal_graph.py index a85616ff7..c879a9fc2 100644 --- a/tinygrad_repo/test/unit/test_metal_graph.py +++ b/tinygrad_repo/test/unit/test_metal_graph.py @@ -1,5 +1,4 @@ import unittest -from unittest.mock import MagicMock from tinygrad import Device from tinygrad.uop.ops import Ops, UOp from tinygrad.dtype import dtypes @@ -11,36 +10,27 @@ class TestMetalGraph(unittest.TestCase): self.MetalGraph = MetalGraph self.dev = Device[Device.DEFAULT] - def metal_buf(self, offset): - buf = MagicMock() - if offset > 0: - buf.op = Ops.SLICE - src = MagicMock() - src.dtype = dtypes.uint8 - buf.src = (src, UOp.const(offset)) - buf.dtype = dtypes.uint8 - else: - buf.op = Ops.BUFFER - buf.device = Device.DEFAULT - return buf + def metal_buf(self, offset, bitcast=False): + size = 4 if bitcast else 1 + buf = UOp.new_buffer(Device.DEFAULT, offset+size, dtypes.uint8) + if offset: buf = buf[offset:offset+size] + return buf.bitcast(dtypes.float32) if bitcast else buf - def call(self, *bufs): - c = MagicMock() - c.src = (MagicMock(op=Ops.PROGRAM),) + tuple(bufs) - return c + def supports_uop(self, *bufs): + return self.MetalGraph.supports_uop([self.dev], UOp(Ops.PROGRAM, src=(UOp.sink(),)).call(*bufs)) def test_supports_uop_normal_offset(self): - assert self.MetalGraph.supports_uop([self.dev], self.call(self.metal_buf(0), self.metal_buf(100), self.metal_buf(0xFFFFFFFF))) is True + assert self.supports_uop(self.metal_buf(0), self.metal_buf(100), self.metal_buf(0xFFFFFFFF)) is True def test_supports_uop_overflow_offset(self): - assert self.MetalGraph.supports_uop([self.dev], self.call(self.metal_buf(0), self.metal_buf(0x100000000))) is False + assert self.supports_uop(self.metal_buf(0), self.metal_buf(0x100000000)) is False - def test_supports_uop_nonmetal_buf(self): - # non-SLICE ops should not be checked for offset - buf = MagicMock() - buf.op = Ops.BUFFER - buf.device = Device.DEFAULT - self.MetalGraph.supports_uop([self.dev], self.call(buf)) + def test_supports_uop_non_view_buf(self): + assert self.supports_uop(self.metal_buf(0)) is True + + def test_supports_uop_bitcast(self): + assert self.supports_uop(self.metal_buf(0xFFFFFFFF, bitcast=True)) is True + assert self.supports_uop(self.metal_buf(0x100000000, bitcast=True)) is False if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/unit/test_multitensor.py b/tinygrad_repo/test/unit/test_multitensor.py index 139c4588f..8ceebe6e1 100644 --- a/tinygrad_repo/test/unit/test_multitensor.py +++ b/tinygrad_repo/test/unit/test_multitensor.py @@ -390,6 +390,12 @@ class TestMultiTensor(unittest.TestCase): self.assertEqual(out.shape, (rows, 8)) np.testing.assert_equal(out[:3].to(Device.DEFAULT).numpy(), np.ones((3, 8))) + def test_symbolic_broadcast_consumed(self): + rows = Variable("rows", 1, 4).bind(3) + out = (Tensor.ones(rows).to(devices_2) + 1).realize() + self.assertEqual(out.shape, (rows,)) + np.testing.assert_equal(out[:3].to(Device.DEFAULT).numpy(), np.full(3, 2)) + def test_multitensor_jit_in_list(self): # test MULTI tensor inside a list container - exercises the container unpacking + MULTI unpacking @TinyJit @@ -846,82 +852,6 @@ class TestMultiFromUnrenderable(unittest.TestCase): np.testing.assert_equal(ll.numpy(), np.arange(100)+1) @unittest.skipIf(not_support_multi_device(), "need multi") -class TestMultiAssign(unittest.TestCase): - device = tuple(f"{Device.DEFAULT}:{i}" for i in range(2)) - - @needs_second_gpu - def setUp(self): pass - - def test_multi_assign_realized(self): - out = Tensor.zeros(4).shard(self.device, 0).contiguous().realize() - ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(ones).realize() - self.assertListEqual(out.tolist(), [1,1,1,1]) - - def test_multi_assign_unrealized(self): - out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) - ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(ones).realize() - self.assertListEqual(out.tolist(), [1,1,1,1]) - - def test_multi_assign_both_unrealized(self): - out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) - ones = Tensor.ones(4).contiguous().realize().shard(self.device, 0) - out.assign(ones).realize() - self.assertListEqual(out.tolist(), [1,1,1,1]) - - def test_multi_assign_scalar(self): - out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(0).realize() - self.assertListEqual(out.tolist(), [0,0,0,0]) - - def test_multi_assign_const_like(self): - out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(out.const_like(7)).realize() - self.assertListEqual(out.tolist(), [7,7,7,7]) - - def test_multi_assign_piece(self): - out = Tensor.zeros(4,4).shard(self.device, 0).contiguous().realize() - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - out[:, 2:3].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - def test_multi_assign_piece_noncontig(self): - out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - out[:, 2:3].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - @unittest.expectedFailure - def test_multi_assign_piece_unrealized(self): - out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0) - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - out[:, 2:3].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - def test_multi_assign_var_offset(self): - out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - vi = Variable("i", 0, 3).bind(2) - out[:, vi:vi+1].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - def test_multi_assign_var_offset_jit_none(self): self.test_multi_assign_var_offset_jit(None) - def test_multi_assign_var_offset_jit(self, shard_axis=0): - out = Tensor.zeros(4,6).contiguous().realize().shard(self.device, shard_axis).realize() - ones = Tensor.ones(4,1).shard(self.device, shard_axis).contiguous().realize() - - @TinyJit - def f(out:Tensor, vi): - out[:, vi:vi+1].assign(ones).realize() - ones.assign(ones+1).realize() - - vi = Variable("i", 0, 5) - for i in range(1,5): - GlobalCounters.reset() - f(out, vi.bind(i)) - self.assertListEqual(out.tolist(), [[0,1,2,3,4,0]]*4) - @unittest.skipIf(not_support_multi_device(), "need multi") class TestMultiSetitem(unittest.TestCase): device = tuple(f"{Device.DEFAULT}:{i}" for i in range(4)) diff --git a/tinygrad_repo/test/unit/test_schedule_cache.py b/tinygrad_repo/test/unit/test_schedule_cache.py index 078d51214..eae741b57 100644 --- a/tinygrad_repo/test/unit/test_schedule_cache.py +++ b/tinygrad_repo/test/unit/test_schedule_cache.py @@ -1,11 +1,16 @@ import unittest import functools -from tinygrad import Tensor, Variable, UOp +from tinygrad import Tensor, Variable, UOp, function from tinygrad.uop.ops import KernelInfo from tinygrad.schedule import schedule_cache -def custom_set0_kernel(A:UOp, num:int) -> UOp: - return A[0].set(num).sink(arg=KernelInfo(f"custom_set0_{num}")) +def custom_add_kernel(A:UOp, B:UOp, num:int=0) -> UOp: + return A[0].set(B[0] + num).sink(arg=KernelInfo(f"custom_add_{num}")) + +def custom_add_backward(grad_output:UOp, _) -> tuple[None, UOp]: + grad = Tensor.invalids(*grad_output.shape, dtype=grad_output.dtype, device=grad_output.device) + grad = Tensor.custom_kernel(grad, Tensor(grad_output, device=grad_output.device), fxn=functools.partial(custom_add_kernel, num=0))[0] + return None, grad.uop class TestScheduleCache(unittest.TestCase): def test_bound_variable_reuses_cache(self): @@ -25,27 +30,27 @@ class TestScheduleCache(unittest.TestCase): def test_custom_kernel(self): for i in range(4): - a = Tensor.empty(1) - a = Tensor.custom_kernel(a, fxn=functools.partial(custom_set0_kernel, num=i))[0] + a, b = Tensor.empty(1), Tensor.ones(1) + a = Tensor.custom_kernel(a, b, fxn=functools.partial(custom_add_kernel, num=i))[0] a.realize() - self.assertEqual(a.item(), i) + self.assertEqual(a.item(), i+1) def test_same_custom_function_reuses_cache(self): schedule_cache.clear() - fxn = functools.partial(custom_set0_kernel, num=10) + fxn = functools.partial(custom_add_kernel, num=10) # first run - a = Tensor.empty(1) - a = Tensor.custom_kernel(a, fxn=fxn)[0] + a, x = Tensor.empty(1), Tensor.ones(1) + a = Tensor.custom_kernel(a, x, fxn=fxn)[0] a.realize() - self.assertEqual(a.item(), 10) + self.assertEqual(a.item(), 11) cache_size_after_first = len(schedule_cache) # second run with same function should reuse cache - b = Tensor.empty(1) - b = Tensor.custom_kernel(b, fxn=fxn)[0] + b, x = Tensor.empty(1), Tensor.ones(1) + b = Tensor.custom_kernel(b, x, fxn=fxn)[0] b.realize() - self.assertEqual(b.item(), 10) + self.assertEqual(b.item(), 11) self.assertEqual(len(schedule_cache), cache_size_after_first) def test_simple(self): @@ -65,5 +70,29 @@ class TestScheduleCache(unittest.TestCase): print(num) self.assertEqual(len(schedule_cache), start_len_schedule_cache) + def test_simple_precompile(self): + @function(precompile=True, precompile_backward=True) + def f(x:Tensor) -> Tensor: + out = Tensor.invalids(*x.shape, dtype=x.dtype, device=x.device) + out = Tensor.custom_kernel(out, x, fxn=functools.partial(custom_add_kernel, num=10), grad_fxn=custom_add_backward)[0] + return out + x + + # warmup + x = Tensor.ones(1).realize() + out = f(x) + out.backward(x) + self.assertEqual(out.item(), 12) + self.assertEqual(x.grad.item(), 2) + + # use the cache next time function is called + start_len_schedule_cache = len(schedule_cache) + for _ in range(3): + x = Tensor.ones(1).realize() + out = f(x) + out.backward(x) + self.assertEqual(out.item(), 12) + self.assertEqual(x.grad.item(), 2) + self.assertEqual(len(schedule_cache), start_len_schedule_cache) + if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/unit/test_tinyfs.py b/tinygrad_repo/test/unit/test_tinyfs.py deleted file mode 100644 index aa7c0f619..000000000 --- a/tinygrad_repo/test/unit/test_tinyfs.py +++ /dev/null @@ -1,83 +0,0 @@ -import json, math, os, socketserver, threading, unittest -import numpy as np -from tinygrad import Tensor, dtypes -from tinygrad.helpers import CHUNK_SIZE -from tinygrad.nn.state import fs_store, fs_load -from extra.tinyfs.fetch_file import hash_file, _python_hash_1mb - -_chunks: dict[bytes, bytes] = {} - -class _Handler(socketserver.StreamRequestHandler): - def handle(self): - while line := self.rfile.readline(): - cmd = line.decode().strip() - if cmd == "INFO": - self.wfile.write(json.dumps({"node0": ["node0", f"127.0.0.1:{self.server.server_address[1]}"]}).encode() + b"\r\n") - elif cmd.startswith("STORE_IN"): - data = self.rfile.read(int(cmd.split()[1])) - hashes = bytearray() - for i in range(math.ceil(len(data) / CHUNK_SIZE)): - chunk = data[i*CHUNK_SIZE:(i+1)*CHUNK_SIZE].ljust(CHUNK_SIZE, b'\0') - h = _python_hash_1mb(chunk) - _chunks[h] = chunk - hashes.extend(h) - self.wfile.write(hashes) - elif cmd.startswith("LOAD_IN"): - hashes = self.rfile.read(int(cmd.split()[1])) - self.wfile.write(json.dumps(["node0"] * (len(hashes) // 16)).encode() + b"\r\n") - elif cmd.startswith("CHUNK_OUT"): - size = int(cmd.split()[1]) - self.wfile.write(_chunks.get(self.rfile.read(16), bytes(size))[:size]) - self.wfile.flush() - -# regressed in 55d3a5def "preallocate all realized buffers" -class TestTinyFS(unittest.TestCase): - @classmethod - def setUpClass(cls): - _chunks.clear() - cls._server = socketserver.ThreadingTCPServer(('127.0.0.1', 0), _Handler) - cls._server.daemon_threads = True - threading.Thread(target=cls._server.serve_forever, daemon=True).start() - os.environ["TINYFS_ENDPOINT"] = f"127.0.0.1:{cls._server.server_address[1]}" - - @classmethod - def tearDownClass(cls): - _chunks.clear() - os.environ.pop("TINYFS_ENDPOINT", None) - cls._server.shutdown() - cls._server.server_close() - - def test_store(self): - h = fs_store(Tensor([1.0, 2.0, 3.0, 4.0])).realize() - self.assertEqual(h.shape, (16,)) - self.assertEqual(h.dtype, dtypes.uint8) - - def test_store_deterministic(self): - a = fs_store(Tensor([1.0, 2.0, 3.0, 4.0])).realize() - b = fs_store(Tensor([1.0, 2.0, 3.0, 4.0])).realize() - np.testing.assert_array_equal(a.numpy(), b.numpy()) - - def test_store_different_data(self): - a = fs_store(Tensor([1.0, 2.0, 3.0, 4.0])).realize() - b = fs_store(Tensor([5.0, 6.0, 7.0, 8.0])).realize() - self.assertNotEqual(a.tolist(), b.tolist()) - - def test_roundtrip_uint8(self): - arr = np.arange(256, dtype=np.uint8) - loaded = fs_load(fs_store(Tensor(arr)).realize(), len(arr)).to("CPU") - np.testing.assert_array_equal(loaded.numpy(), arr) - - def test_roundtrip_multichunk_uint8(self): - arr = np.random.default_rng(42).integers(0, 256, size=CHUNK_SIZE + 1024, dtype=np.uint8) - loaded = fs_load(fs_store(Tensor(arr)).realize(), len(arr)).to("CPU") - np.testing.assert_array_equal(loaded.numpy(), arr) - - def test_hash_matches_python_impl(self): - arr = np.arange(256, dtype=np.uint8) - h = fs_store(Tensor(arr)).realize() - # the hash from fs_store should match the pure-Python hash_file reference - padded = arr.tobytes().ljust(CHUNK_SIZE, b'\0') - self.assertEqual(h.data().tobytes(), hash_file(padded)) - -if __name__ == "__main__": - unittest.main() diff --git a/tinygrad_repo/tinygrad/codegen/__init__.py b/tinygrad_repo/tinygrad/codegen/__init__.py index 7d8c0dbd3..da035b337 100644 --- a/tinygrad_repo/tinygrad/codegen/__init__.py +++ b/tinygrad_repo/tinygrad/codegen/__init__.py @@ -1,9 +1,9 @@ from dataclasses import replace, dataclass import itertools, functools -from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC -from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TracingKey, Context, panic +from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, USE_TC +from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT, TC_MIN_GLOBALS, TracingKey, Context, panic from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType -from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak, pm_cast_weak +from tinygrad.uop.weak import pm_lower_weak, pm_commit_weak, pm_cast_const from tinygrad.uop.render import pyrender from tinygrad.uop.spec import type_verify, spec_tensor, spec_program from tinygrad.renderer import Renderer, Estimates @@ -12,7 +12,7 @@ from tinygrad.dtype import dtypes, AddrSpace # import all pattern matchers here from tinygrad.codegen.gpudims import pm_add_gpudims -from tinygrad.uop.symbolic import sym, symbolic_simple, symbolic, pm_fold_cast_const, pm_move_where_on_load, pm_clean_up_group_sink, pm_remove_invalid +from tinygrad.uop.symbolic import sym, symbolic_simple, symbolic, pm_move_where_on_load, pm_clean_up_group_sink, pm_remove_invalid, invalid_gate from tinygrad.uop.movement import mop_cleanup from tinygrad.codegen.decomp.dtype import pm_dtype_decomps from tinygrad.codegen.decomp.op import get_late_rewrite_patterns, get_simplifying_rewrite_patterns @@ -22,11 +22,11 @@ from tinygrad.codegen.opt.postrange import apply_opts from tinygrad.codegen.late.gater import pm_move_gates_from_index from tinygrad.codegen.simplify import pm_simplify_ranges, pm_flatten_range, pm_split_ranges, pm_load_collapse, pm_reduce_unparented from tinygrad.schedule.multi import multi_pm -from tinygrad.schedule.rangeify import pm_mops +from tinygrad.schedule.prepare import pm_mops from tinygrad.codegen.late.linearizer import CFGContext, pm_split_ends, pm_add_control_flow, linearize from tinygrad.codegen.late.regalloc import LinearScanRegallocContext, pm_regalloc_rewrite from tinygrad.codegen.late.coalesce import memory_coalescing, pm_simplify_add_image -from tinygrad.helpers import all_same, flatten, argsort, partition +from tinygrad.helpers import all_same, all_int, flatten, argsort, partition from tinygrad.uop.ops import _broadcast_shape, identity_element from tinygrad.schedule.rangeify import BufferizeOpts @@ -126,7 +126,7 @@ def do_devectorize(b:UOp): if not all(x.shape == b.shape or x.base.is_invalid for x in b.src): return None src = [] for idx_c in itertools.product(*[[UOp.const(i) for i in range(x)] for x in b.shape]): - src.append(b.replace(dtype=None, src=tuple(x.base if x.base.is_invalid else x.index(*idx_c) for x in b.src))) + src.append(b.replace(src=tuple(x.base if x.base.is_invalid else x.index(*idx_c) for x in b.src))) return UOp.stack(*src).reshape(b.shape) if b.op is not Ops.STORE else UOp.group(*src) def do_stack_wmma(u:UOp): @@ -153,8 +153,8 @@ devectorizer2 = mop_cleanup+pm_mops+PatternMatcher([ # unpack WMMA (UPat(Ops.WMMA, name="u"), do_stack_wmma), # stacked INDEX is many INDEX - (UPat(Ops.INDEX, src=(UPat((Ops.PARAM, Ops.BUFFER), name="b"), UPat(Ops.STACK, name="s"))), - lambda b,s: UOp.stack(*[b.index(u) for u in s.src])), + (UPat(Ops.INDEX, src=(UPat((Ops.PARAM, Ops.BUFFER), name="b"), UPat(Ops.STACK, name="s")), name="x"), + lambda b,s,x: UOp.stack(*[x.replace(src=(b,u)) for u in s.src])), # INDEX into RESHAPE moves the RESHAPE (UPat(Ops.INDEX, src=(UPat((Ops.PARAM, Ops.BUFFER), name="b"), UPat(Ops.RESHAPE, name="s"))), lambda b,s: b.index(s.src[0]).reshape(s.shape)), @@ -162,9 +162,10 @@ devectorizer2 = mop_cleanup+pm_mops+PatternMatcher([ (UPat(Ops.RESHAPE, dtype=dtypes.void, name="x"), lambda x: x.src[0]), # reshape of a single element shaped value to scalar is an index (UPat(Ops.RESHAPE, name="x"), lambda x: x.src[0].index(0) if x.marg == () and x.src[0].shape == (1,) else None), - # EXPAND on scalar -> STACK + # EXPAND on scalar -> nested STACKs with the same shape (UPat(Ops.EXPAND, src=(UPat.var("x"), UPat()), name="out"), - lambda x,out: UOp.stack(*([x]*out.max_numel())) if x.shape == () and out.shape == (out.max_numel(),) else None), + lambda x,out: functools.reduce(lambda x,s: UOp.stack(*([x]*s)), reversed(out.shape), x) + if x.shape == () and all_int(out.shape) and 0 not in out.shape else None), ]) def fix_group_for_reduce(x:UOp): @@ -172,7 +173,7 @@ def fix_group_for_reduce(x:UOp): if len(reduce_gfr) == 0: return None # NOTE: if there's other locals here, we need them in the buffer too - upstream_locals = [u for u in x.toposort() if u.op is Ops.RANGE and u.arg[1] == AxisType.LOCAL] + upstream_locals = [u for u in x.toposort() if u.op is Ops.RANGE and u.arg[1] in (AxisType.WARP, AxisType.LOCAL)] # do only the non grouped reduces early ret = x.replace(src=(x.src[0],)+tuple(reduce_r)) @@ -224,6 +225,16 @@ def expand_horizontal_reduce(r:UOp): vals = [inp.index(*idx) for idx in itertools.product(*[range(inp.max_shape[a]) for a in range(r.arg[1])])] return functools.reduce(lambda x,y: x.alu(r.arg[0], y), vals) +# an Invalid in a REDUCE source is that reduce's identity. a WMMA is a rangeless reduce, so it takes the ADD identity +pm_reduce_identity = PatternMatcher([ + (invalid_gate.reduce(allow_any_len=True, name="red"), lambda red,cond,x,i: + red.replace(src=(cond.where(x, x.const_like(identity_element(red.arg[0], red.dtype))),)+red.src[1:])), + (UPat(Ops.WMMA, src=(invalid_gate, UPat.var("b"), UPat.var("acc")), name="w"), + lambda w,cond,x,i,b,acc: w.replace(src=(cond.where(x, x.const_like(0)), b, acc))), + (UPat(Ops.WMMA, src=(UPat.var("a"), invalid_gate, UPat.var("acc")), name="w"), + lambda w,cond,x,i,a,acc: w.replace(src=(a, cond.where(x, x.const_like(0)), acc))), +]) + pm_reduce_local = pm_wmma_add+PatternMatcher([ # fix group for reduce (UPat(Ops.REDUCE, name="x"), fix_group_for_reduce), @@ -233,10 +244,11 @@ pm_reduce_local = pm_wmma_add+PatternMatcher([ (UPat(Ops.SINK, name="sink"), merge_reduce_ends), ])+pm_clean_up_group_sink +def is_shape_changing_bitcast(u:UOp): return u.op is Ops.BITCAST and u.shape != u.src[0].shape def maybe_load(u:UOp): return u.load() if u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL, AddrSpace.REG) else u pm_add_loads = PatternMatcher([ - # BITCAST? - (UPat(GroupOp.Elementwise|{Ops.REDUCE,Ops.WMMA,Ops.STACK}, name="x"), lambda x: x.replace(src=tuple([maybe_load(u) for u in x.src]))), + (UPat(GroupOp.Elementwise|{Ops.REDUCE,Ops.WMMA,Ops.STACK}, name="x"), + lambda x: None if is_shape_changing_bitcast(x) else x.replace(src=tuple(map(maybe_load, x.src)))), (UPat(Ops.STORE, name="x"), lambda x: x.replace(src=(x.src[0], maybe_load(x.src[1]))+x.src[2:])), ]) @@ -301,7 +313,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_split_ranges+pm_flatten_range, ctx={}, name="split ranges") # symbolic (NOTE: this is a requirement for pm_simplify_ranges to be correct) - sink = graph_rewrite(sink, sym+pm_fold_cast_const+pm_flatten_range, name="initial symbolic") + sink = graph_rewrite(sink, sym+pm_flatten_range, name="initial symbolic") # optimize (schedule) the AST sink = graph_rewrite(sink, pm_flatten_range+pm_simplify_ranges, ctx={}, name="simplify ranges") @@ -311,7 +323,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # ** expander (expand_rewrite) ** # reduce_unparented: a REDUCE whose src folded to a CONST (e.g. x*0) has no parented ranges, collapse it before the expander - sink = graph_rewrite(sink, sym+pm_move_where_on_load+pm_flatten_range+pm_reduce_unparented, name="postopt symbolic") + sink = graph_rewrite(sink, sym+pm_move_where_on_load+pm_flatten_range+pm_reduce_unparented+pm_reduce_identity, name="postopt symbolic") # expand sink = graph_rewrite(sink, expander2, ctx=build_range_map(sink), name="expander") @@ -342,11 +354,13 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # extra symbolic before decomp. crashes without this? # NOTE: also run indexing_simplify here, while the index is still weakint and (x+y)*c -> x*c+y*c applies - sink = graph_rewrite(sink, sym+indexing_simplify, name="extra symbolic") + # commit widths minted in this fixpoint before lowering inspects INDEX shapes + sink = graph_rewrite(sink, sym+indexing_simplify+pm_commit_weak, name="extra symbolic") - # lower index dtype + # the boundary: required compute dtypes settle here; derivable const edges may stay bare # NOTE: we need indexing_simplify to remove the cast to long using the Invalid - sink = graph_rewrite(sink, symbolic_simple+pm_fold_cast_const+pm_lower_index_dtype+indexing_simplify, ctx={}, name="lower all index dtypes") + # NOTE: symbolic must NOT be composed here -- pm_data_invalid pushes the weak result CAST into a gated WHERE, remaking the weak node, and it cycles + sink = graph_rewrite(sink, pm_lower_weak+indexing_simplify, name="lower all index dtypes") # final symbolic before decomp sink = graph_rewrite(sink, symbolic, name="final symbolic") @@ -357,7 +371,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # floordiv+mod / dtype decomp (early) supported_ops = tuple(ren.code_for_op.keys()) - pm_decomp = symbolic_simple+pm_fold_cast_const+get_simplifying_rewrite_patterns(supported_ops) + pm_decomp = symbolic_simple+get_simplifying_rewrite_patterns(supported_ops) sink = graph_rewrite(sink, pm_decomp, name="early decompositions") # late decomps + move gates from unrenderable INVALID where @@ -370,9 +384,12 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # final rules for the renderer (without sym) extra_matcher = ren.extra_matcher if ren.extra_matcher is not None else PatternMatcher([]) - pm_final_rewrite = pm_commit_weak+pm_cast_weak+pm_decomp+extra_matcher+pm_split_ends + pm_final_rewrite = pm_commit_weak+pm_decomp+extra_matcher+pm_split_ends sink = graph_rewrite(sink, pm_final_rewrite+pm_remove_invalid, ctx=ren, name="final rewrite") + # commit every const still bare so no renderer reads one + sink = graph_rewrite(sink, pm_cast_const, name="cast consts") + # add implicit barriers (stores/loads through LOCAL memory ordered by AFTER or across loop iterations need workgroup barriers) sink = graph_rewrite(sink, pm_implicit_barriers, name="add implicit barriers") @@ -384,7 +401,15 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_number_params, ctx=[num_params], name="number params with -1", walk=True) if VIZ: graph_rewrite(sink, PatternMatcher([]), name="View Output AST") - if SPEC: type_verify(sink, spec_program) + if SPEC: + import os + if os.environ.get("DBGTV"): + try: type_verify(sink, spec_program) + except RuntimeError: + from tinygrad.uop.render import print_uops + print_uops(list(sink.toposort())) + raise + else: type_verify(sink, spec_program) # return the rewritten sink return sink @@ -428,7 +453,7 @@ def do_estimates(prg:UOp, sink:UOp, lin:UOp) -> UOp|None: return prg.replace(src=(sink.replace(arg=replace(sink.arg, estimates=Estimates.from_uops(lin.src, ignore_indexing=True))),)+prg.src[1:]) def do_assemble(ctx:Renderer, prg:UOp, lin:UOp) -> UOp: - src = "\n".join(str(u.arg) for u in lin.src) + src = "\n".join(str(u.arg[0]) for u in lin.src) if DEBUG >= 4: print(src) binary = ctx.asm(prg, lin) return prg.replace(src=prg.src[:2]+(UOp(Ops.SOURCE, arg=src), UOp(Ops.BINARY, arg=binary))) @@ -451,7 +476,7 @@ pm_to_program = PatternMatcher([ (UPat(Ops.PROGRAM, src=(UPat(), UPat(Ops.LINEAR), UPat(Ops.SOURCE, name="source")), name="prg"), do_compile), ]) -@rewrite_group(name=lambda ast,renderer,ret,**kwargs: TracingKey(ret.src[0].arg.name,(ret.src[0].arg.function_name, ast), ret=renderer), replay=True) +@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(k.function_name, ast, ret.key),ret=renderer), replay=True) @Context(ALLOW_DEVICE_USAGE=0) def do_to_program(ast:UOp, renderer:Renderer) -> UOp: """ @@ -480,9 +505,14 @@ def do_to_program(ast:UOp, renderer:Renderer) -> UOp: if VIZ: graph_rewrite(prg, PatternMatcher([]), name="View Program") return prg +# config affects generated programs and cache keys; context also carries compile-only behavior to workers +to_program_config = (NOOPT, EMULATED_DTYPES, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, + DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT, TC_MIN_GLOBALS) +to_program_context = (*to_program_config, SPEC, DEBUG) +def to_program_key(ast:UOp, renderer:Renderer) -> tuple: + return (ast.key, type(renderer), renderer.target, *[x.value for x in to_program_config]) + to_program_cache: dict[tuple, UOp] = {} def to_program(ast:UOp, renderer:Renderer) -> UOp: - config = (NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT) - key = (ast.key, type(renderer), renderer.target, *[x.value for x in config]) - if (prg:=to_program_cache.get(key)) is None: to_program_cache[key] = prg = do_to_program(ast, renderer) + if (prg:=to_program_cache.get(key:=to_program_key(ast, renderer))) is None: to_program_cache[key] = prg = do_to_program(ast, renderer) return prg diff --git a/tinygrad_repo/tinygrad/codegen/decomp/dtype.py b/tinygrad_repo/tinygrad/codegen/decomp/dtype.py index 52b8e4630..f02e781d0 100644 --- a/tinygrad_repo/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad_repo/tinygrad/codegen/decomp/dtype.py @@ -1,8 +1,9 @@ from dataclasses import replace -from tinygrad.dtype import dtypes, DType, truncate -from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES, Context, SPEC +from tinygrad.dtype import dtypes, DType, AddrSpace, truncate +from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES from tinygrad.uop import GroupOp -from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite, ParamArg +from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite +from tinygrad.uop.weak import commit_weak_consts from tinygrad.renderer import Renderer from tinygrad.codegen.decomp.transcendental import exponent_bias, shl, shr @@ -25,15 +26,16 @@ def l2i(op: Ops, dt: DType, *uops:UOp): match op: case Ops.NEG: return l2i(Ops.SUB, dt, zero, zero, *uops) case Ops.CAST if dt in (dtypes.long, dtypes.ulong) and uops[0].dtype not in dtypes.floats: - # the high word is the sign extension; bool has no sign, test the already-cast low word instead (bool < 0 would promote to weakint) + # the high word is the sign extension, and unsigned and bool sources zero extend x, lo = uops[0], uops[0].cast(l2i_dt[dt]) - sign = lo if x.dtype is dtypes.bool else x - return lo, (sign < sign.const_like(0)).where(lo.const_like(-1), lo.const_like(0)) + if x.dtype is dtypes.bool or x.dtype in dtypes.uints: return lo, lo.const_like(0) + return lo, (x < x.const_like(0)).where(lo.const_like(-1), lo.const_like(0)) case Ops.CAST if dt in (dtypes.long, dtypes.ulong): return (lo:=uops[0].cast(l2i_dt[dt])), (uops[0] / 2**32).cast(l2i_dt[dt]) - ((uops[0] < 0) & lo.ne(0)) case Ops.CAST if dt in dtypes.floats: small = (a1.eq(0) & (a0 >= 0)) | (a1.eq(-1) & (a0 < 0)) - return small.where(a0.cast(dt), ((a1.cast(dtypes.float32) * (2**32)) + a0.bitcast(dtypes.uint).cast(dtypes.float32)).cast(dt)) + cdt = dt if dt == dtypes.float64 else dtypes.float32 + return small.where(a0.cast(dt), ((a1.cast(cdt) * (2**32)) + a0.bitcast(dtypes.uint).cast(cdt)).cast(dt)) case Ops.CAST: return a0.bitcast(dtypes.uint).cast(dt) case Ops.BITCAST: return a0.bitcast(dt), a1.bitcast(dt) case Ops.SHL: @@ -78,6 +80,11 @@ def l2i(op: Ops, dt: DType, *uops:UOp): case Ops.MAX: return l2i(Ops.WHERE, dt, l2i(Ops.CMPLT, dt, *uops), b0, b1, a0, a1) case _: raise NotImplementedError(f"long decomposition of {op} unsupported") +def l2i_define(x:UOp) -> UOp: + # cannot decomp a Variable + if x.addrspace == AddrSpace.ALU: raise RuntimeError(f"long decomposition of variable {x.arg.name} unsupported") + return UOp(x.op, arg=replace(x.arg, dtype=l2i_dt[x.dtype], size=None if x.arg.size is None else x.arg.size*2), tag=x.tag) + def split_l2i(ctx:dict, op: Ops, dt: DType, *uops:UOp): # l2i does arithmetic on its inputs; rules enter here to split them to 32-bit words first, l2i recurses on itself. # both word halves of a node ask for the same split, so ctx memos it for the pass @@ -126,19 +133,21 @@ def f2f_clamp(val:UOp, dt:DType, sat=True) -> UOp: return val.ne(val).where(val, (val < -mx).where(-sat, (mx < val).where(sat, val))) def f2f_load(x: UOp, fr:DType, to:DType) -> UOp: - if (n:=x.max_numel()) == 1: return f2f(x.replace(dtype=f2f_dt[fr]), fr, to) - return UOp(Ops.STACK, src=tuple(f2f(x.replace(dtype=f2f_dt[fr], src=(reindex(x.src[0], i, 1),)), fr, to) for i in range(n))) + storage_idx = graph_rewrite(x.src[0], pm_float_decomp, ctx=(fr, to), bottom_up=True) + if (n:=x.max_numel()) == 1: return f2f(storage_idx.load(*x.src[1:]), fr, to) + return UOp(Ops.STACK, src=tuple(f2f(reindex(storage_idx, i, 1).load(*x.src[1:]), fr, to) for i in range(n))) def f2f_store(st, idx, val, fr:DType, to:DType): if (n:=val.max_numel()) == 1: return st.replace(src=(idx, f2f(val.bitcast(f2f_dt[to]), to, fr))) return UOp.group(*(st.replace(src=(reindex(idx, i, 1), f2f(val.index(i).bitcast(f2f_dt[to]), to, fr))) for i in range(n))) # tag is the 32-bit word this node becomes - (0 for the low word, 1 for the high, the dtype the consumer wants) -pm_long_decomp = PatternMatcher([ - (UPat(GroupOp.Defines, src=(UPat.var("sz"),), name="x"), lambda x,sz: - x.replace(dtype=l2i_dt[x.dtype], arg=replace(x.arg, dtype=l2i_dt[x.dtype]), src=(sz*2,)) if x.dtype in l2i_dt else None), +pm_long_decomp: PatternMatcher = PatternMatcher([ + # the decomp's own bottom-up rewrite can mint bare consts mid-flight: word splitting commits them at the long sibling's dtype + (UPat(GroupOp.All, name='x'), lambda x: commit_weak_consts(x, next((s.dtype for s in x.src if s.dtype in l2i_dt), None))), + (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), name="x"), l2i_define), (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: - reindex(x, x.tag[0]).replace(dtype=x.tag[1], tag=None) if x.tag is not None else None), + reindex(x, x.tag[0]).replace(tag=None) if x.tag is not None else None), (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val')), name='st'), lambda st,idx,val: st.replace(src=(idx.rtag((0, dt:=l2i_dt[idx.dtype])), val.rtag((0, dt)))).group( st.replace(src=(idx.rtag((1, dt)), val.rtag((1, dt))))) if val.tag is None else None), @@ -146,6 +155,9 @@ pm_long_decomp = PatternMatcher([ split_l2i(ctx, x.op, dt:=l2i_dt[a.dtype], *flatten((s.rtag((0, dt)), s.rtag((1, dt))) for s in x.src))), (UPat(Ops.CAST, tuple(l2i_dt.keys()), src=(UPat.var('a', tuple(l2i_dt.keys())),), name="x"), lambda ctx,a,x: split_l2i(ctx, Ops.BITCAST, l2i_dt[x.dtype], a.rtag((0, dt:=l2i_dt[a.dtype])), a.rtag((1, dt)))[x.tag[0]]), + # a const splits by value; the general CAST arm below would drop its high word + (UPat(Ops.CAST, src=(UPat(Ops.CONST, name='c'),), tag={(w, dt) for w in (0, 1) for dt in l2i_dt.values()}, name='x'), + lambda x,c: UOp.const(truncate[x.tag[1]](c.val >> (32*x.tag[0])), x.tag[1])), (UPat(Ops.CAST, tuple(l2i_dt.keys()), src=(UPat.var('a'),), name="x"), lambda ctx,a,x: split_l2i(ctx, x.op, x.dtype, a)[x.tag[0]] if x.tag is not None else None), (UPat(Ops.CAST, src=(UPat.var('a', tuple(l2i_dt.keys())),), name="x"), lambda ctx,a,x: @@ -158,21 +170,22 @@ pm_long_decomp = PatternMatcher([ (UPat((*(GroupOp.ALU - GroupOp.Comparison - {Ops.SHL, Ops.SHR, Ops.WHERE}), Ops.BITCAST), tuple(l2i_dt.keys()), name="x"), lambda ctx,x: split_l2i(ctx, x.op, l2i_dt[x.dtype], *flatten((a.rtag((0, l2i_dt[x.dtype])), a.rtag((1, l2i_dt[x.dtype]))) for a in x.src))[x.tag[0]] if x.tag is not None else None), - (UPat(Ops.LOAD, tuple(l2i_dt.keys()), src=(UPat.var('idx'),), name='x'), lambda x,idx: - x.replace(dtype=l2i_dt[x.dtype], src=(reindex(idx, x.tag[0]).replace(dtype=l2i_dt[x.dtype], tag=None),), tag=None) if x.tag is not None else None), - (UPat(Ops.CONST, tag={(w, dt) for w in (0, 1) for dt in l2i_dt.values()}, name='x'), lambda x: - UOp.const(truncate[x.tag[1]]((x.val >> 32) if x.tag[0] == 1 else (x.val & 0xFFFFFFFF)), x.tag[1])) + (UPat(Ops.LOAD, tuple(l2i_dt.keys()), src=(UPat.var('idx'),), name='x'), lambda ctx,x,idx: + reindex(graph_rewrite(idx, pm_long_decomp, ctx=ctx, bottom_up=True), x.tag[0]).replace(tag=None).load() if x.tag is not None else None) ]) # float decomposition patterns - ctx is (fr, to) tuple -pm_float_decomp = PatternMatcher([ - (UPat((*GroupOp.Defines, Ops.INDEX, Ops.SHRINK), name="x"), lambda ctx,x: - x.replace(dtype=f2f_dt[ctx[0]], arg=replace(x.arg, dtype=f2f_dt[ctx[0]]) if isinstance(x.arg, ParamArg) else x.arg, tag=ctx[0]) - if x.dtype == ctx[0] and (x.op is not Ops.INDEX or x.src[0].op not in {Ops.LOAD, Ops.STACK}) else None), +pm_float_decomp: PatternMatcher = PatternMatcher([ + (UPat(GroupOp.Defines, name="x"), lambda ctx,x: + UOp(x.op, src=x.src, arg=replace(x.arg, dtype=f2f_dt[ctx[0]]), tag=ctx[0]) if x.dtype == ctx[0] else None), + # INDEX into a LOAD/STACK selects a lane of an already converted value, the load rules below own those + (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(GroupOp.All-{Ops.LOAD, Ops.STACK}),), allow_any_len=True, name="x"), lambda ctx,x: + UOp(x.op, src=(graph_rewrite(x.src[0], pm_float_decomp, ctx=ctx, bottom_up=True), *x.src[1:]), arg=x.arg, tag=ctx[0]) + if x.dtype == ctx[0] else None), (UPat(Ops.LOAD, dtypes.floats, name="x"), lambda ctx,x: f2f_load(x, *ctx) if x.dtype == ctx[0] else None), # bitcasted load should just replace load (UPat(Ops.BITCAST, src=(UPat(Ops.LOAD, name="ld"),), name="bc"), lambda ctx,bc,ld: - ld.replace(dtype=f2f_dt[ctx[0]]).bitcast(bc.dtype) if ld.dtype == ctx[0] else None), + graph_rewrite(ld.src[0], pm_float_decomp, ctx=ctx, bottom_up=True).load(*ld.src[1:]).bitcast(bc.dtype) if ld.dtype == ctx[0] else None), # bitcast from (UPat(Ops.BITCAST, src=(UPat.var("x", dtypes.floats),), name="bc"), lambda ctx,bc,x: bc.replace(src=(f2f(x.bitcast(f2f_dt[ctx[1]]), ctx[1], ctx[0]),)) if x.dtype == ctx[1] and bc.dtype.bitsize == ctx[0].bitsize else None), @@ -181,26 +194,21 @@ pm_float_decomp = PatternMatcher([ f2f(x.bitcast(f2f_dt[ctx[0]]), ctx[0], ctx[1]) if bc.dtype == ctx[0] else None), (UPat(Ops.CAST, dtypes.floats, src=(UPat.var("val"),), name="x"), lambda ctx,x,val: f2f_clamp(val.cast(ctx[1]), ctx[0]) if x.dtype == ctx[0] else None), - # a CONST has no srcs to cast, it restates its value at the emulating dtype - (UPat(Ops.CONST, dtypes.floats, name="x"), lambda ctx,x: UOp.const(x.val, ctx[1]) if x.dtype == ctx[0] else None), (UPat(GroupOp.All-GroupOp.Defines-{Ops.CAST, Ops.BITCAST, Ops.CONST}, dtypes.floats, name="x"), lambda ctx,x: - x.replace(dtype=ctx[1], src=tuple(s.cast(ctx[1]) if s.dtype == ctx[0] else s for s in x.src)) - if x.dtype == ctx[0] else None), + UOp(x.op, src=tuple(s.cast(ctx[1]) if s.dtype == ctx[0] else s for s in x.src), arg=x.arg, tag=x.tag) if x.dtype == ctx[0] else None), (UPat(Ops.STORE, src=(UPat.var("idx"), UPat(Ops.BITCAST, dtypes.floats, name="val")), name='st'), lambda ctx,st,idx,val: - st.replace(src=(idx, val.replace(dtype=f2f_dt[ctx[0]]))) if val.dtype == ctx[0] and idx.tag == ctx[0] else None), - (UPat(Ops.STORE, src=(UPat.var("idx"), UPat.var("val", dtypes.floats)), name='st'), lambda ctx,st,idx,val: - f2f_store(st, idx, val, *ctx) if val.dtype == ctx[1] and (idx:=idx.src[0] if idx.op == Ops.CAST else idx).tag == ctx[0] else None), + st.replace(src=(idx, val.src[0].bitcast(f2f_dt[ctx[0]]))) if val.dtype == ctx[0] and idx.tag == ctx[0] else None), + (UPat(Ops.STORE, src=(UPat.var("idx").or_casted(), UPat.var("val", dtypes.floats)), name='st'), lambda ctx,st,idx,val: + f2f_store(st, idx, val, *ctx) if val.dtype == ctx[1] and idx.tag == ctx[0] else None), ]) def do_dtype_decomps(sink:UOp, ctx:tuple[set[DType], Renderer]) -> UOp: def _should_emulate(dt): return dt in EMULATED_DTYPES.tolist(dtypes) or dt not in ctx[1].supported_dtypes() - # NOTE: dtype decomp creates intermediate UOps that don't follow the spec (e.g. half LOAD on ushort BUFFER) - with Context(SPEC=min(SPEC.value, 1)): - for fr in sorted(filter(_should_emulate, ctx[0])): - to = dtypes.int if fr == dtypes.long else dtypes.half if not _should_emulate(dtypes.half) and fr in dtypes.fp8s else dtypes.float - if DEBUG >= 2: print(f"emulating {fr} as {to}") - pm = pm_float_decomp if fr in dtypes.floats else pm_long_decomp - sink = graph_rewrite(sink, pm, name=f"decomp {fr} -> {to}", ctx={} if pm is pm_long_decomp else (fr, to), bottom_up=True) + for fr in sorted(filter(_should_emulate, ctx[0])): + to = dtypes.int if fr == dtypes.long else dtypes.half if not _should_emulate(dtypes.half) and fr in dtypes.fp8s else dtypes.float + if DEBUG >= 2: print(f"emulating {fr} as {to}") + pm = pm_float_decomp if fr in dtypes.floats else pm_long_decomp + sink = graph_rewrite(sink, pm, name=f"decomp {fr} -> {to}", ctx={} if pm is pm_long_decomp else (fr, to), bottom_up=True) ctx[0].clear() return sink diff --git a/tinygrad_repo/tinygrad/codegen/decomp/op.py b/tinygrad_repo/tinygrad/codegen/decomp/op.py index 6a48cdca5..160432122 100644 --- a/tinygrad_repo/tinygrad/codegen/decomp/op.py +++ b/tinygrad_repo/tinygrad/codegen/decomp/op.py @@ -75,12 +75,16 @@ powers_of_two: dict[int, int] = {2**i:i for i in range(64)} @functools.cache def get_simplifying_rewrite_patterns(ops:tuple[Ops, ...]) -> PatternMatcher: # these are rewrites that make things simpler - pat: list[tuple[UPat, Callable]] = [(UPat.var("a")//UPat.var("b"), floordiv_to_idiv)] + pat: list[tuple[UPat, Callable]] = [] + # FLOORDIV by 2**y -> x >> y (an arithmetic shift is exactly floor division for any sign); fires before floordiv_to_idiv + if Ops.SHR in ops: pat.append((UPat.var("x", dtypes.ints)//UPat.cvar("c"), + lambda x,c: x >> v if (v:=powers_of_two.get(c.val, 0)) else None)) + pat.append((UPat.var("a")//UPat.var("b"), floordiv_to_idiv)) # FLOORMOD by 2**y -> x & (2**y-1) (correct floor mod for any sign in two's complement); fires before floormod_to_mod if Ops.AND in ops: pat.append((UPat.var("x", dtypes.ints)%UPat.cvar("c"), lambda x,c: x & (c.val-1) if c.val in powers_of_two else None)) pat.append((UPat.var("a")%UPat.var("b"), floormod_to_mod)) # no real hardware supports THREEFRY, but NullRenderer does - if Ops.THREEFRY not in ops: pat.append((UPat(Ops.THREEFRY, dtype=dtypes.uint64, src=(UPat.var("x"), UPat.var("key"))), threefry2x32)) + if Ops.THREEFRY not in ops: pat.append((UPat(Ops.THREEFRY, src=(UPat.var("x"), UPat.var("key"))), threefry2x32)) # MAX can be rewritten as CMPLT + WHERE (max function is annoying on many cstyle backends) if Ops.MAX not in ops and Ops.CMPLT in ops: pat.append((UPat(Ops.MAX, name="m"), lambda m: (m.src[0] < m.src[1]).where(m.src[1], m.src[0]))) return PatternMatcher(pat) @@ -128,6 +132,6 @@ def get_late_rewrite_patterns(ops:tuple[Ops, ...], disable_fast_idiv:bool) -> Pa if Ops.SHL in ops: pat += [(UPat.var('x').alu(Ops.SHL, UPat.cvar('n'))+UPat.var('c'), lambda x,n,c: x.alu(Ops.MULACC, x.const_like(1< a/b if Ops.FDIV in ops: - pat += [(UPat.var("x").reciprocal(), lambda x: x.const_like(1).alu(Ops.FDIV, x))] - pat += [(UPat.var("a", dtypes.floats) * UPat(Ops.FDIV, dtypes.floats, src=(UPat.const(1), UPat.var("b"))), lambda a,b: a.alu(Ops.FDIV, b))] + pat += [(UPat.var("x").reciprocal(), lambda x: UOp.const(1.0).alu(Ops.FDIV, x))] + pat += [(UPat.var("a") * UPat(Ops.FDIV, dtypes.floats, src=(UPat.const(1), UPat.var("b"))), lambda a,b: a.alu(Ops.FDIV, b))] return PatternMatcher(pat) diff --git a/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py b/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py index e4e66fbd5..62c319175 100644 --- a/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py +++ b/tinygrad_repo/tinygrad/codegen/decomp/transcendental.py @@ -90,8 +90,8 @@ def payne_hanek_reduction(d:UOp) -> tuple[UOp, UOp]: if count+offset < len(two_over_pi_f) - 1: an = i.ne(count).where(_take(an, offset, count=count+1), an.const_like(two_over_pi_f[count+offset])) return an - def _shl_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) * pow2if(y, d.dtype).cast(dtypes.uint64)).cast(dtypes.uint32) - def _shr_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) // pow2if(y, d.dtype).cast(dtypes.uint64)).cast(dtypes.uint32) + def _shl_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) << y.cast(dtypes.uint64)).cast(dtypes.uint32) + def _shr_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) >> y.cast(dtypes.uint64)).cast(dtypes.uint32) a = [_take(UOp.const(0, dtypes.uint32), i) for i in range(4)] # (two_over_pi_f[Int(i) + n] << e) | (two_over_pi_f[Int(i) + n+1] >> (nbits - e)) diff --git a/tinygrad_repo/tinygrad/codegen/gpudims.py b/tinygrad_repo/tinygrad/codegen/gpudims.py index 0a22c5687..f2e61f6a7 100644 --- a/tinygrad_repo/tinygrad/codegen/gpudims.py +++ b/tinygrad_repo/tinygrad/codegen/gpudims.py @@ -1,6 +1,6 @@ import math -from tinygrad.uop.ops import UOp, Ops, sint, PatternMatcher, UPat, KernelInfo, ssimplify, AxisType -from tinygrad.dtype import dtypes, AddrSpace +from tinygrad.uop.ops import UOp, Ops, sint, PatternMatcher, UPat, ssimplify, AxisType +from tinygrad.dtype import AddrSpace from tinygrad.renderer import Renderer def _dim_max(d:sint) -> int: return d if isinstance(d, int) else int(d.vmax) @@ -47,7 +47,7 @@ def add_gpudims(ctx:Renderer, s:UOp): all_ranges = {x.arg[0:-1]:x for x in s_topo if x.op is Ops.RANGE} # extract global/local dims - global_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] in (AxisType.GLOBAL, AxisType.THREAD)]) + global_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] is AxisType.GLOBAL]) local_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] in (AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE)]) if not global_dims and not local_dims: return None @@ -55,25 +55,21 @@ def add_gpudims(ctx:Renderer, s:UOp): global_shape = tuple(ssimplify(all_ranges[r].src[0]) for r in global_dims) local_shape = tuple(ssimplify(all_ranges[r].src[0]) for r in local_dims) - # get the idxs - ki: KernelInfo = s.arg - if ctx.has_threads: idxs = [UOp.variable("core_id", 0, int(global_shape[0])-1, dtypes.int).cast(dtypes.weakint)] - elif ki.dont_use_locals: - assert not local_dims, "can't use locals if there's no local dims" - idxs = get_grouped_dims("idx", global_shape, ctx.global_max, reverse=True) - else: - # define indexes for GPU-like execution - local_idxs = get_grouped_dims("lidx", local_shape, ctx.local_max) - hw_local = [_dim_max(u.src[0]) for u in local_idxs if u.op is Ops.SPECIAL] - global_max = ctx.global_max if ctx.global_prod_max is None else \ - tuple(min(gm, pm//l) for gm,pm,l in zip(ctx.global_max or ctx.global_prod_max, ctx.global_prod_max, hw_local+[1]*3)) - idxs = get_grouped_dims("gidx", global_shape, global_max, reverse=True) + local_idxs + # define indexes for GPU-like execution + # if we got a WARP, set the local_max to it so it does not fold with other dims + local_max = (local_shape[0],)+ctx.local_max[1:] if ctx.local_max is not None and local_dims and \ + all_ranges[local_dims[0]].arg[-1] is AxisType.WARP else ctx.local_max + local_idxs = get_grouped_dims("lidx", local_shape, local_max) + hw_local = [_dim_max(u.src[0]) for u in local_idxs if u.op is Ops.SPECIAL] + global_max = ctx.global_max if ctx.global_prod_max is None else \ + tuple(min(gm, pm//l) for gm,pm,l in zip(ctx.global_max or ctx.global_prod_max, ctx.global_prod_max, hw_local+[1]*3)) + idxs = get_grouped_dims("gidx", global_shape, global_max, reverse=True) + local_idxs # apply to multiple ranges subs = {} for r in s_topo: # look for local INDEXes that are not used in the GLOBAL store, then add them as an INVALID - if r.op is Ops.STORE and (idx := r.src[0]).src[0].addrspace == AddrSpace.GLOBAL: + if r.op is Ops.STORE and len((idx := r.src[0]).src) and idx.src[0].addrspace == AddrSpace.GLOBAL: missing_locals = [all_ranges[rng] for rng in local_dims if all_ranges[rng] not in idx.ranges] if len(missing_locals): assert len(idx.src) == 2, "index has 2 sources" @@ -89,7 +85,7 @@ def add_gpudims(ctx:Renderer, s:UOp): pm_device_to_var = PatternMatcher([ # the DEVICE axis is not a program axis, it's bound per device at launch. lower it to the _device_num variable (like SPECIAL for devices) - (UPat(Ops.RANGE, name="r"), lambda r: UOp.variable("_device_num", 0, r.vmax, dtype=r.dtype) if r.arg[-1] is AxisType.DEVICE else None), + (UPat(Ops.RANGE, name="r"), lambda r: UOp.variable("_device_num", 0, r.vmax, dtype=r.dtype, param=True) if r.arg[-1] is AxisType.DEVICE else None), # ENDs that closed a DEVICE range no longer close it (UPat(Ops.END, name="e"), lambda e: e.replace(src=(e.src[0],)+tuple(s for s in e.src[1:] if s.op is not Ops.PARAM)) if any(s.op is Ops.PARAM and s.arg.name == '_device_num' for s in e.src[1:]) else None), diff --git a/tinygrad_repo/tinygrad/codegen/late/coalesce.py b/tinygrad_repo/tinygrad/codegen/late/coalesce.py index f72c951c1..d62997865 100644 --- a/tinygrad_repo/tinygrad/codegen/late/coalesce.py +++ b/tinygrad_repo/tinygrad/codegen/late/coalesce.py @@ -1,7 +1,8 @@ import itertools, functools from collections import defaultdict +from dataclasses import replace from tinygrad.dtype import dtypes, AddrSpace, Invalid, DType -from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat, GroupOp, shape_to_shape_arg, graph_rewrite +from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat, GroupOp, graph_rewrite from tinygrad.uop.symbolic import uop_given_valid, parse_valid, invalid_gate, sym from tinygrad.helpers import getenv, IMAGE, OSX, ceildiv, is_image_shape from tinygrad.renderer import Renderer @@ -26,7 +27,7 @@ def _drop_valid_stmts(valid:UOp, idx:UOp, height:int, width:int) -> list[UOp]: # check if idx is out of bound when X is on the wrong side of the bound: X in [c+1, vmax] or [vmin, c-1] lo, hi = (c + 1, X.vmax) if is_upper_bound else (X.vmin, c - 1) if lo <= hi: - fake = UOp.variable(f"fake{i}", lo, hi, X.dtype) + fake = UOp.variable(f"fake{i}", lo, hi, X.dtype, param=True) subs = [{X: fake}] # idx may not have X itself, so also substitute a term of X: v -> fake - (X - v) terms = list(X.split_uop(Ops.ADD)) @@ -51,8 +52,8 @@ def simplify_valid_image_load(buf:UOp, idx_y:UOp, idx_x:UOp, valid:UOp) -> UOp|N if not drop_stmt and idx is start_idx: return None new_valid = UOp.uprod(*ss) if (ss:=[s for s in valid.split_uop(Ops.AND) if s not in drop_stmt]) else None idx_y, idx_x = idx.index(1), idx.index(0) - if new_valid is not None: return buf.index(idx_y.valid(new_valid), idx_x.valid(new_valid), dtype=dtypes.float) - return buf.index(idx_y, idx_x, dtype=dtypes.float) + if new_valid is not None: return buf.index(idx_y.valid(new_valid), idx_x.valid(new_valid)) + return buf.index(idx_y, idx_x) indexing_simplify = PatternMatcher([ # image load valid idx simplification @@ -85,17 +86,17 @@ def transform_to_image(ctx, buf:UOp, x:UOp) -> UOp|None: if len(cands) == 0: return None # and tiebreak with indexing complexity (ie. number of nodes) h, w, cidx = cands[0] if len(cands) == 1 else min(cands, key=lambda cand: len(cand[2].index(1).simplify().backward_slice)) - buf = buf.replace(src=(shape_to_shape_arg((h, w, 4)),)) + # the image dims are stored in the param's arg, the size stays the flat buffer len + buf = buf.replace(arg=replace(buf.arg, image=(h, w))) shapes[buf.arg.slot] = (h, w) if valid.op is not Ops.CONST or valid.val is not True: - return buf.index(cidx.src[1].valid(valid), cidx.src[0].valid(valid), dtype=dtypes.float) + return buf.index(cidx.src[1].valid(valid), cidx.src[0].valid(valid)) else: - return buf.index(cidx.src[1], cidx.src[0], dtype=dtypes.float) + return buf.index(cidx.src[1], cidx.src[0]) pm_simplify_add_image = PatternMatcher([ (UPat(Ops.SHRINK, src=(UPat(Ops.PARAM, name="buf"), UPat(name="x"), UPat(arg=4))), transform_to_image), # image load/store is always float - (UPat(Ops.INDEX, dtype=dtypes.float, name="x").load(dtype=dtypes.half), lambda x: x.load().cast(dtypes.half)), (UPat(Ops.INDEX, dtype=dtypes.float, name="x").store(UPat(name="d", dtype=dtypes.half)), lambda x,d: x.store(d.cast(dtypes.float))), (UPat.var("x", dtype=dtypes.float).cast(dtypes.half).cast(dtypes.float), lambda x: x), ]) @@ -104,7 +105,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: if getenv("DMC"): return sink # collect - memory: defaultdict[tuple[Ops, UOp, UOp|str, UOp], dict[int, list[UOp]]] = defaultdict(dict) + memory: defaultdict[tuple[Ops, UOp, UOp|str, UOp, object], dict[int, list[UOp]]] = defaultdict(dict) for u in sink.toposort(): # TODO: this should handle images too, it's just memory coalescing if u.op in {Ops.LOAD, Ops.STORE}: @@ -119,11 +120,12 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: elif idx.op is Ops.CONST and idx.val is Invalid: root_src, arg = "INVALID", 0 elif idx.op is Ops.CONST: root_src, arg = "CONST", idx.val else: root_src, arg = idx, 0 - memory[(u.op, buf, root_src, valid)].setdefault(arg, []).append(u) + # loads/stores only coalesce with others carrying the same arg (e.g. the nontemporal flag) + memory[(u.op, buf, root_src, valid, u.arg)].setdefault(arg, []).append(u) # build replacements replacements = {} - for (op,buf,base,valid),offsets in memory.items(): + for (op,buf,base,valid,ld_arg),offsets in memory.items(): # allowed lengths (copied in) lengths = [] must_divide = True @@ -158,7 +160,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: store = idx.store(UOp.stack(*datas) if len(datas) > 1 else datas[0]) for i,g in enumerate(grp): replacements[offsets[g][0]] = store else: - ld = idx.load() + ld = idx.load(arg=ld_arg) for i,g in enumerate(grp): for oo in offsets[g]: replacements[oo] = ld.index(i) if len(grp) > 1 else ld diff --git a/tinygrad_repo/tinygrad/codegen/late/gater.py b/tinygrad_repo/tinygrad/codegen/late/gater.py index b9b3a4efb..319a60c32 100644 --- a/tinygrad_repo/tinygrad/codegen/late/gater.py +++ b/tinygrad_repo/tinygrad/codegen/late/gater.py @@ -3,17 +3,17 @@ from tinygrad.uop.ops import PatternMatcher, UPat, Ops from tinygrad.dtype import Invalid, dtypes def move_where_load(gate, l, a, w): - return l.replace(src=(l.src[0], l.vconst_like(0) if a.is_invalid else + return l.replace(src=(l.src[0], l.vconst_like(0) if a.is_invalid else l.const_like(a.val) if a.op is Ops.CONST else a.src[0] if a.op is Ops.CAST and a.src[0].dtype == l.dtype else a.cast(l.dtype), l.src[2])).cast(w.dtype) pm_move_gates_from_index = PatternMatcher([ # for image idx (must be first) (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).load(name="l"), - lambda buf,gate,idx_y,idx_x,l: buf.index(idx_y, idx_x, dtype=dtypes.float).load(l.vconst_like(0), gate)), + lambda buf,gate,idx_y,idx_x,l: buf.index(idx_y, idx_x).load(l.vconst_like(0), gate)), (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).store(UPat.var("data")), - lambda buf,gate,idx_y,idx_x,data: buf.index(idx_y, idx_x, dtype=dtypes.float).store(data, gate)), + lambda buf,gate,idx_y,idx_x,data: buf.index(idx_y, idx_x).store(data, gate)), # here we create the alt value for load to be 0s and remove the where Invalid (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(), UPat.var("gate").where(UPat.var("idx"), UPat(arg=Invalid)),), name="mop", allow_any_len=True) \ diff --git a/tinygrad_repo/tinygrad/codegen/late/regalloc.py b/tinygrad_repo/tinygrad/codegen/late/regalloc.py index 0d675e62e..611f2780c 100644 --- a/tinygrad_repo/tinygrad/codegen/late/regalloc.py +++ b/tinygrad_repo/tinygrad/codegen/late/regalloc.py @@ -4,7 +4,7 @@ from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat from tinygrad.renderer.isa import ISARenderer, Register, greg from tinygrad.dtype import dtypes -PSEUDO_OPS = {Ops.CONST, Ops.NOOP, Ops.AFTER, Ops.BARRIER, Ops.GROUP, Ops.STACK} +PSEUDO_OPS = {Ops.CONST, Ops.CAST, Ops.BITCAST, Ops.NOOP, Ops.AFTER, Ops.BARRIER, Ops.GROUP, Ops.STACK} class LinearScanRegallocContext: # returns the uop that defines the virtual register @@ -52,7 +52,7 @@ class LinearScanRegallocContext: # the value of a BUFFER is its 64bit address, XMM registers need 16 bytes sz = 16 if v.cons[0].size == 16 else (8 if self.vdef(v).op is Ops.BUFFER else self.vdef(v).dtype.itemsize) offset = self.stack_size + (sz - self.stack_size % sz) % sz - self.spills[v] = UOp.const(offset, dtypes.int32) + self.spills[v] = UOp.cconst(offset, dtypes.int32) self.stack_size = offset + sz r = alloc(cons if cons is not None else v.cons, i) self.insert_before.setdefault(i, []).append((v, r)) @@ -84,7 +84,7 @@ class LinearScanRegallocContext: # allocate stack array if u.op is Ops.BUFFER: - self.locals[u] = UOp.const(self.stack_size, dtypes.int32) + self.locals[u] = UOp.cconst(self.stack_size, dtypes.int32) self.stack_size += u.max_numel() * u.dtype.itemsize # loop prologue, avoid loading inside the loop @@ -125,7 +125,7 @@ def regalloc_rewrite(ctx:LinearScanRegallocContext, x:UOp): # alloc/dealloc stack if ctx.stack_size > 0: sp = ctx.ren.stack_pointer() - offset = UOp.const(ctx.stack_size, sp.dtype) + offset = UOp.cconst(ctx.stack_size, sp.dtype) if i == 0: before = [ctx.ren.isel_matcher.rewrite(UOp(Ops.SUB, src=(sp, offset), tag=sp.tag))] + before elif i == len(ctx.uops) - 2: before += [ctx.ren.isel_matcher.rewrite(UOp(Ops.ADD, src=(sp, offset), tag=sp.tag))] diff --git a/tinygrad_repo/tinygrad/codegen/opt/__init__.py b/tinygrad_repo/tinygrad/codegen/opt/__init__.py index fb4b84dde..ec637b203 100644 --- a/tinygrad_repo/tinygrad/codegen/opt/__init__.py +++ b/tinygrad_repo/tinygrad/codegen/opt/__init__.py @@ -4,8 +4,7 @@ from enum import Enum, auto from dataclasses import dataclass class OptOps(Enum): - TC = auto(); UPCAST = auto(); UNROLL = auto(); LOCAL = auto(); THREAD = auto() # noqa: E702 - GROUP = auto(); GROUPTOP = auto(); NOLOCALS = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 + TC = auto(); SPLIT = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 def __lt__(self, x:OptOps): return self.value < x.value @dataclass(frozen=True, order=True) diff --git a/tinygrad_repo/tinygrad/codegen/opt/heuristic.py b/tinygrad_repo/tinygrad/codegen/opt/heuristic.py index f78dcb2b7..8b25e5304 100644 --- a/tinygrad_repo/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad_repo/tinygrad/codegen/opt/heuristic.py @@ -1,6 +1,6 @@ import itertools from tinygrad.codegen.opt import Opt, OptOps, KernelOptError -from tinygrad.helpers import getenv, DEBUG, prod, NOLOCALS, TC_OPT, TC_SELECT, USE_TC, IMAGE +from tinygrad.helpers import getenv, DEBUG, prod, TC_OPT, TC_SELECT, TC_MIN_GLOBALS, USE_TC, IMAGE from tinygrad.uop.ops import Ops, resolve, AxisType from tinygrad.codegen.late.coalesce import image_valid_dims from tinygrad.codegen.opt.postrange import Scheduler @@ -10,19 +10,19 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: """ Attempts to apply a tensor core optimization to the kernel. If one exists and applies properly, return true, otherwise return false. Tensor cores are optimized instructions that matrix multiply-accumulate across a wave of threads: D(M, N) = A(M, K) * B(K, N) + C(M, N). - Keyword arguments: - use_tensor_cores -- controls how tensor cores are applied (default 1) + ContextVars: + USE_TC -- controls how tensor cores are applied (default 1) 0: will disable any tensor core matching 1: enable tensor cores 2: apply tensor core shape but don't use UOp.WMMA - extra_opts -- additional Opt's to apply after the tensor core instead of the hand-coded additional Opt's (default None) - tc_select -- specifies which tensor core(s) to use for optimization (default -1) + TC_SELECT -- specifies which tensor core(s) to use for optimization (default -1) -1: iterates through all available tensor cores in order and uses the first one that matches the requirements (dims and dtypes) [0-N]: uses only the n'th tensor core available; useful for search - tc_opt -- controls which kinds of kernels may be eligible for tensor cores application (default 2 during BEAM, 0 otherwise) + TC_OPT -- controls which kinds of kernels may be eligible for tensor cores application (default 2 during BEAM, 0 otherwise) 0: applies to only kernels with a single reduce axis and direct Ops.LOAD into Ops.MUL 1: allows kernels with multiple reduce axes and also multiplication of Ops.CAST'd buffers 2: allows kernels with M, N, K axes that are not multiples of the tensor core dimensions by applying padding those axes as needed + TC_MIN_GLOBALS -- do not upcast N when it would drop the specified global count """ # NOTE: unless TC_OPT is > 0, we only trigger tensor cores if there's only one reduce axis if USE_TC > 0 and (len(k.axes_of(AxisType.GROUP_REDUCE, AxisType.REDUCE)) == 1 or (TC_OPT.value >= 1)): @@ -31,13 +31,16 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: # check TC first and apply hand-coded opts if successful try: rngs = tk.apply_opt(Opt(OptOps.TC, axis, (TC_SELECT.value, TC_OPT.value, USE_TC.value))) except KernelOptError: continue - for tc_dim in [1,0]: # attempt to upcast M and N - szs = [sz for sz in [5,4,3,2] if rngs[tc_dim].src[0].divides(sz) is not None] - if szs: - # set it to the replaced range - rngs[tc_dim] = tk.apply_opt(Opt(OptOps.UPCAST, tk.rngs.index(rngs[tc_dim]), szs[0]))[0] - if (szs := [sz for sz in [4,2] if rngs[0].src[0].divides(sz) is not None]): # attempt to local N - tk.apply_opt(Opt(OptOps.LOCAL, tk.rngs.index(rngs[0]), szs[0])) + def split(idx, size, atype): rngs[idx] = tk.apply_opt(Opt(OptOps.SPLIT, tk.rngs.index(rngs[idx]), (size, atype)))[0] + if TC_MIN_GLOBALS: # attempt to upcast M, local N, upcast N, skipping upcast N if we'd end up with too few globals + if (size:=next(filter(lambda sz: rngs[1].src[0].divides(sz) is not None, [5,4,3,2]), None)) is not None: split(1, size, AxisType.UPCAST) + if (size:=next(filter(lambda sz: rngs[0].src[0].divides(sz) is not None, [4,2]), None)) is not None: split(0, size, AxisType.LOCAL) + if ((size:=next(filter(lambda sz: rngs[0].src[0].divides(sz) is not None, [5,4,3,2]), None)) is not None and + resolve(prod(tk.full_shape[i] for i in tk.axes_of(AxisType.GLOBAL)) >= size*TC_MIN_GLOBALS.value, False)): split(0, size, AxisType.UPCAST) + else: # attempt to upcast M, N, local N + for i in [1,0]: + if (size:=next(filter(lambda sz: rngs[i].src[0].divides(sz) is not None, [5,4,3,2]), None)) is not None: split(i, size, AxisType.UPCAST) + if (size:=next(filter(lambda sz: rngs[0].src[0].divides(sz) is not None, [4,2]), None)) is not None: split(0, size, AxisType.LOCAL) return tk # make a copy so it does not mutate the input @@ -52,10 +55,8 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: unit_stride_axes_mul_4 = [k.rngs.index(c) for c in idx.get_idx().split_uop(Ops.ADD) if c.op is Ops.RANGE and (c.vmax+1)%4 == 0 and c not in idx.get_valid().backward_slice] if len(unit_stride_axes_mul_4): - if (axis:=unit_stride_axes_mul_4[0]) in k.upcastable_dims: - k.apply_opt(Opt(OptOps.UPCAST, axis, 4)) - elif axis in k.unrollable_dims: - k.apply_opt(Opt(OptOps.UNROLL, k.unrollable_dims.index(axis), 4)) + if (axis:=unit_stride_axes_mul_4[0]) in (upd:=k.upcastable_dims)+k.unrollable_dims: + k.apply_opt(Opt(OptOps.SPLIT, axis, (4, AxisType.UPCAST if axis in upd else AxisType.UNROLL))) # should use matvec - TODO: adjust/tune based on the wide vs tall/large vs small mat MV_BLOCKSIZE, MV_THREADS_PER_ROW, MV_ROWS_PER_THREAD = getenv("MV_BLOCKSIZE", 4), getenv("MV_THREADS_PER_ROW", 8), getenv("MV_ROWS_PER_THREAD", 4) @@ -71,17 +72,17 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: if DEBUG >= 3: print(f"MATVEC: {k.full_shape=} {first_reduce_rng.render()} {MV_BLOCKSIZE=} {MV_THREADS_PER_ROW=} {MV_ROWS_PER_THREAD=}") try: - if MV_THREADS_PER_ROW > 1: k.apply_opt(Opt(OptOps.GROUP, 0, MV_THREADS_PER_ROW)) + if MV_THREADS_PER_ROW > 1: k.apply_opt(Opt(OptOps.SPLIT, k.axes_of(AxisType.REDUCE)[0], (MV_THREADS_PER_ROW, AxisType.GROUP_REDUCE))) except KernelOptError: pass - if MV_BLOCKSIZE > 1: k.apply_opt(Opt(OptOps.LOCAL, global_idx, MV_BLOCKSIZE)) - if MV_ROWS_PER_THREAD > 1: k.apply_opt(Opt(OptOps.UPCAST, global_idx, MV_ROWS_PER_THREAD)) + if MV_BLOCKSIZE > 1: k.apply_opt(Opt(OptOps.SPLIT, global_idx, (MV_BLOCKSIZE, AxisType.LOCAL))) + if MV_ROWS_PER_THREAD > 1: k.apply_opt(Opt(OptOps.SPLIT, global_idx, (MV_ROWS_PER_THREAD, AxisType.UPCAST))) return k # are we grouping? (requires local shape support) - if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= (240 if NOLOCALS else 2048), False): - for axis, sz in itertools.product((0, 1, 2), (16,)): + if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= (240 if k.ren.target.device == "QCOM" else 2048), False): + for axis, sz in itertools.product(k.axes_of(AxisType.REDUCE)[:3], (16,)): try: - k.apply_opt(Opt(OptOps.GROUPTOP, axis, sz)) + k.apply_opt(Opt(OptOps.SPLIT, axis, (sz, AxisType.GROUP_REDUCE, True))) break except KernelOptError: pass @@ -106,7 +107,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: if resolve(global_items_after < getenv("OCCUPANCY_FLOOR", 4096), False): continue if DEBUG >= 4: print(f"upcasting masked axis : {axis}") to_upcast.append(axis) - for axis in to_upcast[::-1]: k.apply_opt(Opt(OptOps.UPCAST, axis, 0)) + for axis in to_upcast[::-1]: k.apply_opt(Opt(OptOps.SPLIT, axis, (0, AxisType.UPCAST))) # potentially do more upcasts of non reduce axes based on a heuristic is_dsp = k.ren is not None and k.ren.target.device == "DSP" @@ -132,7 +133,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: if xb_choices: xb_choices = sorted(xb_choices) if DEBUG >= 4: print(f"more upcast axis : {xb_choices}") - k.apply_opt(Opt(OptOps.UPCAST, xb_choices[0][2], xb_choices[0][3])) + k.apply_opt(Opt(OptOps.SPLIT, xb_choices[0][2], (xb_choices[0][3], AxisType.UPCAST))) upcasted_axis.add(xb_choices[0][2]) else: break @@ -141,27 +142,38 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: try: if k.unrollable_dims and (k.upcast_size() <= 4 or not k.axes_of(AxisType.UNROLL)) and (k.upcast_size() < 64): if (s:=k.full_shape[k.unrollable_dims[-1]]) <= 32: - k.apply_opt(Opt(OptOps.UNROLL, len(k.unrollable_dims)-1, 0)) + k.apply_opt(Opt(OptOps.SPLIT, k.unrollable_dims[-1], (0, AxisType.UNROLL))) # if it's small, upcast a second reduce dimension too if k.unrollable_dims and s <= 3 and k.full_shape[k.unrollable_dims[-1]] <= 3: - k.apply_opt(Opt(OptOps.UNROLL, len(k.unrollable_dims)-1, 0)) + k.apply_opt(Opt(OptOps.SPLIT, k.unrollable_dims[-1], (0, AxisType.UNROLL))) else: for splits in [4]: if k.full_shape[axis:=k.unrollable_dims[-1]]%splits == 0: - k.apply_opt(Opt(OptOps.UNROLL, len(k.unrollable_dims)-1, splits)) + k.apply_opt(Opt(OptOps.SPLIT, axis, (splits, AxisType.UNROLL))) break except KernelOptError: pass # if nothing at all is upcasted and it's easy to, do an upcast for splits in [4]: if not k.upcasted and k.upcastable_dims and k.full_shape[k.upcastable_dims[-1]] % splits == 0: - k.apply_opt(Opt(OptOps.UPCAST, k.upcastable_dims[-1], splits)) + k.apply_opt(Opt(OptOps.SPLIT, k.upcastable_dims[-1], (splits, AxisType.UPCAST))) # **** local groups **** if k.ren.has_local: - if NOLOCALS: - k.apply_opt(Opt(OptOps.NOLOCALS)) + if k.ren.target.device == "QCOM": + # for openpilot: use 32..128 threads per workgroup, at most 8 on the innermost axis + # apply innermost global axes first so the leading hardware local dims hold the trailing global axes, like gidx + workgroup = 1 + opts: list[tuple[int, int]] = [] + for axis in [a for a in k.axes_of(AxisType.GLOBAL, AxisType.WEAK) if k.rngs[a].src[0].op is Ops.CONST][-3:][::-1]: + if (sz:=max(x for x in range(1, min(int(k.full_shape[axis]), 128 // workgroup if opts else 8) + 1) if int(k.full_shape[axis]) % x == 0)) > 1: + opts.append((axis, sz)) + workgroup *= sz + if opts and workgroup < 32: # fill at least one wave: grow the innermost local as much as possible + axis, sz = opts[0] + opts[0] = axis, max(x for x in range(1, min(int(k.full_shape[axis]), 128 * sz // workgroup) + 1) if int(k.full_shape[axis]) % x == 0) + for axis, sz in opts: k.apply_opt(Opt(OptOps.SPLIT, axis, (sz, AxisType.LOCAL))) else: # prioritize making expand axes local local_axis_ranking = [(any(k.rngs[axis] not in b.src[1].get_idx().backward_slice for b in k.bufs), axis) \ @@ -175,20 +187,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: for axis, local_sz in sorted(to_local[:3]): axis = axis - deleted_shape will_delete_shape = local_sz == k.full_shape[axis] - k.apply_opt(Opt(OptOps.LOCAL, axis, local_sz)) + k.apply_opt(Opt(OptOps.SPLIT, axis, (local_sz, AxisType.LOCAL))) if will_delete_shape: deleted_shape += 1 - # **** threading **** - - if k.ren.has_threads and k.ren.global_max is not None: - for threads in [32,16,12,8,6,5,4,3,2]: - # Skip if too many threads. Heuristic: use about 128K ops per thread - if threads > k.ren.global_max[0] or resolve(prod(k.full_shape) // (128 << 10) < threads): continue - for axis in k.axes_of(AxisType.WEAK): - if k.full_shape[axis] % threads == 0: - try: k.apply_opt(Opt(OptOps.THREAD, axis, threads)) - except KernelOptError: pass - break - if k.applied_opts and k.applied_opts[-1].op is OptOps.THREAD: break - return k diff --git a/tinygrad_repo/tinygrad/codegen/opt/postrange.py b/tinygrad_repo/tinygrad/codegen/opt/postrange.py index f5f682c27..4c29be130 100644 --- a/tinygrad_repo/tinygrad/codegen/opt/postrange.py +++ b/tinygrad_repo/tinygrad/codegen/opt/postrange.py @@ -1,21 +1,22 @@ from __future__ import annotations import math, itertools -from collections import defaultdict -from typing import cast, Final +from typing import cast from tinygrad.uop.ops import Ops, UOp, KernelInfo, graph_rewrite, AxisType, ssimplify, remove_all_tags from tinygrad.uop.ops import axis_letters, axis_colors, axis_to_pos from tinygrad.device import Buffer from tinygrad.dtype import dtypes, Invalid -from tinygrad.helpers import colored, getenv, DEBUG, to_function_name, NOOPT, argsort, round_up, prod, merge_dicts, get_single_element, flatten +from tinygrad.helpers import colored, getenv, DEBUG, NOOPT, argsort, round_up, prod, merge_dicts, get_single_element, flatten from tinygrad.helpers import ALLOW_TF32, count, Context from tinygrad.codegen.opt import Opt, OptOps, KernelOptError, check from tinygrad.codegen.simplify import pm_flatten_range from tinygrad.renderer import Renderer +split_targets = {AxisType.UPCAST: (AxisType.GLOBAL, AxisType.LOCAL, AxisType.WEAK), AxisType.UNROLL: (AxisType.REDUCE, AxisType.GROUP_REDUCE), + AxisType.LOCAL: (AxisType.GLOBAL, AxisType.WEAK), AxisType.GROUP_REDUCE: (AxisType.REDUCE,)} + class Scheduler: def __init__(self, ast:UOp, ren:Renderer): self.ast, self.ren = ast, ren - self.dont_use_locals = self.ast.arg.dont_use_locals if self.ast.arg is not None else False self.applied_opts = list(self.ast.arg.applied_opts) if self.ast.arg is not None else [] self.opt_range = count(start=max([x.arg[0] for x in self.rngs], default=0)+1) @@ -43,12 +44,10 @@ class Scheduler: def copy(self) -> Scheduler: ret = Scheduler(self.ast, self.ren) - ret.dont_use_locals = self.dont_use_locals ret.applied_opts = self.applied_opts[:] if hasattr(self, 'tensor_core'): ret.tensor_core = self.tensor_core return ret - kernel_cnt: Final[defaultdict[str, int]] = defaultdict(int) def get_optimized_ast(self, name_override:str|None=None) -> UOp: if name_override is not None: name = name_override else: @@ -56,11 +55,8 @@ class Scheduler: special_uops = sorted([x for x in self.ast.toposort() if x.op is Ops.SPECIAL], key=lambda x: x.arg) special_ops = [colored(str(x.vmax+1), "blue" if x.arg[0] == "g" else "cyan") for x in special_uops] name = k_type + colored('_', 'BLACK').join(['']+special_ops+[colored(x.src[0].render(), color) for x,color in zip(self.rngs, self.colors())]) - Scheduler.kernel_cnt[(function_name := to_function_name(name))] += 1 - num = f"n{Scheduler.kernel_cnt[function_name]-1}" if Scheduler.kernel_cnt[function_name] > 1 else "" - name += colored(num, 'BLACK') self.ast = graph_rewrite(self.ast, pm_flatten_range, name="flatten range") - return self.ast.replace(arg=KernelInfo(name=name, applied_opts=tuple(self.applied_opts), dont_use_locals=self.dont_use_locals), tag=1) + return self.ast.replace(arg=KernelInfo(name=name, applied_opts=tuple(self.applied_opts)), tag=1) def _output_rngs(self) -> list[UOp]: return flatten([[r for r in UOp.sink(*s.src[1:]).ranges if r.arg[-1] != AxisType.REDUCE] for s in self.ast.src if s.op is Ops.END]) @@ -85,8 +81,7 @@ class Scheduler: globalizible_rngs = self._globalizable_rngs() ret = [] for x,r in zip(self.axis_types, self.rngs): - if self.dont_use_locals and x == AxisType.GLOBAL: ret.append("BLUE") - elif r not in output_rngs and x == AxisType.WEAK: ret.append("BLACK") + if r not in output_rngs and x == AxisType.WEAK: ret.append("BLACK") elif r not in globalizible_rngs and x == AxisType.WEAK: ret.append("white") else: ret.append(axis_colors[x]) return ret @@ -106,7 +101,6 @@ class Scheduler: def upcast_size(self): return prod(self.full_shape[a] for a in self.axes_of(AxisType.UPCAST, AxisType.UNROLL)) - # copied from kernel.py @property def upcastable_dims(self) -> list[int]: return [i for i in self.axes_of(AxisType.GLOBAL, AxisType.LOCAL, AxisType.WEAK) \ if isinstance(s:=self.full_shape[i], int) and s > 1] @@ -115,69 +109,40 @@ class Scheduler: if isinstance(s:=self.full_shape[i], int) and s > 1] def real_axis(self, op:OptOps, axis:int|None) -> int: - try: - if axis is None or op is OptOps.TC: return -1 - if op is OptOps.UNROLL: return self.unrollable_dims[axis] - if op in {OptOps.GROUP, OptOps.GROUPTOP}: return self.axes_of(AxisType.REDUCE)[axis] - check(axis < self.shape_len, f"invalid axis on {axis=} {op=} {self.shape_len=}") - return axis - except IndexError as e: raise KernelOptError from e + if axis is None or op is OptOps.TC: return -1 + check(0 <= axis < self.shape_len, f"invalid axis on {axis=} {op=} {self.shape_len=}") + return axis def apply_opt(self, opt:Opt, append_opt:bool=True): - if opt.op is OptOps.NOLOCALS: - check(all(x not in {AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE} for x in self.axis_types), "no locals can't have locals") - if append_opt: self.applied_opts.append(opt) - self.dont_use_locals = True - return - - if opt.op in {OptOps.LOCAL, OptOps.GROUP, OptOps.GROUPTOP}: - check(self.ren.has_local, "locals needed for opt") - rng = self.rngs[real_axis] if (real_axis:=self.real_axis(opt.op, opt.axis)) >= 0 else UOp(Ops.NOOP) - opt_to_at = { - OptOps.LOCAL: AxisType.LOCAL, OptOps.UPCAST: AxisType.UPCAST, - OptOps.UNROLL: AxisType.UNROLL, OptOps.GROUP: AxisType.GROUP_REDUCE, - OptOps.GROUPTOP: AxisType.GROUP_REDUCE, OptOps.THREAD: AxisType.THREAD} - ret = None - if opt.op in opt_to_at: - amt:int = int(rng.vmax+1) if opt.arg == 0 else cast(int, opt.arg) + if opt.op is OptOps.SPLIT: + check(isinstance(opt.arg, tuple) and len(opt.arg) in (2, 3), f"split arg is (amt, target) or (amt, target, top), not {opt.arg}") + amt, new_type, top = (*cast(tuple, opt.arg), False)[0:3] + check(type(amt) is int and (amt == 0 or amt > 1) and isinstance(new_type, AxisType) and new_type in split_targets and isinstance(top, bool), + f"invalid split arg {opt.arg}") + check(not top or new_type is AxisType.GROUP_REDUCE, "top is only for group reduce") + if new_type in (AxisType.LOCAL, AxisType.GROUP_REDUCE): check(self.ren.has_local, "locals needed for opt") + check(rng.arg[-1] in split_targets[new_type], f"{new_type} is from {split_targets[new_type]}, not {rng.arg[-1]}") - # copied from kernel.py. prevents METAL compiler hangs - if self.reduceop is not None and (opt.op in {OptOps.GROUP, OptOps.GROUPTOP} or \ - (self.group_for_reduces and opt.op not in {OptOps.NOLOCALS, OptOps.PADTO})): + if amt == 0: amt = int(rng.vmax+1) + if new_type is AxisType.UNROLL: check(amt <= 32, "don't unroll more than 32") + if new_type is AxisType.UPCAST: check(self.ren.target.device == "DSP" or amt <= 16, "don't upcast more than 16") + # prevents METAL compiler hangs + if self.reduceop is not None and (new_type is AxisType.GROUP_REDUCE or self.group_for_reduces): upcast_local_sz = prod([self.full_shape[a] for a in self.axes_of(AxisType.UPCAST, AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE)]) smem_sz = amt*upcast_local_sz*self.reduceop.dtype.itemsize check(smem_sz <= self.ren.shared_max, f"exceeds maximum shared memory size: needs {smem_sz}, max {self.ren.shared_max}") - if self.reduceop is not None and (opt.op in {OptOps.GROUP, OptOps.GROUPTOP}): + if self.reduceop is not None and new_type is AxisType.GROUP_REDUCE: # We currently dont support a group within another rudece, TODO: fix if-contexts reduce = [u for u in self.ast.backward_slice if u.op is Ops.REDUCE and rng in merge_dicts([r.ranges for r in u.src[1:]])][0] check(not any(u.arg[-1] in (AxisType.REDUCE, AxisType.UNROLL, AxisType.GROUP_REDUCE) for u in reduce.ranges), "cannot have a GROUP_REDUCE inside another reduce") - - if opt.op is OptOps.UNROLL: - check(amt <= 32, "don't unroll more than 32") - check(rng.arg[-1] in {AxisType.GROUP_REDUCE, AxisType.REDUCE}, "unroll is for GROUP_REDUCE/REDUCE") - if opt.op is OptOps.UPCAST: - check((self.ren is not None and self.ren.target.device == "DSP") or amt <= 16, "don't upcast more than 16") - check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.LOCAL, AxisType.WEAK}, f"upcast is for GLOBAL/LOCAL/LOOP, not {rng.arg[-1]}") - if opt.op is OptOps.LOCAL: - check(not self.dont_use_locals, "can't use locals") - check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.WEAK}, "local is for globals") - if opt.op is OptOps.THREAD: - check(self.ren is not None and self.ren.has_threads, "target does not support threads") - check(self.ren is not None and self.ren.global_max is not None and amt <= self.ren.global_max[0], "too many threads") - check(all(x is not AxisType.THREAD for x in self.axis_types), "already threaded") - check(rng in self._globalizable_rngs(), "can't apply range to this dim") - if opt.op in {OptOps.GROUP, OptOps.GROUPTOP}: - check(all(x.op is not OptOps.TC for x in self.applied_opts), "no grouping with tensor cores") # TODO: why is this wrong? - check(not self.dont_use_locals, "can't use locals") - check(rng.arg[-1] == AxisType.REDUCE, "group is for reduce") - ret = self.shift_to(rng, amt, opt_to_at[opt.op], top=opt.op in {OptOps.GROUPTOP, OptOps.THREAD}) + ret = self.shift_to(rng, amt, new_type, top=top) elif opt.op is OptOps.TC: check(len(self.applied_opts) == 0, "tensor core opts must be first") # TODO: remove the need for this by having warps - check(opt.axis is not None, "tensor core opts must have an axis") + check(opt.axis is not None and opt.axis >= 0, "tensor core opts must have an axis") check(opt.arg is not None and isinstance(opt.arg, tuple) and len(opt.arg) == 3, "tensor core opts must have valid arg") check(-1 <= (tc_select:=cast(tuple, opt.arg)[0]) < len(self.ren.tensor_cores), "tensor core opts must have valid tc_select") check(0 <= (tc_opt:=cast(tuple, opt.arg)[1]) <= 2, "tensor core opts must have valid tc_opt") @@ -186,9 +151,9 @@ class Scheduler: except ValueError as e: raise KernelOptError(str(e)) check(ret is not None, "no tensor core available") elif opt.op is OptOps.PADTO: + check(type(opt.arg) is int and opt.arg > 1, f"padto arg is a multiple > 1, not {opt.arg}") check(rng.src[0].op is Ops.CONST, "only pad const axes") check(rng.arg[-1] not in {AxisType.UPCAST, AxisType.UNROLL}, "cannot pad upcasted") # TODO: why is this wrong? - check(rng.arg[-1] is not AxisType.THREAD, "cannot pad thread") new_sz = round_up(int(rng.vmax+1), cast(int, opt.arg)) check(rng.vmax+1 > new_sz//4, "pad adds more than quadruple the work") replaced_rng = UOp.range(new_sz, *rng.arg, dtype=rng.dtype) @@ -198,7 +163,7 @@ class Scheduler: for b in self.bufs: if rng in (i:=b.src[1].get_idx()).backward_slice_with_self: nb = b.replace(src=(b.src[0], i.valid(valid&b.src[1].get_valid()))) - replaces[b] = nb if b in store_targets else valid.where(nb, UOp.const(Invalid, b.dtype)) + replaces[b] = nb if b in store_targets else valid.where(nb, UOp.const(Invalid)) self.ast = self.ast.substitute(replaces, f"padto {rng.arg[:-1]} {opt.arg}") elif opt.op is OptOps.SWAP: try: diff --git a/tinygrad_repo/tinygrad/codegen/opt/search.py b/tinygrad_repo/tinygrad/codegen/opt/search.py index c8c7a3680..c53e0a2a9 100644 --- a/tinygrad_repo/tinygrad/codegen/opt/search.py +++ b/tinygrad_repo/tinygrad/codegen/opt/search.py @@ -1,28 +1,26 @@ -import math, time, multiprocessing, traceback, signal, atexit +import math, time, traceback, signal from dataclasses import replace from tinygrad.uop.ops import sym_infer, AxisType, UOp, Ops from tinygrad.uop.render import pyrender from tinygrad.device import Device, Buffer -from tinygrad.helpers import prod, flatten, DEBUG, CACHELEVEL, diskcache_get, diskcache_put, getenv, Context, colored, time_to_str +from tinygrad.helpers import prod, flatten, DEBUG, CACHELEVEL, diskcache_get, diskcache_put, getenv, colored, time_to_str from tinygrad.helpers import IGNORE_BEAM_CACHE from tinygrad.codegen.opt import Opt, OptOps, KernelOptError from tinygrad.engine.realize import time_call +from tinygrad.engine.worker import get_worker_pool, terminate_worker_pool from tinygrad.codegen import to_program from tinygrad.codegen.opt.postrange import Scheduler -actions = [Opt(op=OptOps.UPCAST, axis=axis, arg=amt) for amt in [0,2,3,4,5,7] for axis in range(8)] -actions += [Opt(op=OptOps.UNROLL, axis=axis, arg=amt) for amt in [0,4,7] for axis in range(5)] -actions += [Opt(op=OptOps.LOCAL, axis=axis, arg=amt) for amt in [2,3,4,8,13,16,29] for axis in range(6)] -actions += [Opt(op=OptOps.GROUPTOP, axis=axis, arg=amt) for amt in [13,16,28,29,32,49,64,256] for axis in range(3)] -actions += [Opt(op=OptOps.GROUP, axis=axis, arg=amt) for amt in [0,4,8,16] for axis in range(3)] +actions = [Opt(op=OptOps.SPLIT, axis=axis, arg=(amt, at)) for at in (AxisType.UPCAST, AxisType.UNROLL) for amt in [0,2,3,4,5,7] for axis in range(10)] +actions += [Opt(op=OptOps.SPLIT, axis=axis, arg=(amt, at)) for at in (AxisType.LOCAL, AxisType.GROUP_REDUCE) + for amt in [0,2,3,4,8,13,16,29] for axis in range(8)] +actions += [Opt(op=OptOps.SPLIT, axis=axis, arg=(amt, AxisType.GROUP_REDUCE, True)) for amt in [13,16,28,29,32,49,64,256] for axis in range(8)] if getenv("BEAM_PADTO", 0): actions += [Opt(op=OptOps.PADTO, axis=axis, arg=amt) for amt in [32] for axis in range(7)] -actions += [Opt(op=OptOps.LOCAL, axis=0, arg=32), Opt(op=OptOps.LOCAL, axis=6, arg=2)] +actions += [Opt(op=OptOps.SPLIT, axis=0, arg=(32, at)) for at in (AxisType.LOCAL, AxisType.GROUP_REDUCE)] actions += [Opt(op=OptOps.TC, axis=0, arg=(-1, 0, getenv("TC", 1)))] # covers resnet kernels (3 global * 3 reduce) actions += [Opt(op=OptOps.TC, axis=axis, arg=(-1, getenv("TC_OPT", 2), getenv("TC", 1))) for axis in range(9)] actions += [Opt(op=OptOps.SWAP, axis=axis_0, arg=axis_1) for axis_0 in range(5) for axis_1 in range(axis_0+1, 5)] -actions += [Opt(op=OptOps.THREAD, axis=axis, arg=amt) for amt in [2,3,4,5,8,12,16,24,32,64] for axis in range(3)] -if getenv("NOLOCALS"): actions += [Opt(op=OptOps.NOLOCALS)] def get_test_global_size(global_size, max_global_size, var_vals): test_global_size = [sym_infer(sz, var_vals) for sz in global_size] @@ -42,9 +40,9 @@ def _time_program(prg:UOp, var_vals:dict[str, int], rawbufs:list[Buffer], early_ global_size, factor = get_test_global_size(prg.arg.global_size, max_global_size, var_vals) prg = prg.replace(arg=replace(prg.arg, global_size=tuple(global_size))) call = prg.call(*[UOp.from_buffer(b) for b in rawbufs]) - tms = [] + tms, timer = [], time_call(call, var_vals, timeout=timeout, clear_l2=clear_l2) for _ in range(cnt): - try: tms.append(time_call(call, var_vals, timeout=timeout, clear_l2=clear_l2) * factor) + try: tms.append(next(timer) * factor) except AssertionError: return [math.inf] * cnt if early_stop is not None and early_stop < min(tms): break return tms @@ -78,11 +76,6 @@ def _try_compile(x:tuple[int,Scheduler]) -> tuple[int, tuple[UOp, float]|None]: if hasattr(signal, "alarm"): signal.alarm(0) return x[0], ret -# workers should not open devices and should ignore ctrl c and should not launch VIZ -def _init_worker(): - Context(ALLOW_DEVICE_USAGE=0, VIZ=0, TRACK_MATCH_STATS=0).__enter__() - signal.signal(signal.SIGINT, signal.SIG_IGN) - def _ensure_buffer_alloc(bufs:list[Buffer]) -> list[Buffer]: return [buf.ensure_allocated() if buf is not None else buf for buf in bufs] # *** external API *** @@ -96,7 +89,8 @@ def get_kernel_actions(s:Scheduler, include_0=True, max_up:int|None=None) -> dic if a.axis is not None and a.op is not OptOps.TC: try: ax = s.real_axis(a.op, a.axis) except KernelOptError: continue - if (ax >= s.shape_len) or (s.full_shape[ax] == a.arg and Opt(a.op, a.axis, 0) in kernel_actions): continue + if (ax >= s.shape_len) or (a.op is OptOps.SPLIT and isinstance(arg:=a.arg, tuple) and s.full_shape[ax] == arg[0] + and replace(a, arg=(0,)+arg[1:]) in kernel_actions): continue s2 = s.copy() try: s2.apply_opt(a) @@ -111,9 +105,8 @@ def get_kernel_actions(s:Scheduler, include_0=True, max_up:int|None=None) -> dic except KernelOptError: pass return acted -beam_pool, BEAM_DEBUG = None, getenv("BEAM_DEBUG") +BEAM_DEBUG = getenv("BEAM_DEBUG") def beam_search(s:Scheduler, rawbufs:list[Buffer], var_vals:dict[str,int], amt:int, allow_test_size=True, disable_cache=IGNORE_BEAM_CACHE.value): - global beam_pool key = {"ast": s.ast.key, "amt": amt, "allow_test_size": allow_test_size, "device": s.ren.target.device, "suffix": s.ren.suffix} if not disable_cache and CACHELEVEL >= 1 and (val:=diskcache_get("beam_search", key)) is not None: ret = s.copy() @@ -123,11 +116,7 @@ def beam_search(s:Scheduler, rawbufs:list[Buffer], var_vals:dict[str,int], amt:i beam: list[tuple[Scheduler, float]] = [(s, float("inf"))] seen_libs = set() - default_parallel = multiprocessing.cpu_count() if s.ren.target.device in {"CUDA", "AMD", "NV", "METAL", "HIP"} else 0 - if beam_pool is None and (workers := getenv("PARALLEL", default_parallel)): - beam_pool = multiprocessing.get_context("spawn").Pool(workers, _init_worker, (), getenv("BEAM_MAX_TASKS_PER_CHILD", 16)) - @atexit.register - def close_pool(): beam_pool.close() + pool = get_worker_pool() min_progress = getenv("BEAM_MIN_PROGRESS", 0.01)/1e6 if BEAM_DEBUG: @@ -143,7 +132,7 @@ def beam_search(s:Scheduler, rawbufs:list[Buffer], var_vals:dict[str,int], amt:i candidates: list[Scheduler] = flatten([get_kernel_actions(si, include_0=False).values() for si,_ in beam]) timed: list[tuple[Scheduler, float]] = [] least_compute_ops = math.inf - for i, proc in ((map if beam_pool is None else beam_pool.imap_unordered)(_try_compile, enumerate(candidates))): + for i, proc in ((map if pool is None else pool.imap_unordered)(_try_compile, enumerate(candidates))): if proc is None: continue prg, compile_et = proc if (lib:=prg.src[3].arg) in seen_libs: continue @@ -179,7 +168,7 @@ def beam_search(s:Scheduler, rawbufs:list[Buffer], var_vals:dict[str,int], amt:i print(f"\r{time.perf_counter() - st:7.2f}s:", colored(time_to_str(beam[0][1], w=12), "green" if exiting else None), f"from {len(candidates):3d} -> {len(opts):3d} actions\033[K", beam[0][0].colored_shape()) except KeyboardInterrupt as e: - if beam_pool is not None: beam_pool.terminate() + terminate_worker_pool() raise e if CACHELEVEL >= 1: diskcache_put("beam_search", key, beam[0][0].applied_opts) diff --git a/tinygrad_repo/tinygrad/codegen/simplify.py b/tinygrad_repo/tinygrad/codegen/simplify.py index 879dd0e6c..3fe79b5a7 100644 --- a/tinygrad_repo/tinygrad/codegen/simplify.py +++ b/tinygrad_repo/tinygrad/codegen/simplify.py @@ -1,7 +1,7 @@ import itertools from typing import Callable from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, graph_rewrite, _substitute, range_start, AxisType -from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const, invalid_gate +from tinygrad.uop.symbolic import symbolic from tinygrad.helpers import partition from tinygrad.dtype import dtypes @@ -32,13 +32,13 @@ def simplify_merge_adjacent(u:UOp) -> UOp|None: s0, s1 = r0.src[0], r1.src[0] # do the merge new_range = r0.replace(src=(s0*s1,)) - nidx = graph_rewrite(u, _substitute+symbolic+pm_fold_cast_const+pm_flatten_range, ctx={r0:new_range//s1, r1:new_range%s1}, + nidx = graph_rewrite(u, _substitute+symbolic+pm_flatten_range, ctx={r0:new_range//s1, r1:new_range%s1}, name=f"check_merge_{r0.arg[0]}_{r1.arg[0]}") - # check if it simplifies - if count_divmod(nidx) <= count_divmod(u): - u = nidx - return u + # check if it simplifies. return after one merge so the next rewrite uses the new ranges, + # rather than continuing with stale pairs from the original ended_ranges. + if count_divmod(nidx) <= count_divmod(u): return nidx + return None def mark_gated(ctx, idx): if len(idx.src) > 1 and idx.src[1].op is Ops.WHERE: @@ -84,7 +84,7 @@ def reduce_unparented(red:UOp) -> UOp|None: assert all(x.op is Ops.RANGE for x in red.src[1:]), "some reduce srcs aren't ranges" reduce_parented, reduce_unparented = partition(red.src[1:], lambda x: x in red.src[0].ranges) if len(reduce_unparented) == 0: return None - ret = red.replace(src=(red.src[0],)+tuple(reduce_parented)) if len(reduce_parented) or red.dtype != red.src[0].dtype else red.src[0] + ret = red.replace(src=(red.src[0],)+tuple(reduce_parented)) if len(reduce_parented) else red.src[0] if red.arg[0] is Ops.ADD: for r in reduce_unparented: ret = ret * r.src[0] if red.arg[0] is Ops.MUL: @@ -110,8 +110,6 @@ pm_reduce_collapse = pm_reduce_unparented + PatternMatcher([ ).reduce(UPat.var("r"), arg=Ops.ADD), lambda r,val,lower=None,upper=None: ((upper.minimum(r.src[0]) if upper is not None else r.src[0]) - (lower.maximum(0) if lower is not None else r.const_like(0))).maximum(0).minimum(r.src[0]) * val if no_range(val) else None), - (invalid_gate.reduce(arg=Ops.ADD, allow_any_len=True, name="r"), - lambda cond,x,i,r: cond.where(x.reduce(*r.src[1:], arg=Ops.ADD), i) if no_range(cond) else None), ((UPat.var("x")+UPat.var("y")).reduce(arg=Ops.ADD, allow_any_len=True, name="r"), lambda x,y,r: x.reduce(*r.src[1:], arg=Ops.ADD) + y.reduce(*r.src[1:],arg=Ops.ADD)), # AND on WHERE @@ -137,7 +135,7 @@ def reduce_collapse(red:UOp, u:UOp, pm:PatternMatcher=pm_reduce_collapse) -> UOp for u in included: for s in u.src: if s in included or s in replaces or s.op in {Ops.CONST, Ops.PARAM, Ops.BUFFER}: continue - replaces[s] = UOp.variable(f'in{len(replaces)}', s.vmin, s.vmax, s.dtype) + replaces[s] = UOp.variable(f'in{len(replaces)}', s.vmin, s.vmax, s.dtype, param=True) collapse_fxn = u.substitute(replaces).reduce(r, arg=Ops.ADD) sink = graph_rewrite(collapse_fxn, pm, name="reduce_collapse") if not no_range(sink): return None diff --git a/tinygrad_repo/tinygrad/device.py b/tinygrad_repo/tinygrad/device.py index 570cd7981..4d9041758 100644 --- a/tinygrad_repo/tinygrad/device.py +++ b/tinygrad_repo/tinygrad/device.py @@ -1,8 +1,8 @@ from __future__ import annotations from dataclasses import dataclass, replace from collections import defaultdict -from typing import Any, Generic, TypeVar, Iterator, Generator, Self, TYPE_CHECKING -import importlib, inspect, functools, pathlib, os, contextlib, re, atexit, pickle, decimal +from typing import Any, Callable, Generic, TypeVar, Iterator, Generator, Self, TYPE_CHECKING +import importlib, inspect, functools, pathlib, os, contextlib, re, atexit, pickle, decimal, subprocess, struct from tinygrad.helpers import LRU, getenv, diskcache_get, diskcache_put, DEBUG, GlobalCounters, PROFILE, temp, colored from tinygrad.helpers import Context, CCACHE, ALLOW_DEVICE_USAGE, MAX_BUFFER_SIZE, cpu_events, ProfileEvent, ProfilePointEvent, suppress_finalizing from tinygrad.helpers import select_by_name, select_first_inited, DEV, TracingKey, size_to_str, pluralize, Target, unwrap, round_up @@ -22,7 +22,7 @@ class _Device: def canonicalize(self, device:str|None) -> str: return self._canonicalize(device if device is not None else Device.DEFAULT) def __getitem__(self, ix:str) -> Compiled: ix = self.canonicalize(ix) - assert ALLOW_DEVICE_USAGE or ix.split(":")[0] in ["DISK", "TINYFS", "NPY", "PYTHON"], f"usage of device {ix} disallowed" + assert ALLOW_DEVICE_USAGE or ix.split(":")[0] in ["DISK", "NPY", "PYTHON"], f"usage of device {ix} disallowed" return self.__get_canonicalized_item(ix) @functools.cache # this class is a singleton, pylint: disable=method-cache-max-size-none def get_class(self, ix:str): @@ -46,7 +46,7 @@ class _Device: def DEFAULT(self, v): raise AttributeError(f'setting Device.DEFAULT is deprecated, use "with Context(DEV={v!r})" or "DEV.value = {v!r}"') @functools.cached_property def _select_device(self) -> str: - assert (dev:=next((d for d in self._devices if d not in ["DISK", "TINYFS", "NPY"] and getenv(d) == 1), None)) is None, \ + assert (dev:=next((d for d in self._devices if d not in ["DISK", "NPY"] and getenv(d) == 1), None)) is None, \ f"{dev}=1 is deprecated, use DEV={dev} instead" try: device = next(self.get_available_devices()) @@ -54,7 +54,7 @@ class _Device: return device except StopIteration as exc: raise RuntimeError("no usable devices") from exc Device: _Device = _Device() -atexit.register(lambda: [Device[dn].finalize() for dn in Device._opened_devices]) +atexit.register(lambda: [Device[dn].finalize() for dn in tuple(Device._opened_devices)]) def canonicalize_device(device:str|tuple|list|None) -> str|tuple[str, ...]: if not isinstance(device, (tuple, list)): return Device.canonicalize(device) @@ -66,10 +66,10 @@ def canonicalize_device(device:str|tuple|list|None) -> str|tuple[str, ...]: class ProfileDeviceEvent(ProfileEvent): device:str; tdiff:decimal.Decimal=decimal.Decimal(0); props:dict[str,Any]|None=None # noqa: E702 @dataclass(frozen=True) -class ProfileProgramEvent(ProfileEvent): device:str; name:str; lib:bytes|None; base:int|None; tag:int|None=None # noqa: E702 +class ProfileProgramEvent(ProfileEvent): device:str; name:str; lib:bytes|None; base:int|None; tag:int|None=None; profile_key:bytes|None=None # noqa: E702 @dataclass(frozen=True) -class ProfileGraphEntry: device:str; name:str|TracingKey; st_id:int; en_id:int # noqa: E702 +class ProfileGraphEntry: device:str; name:str|TracingKey; st_id:int; en_id:int; profile_key:bytes|None=None # noqa: E702 @dataclass(frozen=True) class ProfileGraphEvent(ProfileEvent): ents:list[ProfileGraphEntry]; deps:list[list[int]]; sigs:list[decimal.Decimal] # noqa: E702 @@ -83,6 +83,7 @@ class BufferSpec: cpu_access: bool = False host: bool = False nolru: bool = False + zero: bool = False external_ptr: int|None = None class MultiBuffer: @@ -103,7 +104,7 @@ class Buffer: def __init__(self, device:str, size:int, dtype:DType, opaque:Any=None, options:BufferSpec|None=None, initial_value:bytes|pickle.PickleBuffer|None=None, uop_refcount=0, base:Buffer|None=None, offset:int=0, preallocate=False): assert isinstance(dtype, DType) - self.device, self.size, self.dtype, self.options, self.offset, self.allocated_views = device, size, dtype, options, offset, 0 + self.device, self.size, self.dtype, self.options, self.offset, self.allocated_views = Device.canonicalize(device), size, dtype, options, offset, 0 self._bufs: dict[str, Any] = {} if base is None: assert offset == 0, "base buffers can't have offset" @@ -116,7 +117,7 @@ class Buffer: if isinstance(initial_value, pickle.PickleBuffer): initial_value.release() else: assert base._base is None, "base can't have a base" - assert device == base.device, "base must have the same device" + assert self.device == base.device, "base must have the same device" self._base = base if preallocate: self.allocate() @property @@ -133,7 +134,7 @@ class Buffer: # check if the underlying buffer is allocated, possibly from the base object def is_allocated(self) -> bool: return self.base.is_allocated() if self._base is not None else self.device in self._bufs def get_buf(self, device: str) -> Any: - if device not in self._bufs: + if device not in self._bufs and (device:=Device.canonicalize(device)) not in self._bufs: allocator = Device[device].allocator if device == self.device: self.ensure_allocated() elif self._base is not None: self._bufs[device] = allocator._offset(self._base.get_buf(device), self.nbytes, self.offset) @@ -265,7 +266,7 @@ class LRUAllocator(Allocator, Generic[DeviceType]): for opaque in opaques: super().free(opaque, sz, options) opaques.clear() def free(self, opaque:Any, size:int, options:BufferSpec|None=None): - if LRU and (options is None or (not options.nolru and options.external_ptr is None)): self.cache[(size, options)].append(opaque) + if LRU and (options is None or (not (options.nolru or options.zero) and options.external_ptr is None)): self.cache[(size, options)].append(opaque) else: super().free(opaque, size, options) class DepsTracker: @@ -310,6 +311,14 @@ class Compiler: if self.cachekey is not None: diskcache_put(self.cachekey, src, lib) return lib def disassemble(self, lib:bytes): pass + def server(self, cmd:str, arch:str, *args) -> subprocess.Popen: + argv = f"{cmd} {pathlib.Path(__file__).parent}/runtime/support/compileserver.py {type(self).__module__}:{type(self).__name__} {arch}" + return subprocess.Popen(argv.split() + [str(a) for a in args], stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) + def compile_server(self, src:str, proc:subprocess.Popen) -> bytes: + unwrap(proc.stdin).write(struct.pack("I", len(src.encode())) + src.encode()) + if (lib:=unwrap(proc.stdout).read(struct.unpack("I", unwrap(proc.stdout).read(4))[0])): return lib + raise CompileError("Compilation Error") + @dataclass class TinyELF: @@ -318,6 +327,7 @@ class TinyELF: target: Target # tuple of (name, slot, dtype, shape) signature: tuple[tuple[str|None, int, DType, tuple], ...] + profile_key: bytes|None = None @staticmethod def iter_sig(signature:tuple[tuple[str|None, int, DType, tuple], ...], offset:int=0) -> Generator[tuple[int, DType], None, None]: @@ -331,15 +341,19 @@ class Program(Generic[DeviceType]): wait=False) -> float|None: pass class Compiled: + ifaces:list[Callable] = [] profile_events:list[ProfileEvent] = [ProfileDeviceEvent("CPU")] # NOTE: CPU is the default device. - pm_lower:Any = None + has_copy_queue:bool = True + + pm_encode:Any = None # per queue kind: queue ops -> flat command words + pm_lower:Any = None # per queue kind: custom_function(submit, cmdbuf) -> the queue push pm_bufferize:Any = None def __init__(self, device:str, allocator:Allocator, renderers:list[type[Renderer]], runtime:type[Program[Self]]|None, graph=None, arch=None): from tinygrad.renderer import Renderer self.device, self.allocator, self.runtime_t, self.graph, self.renderers = device, allocator, runtime, graph, renderers or [Renderer] - self.arch = arch + self.device_id, self.arch = (int(idx) if ":" in device and (idx:=device.split(":")[1]).isdigit() else 0), arch self.cached_renderer:dict[Any, Renderer] = {} @property @@ -362,11 +376,21 @@ class Compiled: return select_first_inited(select_by_name(self.renderers, self._renderer_name, t.renderer, f"{self.device} has no renderer {t.renderer!r}"), f"No renderer for {self.device} is available", self.cached_renderer, t) + def _select_iface(self, device:str): + self.device_id = int(device.split(":")[1]) if ":" in device else 0 + assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \ + f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead" + t = DEV.target(dev:=type(self).__name__[:-6]) + filtered = select_by_name(self.ifaces, lambda i: i.__name__[:-5], t.interface, f"{dev} has no interface {t.interface!r}") + filtered = [i for i in filtered if t.interface.startswith("MOCK") or not i.__name__[:-5].startswith("MOCK")] # never fallback to mock ifaces + return select_first_inited([functools.partial(iface, self, self.device_id) for iface in filtered], + f"No interface for {dev}:{self.device_id} is available") + def count(self) -> int: """ Returns the number of physical accelerators available to the runtime. """ - return 1 + return self.iface.count if hasattr(self, 'iface') else 1 def synchronize(self): """ @@ -384,7 +408,7 @@ class Compiled: """ Called at the end of process lifetime to allow the device to finalize. """ - # override this in your device implementation + if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini() if PROFILE: @atexit.register @@ -406,7 +430,7 @@ def enumerate_devices_str() -> Generator[str, None, None]: ren_results, iface_results = [], [] try: d = Device[device] - for iface in [i for i in getattr(d, 'ifaces', []) if not i.__name__.startswith("MOCK")]: + for iface in [i for i in d.ifaces if not i.__name__.startswith("MOCK")]: try: name = iface.__name__[:-5] default_text, count = ("(default)", d.count()) if type(d.iface) is iface else (f"(DEV={name}+{device} to make default)", iface(d, 0).count) # type: ignore diff --git a/tinygrad_repo/tinygrad/dtype.py b/tinygrad_repo/tinygrad/dtype.py index 9c0b336a5..edbce227d 100644 --- a/tinygrad_repo/tinygrad/dtype.py +++ b/tinygrad_repo/tinygrad/dtype.py @@ -66,7 +66,6 @@ class DType(metaclass=DTypeMetaClass): def __reduce__(self): return type(self), tuple(getattr(self, f.name) for f in fields(self)) def __repr__(self): return f"dtypes.{INVERSE_DTYPES_DICT[self.name]}" def __lt__(self, o:DType): return (self.priority, self.bitsize, self.name, self.fmt) < (o.priority, o.bitsize, o.name, o.fmt) - def scalar(self) -> DType: return self @functools.cached_property def min(self): if dtypes.is_int(self): return 0 if dtypes.is_unsigned(self) else -2**(self.bitsize-1) @@ -102,7 +101,11 @@ class DTypes: if isinstance(x, float): return dtypes.weakfloat if isinstance(x, int): return dtypes.weakint # put this in the last is faster because there are more items than lists/tuples to check - if isinstance(x, (list, tuple)): return strong_dtype(max(dtypes.from_py(xi) for xi in x)) if x else dtypes.default_float + if isinstance(x, (list, tuple)): + dt = max(dtypes.from_py(xi) for xi in x) if x else dtypes.weakfloat + if dt is not dtypes.weakint: return strong_dtype(dt) + ints = [xi for xi in x if isinstance(xi, int)] # a vconst also holds Invalid + return commit_int(min(ints), max(ints)) raise RuntimeError(f"Could not infer dtype of {x} with type {type(x)}") @staticmethod def finfo(dtype:DType) -> tuple[int, int]: @@ -121,8 +124,6 @@ class DTypes: uint32: Final[DType] = DType.new(6, 32, "unsigned int", 'I') int64: Final[DType] = DType.new(7, 64, "long", 'q') uint64: Final[DType] = DType.new(8, 64, "unsigned long", 'Q') - _uint128: Final[DType] = DType.new(8, 128, "uint128", None) - _uint256: Final[DType] = DType.new(8, 256, "uint256", None) weakfloat: Final[DType] = DType.new(9, 800, "weakfloat", None) fp8e4m3: Final[DType] = DType.new(10, 8, "float8_e4m3", None) fp8e5m2: Final[DType] = DType.new(11, 8, "float8_e5m2", None) @@ -165,6 +166,10 @@ assert dtypes.is_float(dtypes.default_float), f"{DEFAULT_FLOAT.value} is not a f assert dtypes.is_int(dtypes.default_int), f"{DEFAULT_INT.value} is not an int dtype" def strong_dtype(dtype:DType) -> DType: return {dtypes.weakint: dtypes.default_int, dtypes.weakfloat: dtypes.default_float}.get(dtype, dtype) +def commit_int(lo:int|float, hi:int|float, default_int:DType|None=None) -> DType: + if lo == hi and not dtypes.long.min <= lo <= dtypes.ulong.max: raise OverflowError(f"{lo} does not fit any int") + ladder = (dtypes.default_int if default_int is None else default_int, dtypes.int, dtypes.long, dtypes.ulong) + return next((dt for dt in ladder if dt.min <= lo and hi <= dt.max), dtypes.long) def weak_dtype(dtype:DType) -> DType: return dtypes.weakfloat if dtypes.is_float(dtype) else dtypes.weakint if dtypes.is_int(dtype) else dtype @@ -222,7 +227,7 @@ def float_to_fp16(x): def float_to_bf16(x): if not math.isfinite(x): return x - u = struct.unpack('I', struct.pack('f', x))[0] + u = struct.unpack('I', struct.pack('f', truncate[dtypes.float](x)))[0] u = (u + 0x7FFF + ((u >> 16) & 1)) & 0xFFFF0000 return struct.unpack('f', struct.pack('I', u))[0] diff --git a/tinygrad_repo/tinygrad/engine/jit.py b/tinygrad_repo/tinygrad/engine/jit.py index fe11805c1..1ad4c5e79 100644 --- a/tinygrad_repo/tinygrad/engine/jit.py +++ b/tinygrad_repo/tinygrad/engine/jit.py @@ -7,7 +7,7 @@ from tinygrad.dtype import DType from tinygrad.uop.ops import UOp, PatternMatcher, Variable, sym_infer, Ops, buffers, rewrite_group, graph_rewrite from tinygrad.renderer import Estimates from tinygrad.engine.realize import capturing, compile_linear, link_linear, run_linear, graph_cache, estimate_uop, get_runtime -from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_outs_ins +from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_written_bufs from tinygrad.schedule.memory import memory_plan_rewrite, _collect_bufs from tinygrad.nn.state import get_parameters from tinygrad.uop.movement import mop_cleanup @@ -44,9 +44,7 @@ def graph_split_rewrite(linear:UOp, max_batch_size:int=0) -> UOp: current_batch, current_batch_devs = [], [] for si in linear.src: - if si.src[0].op is Ops.SLICE: continue - - devs = dedup([Device[x] for b in si.src[1:] if b.op is not Ops.BIND for x in (b.device if isinstance(b.device, tuple) else (b.device,))]) + devs = dedup([Device[x] for b in si.src[1:] if not b.is_bound_var for x in (b.device if isinstance(b.device, tuple) else (b.device,))]) graph_t = graph_class(devs[0]) if devs[0].graph is not None else None can_graph = graph_t is not None and graph_t.supports_uop(devs, si) @@ -69,7 +67,7 @@ def jit_lower(linear:UOp, held_bufs:set[UOp], input_uops:list[UOp]) -> UOp: if VIZ: graph_rewrite(linear, PatternMatcher([]), name="View captured linear") # parametrize input buffers: map each input buffer UOp to a PARAM with the correct slot index - linear = linear.substitute({u: UOp.param(i, u.dtype, u.shape, u.device) for i,u in enumerate(input_uops)}, walk=True) + linear = linear.substitute({u: UOp.param(i, u.dtype, u.max_numel(), u.device) for i,u in enumerate(input_uops)}, walk=True) linear = memory_plan_rewrite(linear, held_bufs) linear = compile_linear(linear, beam=getenv("JITBEAM", BEAM.value)) if JIT < 2: linear = graph_split_rewrite(linear, max_batch_size=JIT_BATCH_SIZE.value) @@ -108,18 +106,17 @@ class GraphRunner: def is_sym_dim(dim) -> bool: return not all(isinstance(d, (int, float)) for d in dim) crs = [(j, self.calls[j][1].arg, self.calls[j][3]) for j in range(len(self.calls)) if self.calls[j][1].op is Ops.PROGRAM] - self.vars = sorted({v.expr for _,p,dv in crs for v in p.vars if v.expr not in dv | p.runtimevars}) - self.symbolic_dims = dedup(tuple(d) for _,p,_ in crs for d in (p.local_size, p.global_size) if d and is_sym_dim(d)) + self.vars = sorted({v.expr for _,p,dv in crs for v in p.vars if v.expr not in dv}) + self.symbolic_dims = dedup(tuple(d) for _,p,_ in crs for d in (p.local_size, p.global_size) if is_sym_dim(d)) - def find_symbolic_dim(dim): return self.symbolic_dims.index(tuple(dim)) if dim is not None and tuple(dim) in self.symbolic_dims else None + def find_symbolic_dim(dim:tuple[int,int,int]): return self.symbolic_dims.index(tuple(dim)) if tuple(dim) in self.symbolic_dims else None for j,p,dv in crs: - if (replace:=[(i, self.vars.index(v.expr)) for i, v in enumerate(p.vars) if v.expr not in dv | p.runtimevars]): + if (replace:=[(i, self.vars.index(v.expr)) for i, v in enumerate(p.vars) if v.expr not in dv]): self.var_vals_replace[j] = replace global_dim_idx, local_dim_idx = find_symbolic_dim(p.global_size), find_symbolic_dim(p.local_size) if global_dim_idx is not None or local_dim_idx is not None: self.launch_dims_replace[j] = (global_dim_idx, local_dim_idx) - assert p.local_size is not None self.launch_dims_base[j] = (tuple(p.global_size), tuple(p.local_size)) estimates = sum((estimate_uop(call) for call in self.linear.src), Estimates()) @@ -175,13 +172,7 @@ class CapturedJit(Generic[ReturnType]): @functools.cached_property def _written_uops(self) -> set[UOp]: - out: set[UOp] = set() - for call in self.linear.toposort(): - if call.op is not Ops.CALL: continue - arg_uops = get_call_arg_uops(call) - outs, ins = get_call_outs_ins(call) - out |= {arg_uops[k] for k in set(outs) - set(ins) if arg_uops[k].op in (Ops.BUFFER, Ops.SLICE)} - return out + return {b for call in self.linear.toposort() if call.op is Ops.CALL for b in get_call_written_bufs(call)} def __call__(self, input_uops:list[UOp], var_vals:dict[str, int]) -> ReturnType: concrete = tuple(_copy_input(u) if u in self._written_uops else u for u in input_uops) @@ -213,7 +204,7 @@ def _prepare_jit_inputs(args, kwargs): # collect buffer UOps (including MultiBuffer) input_buf_uops: list[UOp] = [u.base for u in input_uops if u.base.realized is not None] if len(set(input_buf_uops)) != len(input_buf_uops): raise JitError("duplicate inputs to JIT") - inputs = [(*(u.substitute({u.base:UOp(Ops.NOOP, u.base.dtype)}, extra_pm=mop_cleanup).unbind_all()), u.dtype, u.device) for u in input_uops] + inputs = [(*(u.substitute({u.base:UOp(Ops.NOOP)}, extra_pm=mop_cleanup).unbind_all()), u.dtype, u.device) for u in input_uops] _var_vals = merge_dicts([x[1] for x in inputs] + [dict(v.unbind() for v in (args + tuple(kwargs.values())) if isinstance(v, UOp))]) var_vals = {k.expr:v for k,v in _var_vals.items()} expected_input_info = [(x[0], tuple(sorted(x[1].keys(), key=lambda v: v.expr)), x[2], x[3]) for x in inputs] @@ -271,10 +262,14 @@ class _TinyJit(Generic[ReturnType]): big_linear, onetime_linear = prune_linear(big_linear, set(input_buf_uops)) if DEBUG >= 1: print(f"pruned from {len(big_linear.src) + len(onetime_linear.src)} -> {len(big_linear.src)} kernels") run_linear(onetime_linear, var_vals) + del onetime_linear # hold all buffers reachable from live Tensors (e.g. lazy .grad created during capture), the memory planner can't suballocate those held_bufs = set(buffers) | {u for tref in list(all_tensors) if (t:=tref()) is not None for u in t.uop.toposort() if u.op is Ops.BUFFER} linear = jit_lower(big_linear, held_bufs, input_buf_uops) + # drop the pre-planning graph: it keeps the whole capture-time working set allocated (big_linear) or referenced (held_bufs). + # the planned linear only uses the arena/held buffers, so the intermediates must be freed before linking and first exec + del big_linear, held_bufs self.captured = CapturedJit(ret, linear, names, expected_input_info) ret = self.captured(input_buf_uops, var_vals) elif self.cnt >= 2: diff --git a/tinygrad_repo/tinygrad/engine/realize.py b/tinygrad_repo/tinygrad/engine/realize.py index 92006e87d..062ad32fc 100644 --- a/tinygrad_repo/tinygrad/engine/realize.py +++ b/tinygrad_repo/tinygrad/engine/realize.py @@ -1,35 +1,46 @@ from __future__ import annotations from typing import cast, Iterator, Any, Sequence -import time, random, itertools, math, contextlib, weakref, array +import weakref, array, decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansilen, all_int, prod, flatten, Context, getenv, to_tuple -from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, wait_cond -from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, buffers, graph_rewrite -from tinygrad.device import Device, Buffer, MultiBuffer -from tinygrad.renderer import Estimates -from tinygrad.codegen import to_program -from tinygrad.codegen.opt.postrange import args_from_ast +from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, prod, flatten, Context, to_tuple, tqdm, dedup, to_mv +from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, HCQ2, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us +from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo +from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry +from tinygrad.renderer import Estimates, Renderer +from tinygrad.codegen import to_program, to_program_cache, to_program_key, to_program_context +from tinygrad.engine.worker import get_worker_pool, terminate_worker_pool # **************** Helpers **************** -def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call.src[1:] if s.op is not Ops.BIND) +def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call.src[1:] if not s.is_bound_var) +def get_call_var_uops(call:UOp, prg:UOp) -> list[UOp]: + bound = {s.src[0].expr: s.src[1].src[1] for s in call.src[1:] if s.is_bound_var} + return [bound.get(v.expr, v) for v in prg.arg.vars] def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: ast = call.src[0] if ast.op is Ops.PROGRAM: return tuple(ast.arg.outs), tuple(ast.arg.ins) - if ast.op in (Ops.COPY, Ops.SLICE): return (0,), (1,) + if ast.op is Ops.COPY: return (0,), (1,) if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec": return (0,), tuple(range(1, len(get_call_arg_uops(call)))) return (), () +def get_call_written_bufs(call:UOp) -> list[UOp]: + arg_uops, (outs, ins) = get_call_arg_uops(call), get_call_outs_ins(call) + return dedup([b for k in outs if k not in ins and (b:=u if (cv:=(u:=arg_uops[k]).contiguous_view()) is None else cv[0]).op is Ops.BUFFER]) + +def get_call_kernels(call:UOp) -> list[tuple[str, UOp, tuple[str, Estimates, bytes]|None]]: + if (ast:=call.src[0]).op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq": + return [(d, call, (name, estimates, profile_key)) for devices,name,estimates,_,profile_key in call.arg.aux.kernels for d in devices] + if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return [(to_tuple(ast.device)[0], call, None)] + if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "validate": return [] + return [(d, call, None) for d in to_tuple(call.src[1].device)] + def get_call_name(call:UOp, bufs:Sequence[Buffer|UOp], var_vals:dict[str, int]|None=None) -> str: def _uop_sz_to_str(uop:UOp) -> str: return size_to_str(sym_infer(prod(uop.shape) * uop.dtype.itemsize, var_vals or {})) def _dev_str(buf:Buffer|UOp) -> str: return ', '.join(d[:7] for d in to_tuple(buf.device)) ast, arg_uops = call.src[0], get_call_arg_uops(call) if ast.op is Ops.PROGRAM: return ast.arg.name - if ast.op is Ops.SLICE: - offset = ast.src[1].val * arg_uops[1].dtype.itemsize - return colored(f"view {_uop_sz_to_str(arg_uops[0]):>10} @ {offset:<10d}", "yellow") if ast.op is Ops.COPY: return colored(f"copy {_uop_sz_to_str(arg_uops[0]):>10}, {_dev_str(bufs[0]):>7s} <- {_dev_str(bufs[1]):7s}", "yellow") if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec": return colored(f"enc/dec {_uop_sz_to_str(arg_uops[0])}", "yellow") if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return colored(f"batched {len(ast.src[0].src)}", "cyan") @@ -39,75 +50,53 @@ def get_call_name(call:UOp, bufs:Sequence[Buffer|UOp], var_vals:dict[str, int]|N # **************** Stat **************** def estimate_uop(call:UOp) -> Estimates: - ast = call.src[0] - if ast.op is Ops.PROGRAM: return ast.src[0].arg.estimates or Estimates() + if (ast:=call.src[0]).op is Ops.PROGRAM: return ast.src[0].arg.estimates or Estimates() if ast.op is Ops.COPY or (ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec"): - nbytes = prod(call.src[1].shape) * call.src[1].dtype.itemsize - return Estimates(lds=nbytes, mem=nbytes) + return Estimates(lds=(nbytes:=prod(call.src[1].shape) * call.src[1].dtype.itemsize), mem=nbytes) if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return get_graph_runtime(ast).estimates if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq": return call.arg.aux.estimates return Estimates() first_run_cache:set[bytes] = set() -@contextlib.contextmanager -def track_stats(ctx:ExecContext, call:UOp, device:str, bufs:list[Buffer], var_vals:dict[str, int]): - if PROFILE: - outputs, inputs = get_call_outs_ins(call) - cpu_events.append(ProfilePointEvent(device, "exec", len(cpu_events), {"var_vals": var_vals, - "bufs": [b.trace_num for b in bufs], "name": get_call_name(call, bufs, var_vals), "outputs": outputs, "inputs": inputs})) - et: list[float|None] = [None] - if DEBUG >= 2: st = time.perf_counter() - yield et - if not ctx.update_stats: return +def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|None]): + if ctx.update_stats: + is_hcq = (ast:=call.src[0]).op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq" + estimates, n = estimate_uop(call), 1 if is_hcq else len(get_call_kernels(call)) + GlobalCounters.kernel_count += len(call.arg.aux.kernels) if is_hcq else n + GlobalCounters.global_ops += n*sym_infer(estimates.ops, ctx.var_vals) + GlobalCounters.global_mem += n*sym_infer(estimates.mem, ctx.var_vals) + GlobalCounters.time_sum_s += sum(et for et in ets if et is not None) + if DEBUG < 2 and not PROFILE: return - if DEBUG >= 2 and et[0] is None: - Device[device].synchronize() - et[0] = time.perf_counter() - st + kernels = get_call_kernels(call) # everything below is the per kernel display: exec events for the profiler and DEBUG=2 lines + args = resolve_params(call, ctx.input_uops) if kernels and kernels[0][2] is None else [] + lanes = list(unwrap_multi(call, [args[g] for g in call.src[0].arg.globals] if call.src[0].op is Ops.PROGRAM else args)) if args else [] + for i, (device, kcall, stats) in enumerate(kernels): + et, bufs = ets[i] if i < len(ets) else None, lanes[i][0] if i < len(lanes) else [] + display_name = get_call_name(kcall, bufs, ctx.var_vals) if stats is None else stats[0] + if PROFILE: # backdate the event to the start of the call, the viz matches a device range with the exec event before it + outputs, inputs = get_call_outs_ins(kcall) + cpu_events.append(ProfilePointEvent(device, "exec", len(cpu_events), {"var_vals": ctx.var_vals, + "bufs": [b.trace_num for b in bufs], "name": display_name, "outputs": outputs, "inputs": inputs}, ts=st)) + if DEBUG < 2 or not ctx.update_stats: continue + if et is None: + Device[device].synchronize() + et, st = float(perf_counter_us() - st)*1e-6, perf_counter_us() + GlobalCounters.time_sum_s += et - estimates = estimate_uop(call) - GlobalCounters.kernel_count += 1 - GlobalCounters.global_ops += (op_est:=sym_infer(estimates.ops, var_vals)) - GlobalCounters.global_mem += (mem_est:=sym_infer(estimates.mem, var_vals)) - if et[0] is not None: GlobalCounters.time_sum_s += et[0] - if DEBUG >= 2: - display_name = get_call_name(call, bufs, var_vals) - lds_est = sym_infer(estimates.lds, var_vals) - header_color = 'magenta' if ctx.jit else ('green' if call.src[0].key not in first_run_cache else None) - ptm = colored(time_to_str(et[0], w=9), "yellow" if et[0] > 0.01 else None) if et[0] is not None else "" - flops, membw, ldsbw = op_est/(et[0] or 1e-20), mem_est/(et[0] or 1e-20), lds_est/(et[0] or 1e-20) + estimates = estimate_uop(kcall) if stats is None else stats[1] + op_est, mem_est, lds_est = (sym_infer(x, ctx.var_vals) for x in (estimates.ops, estimates.mem, estimates.lds)) + key = kcall.src[0].key if stats is None else stats[2] + header_color = 'magenta' if ctx.jit else ('green' if key not in first_run_cache else None) + ptm = colored(time_to_str(et, w=9), "yellow" if et > 0.01 else None) if et is not None else "" + flops, membw, ldsbw = op_est/(et or 1e-20), mem_est/(et or 1e-20), lds_est/(et or 1e-20) flops_str = f"{flops*1e-9:7.0f} GFLOPS" if flops < 1e14 else colored(f"{flops*1e-12:7.0f} TFLOPS", 'green') mem_str = f"{membw*1e-9:4.0f}|{ldsbw*1e-9:<6.0f} GB/s" if membw < 1e13 and ldsbw < 1e15 else \ colored(f"{membw*1e-12:4.0f}|{ldsbw*1e-12:<6.0f} TB/s", 'green') print(f"{colored(f'*** {device[:7]:7s} {GlobalCounters.kernel_count:4d}', header_color)}"+ - f" {display_name+' '*(46-ansilen(display_name))} arg {len(bufs):2d} mem {GlobalCounters.mem_used/1e9:6.2f} GB"+ - ("" if et[0] is None else f" tm {ptm}/{GlobalCounters.time_sum_s*1e3:9.2f}ms ({flops_str} {mem_str})")) - first_run_cache.add(call.src[0].key) - -local_size_cache: dict[bytes, tuple[int, ...]] = {} -def optimize_local_size(call:UOp, prg:UOp) -> UOp|None: - device = to_tuple(prg.device)[0] - if prg.arg.local_size is not None or not Device[device].renderer.has_local or not all_int(prg.arg.global_size): return None - - if (local_size:=local_size_cache.get(prg.key)) is None: - # reuse one loaded runtime across candidates, only launch dims vary - (bufs, var_vals), runtime = args_from_ast(prg.src[0], device), get_runtime(device, prg, cache=False) - bufs = [b.allocate() for b in bufs] - def try_exec(local_size): - try: - new_gs = tuple(g//l if g%l == 0 else g/l for g,l in zip(prg.arg.global_size, local_size)) - return runtime(*[bufs[i].get_buf(device) for i in prg.arg.globals], global_size=new_gs, local_size=(*local_size,), - vals=prg.arg.vals(var_vals), wait=True) - except Exception: return float('inf') - - MAX_WORKGROUP = 1024 - local_dims = [[x for x in set([sz, 1, 2, 4, 8, 16, 32, 64, 128, 256, MAX_WORKGROUP]) if x<=sz] for sz in prg.arg.global_size] - local_sizes = [list(x) for x in itertools.product(*local_dims) if prod(x) <= MAX_WORKGROUP] * 2 # try each valid size twice - best_time, best = min([(try_exec(ls), ls) for ls in random.sample(local_sizes, len(local_sizes))]) - assert not math.isinf(best_time), "all optimize_local_size exec failed" - local_size = local_size_cache[prg.key] = tuple(best) - - new_global = tuple(g//l if g%l == 0 else g/l for g,l in zip(prg.arg.global_size, local_size)) - return call.replace(src=(prg.replace(arg=replace(prg.arg, global_size=new_global, local_size=local_size)), *call.src[1:])) + f" {ansipad(display_name, 46)} arg {len(bufs):2d} mem {GlobalCounters.mem_used/1e9:6.2f} GB"+ + ("" if et is None else f" tm {ptm}/{GlobalCounters.time_sum_s*1e3:9.2f}ms ({flops_str} {mem_str})")) + first_run_cache.add(key) # **************** runtime cache **************** @@ -140,7 +129,7 @@ class ExecContext: cache: bool = True def _resolve(b:UOp, inputs:tuple[UOp, ...]) -> UOp: - if b.op in (Ops.SLICE, Ops.MSELECT) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) + if b.op in (Ops.MSELECT, Ops.SHRINK) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) if b.op is Ops.MSTACK: return b.replace(src=tuple(_resolve(x, inputs) for x in b.src)) return inputs[b.arg.slot] if b.op is Ops.PARAM else b def resolve_params(call:UOp, inputs:tuple[UOp, ...]) -> list[UOp]: return [_resolve(b, inputs) for b in get_call_arg_uops(call)] @@ -152,41 +141,35 @@ def unwrap_multi(call:UOp, resolved:list[UOp]) -> Iterator[tuple[list[Buffer], d # the DEVICE axis is bound per device at launch: it's a RANGE in the AST and the _device_num variable after codegen has_dnum = any((x.op is Ops.RANGE and x.arg[-1] is AxisType.DEVICE) or (x.op is Ops.PARAM and x.arg.name == '_device_num') for x in call.src[0].toposort()) - for j, per_dev in enumerate(zip(*[cast(MultiBuffer, b).bufs for b in bufs])): yield list(per_dev), {"_device_num": j} if has_dnum else {} + lanes = max(len(b.bufs) for b in bufs if isinstance(b, MultiBuffer)) # a single buffer is shared by every lane + per_lane = [b.bufs if isinstance(b, MultiBuffer) else (b,)*lanes for b in bufs] + for j, per_dev in enumerate(zip(*per_lane)): yield list(per_dev), {"_device_num": j} if has_dnum else {} -def exec_view(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - resolved = resolve_params(call, ctx.input_uops) - bufs = [cast(Buffer, b.buffer) for b in resolved] - bv = bufs[1].view(resolved[0].max_numel(), ast.dtype, ast.src[1].val*bufs[1].dtype.itemsize) - with track_stats(ctx, call, bv.device, [bv, bufs[1]], ctx.var_vals): buffers[resolved[0]] = bv - return None - -def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: +def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: for bufs, device_vars in unwrap_multi(call, resolve_params(call, ctx.input_uops)): dest, src = bufs[0].ensure_allocated(), bufs[1].ensure_allocated() - with track_stats(ctx, call, dest.device, [dest, src], ctx.var_vals): - if hasattr(dest.allocator,'_transfer') and dest.allocator.supports_transfer and dest.device.split(":")[0] == src.device.split(":")[0]: - dest.allocator._transfer(dest._buf, src._buf, dest.nbytes, src_dev=src.allocator.dev, dest_dev=dest.allocator.dev) - elif src.device.startswith("DISK") and getattr(src.allocator.dev, 'fd', None) is not None \ - and hasattr(dest.allocator, 'copy_from_disk') and src.nbytes >= 4096 and dest.allocator.supports_copy_from_disk: - dest.allocator.copy_from_disk(dest._buf, src._buf, src.nbytes) - elif hasattr(dest.allocator, '_as_buffer'): src.allocator._copyout(dest.as_memoryview(force_zero_copy=True), src._buf) - else: dest.allocator._copyin(dest._buf, src.as_memoryview(allow_zero_copy=True)) - return None + if hasattr(dest.allocator,'_transfer') and dest.allocator.supports_transfer and dest.device.split(":")[0] == src.device.split(":")[0]: + dest.allocator._transfer(dest._buf, src._buf, dest.nbytes, src_dev=src.allocator.dev, dest_dev=dest.allocator.dev) + elif src.device.startswith("DISK") and getattr(src.allocator.dev, 'fd', None) is not None \ + and hasattr(dest.allocator, 'copy_from_disk') and src.nbytes >= 4096 and dest.allocator.supports_copy_from_disk: + dest.allocator.copy_from_disk(dest._buf, src._buf, src.nbytes) + elif hasattr(dest.allocator, '_as_buffer'): src.allocator._copyout(dest.as_memoryview(force_zero_copy=True), src._buf) + else: dest.allocator._copyin(dest._buf, src.as_memoryview(allow_zero_copy=True)) + return [] -def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - et = None - for device, (bufs, device_vars) in zip(to_tuple(call.src[1].device), unwrap_multi(call, resolve_params(call, ctx.input_uops))): +def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp, devices=None) -> list[float|None]: + ets:list[float|None] = [] + resolved = resolve_params(call, ctx.input_uops) + for device, (bufs, device_vars) in zip(devices or to_tuple(call.src[1].device), unwrap_multi(call, [resolved[i] for i in ast.arg.globals])): var_vals = {**ctx.var_vals, **device_vars} - prg_bufs = [bufs[i].ensure_allocated() for i in ast.arg.globals] + prg_bufs = [b.ensure_allocated() for b in bufs] rt = get_runtime(device, ast, cache=ctx.cache) global_size, local_size = ast.arg.launch_dims(var_vals) - with track_stats(ctx, call, device, prg_bufs, var_vals) as tm: - et = tm[0] = rt(*[b.get_buf(device) for b in prg_bufs], global_size=global_size, local_size=local_size, vals=ast.arg.vals(var_vals), - wait=ctx.wait, timeout=ctx.timeout) - return et + ets.append(rt(*[b.get_buf(device) for b in prg_bufs], global_size=global_size, local_size=local_size, vals=ast.arg.vals(var_vals), + wait=ctx.wait, timeout=ctx.timeout)) + return ets -def exec_validate(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: +def exec_validate(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: import numpy as np for bufs, device_vars in unwrap_multi(call, resolve_params(call, ctx.input_uops)): bufs, dev_bufs = bufs[:len(bufs)//2], bufs[len(bufs)//2:] @@ -195,42 +178,38 @@ def exec_validate(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: global_size, local_size = prg.arg.launch_dims(var_vals) cpu_rt(*[bufs[i].ensure_allocated()._buf for i in prg.arg.globals], global_size=global_size, local_size=local_size, vals=prg.arg.vals(var_vals)) for i in prg.arg.outs: np.testing.assert_allclose(dev_bufs[i].ensure_allocated().numpy(), bufs[i].numpy(), rtol=1e-3, atol=1e-3) - return None + return [] -def exec_encdec(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: +def exec_encdec(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: bufs = [cast(Buffer, b.buffer).ensure_allocated() for b in resolve_params(call, ctx.input_uops)] shape, pos_var = tuple(s.val for s in ast.src if s.op is Ops.CONST), ast.variables()[0].expr - with track_stats(ctx, call, bufs[0].device, bufs, ctx.var_vals): - bufs[0].allocator._encode_decode(bufs[0]._buf, bufs[1]._buf, bufs[2]._buf, [x._buf for x in bufs[3:]], shape, ctx.var_vals[pos_var]) - return None + bufs[0].allocator._encode_decode(bufs[0]._buf, bufs[1]._buf, bufs[2]._buf, [x._buf for x in bufs[3:]], shape, ctx.var_vals[pos_var]) + return [] -def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - rt = get_graph_runtime(ast, ctx.input_uops) - with track_stats(ctx, call, rt.device, [], ctx.var_vals) as t: t[0] = rt(ctx.input_uops, ctx.var_vals, wait=ctx.wait) - return t[0] +def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: + return [get_graph_runtime(ast, ctx.input_uops)(ctx.input_uops, ctx.var_vals, wait=ctx.wait)] -def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - if (inputs:=call.arg.aux.inputs) is not None: - bufs = [_resolve(ctx.input_uops[i], ctx.input_uops).buffer for i in call.arg.aux.input_idxs] - table = call.src[1+inputs].buffer - for j,dev in enumerate(call.arg.aux.device): - addrs = array.array('Q', [(b.bufs[j] if isinstance(b, MultiBuffer) else b).get_buf(dev).va_addr for b in bufs]) - mv = (table.bufs[j] if isinstance(table, MultiBuffer) else table).ensure_allocated()._buf.cpu_view().view(fmt='Q') - wait_cond(lambda: mv[0], value=0, timeout_ms=ctx.timeout or getenv("HCQDEV_WAIT_TIMEOUT_MS", 30000), msg=f"{dev} hang detected") - mv[:len(addrs)] = addrs +def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: + info = call.arg.aux + assert len(ast.arg.globals) == len(info.args), f"{call.arg.name}: an arg is dead in the rendered body, the args after it would mis-bind" - exec_kernel(replace(ctx, update_stats=False), call, ast) + # fill the inputs table with the address of every input the sealed cmdbufs reference + if info.table is not None: + addrs = [cast(Buffer, _resolve(_lane(u, lane), ctx.input_uops).buffer).get_buf(dev).va_addr for u, lane, dev in info.inputs] + tab = cast(Buffer, call.src[info.table].without_after.buffer) + to_mv(tab._buf.va_addr, len(addrs) * 8).cast('Q')[:] = array.array('Q', addrs) - tms:list[float|None] = [] - for e in (aux:=call.arg.aux).prof: cast(Any, Device[e.device]).prof_ents[e.st_id] = e - for d in [cast(Any, Device[x]) for x in aux.device]: - with track_stats(ctx, call, d.device, [], ctx.var_vals) as et: - if ctx.wait: - d.synchronize(timeout=ctx.timeout) - ts = [d.signal(i)._buf.cpu_view().view(fmt='Q')[0] for e in aux.prof if e.device == d.device for i in (e.st_id, e.en_id)] - if ts: et[0] = float(max(ts)-min(ts))/d.timestamp_divider/1e6 - tms += et - return tms[0] + # every lane's body runs on the runtime device, info.device is only the lane count + exec_kernel(replace(ctx, var_vals={**ctx.var_vals, **dict(info.vals)}), call, ast, devices=(HCQ_RUNTIME_DEV.value,)*len(info.device)) + + def _prof_tm(device:str, name:str, prof:tuple[int, ...], profile_key:bytes) -> float|None: + (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, name, prof[0], prof[1], profile_key) + if not ctx.wait: return None + d.synchronize(timeout=ctx.timeout) + st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) + return float(en-st)/d.timestamp_divider/1e6 + return [_prof_tm(device, name, prof, profile_key) for devices,name,_,prof,profile_key in info.kernels + if prof for device in devices] if PROFILE or ctx.wait else [] # flatten LINEAR-in-LINEAR: any nested LINEAR child gets inlined into its parent's src pm_flatten_linear = PatternMatcher([ @@ -251,17 +230,46 @@ pm_beam = PatternMatcher([ lambda ctx,call,sink: call.replace(src=(sink.replace(arg=replace(sink.arg, beam=ctx)), *call.src[1:])) if sink.arg.beam == 0 else None), ]) -pm_compile = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat((Ops.SINK, Ops.PROGRAM), name="ast"),), name="call", allow_any_len=True), lambda call,ast: - call.replace(src=(to_program(ast, Device[call.device if isinstance(call.device, str) else call.device[0]].renderer), *call.src[1:]))), -]) +# **************** parallel lowering + compilation **************** -pm_optimize_local_size = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), optimize_local_size), -]) +def _compile_kernel(x:tuple[int, tuple[UOp, Renderer], dict]) -> tuple[int, UOp]: + with Context(**x[2]): return x[0], to_program(*x[1]) + +def _get_call_to_compile(c:UOp) -> tuple[UOp, Renderer]|None: + ast = a0.src[0] if (a0:=c.src[0]).op is Ops.CUSTOM_FUNCTION and a0.arg == "hcq" else a0 + # a PROGRAM with a ProgramInfo and a BINARY is already compiled + if ast.op is Ops.SINK or (ast.op is Ops.PROGRAM and not (isinstance(ast.arg, ProgramInfo) and ast.src[-1].op is Ops.BINARY)): + return ast, Device[c.device if isinstance(c.device, str) else c.device[0]].renderer + return None + +def lower_and_compile(linear:UOp) -> UOp: + # collect the kernels to lower and compile, deduped by their compile cache key + if not len(ar:={c: a for c in linear.toposort() if c.op is Ops.CALL and (a:=_get_call_to_compile(c)) is not None}): return linear + + # lower and compile what's not cached, in parallel if there's a worker pool + keys = {c: to_program_key(*a) for c, a in ar.items()} + todo = list({keys[c]: a for c, a in ar.items() if keys[c] not in to_program_cache}.items()) + if len(todo): + # kernels that beam search must compile in the parent, beam needs device access to time candidates + + pool = None if len(todo) == 1 or any(getattr(c.src[0].arg, "beam", 0) for c in ar) else get_worker_pool() + ctx = {v.key: v.value for v in to_program_context} + tasks = ((i, ast_ren, ctx) for i, (_, ast_ren) in enumerate(todo)) + try: + with tqdm(total=len(todo), desc="compiling", disable=DEBUG<1) as pbar: + for i, prg in (map if pool is None else pool.imap_unordered)(_compile_kernel, tasks): + pbar.set_description(f"compiling {ansipad(prg.src[0].arg.name, 40)}") + to_program_cache[todo[i][0]] = prg + pbar.update(1) + except KeyboardInterrupt: + if pool is not None: terminate_worker_pool() + raise + + # swap the compiled PROGRAMs into the calls + return linear.substitute({c: c.replace(src=(c.src[0].substitute({a[0]: to_program_cache[keys[c]]}), *c.src[1:])) for c, a in ar.items()}, + name="precompile kernels") pm_exec = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.SLICE, name="ast"),), name="call", allow_any_len=True), exec_view), (UPat(Ops.CALL, src=(UPat(Ops.COPY, name="ast"),), name="call", allow_any_len=True), exec_copy), (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="ast"),), name="call", allow_any_len=True), exec_kernel), (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="encdec", name="ast"),), name="call", allow_any_len=True), exec_encdec), @@ -270,29 +278,30 @@ pm_exec = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="validate", name="ast"),), name="call", allow_any_len=True), exec_validate), ]) -if getenv("HCQ2"): from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link # noqa: E402 # down here, hcq2 imports the helpers above +from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, _lane, HCQ_RUNTIME_DEV # noqa: E402 # down here, hcq2 imports realize def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:list[UOp]|None=None, profile:bool|None=None) -> UOp: if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) if (beam_val:=BEAM.value if beam is None else beam) >= 1: linear = graph_rewrite(linear, pm_beam, ctx=beam_val, walk=True) - linear = graph_rewrite(linear, pm_compile, name="precompile kernels", walk=True) - if getenv("HCQ2"): linear = hcq_compile(linear, input_uops, bool(PROFILE) if profile is None else profile) - return graph_rewrite(linear, pm_optimize_local_size, name="optimize local size", walk=True) + linear = lower_and_compile(linear) + if HCQ2: linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile) + return linear -def link_linear(linear:UOp, cache=True) -> UOp: return hcq_link(linear, cache=cache) if getenv("HCQ2") else linear +def link_linear(linear:UOp, cache=True) -> UOp: return hcq_link(linear, cache=cache) if HCQ2 else linear def run_linear(linear:UOp, var_vals:dict[str, int]|None=None, input_uops:Sequence[UOp]=(), update_stats=True, jit=False, wait=False): inputs = list(input_uops) if not jit: linear = link_linear(compile_linear(linear, validate=VALIDATE_WITH_CPU, input_uops=inputs)) ctx = ExecContext(var_vals or {}, tuple(inputs), update_stats, jit, wait or DEBUG>=2) - for call in linear.src: pm_exec.rewrite(call, ctx) + for call in linear.src: track_stats(ctx, call, perf_counter_us(), pm_exec.rewrite(call, ctx)) -def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None, clear_l2:bool=False) -> float: - if clear_l2: - if hasattr(dev:=Device[call.src[1].device], 'invalidate_caches'): dev.invalidate_caches() - else: - from tinygrad.tensor import Tensor - with Context(DEBUG=0, BEAM=0, CAPTURING=0, TRACK_MATCH_STATS=0): Tensor.ones(1024, 1024).contiguous().realize(do_update_stats=False) +def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None, clear_l2:bool=False) -> Iterator[float]: ctx = ExecContext(var_vals or {}, update_stats=False, wait=True, timeout=timeout, cache=False) linear = link_linear(compile_linear(UOp(Ops.LINEAR, src=(call,)), beam=0, profile=True), cache=ctx.cache) - return max(pm_exec.rewrite(c, ctx) or 0.0 for c in linear.src) + while True: + if clear_l2: + if hasattr(dev:=Device[call.src[1].device], 'invalidate_caches'): dev.invalidate_caches() + else: + from tinygrad.tensor import Tensor + with Context(DEBUG=0, BEAM=0, CAPTURING=0, TRACK_MATCH_STATS=0): Tensor.ones(1024, 1024).contiguous().realize(do_update_stats=False) + yield max(et for c in linear.src for et in pm_exec.rewrite(c, ctx) or [0.0]) diff --git a/tinygrad_repo/tinygrad/engine/worker.py b/tinygrad_repo/tinygrad/engine/worker.py new file mode 100644 index 000000000..17fb7f92a --- /dev/null +++ b/tinygrad_repo/tinygrad/engine/worker.py @@ -0,0 +1,49 @@ +import multiprocessing, atexit, signal, sys, threading, contextlib +from multiprocessing.context import SpawnContext, SpawnProcess +from tinygrad.helpers import Context, getenv, PARALLEL + +# generic pool of worker processes for parallel compilation, shared by kernel lowering and BEAM search + +# workers should not open devices and should ignore ctrl c and should not launch VIZ +def _init_worker(): + Context(ALLOW_DEVICE_USAGE=0, VIZ=0, TRACK_MATCH_STATS=0).__enter__() + signal.signal(signal.SIGINT, signal.SIG_IGN) + +# spawn normally reimports the user's __main__ before _init_worker. This replays top-level code and can recursively create pools. There is no public +# multiprocessing switch to skip that import, so hide the two attributes used to locate __main__ while each worker (including replacements) starts. +_spawn_lock, _missing = threading.Lock(), object() +@contextlib.contextmanager +def _without_main(): + main = sys.modules.get("__main__") + if main is None: + yield + return + with _spawn_lock: + saved = {name:getattr(main, name, _missing) for name in ("__file__", "__spec__")} + try: + for name in saved: setattr(main, name, None) + yield + finally: + for name,value in saved.items(): delattr(main, name) if value is _missing else setattr(main, name, value) + +class _WorkerProcess(SpawnProcess): + @staticmethod + def _Popen(process_obj): + with _without_main(): return SpawnProcess._Popen(process_obj) + +class _WorkerContext(SpawnContext): Process = _WorkerProcess + +worker_pool = None +def get_worker_pool(): + global worker_pool + if multiprocessing.current_process().daemon or PARALLEL == 0: return None + if worker_pool is None: + worker_pool = _WorkerContext().Pool(PARALLEL.value, _init_worker, (), getenv("BEAM_MAX_TASKS_PER_CHILD", 16)) + @atexit.register + def close_pool(pool=worker_pool): pool.close() + return worker_pool + +def terminate_worker_pool(): + global worker_pool + if worker_pool is not None: worker_pool.terminate() + worker_pool = None diff --git a/tinygrad_repo/tinygrad/function.py b/tinygrad_repo/tinygrad/function.py index edb1691d9..4ba84d7c0 100644 --- a/tinygrad_repo/tinygrad/function.py +++ b/tinygrad_repo/tinygrad/function.py @@ -1,4 +1,5 @@ import functools, time +from dataclasses import replace from typing import Generic, TypeVar, Callable, cast, overload from tinygrad.helpers import Context, dedup, getenv, DEBUG from tinygrad.uop.ops import UOp, Ops, graph_rewrite, PatternMatcher, UPat @@ -12,7 +13,7 @@ def add_to_ctx(ctx, x:UOp): return ret pm_ctx = PatternMatcher([ - (UPat((Ops.BUFFER, Ops.BIND), name="x"), add_to_ctx), + (UPat(Ops.BUFFER, name="x"), add_to_ctx), (UPat((Ops.AFTER, Ops.CONTIGUOUS), name="x"), lambda ctx,x: add_to_ctx(ctx,x) if not x.op_in_backward_slice_with_self(Ops.PARAM) and x.op_in_backward_slice_with_self(Ops.BUFFER) else None), ]) @@ -23,6 +24,10 @@ def invalid_outputs(uret:UOp) -> set[UOp]: return {u.src[0].buf_uop for u in uret.backward_slice_with_self if u.op is Ops.STORE and u.src[1].base.is_invalid and not u.src[0].buf_uop.is_realized} +def renumber_invalid_outputs(uret:UOp) -> UOp: + return uret.substitute({b:b.replace(arg=replace(b.arg, slot=i)) + for i,b in enumerate(x for x in uret.toposort(enter_calls=False) if x in invalid_outputs(uret))}) + ReturnType = TypeVar('ReturnType') class _function(Generic[ReturnType]): depth = 0 @@ -54,7 +59,7 @@ class _function(Generic[ReturnType]): if isinstance(ret, Tensor): uret = ret.uop elif isinstance(ret, tuple) and all(isinstance(x, Tensor) for x in ret): - uret = UOp.maketuple(*[x.uop for x in ret]) + uret = UOp.sink(*[x.uop for x in ret]) else: raise RuntimeError(f"function return type {type(ret)} not supported") @@ -65,6 +70,7 @@ class _function(Generic[ReturnType]): # the BUFFERs that are left are the implicit inputs num_explicit = len(call_uops) uret = graph_rewrite(uret, pm_ctx, (call_uops, invalid_outputs(uret)), bottom_up=True, name="get_implicit_inputs") + uret = renumber_invalid_outputs(uret) name = getattr(self.fxn, '__qualname__', None) or type(self.fxn).__qualname__ if not self.allow_implicit: implicit_buffers = [x for x in call_uops[num_explicit:] if x.op is Ops.BUFFER] @@ -72,16 +78,17 @@ class _function(Generic[ReturnType]): buf_strs = '\n '.join(f"{i}: dtype={b.dtype}, size={b.max_numel()}, device={b.device}" for i,b in enumerate(implicit_buffers)) raise RuntimeError(f"function {name} has {len(implicit_buffers)} implicit buffer(s), but allow_implicit=False\n {buf_strs}") - fret = uret.call(*call_uops, grad_fxn=self.grad_fxn, name=name, precompile=self.precompile, - precompile_backward=self.precompile_backward) + fret = UOp.call_outputs(uret.src if isinstance(ret, tuple) else (uret,), *call_uops, grad_fxn=self.grad_fxn, name=name, + precompile=self.precompile, precompile_backward=self.precompile_backward) if DEBUG >= 2: print(" "*_function.depth+f"function {uret.key.hex()[:8]} in {(time.perf_counter()-st)*1000:8.2f} ms: {name}") + outs = fret.returned_outputs if isinstance(ret, tuple): - return cast(ReturnType, tuple(Tensor(fret.gettuple(i)) for i in range(len(ret)))) + return cast(ReturnType, tuple(Tensor(o) for o in outs)) else: - return cast(ReturnType, Tensor(fret.gettuple(0))) + return cast(ReturnType, Tensor(outs[0])) # overload signatures support both @function and @function(precompile=True) syntax @overload diff --git a/tinygrad_repo/tinygrad/helpers.py b/tinygrad_repo/tinygrad/helpers.py index acc3af11a..dfd5dc197 100644 --- a/tinygrad_repo/tinygrad/helpers.py +++ b/tinygrad_repo/tinygrad/helpers.py @@ -1,9 +1,9 @@ from __future__ import annotations import time START_TIME = time.perf_counter() -import os, functools, platform, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc, io +import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc, threading from collections import defaultdict -import subprocess, shutil, math, types, copyreg, inspect, importlib, decimal, itertools, difflib +import shutil, math, types, copyreg, inspect, importlib, decimal, itertools, difflib from dataclasses import dataclass, field, replace from typing import ClassVar, Iterable, Any, TypeVar, Callable, Sequence, TypeGuard, Iterator, Generic, Generator, cast, overload @@ -13,8 +13,7 @@ U = TypeVar("U") def prod(x:Iterable[T]) -> T|int: return functools.reduce(operator.mul, x, 1) # NOTE: helpers is not allowed to import from anything else in tinygrad -OSX, WIN = platform.system() == "Darwin", sys.platform == "win32" -ARCH_X86 = any(x in platform.processor() for x in ("Intel", "i386", "x86_64")) +OSX, WIN = sys.platform == "darwin", sys.platform == "win32" BASEDIR = pathlib.Path(__file__).parent # fix colors on Windows, https://stackoverflow.com/questions/12492810/python-how-can-i-make-the-ansi-escape-codes-to-work-also-in-windows @@ -45,6 +44,7 @@ def time_to_str(t:float, w=8) -> str: return next((f"{t * d:{w}.2f}{pr}" for d,p def size_to_str(s:int) -> str: return next((f"{s / d:.2f} {pr}" for d,pr in [(1<<30, "GB"),(1<<20, "MB"),(1<<10, "KB")] if s >= d), f"{s} B") def ansistrip(s:str): return re.sub('\x1b\\[(K|.*?m)', '', s) def ansilen(s:str): return len(ansistrip(s)) +def ansipad(s:str, w:int): return s+' '*max(w-ansilen(s), 0) def make_tuple(x:int|Sequence[int], cnt:int) -> tuple[int, ...]: return (x,)*cnt if isinstance(x, int) else tuple(x) def to_tuple(x:T|tuple[T, ...]) -> tuple[T, ...]: return x if isinstance(x, tuple) else (x,) def flatten(l:Iterable[Iterable[T]]): return [item for sublist in l for item in sublist] @@ -231,16 +231,15 @@ class _DEV(ContextVar): DEV, DEBUG, BEAM, NOOPT = _DEV("DEV", ""), ContextVar("DEBUG", 0), ContextVar("BEAM", 0), ContextVar("NOOPT", 0) IMAGE, FLOAT16, OPENPILOT_HACKS = ContextVar("IMAGE", 0), ContextVar("FLOAT16", 0), ContextVar("OPENPILOT_HACKS", 0) -JIT, JIT_BATCH_SIZE = ContextVar("JIT", 2 if OSX and ARCH_X86 else 1), ContextVar("JIT_BATCH_SIZE", 32) -CHUNK_SIZE = 2**20 # TinyFS content-addressed store: blob chunk + hash-tree node granularity +JIT, JIT_BATCH_SIZE = ContextVar("JIT", 1), ContextVar("JIT_BATCH_SIZE", 32) WINO, CAPTURING, TRACEMETA, NO_COLOR = ContextVar("WINO", 0), ContextVar("CAPTURING", 1), ContextVar("TRACEMETA", 1), ContextVar("NO_COLOR", 0) TRAINING = ContextVar("TRAINING", 0) -USE_TC, TC_SELECT, TC_OPT = ContextVar("TC", 1), ContextVar("TC_SELECT", -1), ContextVar("TC_OPT", 0) -TRANSCENDENTAL, NOLOCALS = ContextVar("TRANSCENDENTAL", 1), ContextVar("NOLOCALS", 0) +USE_TC, TC_SELECT, TC_OPT, TC_MIN_GLOBALS = ContextVar("TC", 1), ContextVar("TC_SELECT", -1), ContextVar("TC_OPT", 0), ContextVar("TC_MIN_GLOBALS", 0) +TRANSCENDENTAL = ContextVar("TRANSCENDENTAL", 1) SPLIT_REDUCEOP, NO_MEMORY_PLANNER, LRU = ContextVar("SPLIT_REDUCEOP", 1), ContextVar("NO_MEMORY_PLANNER", 0), ContextVar("LRU", 1) RING, ALL2ALL, ALLREDUCE_CAST = ContextVar("RING", 1), ContextVar("ALL2ALL", 0), ContextVar("ALLREDUCE_CAST", 1) CACHELEVEL, IGNORE_BEAM_CACHE = ContextVar("CACHELEVEL", 2), ContextVar("IGNORE_BEAM_CACHE", 0) -VALIDATE_WITH_CPU = ContextVar("VALIDATE_WITH_CPU", 0) +VALIDATE_WITH_CPU, HCQ2 = ContextVar("VALIDATE_WITH_CPU", 0), ContextVar("HCQ2", 0) # TODO: this is broken for some indexing DISABLE_FAST_IDIV = ContextVar("DISABLE_FAST_IDIV", 1) FUSE_OPTIM = ContextVar("FUSE_OPTIM", 0) @@ -250,25 +249,27 @@ EMULATED_DTYPES = ContextVar("EMULATED_DTYPES", "") DEFAULT_FLOAT, DEFAULT_INT = ContextVar("DEFAULT_FLOAT", "float32"), ContextVar("DEFAULT_INT", "int32") CAPTURE_PROCESS_REPLAY = ContextVar("CAPTURE_PROCESS_REPLAY", 0) def _get_cpu_count() -> int: - # os.process_cpu_count (3.13+) respects cgroup limits - if hasattr(os, "process_cpu_count"): return max(1, os.process_cpu_count() or 1) - # cgroup v2 (containers with --cpus=N) + # os.process_cpu_count is available in 3.13+, then try affinity, then fallback to cpu_count + count = (os.process_cpu_count() if hasattr(os, "process_cpu_count") else + len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count()) or 1 + # limit with cgroup v2 (containers with --cpus=N) try: with open("/sys/fs/cgroup/cpu.max") as f: quota, period = f.read().strip().split() - if quota != "max": return max(1, int(quota) // int(period)) + if quota != "max": count = min(count, max(1, int(quota) // int(period))) except (FileNotFoundError, ValueError, ZeroDivisionError): pass - # fall back to affinity (respects taskset but not cgroup quota) - return max(1, len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)) -NUM_CPU_THREADS = ContextVar("NUM_CPU_THREADS", _get_cpu_count()) + return count +CPU_COUNT = _get_cpu_count() NULL_ALLOW_COPYOUT = ContextVar("NULL_ALLOW_COPYOUT", 0) # VIZ implies PROFILE, but you can run PROFILE without VIZ VIZ = ContextVar("VIZ", 0) +# this PARALLEL is for BEAM and compilation, it's currently disabled if you are using VIZ +# pytest-xdist workers share the CPU budget, explicit PARALLEL still overrides this default +PARALLEL = ContextVar("PARALLEL", CPU_COUNT // max(1, getenv("PYTEST_XDIST_WORKER_COUNT", 1)) if VIZ == 0 else 0) PROFILE = ContextVar("PROFILE", abs(VIZ.value)) SPEC = ContextVar("SPEC", 1) # TODO: disable by default due to speed CHECK_OOB = ContextVar("CHECK_OOB", 0) -PCONTIG = ContextVar("PCONTIG", 0) # partial contiguous in rangeify DEBUG_RANGEIFY = ContextVar("DEBUG_RANGEIFY", 0) # set to 1, this uses tuplize in the linearizer sort order TUPLE_ORDER = ContextVar("TUPLE_ORDER", 1) @@ -361,7 +362,8 @@ class TracingKey: class ProfileEvent: pass @dataclass -class ProfileRangeEvent(ProfileEvent): device:str; name:str|TracingKey; st:decimal.Decimal; en:decimal.Decimal|None=None # noqa: E702 +class ProfileRangeEvent(ProfileEvent): + device:str; name:str|TracingKey; st:decimal.Decimal; en:decimal.Decimal|None=None; profile_key:bytes|None=None # noqa: E702 @dataclass(frozen=True) class ProfilePointEvent(ProfileEvent): @@ -369,8 +371,8 @@ class ProfilePointEvent(ProfileEvent): cpu_events:list[ProfileEvent] = [] @contextlib.contextmanager -def cpu_profile(name:str|TracingKey, device="TINY", display=True) -> Generator[ProfileRangeEvent, None, None]: - res = ProfileRangeEvent(device, name, perf_counter_us()) +def cpu_profile(name:str|TracingKey, device="TINY", display=True, profile_key:bytes|None=None) -> Generator[ProfileRangeEvent, None, None]: + res = ProfileRangeEvent(device, name, perf_counter_us(), profile_key=profile_key) try: yield res finally: res.en = perf_counter_us() @@ -394,18 +396,17 @@ cache_dir: str = os.path.join(getenv("XDG_CACHE_HOME", os.path.expanduser("~/Lib CACHEDB: str = getenv("CACHEDB", os.path.abspath(os.path.join(cache_dir, "cache.db"))) VERSION = 22 -_db_connection = None +_db_connection = threading.local() def db_connection(): - global _db_connection - if _db_connection is None: + if (conn:=getattr(_db_connection, "conn", None)) is None: os.makedirs(CACHEDB.rsplit(os.sep, 1)[0], exist_ok=True) - _db_connection = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") + conn = _db_connection.conn = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") # another connection has set it already or is in the process of setting it # that connection will lock the database - with contextlib.suppress(sqlite3.OperationalError): _db_connection.execute("PRAGMA journal_mode=WAL").fetchone() - _db_connection.execute("PRAGMA synchronous=NORMAL") - if DEBUG >= 8: _db_connection.set_trace_callback(print) - return _db_connection + with contextlib.suppress(sqlite3.OperationalError): conn.execute("PRAGMA journal_mode=WAL").fetchone() + conn.execute("PRAGMA synchronous=NORMAL") + if DEBUG >= 8: conn.set_trace_callback(print) + return conn def diskcache_clear(): cur = db_connection().cursor() @@ -454,23 +455,25 @@ def _ensure_downloads_dir() -> pathlib.Path: if pathlib.Path("/etc/tinybox-release").is_file(): # try creating dir with sudo if not (downloads_dir := pathlib.Path("/raid/downloads")).exists(): - subprocess.run(["sudo", "mkdir", "-p", downloads_dir], check=True) - subprocess.run(["sudo", "chown", "tiny:root", downloads_dir], check=True) - subprocess.run(["sudo", "chmod", "775", downloads_dir], check=True) + system(f"sudo mkdir -p {downloads_dir}") + system(f"sudo chown tiny:root {downloads_dir}") + system(f"sudo chmod 775 {downloads_dir}") return downloads_dir return pathlib.Path(cache_dir) / "downloads" def fetch(url:str, name:pathlib.Path|str|None=None, subdir:str|None=None, gunzip:bool=False, allow_caching=not getenv("DISABLE_HTTP_CACHE"), - headers:dict[str, str]={}, sha256:str|None=None) -> pathlib.Path: + headers:dict[str, str]={}, sha256:str|None=None, extract:bool=False) -> pathlib.Path: import urllib.request if url.startswith(("/", ".")): return pathlib.Path(url) if name is not None and (isinstance(name, pathlib.Path) or '/' in name): fp = pathlib.Path(name) else: hh = "_"+hashlib.md5(("\n".join(f"{k.strip()}:{v.strip()}" for k,v in sorted(headers.items()))).encode("utf-8")).hexdigest() if headers else "" fp = _ensure_downloads_dir() / (subdir or "") / ((name or hashlib.md5(url.encode('utf-8')).hexdigest()) + hh + (".gunzip" if gunzip else "")) + extract_dir = fp.parent / f"{fp.name}.extract" if not fp.is_file() or not allow_caching or (sha256 and hashlib.sha256(fp.read_bytes()).hexdigest() != sha256): + if extract: shutil.rmtree(extract_dir, ignore_errors=True) (_dir := fp.parent).mkdir(parents=True, exist_ok=True) - with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "tinygrad 0.13.0", **headers}), timeout=10) as r: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "tinygrad 0.14.0", **headers}), timeout=10) as r: assert r.status in {200, 206}, r.status length = int(r.headers.get('content-length', 0)) if not gunzip else None readfile = gzip.GzipFile(fileobj=r) if gunzip else r @@ -485,25 +488,30 @@ def fetch(url:str, name:pathlib.Path|str|None=None, subdir:str|None=None, gunzip pathlib.Path(f.name).rename(fp) progress_bar.update(close=True) if length and (file_size:=os.stat(fp).st_size) < length: raise RuntimeError(f"fetch size incomplete, {file_size} < {length}") + if extract: + if not extract_dir.is_dir(): + import tarfile + tmpdir = tempfile.mkdtemp(dir=fp.parent) + try: + with tarfile.open(fp) as t: t.extractall(tmpdir, filter="data") + try: os.rename(tmpdir, extract_dir) # rename is atomic, so concurrent fetches can't see a partial extraction + except OSError: + if not extract_dir.is_dir(): raise + finally: shutil.rmtree(tmpdir, ignore_errors=True) + return extract_dir return fp -def _decompress_zstd(data:bytes) -> bytes: - if sys.version_info >= (3,14): - from compression.zstd import decompress - return decompress(data) - from zstandard import ZstdDecompressor - with ZstdDecompressor().stream_reader(io.BytesIO(data)) as reader: - return reader.read() - def fetch_fw(path:str, name:str, sha256:str) -> bytes: - if (p:=pathlib.Path(f"/lib/firmware/{path}/{name}.zst")).is_file(): - if hashlib.sha256(b:=_decompress_zstd(p.read_bytes())).hexdigest() == sha256: return b - return fetch(f"https://gitlab.com/kernel-firmware/linux-firmware/-/raw/1e2c15348485939baf1b6d1f5a7a3b799d80703d/{path}/{name}", + if sys.version_info >= (3,14) and (p:=pathlib.Path(f"/lib/firmware/{path}/{name}.zst")).is_file(): + from compression.zstd import decompress + if hashlib.sha256(b:=decompress(p.read_bytes())).hexdigest() == sha256: return b + return fetch(f"https://gitlab.com/kernel-firmware/linux-firmware/-/raw/0a6871b19abf5d6e024b5d208b101ae53e7fa0de/{path}/{name}", subdir="fw", sha256=sha256).read_bytes() # *** Exec helpers def system(cmd:str, **kwargs) -> str: + import subprocess st = time.perf_counter() try: ret = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, **kwargs).decode().strip() except subprocess.CalledProcessError as e: @@ -592,9 +600,9 @@ class tqdm(Generic[T]): est_text = f'<{HMS(elapsed/prog-elapsed) if self.n else "?"}' if self.t else '' it_text = (SI(self.n/elapsed) if self.unit_scale else f"{self.n/elapsed:5.2f}") if self.n else "?" suf = f'{prog_text} [{HMS(elapsed)}{est_text}, {it_text}{self.unit}/s]' - sz = max(ncols-len(self.desc)-3-2-2-len(suf), 1) + sz = max(ncols-ansilen(self.desc)-3-2-2-len(suf), 1) bar = '\r' + self.desc + (f'{100*prog:3.0f}%|{("█"*int(num:=sz*prog)+" ▏▎▍▌▋▊▉"[int(8*num)%8].strip()).ljust(sz," ")}| ' if self.t else '') + suf - print(bar[:ncols+1], flush=True, end='\n'*close, file=sys.stderr) + print(bar, flush=True, end='\n'*close, file=sys.stderr) @classmethod def write(cls, s:str): print(f"\r\033[K{s}", flush=True, file=sys.stderr) diff --git a/tinygrad_repo/tinygrad/llm/cli.py b/tinygrad_repo/tinygrad/llm/cli.py index d0865e26b..417ffa751 100644 --- a/tinygrad_repo/tinygrad/llm/cli.py +++ b/tinygrad_repo/tinygrad/llm/cli.py @@ -12,22 +12,29 @@ class SimpleTokenizer: def __init__(self, normal_tokens:dict[str, int], special_tokens:dict[str, int], preset:str="llama3", bos_id:int|None=None, eos_id:int=0, eot_id:int|None=None): preset = {"qwen35":"qwen2","qwen35moe":"qwen2"}.get(preset, preset) - if preset not in ("llama3","llama-v3","llama-bpe","qwen2","olmo","kimi-k2","tekken","glm4"): + if preset not in ("llama3","llama-v3","llama-bpe","qwen2","olmo","kimi-k2","tekken","glm4","gpt-4o"): raise ValueError(f"Invalid tokenizer preset '{preset}'") # https://github.com/openai/gpt-2/blob/9b63575ef42771a015060c964af2c3da4cf7c8ab/src/encoder.py#L9 bs = [*range(33, 127), *range(161, 173), *range(174, 256)] # bytes that map to themselves self._byte_decoder = {chr(b): b for b in bs} | {chr(256+i): b for i,b in enumerate(b for b in range(256) if b not in bs)} # https://github.com/ggml-org/llama.cpp/blob/94933c8c2eeaa9a7983e3f6c08af76bd86724094/src/llama-vocab.cpp#L286 - # 0x323b0 is one past the max codepoint in unicode categories L/N/Z (0x323af is max L) + # each limit is one past the category's max codepoint (Z→U+3000, N→U+1FBF9, L→U+323AF, M→U+E01EF) # compact adjacent codepoints into ranges: listing them all makes re spend seconds on large prompts - def ucat_range(pre:str) -> str: - cps = enumerate(cp for cp in range(0x323b0) if unicodedata.category(chr(cp)).startswith(pre)) + def ucat_range(pre:str|tuple[str, ...]) -> str: + limits = {"Z": 0x3001, "N": 0x1fbfa, "L": 0x323b0, "M": 0xe01f0} + limit = max(limits[p if p in limits else p[0]] for p in (pre if isinstance(pre, tuple) else (pre,))) + cps = enumerate(cp for cp in range(limit) if unicodedata.category(chr(cp)).startswith(pre)) runs = [list(g) for _, g in itertools.groupby(cps, lambda e: e[1]-e[0])] return "".join(re.escape(chr(g[0][1])) + (f"-{re.escape(chr(g[-1][1]))}" if len(g) > 1 else "") for g in runs) r_ws, r_p_N, r_p_L = r"\t\n\x0b\x0c\r\x85" + ucat_range("Z"), ucat_range("N"), ucat_range("L") - self._split_to_word = re.compile("(?i:'s|'t|'re|'ve|'m|'ll|'d)|" + \ - f"[^\\r\\n{r_p_N}{r_p_L}]?[{r_p_L}]+|[{r_p_N}]{{1,3}}| ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*|[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+") + contr, r_l, r_n = "(?i:'s|'t|'re|'ve|'m|'ll|'d)", f"[^\\r\\n{r_p_N}{r_p_L}]?", f"[{r_p_N}]" if preset == "tekken" else f"[{r_p_N}]{{1,3}}" + r_p, r_w, r_t = f" ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*", f"{contr}|{r_l}[{r_p_L}]+", f"[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+" + if preset in ("tekken", "gpt-4o"): + r_up, r_lo = ucat_range(("Lu","Lt","Lm","Lo","M")), ucat_range(("Ll","Lm","Lo","M")) + sfx = f"{contr}?" if preset == "gpt-4o" else "" + r_p, r_w = f" ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n/]*", f"{r_l}[{r_up}]*[{r_lo}]+{sfx}|{r_l}[{r_up}]+[{r_lo}]*{sfx}" + self._split_to_word = re.compile(f"{r_w}|{r_n}|{r_p}|{r_t}") self._split_to_sentence = re.compile("|".join(re.escape(tok) for tok in special_tokens.keys()) if special_tokens else r"(?!)") self._normal_tokens = {bytes(self._byte_decoder[c] for c in tok): tid for tok, tid in normal_tokens.items()} @@ -88,6 +95,8 @@ models = { "qwen3.5:9b": "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/main/Qwen3.5-9B-Q4_K_M.gguf", "qwen3.6:27b": "https://huggingface.co/unsloth/Qwen3.6-27B-GGUF/resolve/main/Qwen3.6-27B-Q4_K_M.gguf", "qwen3.6:35b-a3b": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + # pinned to the last revision with the plain IQ4_XS quant: the UD replacement uses Q3_K tensors the loader doesn't support + "qwen3.8:27b": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/b62a80264f8b0c1bb849ee1c9c487415ebeca194/Qwen3.8-27B-IQ4_XS.gguf", "olmoe": "https://huggingface.co/allenai/OLMoE-1B-7B-0924-Instruct-GGUF/resolve/main/olmoe-1b-7b-0924-instruct-q4_k_m.gguf", "moonlight": "https://huggingface.co/gabriellarson/Moonlight-16B-A3B-Instruct-GGUF/resolve/main/Moonlight-16B-A3B-Instruct-Q4_K_M.gguf", "glm-4.7-flash": "https://huggingface.co/unsloth/GLM-4.7-Flash-GGUF/resolve/main/GLM-4.7-Flash-Q4_K_M.gguf", @@ -136,10 +145,12 @@ def main(): parser.add_argument("--serve", nargs='?', type=int, const=8000, metavar="PORT", help="Run OpenAI compatible API (optional port, default 8000)") parser.add_argument("--warmup", action="store_true", help="warmup the JIT") parser.add_argument("--benchmark", nargs='?', type=int, const=20, metavar="COUNT", help="Benchmark tok/s (optional count, default 20)") + parser.add_argument("--no_chat_template", action="store_true", help="Don't use the model's chat template, always use the fallback template") args = parser.parse_args() # load the model - model, kv = Transformer.from_gguf(fetch(models.get(args.model, args.model)), args.max_context) + with Context(DEBUG=max(DEBUG.value, 2 if args.serve else 0)): + model, kv = Transformer.from_gguf(fetch(models.get(args.model, args.model)), args.max_context) model_name = kv.get('general.name') or kv.get('general.basename') or args.model file_sizes = [y.nbytes() for y in UOp.sink(*[x.uop for x in nn.state.get_parameters(model)]).toposort() if y.op is Ops.BUFFER] print(f"using model \"{model_name}\" with {sum(file_sizes):,} bytes and {sum(x.numel() for x in nn.state.get_parameters(model)):,} params, " @@ -150,7 +161,7 @@ def main(): # use the model's chat template if jinja2 is available (enables model-specific formatting) template: jinja2.Template|FallbackTemplate = FallbackTemplate(tok) - if (ct := kv.get('tokenizer.chat_template')) is not None: + if not args.no_chat_template and (ct := kv.get('tokenizer.chat_template')) is not None: try: import jinja2 env = jinja2.Environment() diff --git a/tinygrad_repo/tinygrad/llm/gguf.py b/tinygrad_repo/tinygrad/llm/gguf.py index 49772dabf..60f937d3a 100644 --- a/tinygrad_repo/tinygrad/llm/gguf.py +++ b/tinygrad_repo/tinygrad/llm/gguf.py @@ -18,7 +18,9 @@ _GGML_NATIVE = {0: dtypes.float32, 1: dtypes.float16, 24: dtypes.int8, 25: dtype # quant types {ggml_type: (number of elements, number of bytes)} _GGML_QUANT = {2:(32,18), 3:(32,20), 6:(32,22), 7:(32,24), 8:(32,34), - 12:(256,144), 13:(256,176), 14:(256,210), 18:(256,98), 21:(256,110), 22:(256,82), 23:(256,136), 39:(32,17), 41:(128,18)} + 10:(256,84), 11:(256,110), 12:(256,144), 13:(256,176), 14:(256,210), + 16:(256,66), 17:(256,74), 18:(256,98), 19:(256,50), 20:(32,18), 21:(256,110), 22:(256,82), 23:(256,136), + 29:(256,56), 39:(32,17), 41:(128,18)} def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor: """ @@ -27,8 +29,9 @@ def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor: Supported native types: float32 (id: 0), float16 (id: 1), int8 (id: 24), int16 (id: 25), int32 (id: 26), int64 (id: 27), float64 (id: 28), bfloat16 (id: 30) Supported quantized types: Q4_0 (id: 2), Q4_1 (id: 3), Q5_0 (id: 6), - Q5_1 (id: 7), Q8_0 (id: 8), Q4_K (id: 12), Q5_K (id: 13), - Q6_K (id: 14), IQ3_XXS (id: 18), IQ3_S (id: 21), IQ2_S (id: 22), IQ4_XS (id: 23), MXFP4 (id: 39), Q1_0 (id: 41) + Q5_1 (id: 7), Q8_0 (id: 8), Q2_K (id: 10), Q3_K (id: 11), Q4_K (id: 12), Q5_K (id: 13), + Q6_K (id: 14), IQ2_XXS (id: 16), IQ2_XS (id: 17), IQ3_XXS (id: 18), IQ1_S (id: 19), + IQ4_NL (id: 20), IQ3_S (id: 21), IQ2_S (id: 22), IQ4_XS (id: 23), IQ1_M (id: 29), MXFP4 (id: 39), Q1_0 (id: 41) """ # https://github.com/ggerganov/ggml/blob/323951f1bdcdfbd5b5ff3a9a7c3770e63b1a560e/include/ggml.h#L356 @@ -54,6 +57,19 @@ def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor: q = q_to_uint8(blocks[:,qh_off+4:], 4).bitcast(dtypes.int8) + qh * 16 return q * d + (blocks[:,2:4].bitcast(dtypes.float16).cast(dtypes.float32) if ggml_type == 7 else -16 * d) if ggml_type == 8: return blocks[:,:2].bitcast(dtypes.float16).cast(dtypes.float32) * blocks[:,2:].bitcast(dtypes.int8) + # Q2_K: 256 elements per 84-byte block (scales:16, qs:64, d:2, dmin:2) + if ggml_type == 10: + d, dmin = (blocks[:,i:i+2].bitcast(dtypes.float16).cast(dtypes.float32).unsqueeze(-1) for i in [80, 82]) + sc = blocks[:, :16] + q = q_to_uint8(blocks[:, 16:80].reshape((-1, 2, 32)), 2).reshape((-1, 16, 16)) + return (d * sc.bitwise_and(0xF).unsqueeze(-1) * q - dmin * sc.rshift(4).unsqueeze(-1)).flatten(-2) + # Q3_K: 256 elements per 110-byte block (hmask:32, qs:64, scales:12, d:2) + if ggml_type == 11: + d = blocks[:,-2:].bitcast(dtypes.float16).cast(dtypes.float32).unsqueeze(-1) + sc = q_to_uint8(blocks[:,96:104], 4).bitwise_or(q_to_uint8(blocks[:,104:108], 2).lshift(4)).bitcast(dtypes.int8) - 32 + q = q_to_uint8(blocks[:,32:96].reshape((-1, 2, 32)), 2).reshape((-1, 16, 16)) + qh = q_to_uint8(blocks[:,:32], 1).reshape((-1, 16, 16)) + return (d * sc.unsqueeze(-1) * (q.bitcast(dtypes.int8) - qh.bitwise_xor(1).lshift(2).bitcast(dtypes.int8))).flatten(-2) # Q4_K: 256 elements per 144-byte block (d:2, dmin:2, scales:12, qs:128) # Q5_K: 256 elements per 176-byte block (d:2, dmin:2, scales:12, qh:32, qs:128) if ggml_type in (12, 13): @@ -79,6 +95,41 @@ def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor: signs = (q_to_uint8(even_signs[sign_idx].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 8, 4, 8)) grid = _ggml_iq_grid(t.device, _ggml.iq3xxs_grid, (256, 4))[blocks[:, 2:66]].reshape((-1, 8, 4, 8)) return (db * grid * signs).flatten(-3) + # IQ2_XXS: 256 elements per 66-byte block (d:2, qs:64). 8 groups of 32: 4 grid bytes + packed signs/scale. + if ggml_type == 16: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) + qs_u32 = blocks[:, 2:].bitcast(dtypes.uint32).reshape((-1, 8, 2)) + db = d * (qs_u32[:, :, 1].rshift(28).cast(dtypes.float32) + 0.5).reshape((-1, 8, 1, 1)) * 0.25 + sign_idx = qs_u32[:, :, 1].unsqueeze(-1).rshift(Tensor.const((0, 7, 14, 21), dtypes.uint32)) + sign_idx = sign_idx.bitwise_and(0x7F).reshape((-1, 32)).cast(dtypes.int32) + even_signs = Tensor([i | (0x80 if i.bit_count() % 2 else 0) for i in range(128)], dtype=dtypes.uint8, device=t.device) + signs = (q_to_uint8(even_signs[sign_idx].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 8, 4, 8)) + grid = _ggml_iq_grid(t.device, _ggml.iq2xxs_grid, (256, 8))[blocks[:, 2:].reshape((-1, 8, 8))[:, :, :4]].reshape((-1, 8, 4, 8)) + return (db * grid * signs).flatten(-3) + # IQ2_XS: 256 elements per 74-byte block (d:2, qs:64 as uint16, scales:8) + if ggml_type == 17: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) + db = d * (q_to_uint8(blocks[:, 66:74].reshape((-1, 8, 1)), 4).reshape((-1, 16)).cast(dtypes.float32) + 0.5).reshape((-1, 16, 1, 1)) * 0.25 + qs = blocks[:, 2:66].bitcast(dtypes.uint16) + sign_idx = qs.rshift(9).cast(dtypes.int32) + even_signs = Tensor([i | (0x80 if i.bit_count() % 2 else 0) for i in range(128)], dtype=dtypes.uint8, device=t.device) + signs = (q_to_uint8(even_signs[sign_idx].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 16, 2, 8)) + grid = _ggml_iq_grid(t.device, _ggml.iq2xs_grid, (512, 8))[qs.bitwise_and(511)].reshape((-1, 16, 2, 8)) + return (db * grid * signs).flatten(-3) + # IQ1_S: 256 elements per 50-byte block (d:2, qs:32, qh:16). grid bytes are int8 {-1,0,1}. + if ggml_type == 19: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) + qh = blocks[:, 34:50].bitcast(dtypes.uint16) + dl = d * (qh.rshift(12).bitwise_and(7).cast(dtypes.float32) * 2 + 1).reshape((-1, 8, 1, 1)) + delta = (qh.bitwise_and(0x8000) == 0).where(0.125, -0.125).reshape((-1, 8, 1, 1)) + qh_hi = qh.unsqueeze(-1).rshift(Tensor.const((0, 3, 6, 9), dtypes.uint16)).bitwise_and(7).lshift(8) + q = blocks[:, 2:34].cast(dtypes.uint16) + qh_hi.reshape((-1, 32)) + grid = _ggml_iq_grid(t.device, _ggml.iq1s_grid, (2048, 8))[q].reshape((-1, 8, 4, 8)) + grid = (grid > 127).where(grid - 256, grid) + return (dl * (grid + delta)).flatten(-3) + if ggml_type == 20: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32) + return d * Tensor(list(_ggml.kvalues_iq4nl), dtype=dtypes.float32, device=t.device)[q_to_uint8(blocks[:, 2:], 4)] if ggml_type == 21: d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) scales = (1 + 2 * q_to_uint8(blocks[:, 106:110].reshape((-1, 4, 1)), 4).reshape((-1, 8))).cast(dtypes.float32).reshape((-1, 8, 1, 1)) @@ -102,6 +153,20 @@ def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor: scales = (scales_l.bitwise_or(scales_h.lshift(4)).bitcast(dtypes.int8) - 32).cast(dtypes.float32).reshape((-1, 8, 1)) q = (qs:=blocks[:, 8:].reshape((-1, 8, 16))).bitwise_and(0xF).cat(qs.rshift(4), dim=2) return (d * scales * iq4_xs_lut[q]).flatten(-2) + # IQ1_M: 256 elements per 56-byte block (qs:32, qh:16, scales:8). f16 scale packed in high nibbles. + if ggml_type == 29: + sc16 = blocks[:, 48:56].bitcast(dtypes.uint16) + d = sc16.bitwise_and(0xF000).rshift(Tensor.const((12, 8, 4, 0), dtypes.uint16)) + d = d[:, 0:1].bitwise_or(d[:, 1:2]).bitwise_or(d[:, 2:3]).bitwise_or(d[:, 3:4]) + d = d.bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1, 1)) + scales = sc16.unsqueeze(-1).rshift(Tensor.const((0, 3, 6, 9), dtypes.uint16)).bitwise_and(7) + dl = d * (scales.cast(dtypes.float32) * 2 + 1).reshape((-1, 8, 2, 1, 1)) + qh_n = Tensor.stack(blocks[:, 32:48].bitwise_and(0x0F), blocks[:, 32:48].rshift(4), dim=-1).reshape((-1, 32)) + q = blocks[:, :32].cast(dtypes.uint16) + qh_n.bitwise_and(7).cast(dtypes.uint16).lshift(8) + delta = (qh_n.bitwise_and(0x08) == 0).where(0.125, -0.125).reshape((-1, 8, 2, 2, 1)) + grid = _ggml_iq_grid(t.device, _ggml.iq1s_grid, (2048, 8))[q].reshape((-1, 8, 2, 2, 8)) + grid = (grid > 127).where(grid - 256, grid) + return (dl * (grid + delta)).flatten(-4) if ggml_type == 39: e = blocks[:, 0].cast(dtypes.uint32) small_bits = Tensor([0x00200000, 0x00400000], dtype=dtypes.uint32, device=t.device)[e.clip(0, 1).cast(dtypes.int32)] # e = 0 or e = 1 case diff --git a/tinygrad_repo/tinygrad/llm/kernels/amd.py b/tinygrad_repo/tinygrad/llm/kernels/amd.py new file mode 100644 index 000000000..b4764e1ba --- /dev/null +++ b/tinygrad_repo/tinygrad/llm/kernels/amd.py @@ -0,0 +1,654 @@ +from __future__ import annotations +import functools, math +from typing import Callable, cast +from tinygrad import Tensor, UOp, nn, Device, Context +from tinygrad.device import Buffer +from tinygrad.dtype import AddrSpace, dtypes +from tinygrad.helpers import prod +from tinygrad.uop.ops import AxisType, KernelInfo, Ops, resolve + +BLOCK_M, BLOCK_N, WARP_SIZE = 32, 32, 32 +WMMA_M, WMMA_N, WMMA_K = 16, 16, 16 +WAVES_M, WAVES_N, LANES_PER_WAVE_M, LANES_PER_WAVE_N = 2, 2, 2, 16 +WMMA_ACC, THREADS_PER_BLOCK = WMMA_M // LANES_PER_WAVE_M, WARP_SIZE * WAVES_M * WAVES_N +LDS_PAD, WMMA_ARG, LOG2E = 4, ((WMMA_M, WMMA_N, WMMA_K), 'AMD', 32), math.log2(math.e) +Q4_K, Q5_K, Q6_K, IQ4_XS, GGML_BLOCK_SIZE, Q8_GROUP_SIZE, Q4_WORDS, Q5_WORDS, Q6_BYTES, IQ4_WORDS = 12, 13, 14, 23, 256, 32, 36, 44, 210, 34 +Q6_PADDED, Q6_WORDS = 212, 53 # the 210-byte Q6 blocks are padded to 212 bytes so they are word-addressable +QUANT_SIZES = {Q4_K: Q4_WORDS*4, Q5_K: Q5_WORDS*4, Q6_K: Q6_BYTES, IQ4_XS: IQ4_WORDS*4} # bytes per 256-weight block + +def kernel_var(x:UOp) -> UOp: + # a Variable is a 0-d ALU BUFFER in the tensor graph; inside kernels it takes the ALU PARAM form (same name keeps the value binding) + return x.substitute({v: UOp.variable(v.expr, v.vmin, v.vmax, dtype=v.dtype, multiple_of=v.arg.multiple_of, param=True) + for v in x.toposort() if v.is_variable}) + +def _unbind(v:int|UOp) -> int|UOp: return kernel_var(v.unbind_all()[0]) if isinstance(v, UOp) else v + +@functools.cache +def amd_custom_kernels_supported(device:str|tuple[str, ...]|None) -> bool: + # the custom kernels are tuned for RDNA3 (gfx11): the WMMA register layouts don't match gfx12 (RDNA4) + # or CDNA (MFMA-only, wave64), and the dp4a builtins and 32-lane wave ops aren't portable either. + if isinstance(device, tuple): device = device[0] + if device is None or device.split(":")[0] != "AMD": return False + # @function contexts set ALLOW_DEVICE_USAGE=0 (scheduling must not open devices); the device is always open here + with Context(ALLOW_DEVICE_USAGE=1): + return (t:=getattr(Device[device], "target", None)) is not None and t[0] == 11 + +def warp_reduce(val:UOp, maximum:bool=False, full_wave:bool=False) -> UOp: + for offset in ((16, 8, 4, 2, 1) if full_wave else (8, 4, 2, 1)): + if val.op is Ops.INDEX and val.addrspace == AddrSpace.REG: val = val.load() + other = UOp(Ops.CUSTOM, src=(val,), arg= + (f"__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {{0}}), {0x1f | offset<<10}))", dtypes.float)) + val = val.maximum(other) if maximum else val + other + return val + +def _reg(shape:tuple[int, ...], slot:int, value:float, dep:UOp|None=None) -> UOp: + ret = UOp.placeholder(shape, dtypes.float, slot=slot, addrspace=AddrSpace.REG) + return ret.after((ret if dep is None else ret.after(dep)).store(ret.const_like(value))) + +# ******** quant linear: q8-activation kernels over packed ggml weights (Q4_K/Q5_K/Q6_K/IQ4_XS) ******** + +class Linear(nn.Linear): + ggml_type:int|None = None + use_custom_quant = True + def __init__(self, in_features:int, out_features:int, bias=True): + super().__init__(in_features, out_features, bias) + self.in_features, self.out_features = in_features, out_features + def set_quantized(self, decoded:Tensor): + packed_sizes = {decoded.numel() // 256 * type_size:typ for typ,type_size in QUANT_SIZES.items()} + graph = decoded.uop.toposort() + raw = next((u for u in graph if u.op is Ops.SHRINK and u.dtype == dtypes.uint8 and prod(u.shape) in packed_sizes), None) + if raw is None: return + ggml_type = packed_sizes[prod(raw.shape)] + # the packed byte rate alone can't distinguish same-rate formats (Q4_0 vs Q4_K, Q5_0 vs Q5_K, MXFP4 vs IQ4_XS). + # the supported formats are 256-wide superblocks: their decode views the packed bytes at the superblock width + # (ggml_data_to_tensor reshapes to (-1, QUANT_SIZES[type])), while same-rate 32-wide formats reshape to 17-22 + if not any(u.op is Ops.RESHAPE and u.shape[-1:] == (QUANT_SIZES[ggml_type],) for u in graph): return + raw_offset = raw.contiguous_view_offset() + assert raw_offset is not None and raw_offset % 4 == 0 and raw.buf_uop.dtype == dtypes.uint8 + self.ggml_type = ggml_type + # store a typed buffer view: a lazy BITCAST is decomposed into byte-combining ALU before custom-kernel + # scheduling and would copy the entire packed weight on every JIT graph + if self.ggml_type == Q6_K: + # Q6 blocks are 210 bytes, so consecutive blocks are only 2-byte aligned. pad each block to 212 bytes + # the kernel can do all its reads as aligned u32 words + nbytes, nblocks = raw.max_numel(), raw.max_numel() // Q6_BYTES + byte_view = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer).view(nbytes, dtypes.uint8, raw_offset))) + padded = byte_view.reshape((nblocks, Q6_BYTES)).pad_to((nblocks, Q6_PADDED)).bitcast(dtypes.uint32) + self.weight = padded.contiguous().reshape(nblocks * Q6_WORDS) + else: + self.weight = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer) + .view(raw.max_numel() * raw.dtype.itemsize // dtypes.uint32.itemsize, dtypes.uint32, raw_offset))) + def __call__(self, x:Tensor) -> Tensor: + supported = self.use_custom_quant and amd_custom_kernels_supported(self.weight.device) + if self.ggml_type is None and supported: + self.set_quantized(self.weight) + if self.ggml_type is None: + # tiny dense fp16 matmul (e.g. the ssm beta/alpha head rows): single fp16 gemv kernel instead of a + # generic matmul schedule, and realize the densely packed weight once if it is still a lazy ggml view + if self.weight.dtype in (dtypes.half, dtypes.float, dtypes.bfloat16) and self.out_features <= 2048 \ + and self.in_features % (WARP_SIZE*4) == 0: + numel, max_shape = x.numel(), x.max_shape + if isinstance(numel, int) or prod(max_shape) // self.in_features <= 32: + out = f16_gemv(self, x if isinstance(numel, int) else x.pad_to(max_shape)) + return out if isinstance(numel, int) else out.shrink(tuple((0, s) for s in (*x.shape[:-1], self.out_features))) + self.use_custom_quant = supported = False # not a supported quant format + if self.ggml_type in (Q4_K, Q5_K, Q6_K, IQ4_XS) and supported: + if isinstance(x.numel(), int): return q8_linear(self, x) + # symbolic token count: pad to the max chunk size so the kernels see static shapes, garbage rows are sliced off + out = q8_linear(self, x.pad_to(x.max_shape)) + return out.shrink(tuple((0, s) for s in (*x.shape[:-1], self.out_features))) + return super().__call__(x) + +def _amd_dp4a(a:UOp, b:UOp, c:UOp) -> UOp: + # int8 4-wide dot, widened to scalar multiply-adds (2% decode slower than the sudot4 builtin, but portable) + for i in range(4): + av = ((a >> (8*i)) & 255).cast(dtypes.uint8).bitcast(dtypes.int8).int() + bv = ((b >> (8*i)) & 255).cast(dtypes.uint8).bitcast(dtypes.int8).int() + c = c + av*bv + return c + +def _amd_byte_perm(a:UOp, b:UOp, selectors:UOp) -> UOp: + return UOp(Ops.CUSTOMI, src=tuple(x.cast(dtypes.uint32) for x in (a, b, selectors)), arg=("__builtin_amdgcn_perm({}, {}, {})", dtypes.uint32)) + +def _amd_load(ptr:UOp, lanes:int|None=None) -> UOp: + assert ptr.op is Ops.INDEX + # nontemporal scalar load: streamed weights must not evict the activations/KV cache from L2 + if lanes is None: return ptr.load(arg="nontemporal") + buf, coords = ptr.src[0], ptr.src[1:] + idx = sum((coord*math.prod(buf.shape[i+1:]) for i,coord in enumerate(coords)), UOp.const(0)) + return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes))).load() + +def _load_byte(raw:UOp, base:UOp, offset:UOp) -> UOp: return (raw[base + offset//4] >> ((offset&3)*8).cast(dtypes.uint32)) & 255 +def _half(value:UOp) -> UOp: return value.cast(dtypes.uint16).bitcast(dtypes.float16).float() + +def _iq4_bytes(packed:UOp, shift:int) -> UOp: + # the non-linear iq4nl table as a byte lookup: 3 byte_perms beat any arithmetic/select-tree form (~60% decode) + selectors = (packed >> shift) & 0x0f0f0f0f + low = _amd_byte_perm(UOp.const(0xf6eaddcf, dtypes.uint32), UOp.const(0xbfad9881, dtypes.uint32), selectors) + high = _amd_byte_perm(UOp.const(0x71594535, dtypes.uint32), UOp.const(0x26190d01, dtypes.uint32), selectors & 0x07070707) + return _amd_byte_perm(high, low, 0x03020100 | ((selectors & 0x08080808) >> 1)) + +def _q5_scales(raw:UOp, base:UOp, subgroup:UOp) -> tuple[UOp, UOp, UOp, UOp]: + # scales/mins (6-bit each) live in block bytes 4-15: three words total, same for the whole super-block's lanes + w1, w2, w3 = _amd_load(raw[base+1]), _amd_load(raw[base+2]), _amd_load(raw[base+3]) + sb = (subgroup & 3) * 8 # byte within word + byte1, byte2, byte3 = (w1 >> sb) & 255, (w2 >> sb) & 255, (w3 >> sb) & 255 + scale = (subgroup < 4).where(byte1 & 63, (byte3 & 15) | ((byte1 >> 6) << 4)) + minimum = (subgroup < 4).where(byte2 & 63, (byte3 >> 4) | ((byte2 >> 6) << 4)) + d, dmin = (raw[base] & 0xffff).cast(dtypes.uint16), (raw[base] >> 16).cast(dtypes.uint16) + return _half(d), _half(dmin), scale.float(), minimum.float() + +def _iq4_scales(raw:UOp, base:UOp, subgroup:UOp) -> tuple[UOp, UOp]: + low = _load_byte(raw, base, 4 + subgroup//2) + scale = ((low >> (4*(subgroup%2)).cast(dtypes.uint32)) & 15) | ((((raw[base] >> 16) >> (2*subgroup).cast(dtypes.uint32)) & 3) << 4) + return _half(raw[base] & 0xffff), (scale.cast(dtypes.uint8).bitcast(dtypes.int8)-32).float() + +@functools.cache +def iq4_half_lut(device:str) -> Tensor: + from tinygrad.runtime.autogen.ggml_common import kvalues_iq4nl + return Tensor([x for j in range(16) for i in range(16) for x in (kvalues_iq4nl[i], kvalues_iq4nl[j])], + dtype=dtypes.float16, device=device).bitcast(dtypes.uint32).contiguous() + +@functools.cache +def _q8_quantize_kernel(q:UOp, scale:UOp, xsum:UOp, x:UOp, tokens:int, in_features:int) -> UOp: + groups = in_features//Q8_GROUP_SIZE + token_group, lane = UOp.range(tokens*groups, 0, axis_type=AxisType.GLOBAL), UOp.range(32, 1, axis_type=AxisType.LOCAL) + token, group = token_group//groups, token_group%groups + x = x.reshape(tokens, groups, 32) + group_scale = (warp_reduce(x[token, group, lane].float().abs(), maximum=True, full_wave=True) / 127).maximum(1e-8) + word_lane = lane.minimum(7) + xs = tuple(x[token, group, word_lane*4+i].float() for i in range(4)) + qs = tuple((v/group_scale).round().clip(-127, 127).cast(dtypes.int8) for v in xs) + word = sum((v.cast(dtypes.uint8).cast(dtypes.uint32) << (i*8) for i, v in enumerate(qs)), UOp.const(0, dtypes.uint32)) + # per-16 sums of the quantized values (lanes 0-3 / 4-7): Q4_K/Q5_K need the 32-sum, Q6_K the 16-sums + part = (lane < 8).where(sum((v.cast(dtypes.int32) for v in qs), UOp.const(0, dtypes.int32)), UOp.const(0, dtypes.int32)) + gsum = [warp_reduce(((lane & 4).eq(h*4)).where(part, UOp.const(0, dtypes.int32)), full_wave=True) for h in range(2)] + store_half = (lane & 4) >> 2 + stores = (q[token, group, lane.valid(lane < 8)].store(word), + UOp.group(scale[token, group.valid(lane.eq(0))].store(group_scale), + xsum[token, group, store_half.valid(lane.eq(0) | lane.eq(4))].store( + store_half.eq(0).where(gsum[0].float(), gsum[1].float())))) + return UOp.group(*stores).end(token_group, lane).sink(arg=KernelInfo(name="q8_quantize", opts_to_apply=())) + +def q8_quantize(x:Tensor, tokens:int, in_features:int) -> tuple[Tensor, Tensor, Tensor]: + groups = in_features//Q8_GROUP_SIZE + q = Tensor.empty(tokens, groups, 8, dtype=dtypes.uint32, device=x.device) + scale = Tensor.empty(tokens, groups, dtype=dtypes.float32, device=x.device) + xsum = Tensor.empty(tokens, groups, 2, dtype=dtypes.float32, device=x.device) + q, scale, xsum = Tensor.custom_kernel(q, scale, xsum, x, fxn=functools.partial(_q8_quantize_kernel, tokens=tokens, in_features=in_features))[:3] + return q, scale, xsum + +def _decode_linear(out:UOp, out_features:int, group_count:int, group_dot, name:str) -> UOp: + chunks = out.shape[2] + # two-dim global grid instead of one flat grid: no div/mods needed to decompose the gid + token_output = UOp.range(out.shape[0]*out_features, 0, axis_type=AxisType.GLOBAL) + chunk, lane = UOp.range(chunks, 1, axis_type=AxisType.GLOBAL), UOp.range(32, 2, axis_type=AxisType.LOCAL) + token, output = token_output // out_features, token_output % out_features + group = (lane+chunk*32).minimum(group_count-1) + value = group_dot(token, output, group) if chunks*32 == group_count else \ + (lane+chunk*32 < group_count).where(group_dot(token, output, group), UOp.const(0, dtypes.float32)) + total = warp_reduce(value, full_wave=True) + return out[token, output, chunk.valid(lane.eq(0))].store(total.cast(out.dtype)).end(token_output, chunk, lane).sink( + arg=KernelInfo(name=name, opts_to_apply=())) + +@functools.cache +def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, out_features:int, in_features:int, ggml_type:int) -> UOp: + group_count = in_features // Q8_GROUP_SIZE + def group_dot(token:UOp, output:UOp, group:UOp) -> UOp: + block, subgroup = group // 8, group % 8 + xwords = _amd_load(xq[token, group, 0], 8) + if ggml_type in (Q4_K, Q5_K): + base = (output * in_features//GGML_BLOCK_SIZE + block) * (Q4_WORDS if ggml_type == Q4_K else Q5_WORDS) + qs_base, dot = base + (4 if ggml_type == Q4_K else 12) + (subgroup//2)*8, UOp.const(0, dtypes.int32) + # vectorize the 8 packed-weight words and (for Q5_K) the 32-byte high-bit bitmap + qs_pair = (_amd_load(raw[qs_base], 4), _amd_load(raw[qs_base+4], 4)) + if ggml_type == Q5_K: qh_pair = (_amd_load(raw[base+4], 4), _amd_load(raw[base+8], 4)) + for word_idx in range(8): + word = (qs_pair[word_idx//4][word_idx%4] >> ((subgroup&1)*4).cast(dtypes.uint32)) & 0x0f0f0f0f + if ggml_type == Q5_K: word |= ((qh_pair[word_idx//4][word_idx%4] >> subgroup.cast(dtypes.uint32)) & 0x01010101) << 4 + dot = _amd_dp4a(word, xwords[word_idx], dot) + d, dmin, scale, minimum = _q5_scales(raw, base, subgroup) + gsum = xs[token, group, 0].load() + xs[token, group, 1].load() + return (dot.float()*d*scale - gsum*dmin*minimum) * xd[token, group] + if ggml_type == IQ4_XS: + base = (output * in_features//GGML_BLOCK_SIZE + block) * IQ4_WORDS + dot = UOp.const(0, dtypes.int32) + for word_idx in range(8): + packed = _amd_load(raw[base + 2 + subgroup*4 + word_idx%4]) + dot = _amd_dp4a(_iq4_bytes(packed, 4*(word_idx//4)), xwords[word_idx], dot) + d, scale = _iq4_scales(raw, base, subgroup) + return dot.float() * xd[token, group] * d * scale + # the packed rows were padded to 212 bytes (53 words) per 256-block in set_quantized: everything is word-aligned + base = (output*in_features//GGML_BLOCK_SIZE+block)*Q6_WORDS + # the subgroup's 8 ql words and 8 qh words are contiguous: two 16-byte vector loads each + lows = tuple(_amd_load(raw[base + (subgroup//4)*16 + (subgroup%2)*8 + half*4], 4) for half in range(2)) + highs = tuple(_amd_load(raw[base + 32 + (subgroup//4)*8 + half*4], 4) for half in range(2)) + dots = [UOp.const(0, dtypes.int32)] * 2 + for word_idx in range(8): + within = (subgroup*32 + word_idx*4)%128 + low = lows[word_idx//4][word_idx%4] >> ((within//64)*4).cast(dtypes.uint32) + high = highs[word_idx//4][word_idx%4] >> ((within//32)*2).cast(dtypes.uint32) + # 4 values per word: (low nibble) | (2 high bits << 4). values stay positive, so the int8-bitcast/-32 of the + # naive dequant is skipped and the -32 offset is applied later via the per-16 sums of the quantized inputs + word = (low & 0x0f0f0f0f) | ((high & 0x03030303) << 4) + dots[word_idx//4] = _amd_dp4a(word, xwords[word_idx], dots[word_idx//4]) + scales = [((raw[base + 48 + (subgroup*2+i)//4] >> (((subgroup*2+i)%4)*8).cast(dtypes.uint32)) & 255) + .cast(dtypes.uint8).bitcast(dtypes.int8).float() for i in range(2)] + gsum = [xs[token, group, i].load() * 32 for i in range(2)] + return ((dots[0].float() - gsum[0])*scales[0] + (dots[1].float() - gsum[1])*scales[1]) * xd[token, group] * _half(raw[base+52] & 0xffff) + names = {Q4_K: "linear_q4_k", Q5_K: "linear_q5_k", IQ4_XS: "linear_iq4_xs", Q6_K: "linear_q6"} + return _decode_linear(out, out_features, group_count, group_dot, names[ggml_type]) + +def _wmma_layout(out:UOp, out_features:int, token_tile:int, output_tiles:int): + output_waves = 2 if out_features % (32*output_tiles) == 0 else 1 + token_block, output_block = UOp.range(out.shape[0]//token_tile, 0), UOp.range(out_features//(16*output_tiles*output_waves), 1) + # lane is a hardware WARP range (like the flash kernel): the fragment math stays visible without being + # range-split into nested loops, which would scramble the WMMA fragment layout + lane, wave = UOp.range(WARP_SIZE, -1, axis_type=AxisType.WARP), UOp.range(output_waves, 3, axis_type=AxisType.LOCAL) + col, half = lane % 16, lane // 16 + outputs = tuple((output_block*output_waves+wave)*(16*output_tiles) + tile*16 + col for tile in range(output_tiles)) + inputs = tuple(token_block*token_tile + tile*16 + col for tile in range(token_tile//16)) + tokens = tuple(tuple(token_block*token_tile + tile*16 + half*8 + i for i in range(8)) for tile in range(token_tile//16)) + return output_waves, token_block, output_block, lane, wave, half, outputs, inputs, tokens + +def _wmma_stores(out, outputs, tokens, accs, update, half, lane, wave, output_waves): + # the accumulator fragment halves are exchanged between lane pairs (l, l^16) through LDS (a ds_swizzle without CUSTOM) + flat_accs = [acc for output_accs in accs for acc in output_accs] + lds = UOp.placeholder((output_waves, 32, len(flat_accs)*8), dtypes.float32, slot=33, addrspace=AddrSpace.LOCAL) + stores = [lds[wave, lane, a*8+i].store(acc.after(update)[i].load()) for a,acc in enumerate(flat_accs) for i in range(8)] + lds = lds.after(UOp.barrier(UOp.group(*stores))) + def values(ai:int) -> tuple[UOp, ...]: + own = tuple(lds[wave, lane, ai*8+i].load() for i in range(8)) + peer = tuple(lds[wave, lane ^ 16, ai*8+i].load() for i in range(8)) + low = half.eq(0) + return tuple(low.where(own[i], peer[i+4]) if j == 0 else low.where(peer[i], own[i+4]) for i in range(4) for j in range(2)) + tt = len(tokens) + return [out[token, output].store(value) for ot,(output,output_accs) in enumerate(zip(outputs, accs)) + for tile,(tile_tokens,_acc) in enumerate(zip(tokens, output_accs)) for token,value in zip(tile_tokens, values(ot*tt+tile))] + +def _quant_linear_wmma(out, x, out_features, in_features, type_words, layout, dequant, name): + x = x.reshape(out.shape[0], in_features) + output_waves, token_block, output_block, lane, wave, physical_half, outputs, input_tokens, tokens = layout + token_tile, output_tiles = len(tokens)*16, len(outputs) + output_words = in_features // GGML_BLOCK_SIZE * type_words + accs = tuple(tuple(UOp.placeholder((8,), dtypes.float32, slot=ot*(token_tile//16)+tile, addrspace=AddrSpace.REG) + for tile in range(token_tile // 16)) for ot in range(output_tiles)) + accs = tuple(tuple(acc.after(acc.store(acc.const_like(0))) for acc in output_accs) for output_accs in accs) + group = UOp.range(in_features // Q8_GROUP_SIZE, 4, AxisType.REDUCE) + block, subgroup = group // 8, group % 8 + wmma_accs = [list(output_accs) for output_accs in accs] + for half in range(2): + afrags = tuple(UOp.stack(*(x[input_token, group*32 + half*16 + i].cast(dtypes.float16) for i in range(16))) + for input_token in input_tokens) + for output_tile,output in enumerate(outputs): + bfrag = UOp.stack(*dequant(output*output_words + block*type_words, subgroup, half)) + for tile,afrag in enumerate(afrags): + previous = accs[output_tile][tile].after(group) if half == 0 else wmma_accs[output_tile][tile] + wmma_accs[output_tile][tile] = UOp.wmma(afrag, bfrag, previous, *WMMA_ARG) + update = UOp.group(*(acc.store(value) for output_accs,output_values in zip(accs, wmma_accs) + for acc,value in zip(output_accs, output_values))).end(group) + stores = _wmma_stores(out, outputs, tokens, accs, update, physical_half, lane, wave, output_waves) + return UOp.group(*stores).end(token_block, output_block, lane, wave).sink(arg=KernelInfo(name=name, opts_to_apply=())) + +@functools.cache +def _q5_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, out_features:int, in_features:int, ggml_type:int) -> UOp: + token_tile, output_tiles = (64, 1) if out_features <= 1024 and out.shape[0] % 64 == 0 else \ + (64, 2) if out.shape[0] % 64 == 0 else (32 if out.shape[0] % 32 == 0 else 16, 2) + def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: + d, dmin, scale, minimum = _q5_scales(raw, base, subgroup) + qs_base = base + (4 if ggml_type == Q4_K else 12) + (subgroup // 2)*8 + half*4 + words = tuple((raw[qs_base+i] >> ((subgroup&1)*4).cast(dtypes.uint32) & 0x0f0f0f0f) | + (((raw[base+4+half*4+i] >> subgroup.cast(dtypes.uint32) & 0x01010101) << 4) if ggml_type == Q5_K else 0) for i in range(4)) + return tuple(((word >> (byte*8) & 255).float()*d*scale-dmin*minimum).cast(dtypes.float16) for word in words for byte in range(4)) + return _quant_linear_wmma(out, x, out_features, in_features, Q4_WORDS if ggml_type == Q4_K else Q5_WORDS, + _wmma_layout(out, out_features, token_tile, output_tiles), dequant, + f"linear_q{4 if ggml_type == Q4_K else 5}_k_f16_wmma") + +@functools.cache +def _iq4_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, lut:UOp, out_features:int, in_features:int) -> UOp: + token_tile = 32 if out_features <= 1024 and out.shape[0] % 32 == 0 else 64 if out.shape[0] % 64 == 0 and \ + (out_features <= 6144 or out_features == 5120 and in_features > 8192) else 128 if out.shape[0] % 128 == 0 else \ + 32 if out.shape[0] % 32 == 0 else 16 + output_tiles = 1 if out_features <= 1024 else 2 if out_features <= 6144 else 1 if out_features < 8192 else 2 + layout = _wmma_layout(out, out_features, token_tile, output_tiles) + output_waves, _, _, lane, wave, _, _, _, _ = layout + local_lut = UOp.placeholder((256,), dtypes.uint32, slot=32, addrspace=AddrSpace.LOCAL) + tid, lut_items = wave*32+lane, 256//(32*output_waves) + lut = local_lut.after(UOp.group(*(local_lut[tid*lut_items+i].store(lut[tid*lut_items+i]) for i in range(lut_items))).barrier()) + def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: + d, scale = _iq4_scales(raw, base, subgroup) + scale = scale * d + if out_features <= 6144: + pairs = tuple(lut[((raw[base + 2 + subgroup*4 + word] >> (byte*8)) & 255).cast(dtypes.weakint)] + for word in range(4) for byte in range(4)) + return tuple((_half((pair >> (half*16)) & 0xffff)*scale).cast(dtypes.float16) for pair in pairs) + # a subgroup-half gathers the lo (half=0) or hi (half=1) nibbles of byte pairs of each packed word + lut_pairs = (lut[(((raw[base+2+subgroup*4+i] >> (8*j+4*half)) & 15) | + (((raw[base+2+subgroup*4+i] >> (8*j+8+4*half)) & 15) << 4)).cast(dtypes.weakint)] + for i in range(4) for j in (0, 2)) + return tuple((_half((pair >> (i*16)) & 0xffff)*scale).cast(dtypes.float16) for pair in lut_pairs for i in range(2)) + return _quant_linear_wmma(out, x, out_features, in_features, IQ4_WORDS, layout, dequant, "linear_iq4_xs_f16_wmma") + +def q8_linear(layer:Linear, x:Tensor) -> Tensor: + assert layer.ggml_type in (Q4_K, Q5_K, Q6_K, IQ4_XS) + tokens = int(x.numel()) // layer.in_features + raw, out_features, in_features = layer.weight.uop, layer.out_features, layer.in_features + def run(fxn:Callable[..., UOp], out:UOp, *srcs:UOp) -> Tensor: + all_srcs = (out,)+srcs + params = tuple(UOp.placeholder_like(src, slot=i) for i,src in enumerate(all_srcs)) + kernel = fxn(*params, out_features=out_features, in_features=in_features).call(*all_srcs) + result = Tensor(out.after(kernel)) + if len(result.shape) == 3: result = result.sum(-1) + result = result.reshape(*x.shape[:-1], out_features) + return result if layer.bias is None else result + layer.bias + out = Tensor.empty(tokens, out_features, dtype=dtypes.float32, device=x.device).uop + if tokens % 16 == 0 and out_features % 16 == 0 and layer.ggml_type in (Q4_K, Q5_K, IQ4_XS): + fxn = _iq4_linear_f16_wmma_kernel if layer.ggml_type == IQ4_XS else functools.partial(_q5_linear_f16_wmma_kernel, ggml_type=layer.ggml_type) + extra = (iq4_half_lut(str(x.device)).uop,) if layer.ggml_type == IQ4_XS else () + return run(fxn, out, raw, x.cast(dtypes.float16).contiguous().uop, *extra) + xq_, xd, xs = q8_quantize(x, tokens, in_features) + decode = functools.partial(_quant_decode_kernel, ggml_type=layer.ggml_type) + out = Tensor.empty(tokens, out_features, (in_features+1023)//1024, dtype=dtypes.float32, device=x.device).uop + return run(decode, out, raw, xq_.uop, xd.uop, xs.uop) + +# ******** tiny dense fp16 gemv ******** + +@functools.cache +def _amd_f16_gemv_kernel(out:UOp, w:UOp, x:UOp, *rest:UOp, in_features:int, out_features:int, tokens:int) -> UOp: + bias: UOp|None = rest[0] if rest else None + # one block per (token, output row), 32 lanes accumulate 4-wide chunks of the row + lanes, val_chunk = WARP_SIZE, 4 + token, out_row = UOp.range(tokens, 0, AxisType.GLOBAL), UOp.range(out_features, 1, AxisType.GLOBAL) + lane = UOp.range(lanes, 2, axis_type=AxisType.LOCAL) + per = in_features // (lanes * val_chunk) + assert per * lanes * val_chunk == in_features + w = w.reshape((out_features, per, lanes*val_chunk)) + x = x.reshape((tokens, per, lanes*val_chunk)) + acc = UOp.const(0, dtypes.float32) + for i in range(per): + for j in range(val_chunk): + acc = acc + w[out_row, i, lane*val_chunk + j].load().float() * x[token, i, lane*val_chunk + j].load().float() + total = warp_reduce(acc, full_wave=True) + if bias is not None: total = total + bias[token, out_row].load().float() + return out[token, out_row.valid(lane.eq(0))].store(total).end(token, out_row, lane).sink(arg=KernelInfo(name="linear_f16_gemv", opts_to_apply=())) + +def _view_back(t:Tensor) -> Tensor: + """strip top-of-chain CAST(s) from a lazy weight: reading the raw file bytes in the kernel instead of + materializing the cast into a fresh buffer every step""" + uop = t.uop + while uop.op is Ops.CAST: uop = uop.src[0] + return Tensor(uop).reshape(t.shape) + +def f16_gemv(layer:Linear, x:Tensor) -> Tensor: + tokens = prod(x.shape[:-1]) + assert isinstance(tokens, int) + weight = _view_back(layer.weight) + x = x.contiguous() if x.dtype == dtypes.half else x.cast(dtypes.half).contiguous() + out = Tensor.empty(tokens, layer.out_features, dtype=dtypes.float32, device=x.device) + fxn = functools.partial(_amd_f16_gemv_kernel, in_features=layer.in_features, out_features=layer.out_features, tokens=tokens) + srcs = (out, weight.reshape(-1), x.reshape(tokens, layer.in_features)) + (() if layer.bias is None else (_view_back(layer.bias),)) + return Tensor.custom_kernel(*srcs, fxn=fxn)[0].reshape(*x.shape[:-1], layer.out_features) + +# ******** flash attention on the KV cache ******** + +def _vec_load(ptr:UOp, lanes:int) -> tuple[UOp, ...]: + if lanes == 1: return (ptr.load().float(),) + vec = _amd_load(ptr, lanes) + return tuple(vec[i].float() for i in range(lanes)) + +@functools.cache +def _amd_flash_attention_decode_partial(out, stats, q, cache_kv, valid_kv_len, max_kv_len, block_n, waves=4): + valid_kv_len = _unbind(valid_kv_len) + _, B, H_KV, N, D = cast(tuple[int, int, int, int, int], cache_kv.shape) + _, H, M, _ = cast(tuple[int, int, int, int], q.shape) + assert M == 1 and H % H_KV == 0 and D % WARP_SIZE == 0 and max_kv_len <= N and max_kv_len % block_n == 0 + G, CHUNK, DPL, WAVES = H // H_KV, block_n, D // WARP_SIZE, waves + assert CHUNK % WAVES == 0 + SEC = CHUNK // WAVES # keys each wave scans independently + live_chunks = (valid_kv_len+CHUNK-1)//CHUNK + live_chunks = min(live_chunks, out.shape[2]) if isinstance(live_chunks, int) else live_chunks.minimum(out.shape[2]) + block_bhkv, block_chunk = UOp.range(B*H_KV, 0, AxisType.GLOBAL), UOp.range(live_chunks, 1, AxisType.GLOBAL) + lane, wave = UOp.range(WARP_SIZE, -1, axis_type=AxisType.WARP), UOp.range(WAVES, 3, axis_type=AxisType.LOCAL) + b, kv_head = block_bhkv // H_KV, block_bhkv % H_KV + # per-lane query fragments for every GQA head, kept packed in registers; unpacked at use + qf = tuple(_vec_load(q[b, kv_head*G+h, 0, lane*DPL], DPL) for h in range(G)) + zerof = UOp.const(0, dtypes.float) + valids: list[UOp] = [] + scores: list[list[UOp]] = [[zerof]*G for _ in range(SEC)] + vfrags: list[tuple[UOp, ...]] = [()]*SEC + for j in range(SEC): + key = block_chunk*CHUNK + wave*SEC + j + valid = key < valid_kv_len + valids.append(valid) + kfrag = _vec_load(cache_kv[0, b, kv_head, key, lane*DPL], DPL) + # V is prefetched in the score pass so both streams are in flight together + vfrags[j] = tuple(valid.where(v, zerof) for v in _vec_load(cache_kv[1, b, kv_head, key, lane*DPL], DPL)) + for h in range(G): + s = warp_reduce(sum((qf[h][i]*kfrag[i] for i in range(DPL)), UOp.const(0, dtypes.float)), full_wave=True) * (1/math.sqrt(D)) + scores[j][h] = valid.where(s, UOp.const(-math.inf, dtypes.float)) + ninf = UOp.const(-math.inf, dtypes.float) + row_max = [functools.reduce(UOp.maximum, (scores[j][h] for j in range(SEC)), ninf) for h in range(G)] + accs:list[list[UOp]] = [[UOp.const(0, dtypes.float)] * DPL for _ in range(G)] + row_sums:list[UOp] = [UOp.const(0, dtypes.float) for _ in range(G)] + for j in range(SEC): + for h in range(G): + beta = valids[j].where(((scores[j][h]-row_max[h])*LOG2E).exp2(), UOp.const(0, dtypes.float)) + accs[h] = [a + beta*v for a, v in zip(accs[h], vfrags[j])] + row_sums[h] = row_sums[h] + beta + # exchange across the block's waves through LDS (fp16 halves LDS so more blocks fit per CU) + acc_lds = UOp.placeholder((WAVES, G, D), dtypes.half, slot=0, addrspace=AddrSpace.LOCAL) + ml_lds = UOp.placeholder((WAVES, G, 2), dtypes.float, slot=1, addrspace=AddrSpace.LOCAL) + lds_acc = acc_lds.reshape(WAVES, G, WARP_SIZE, DPL) + stores = [lds_acc[wave, h, lane].store(UOp.stack(*accs[h]).cast(dtypes.half)) for h in range(G)] + # NOTE: duplicate stores of the same value from every lane are harmless here + stores += [ml_lds[wave, h, i].store(x) for h in range(G) for i, x in enumerate((row_max[h], row_sums[h]))] + barrier = UOp.barrier(UOp.group(*stores)) + acc_lds, ml_lds = acc_lds.after(barrier), ml_lds.after(barrier) + tid = wave*WARP_SIZE + lane + final_stores:list[UOp] = [] + for i in range(-(-G*D//(WAVES*WARP_SIZE))): + flat = tid + i*WAVES*WARP_SIZE + h, d = flat // D, flat % D + M = functools.reduce(UOp.maximum, (ml_lds[w, h, 0].load() for w in range(WAVES)), ninf) + val = sum((((ml_lds[w, h, 0].load()-M)*LOG2E).exp2() * acc_lds[w, h, d].load().float() for w in range(WAVES)), UOp.const(0, dtypes.float)) + oidx = out[b, kv_head*G + h, block_chunk, d] + if G*D % (WAVES*WARP_SIZE): oidx = out[b, (kv_head*G + h).valid(flat < G*D), block_chunk, d] + final_stores.append(oidx.store(val)) + hstat = tid + M = functools.reduce(UOp.maximum, (ml_lds[w, hstat, 0].load() for w in range(WAVES)), ninf) + L = sum((((ml_lds[w, hstat, 0].load()-M)*LOG2E).exp2() * ml_lds[w, hstat, 1].load() for w in range(WAVES)), UOp.const(0, dtypes.float)) + q_head = (kv_head*G + hstat).valid(hstat < G) if WAVES*WARP_SIZE > G else kv_head*G + hstat + final_stores += [stats[b, q_head, block_chunk, 0].store(M), stats[b, q_head, block_chunk, 1].store(L)] + return UOp.group(*final_stores).end(lane, wave, block_chunk, block_bhkv).sink(arg=KernelInfo(name="flash_decode_partial", opts_to_apply=())) + +@functools.cache +def _amd_flash_decode_combine(o:UOp, partial:UOp, stats:UOp, live:int|UOp) -> UOp: + # one wave per (batch, head, 64-dim tile): every lane redundantly weights its chunks; no cross-lane traffic + live = _unbind(live) + B, H, C, D = cast(tuple[int, int, int, int], partial.shape) + DT = 64 if D % 64 == 0 else WARP_SIZE # dims per block + assert D % DT == 0 + block_bh, block_dt = UOp.range(B*H, 0, AxisType.GLOBAL), UOp.range(D//DT, 1, AxisType.GLOBAL) + lane = UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL) + b, h = block_bh // H, block_bh % H + NPD = DT // WARP_SIZE # output dims per lane + dims = tuple(block_dt*DT + lane*NPD + i for i in range(NPD)) + chunk = UOp.range(live, 100, AxisType.REDUCE) + def iloop(ph, val): return ph.store(ph.const_like(val)) + chunk_max = UOp.placeholder((1,), dtypes.float, slot=0, addrspace=AddrSpace.REG) + chunk_max_i = chunk_max.after(iloop(chunk_max, -math.inf)) + update0 = chunk_max_i.store(chunk_max_i.after(chunk).maximum(stats[b, h, chunk, 0].load())).end(chunk) + chunk_max = chunk_max_i.after(update0) + chunk2 = UOp.range(live, 101, AxisType.REDUCE) + acc = UOp.placeholder((NPD,), dtypes.float, slot=1, addrspace=AddrSpace.REG) + weight_sum = UOp.placeholder((1,), dtypes.float, slot=2, addrspace=AddrSpace.REG) + acc_i, weight_sum_i = acc.after(iloop(acc, 0)), weight_sum.after(iloop(weight_sum, 0)) + w = ((stats[b, h, chunk2, 0].load()-chunk_max)*LOG2E).exp2() + update1 = UOp.group(*[acc_i[i].store(acc_i.after(chunk2)[i].load() + w*partial[b, h, chunk2, d].load()) for i, d in enumerate(dims)], + weight_sum_i[0].store(weight_sum_i.after(chunk2)[0].load() + w*stats[b, h, chunk2, 1].load())).end(chunk2) + acc, weight_sum = acc_i.after(update1), weight_sum_i.after(update1) + inv = 1 / weight_sum[0].load() + return UOp.group(*[o[b, h, 0, d].store(acc[i].load() * inv) for i, d in enumerate(dims)]) \ + .end(lane, block_dt, block_bh).sink(arg=KernelInfo(name="flash_decode_combine", opts_to_apply=())) + +def amd_flash_attention_decode(q:Tensor, cache_kv:Tensor, valid_kv_len:int|UOp, max_kv_len:int) -> Tensor: + B, H, D = cache_kv.shape[1], q.shape[1], cache_kv.shape[4] + chunks = min(256, max_kv_len // 64) + partial = Tensor.empty(B, H, chunks, D, dtype="float32", device=q.device) + stats = Tensor.empty(B, H, chunks, 2, dtype="float32", device=q.device) + fxn = functools.partial(_amd_flash_attention_decode_partial, valid_kv_len=valid_kv_len, max_kv_len=max_kv_len, block_n=64, waves=16) + partial, stats = Tensor.custom_kernel(partial, stats, q, cache_kv, fxn=fxn)[:2] + live = (valid_kv_len+63)//64 + live = min(live, chunks) if isinstance(live, int) else live.minimum(chunks) + out = Tensor.empty(B, H, 1, D, dtype="float32", device=q.device) + fxn = functools.partial(_amd_flash_decode_combine, live=live) + return Tensor.custom_kernel(out, partial, stats, fxn=fxn)[0] + +@functools.cache +def _amd_flash_attention(o:UOp, q:UOp, cache:UOp, valid_kv_len:int|UOp, q_start:int|UOp|None=None) -> UOp: + valid_kv_len, q_start = _unbind(valid_kv_len), _unbind(q_start) if q_start is not None else None + BH, M, D = q.shape + _, B, H_KV, physical_n, cache_dim = cache.shape + k, v = cache[0].reshape(B*H_KV, physical_n, cache_dim), cache[1].reshape(B*H_KV, physical_n, cache_dim) + assert k.shape == v.shape and BH % k.shape[0] == 0 and k.shape[2] == D + gqa_group = BH // k.shape[0] + if isinstance(M, int) and isinstance(valid_kv_len, int): assert M % BLOCK_M == 0 and valid_kv_len % BLOCK_N == 0 + assert isinstance(D, int) and D % WMMA_K == 0 and D % LANES_PER_WAVE_N == 0 + TM, TN, TD, SCALE = BLOCK_M//(WAVES_M*LANES_PER_WAVE_M), BLOCK_N//LANES_PER_WAVE_N, D//(WAVES_N*LANES_PER_WAVE_N), 1/math.sqrt(D) + # query row 0 sits at sequence position q_base (the queries may be padded beyond valid_kv_len - q_base rows) + q_base = valid_kv_len - M if q_start is None else q_start + block_bh, block_m = UOp.range(BH, 0, AxisType.GLOBAL), UOp.range(M // BLOCK_M, 1, AxisType.GLOBAL) + kv_head = block_bh // gqa_group + q, o = (x.reshape(BH, M//BLOCK_M, BLOCK_M, D)[block_bh, block_m] for x in (q, o)) + k, v = k[kv_head], v[kv_head] + wave_m, wave_n, lane = UOp.range(WAVES_M, 2, AxisType.LOCAL), UOp.range(WAVES_N, 3, AxisType.LOCAL), UOp.range(WARP_SIZE, -1, AxisType.WARP) + tid, lane_m, lane_n = (wave_m * WAVES_N + wave_n) * WARP_SIZE + lane, lane // LANES_PER_WAVE_N, lane % LANES_PER_WAVE_N + Q_ELEMS_PER_THREAD, KV_ELEMS_PER_THREAD = BLOCK_M * D // THREADS_PER_BLOCK, BLOCK_N * D // THREADS_PER_BLOCK + QP_lds = UOp.placeholder((BLOCK_M, D + LDS_PAD), dtypes.half, slot=0, addrspace=AddrSpace.LOCAL) + KV_lds = UOp.placeholder((BLOCK_N, D + LDS_PAD), dtypes.half, slot=1, addrspace=AddrSpace.LOCAL)[:, :D] + acc, m_i, l_i = _reg((TM, TD), 2, 0), _reg((TM,), 3, -math.inf), _reg((TM,), 4, 0) + n_tile = UOp.range((q_base + (block_m + 1) * BLOCK_M + BLOCK_N - 1) // BLOCK_N, 100, AxisType.REDUCE) + Q_lds = QP_lds[:, :D] + Q_store = Q_lds.after(n_tile).reshape(THREADS_PER_BLOCK, Q_ELEMS_PER_THREAD)[tid].store(q.reshape(THREADS_PER_BLOCK, Q_ELEMS_PER_THREAD)[tid]) + load_k = UOp.range(KV_ELEMS_PER_THREAD, 90) + kval = k.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_k].float() + K_store = KV_lds.reshape(THREADS_PER_BLOCK, KV_ELEMS_PER_THREAD)[tid, load_k].store(kval).end(load_k) + qk_load_barrier = UOp.barrier(UOp.group(Q_store, K_store)) + Q_lds, KV_lds_k = Q_lds.after(qk_load_barrier), KV_lds.after(qk_load_barrier) + S_reg = _reg((TM, TN), 6, 0, n_tile) + k_qk, tm1, tn1 = UOp.range(D//WMMA_K, 101, AxisType.REDUCE), UOp.range(TM//WMMA_ACC, 200), UOp.range(TN, 201) + S_frag = S_reg.reshape(TM // WMMA_ACC, WMMA_ACC, TN).permute(0, 2, 1)[tm1, tn1] + q_frag = Q_lds.reshape(WAVES_M, TM // WMMA_ACC, WMMA_M, D // WMMA_K, WMMA_K)[wave_m, tm1, lane_n, k_qk] + k_frag = KV_lds_k.reshape(TN, WMMA_N, D // WMMA_K, WMMA_K)[tn1, lane_n, k_qk] + qk_done = S_frag.store(UOp.wmma(q_frag, k_frag, S_frag.after(k_qk), *WMMA_ARG)).end(tm1, tn1).end(k_qk) + S_reg = S_reg.after(qk_done, S_reg.store(S_reg * SCALE)) + rm, rn = UOp.range(TM, 250), UOp.range(TN, 251) + q_idx = q_base + block_m * BLOCK_M + wave_m * WMMA_M + rm * LANES_PER_WAVE_M + lane_m + k_idx = n_tile * BLOCK_N + rn * LANES_PER_WAVE_N + lane_n + S_reg = S_reg.after(S_reg[rm, rn].store((k_idx <= q_idx).where(S_reg[rm, rn], S_reg[rm, rn].const_like(-math.inf))).end(rm, rn)) + m_ij, rm2 = _reg((TM,), 7, -math.inf, n_tile), UOp.range(TN, 261, AxisType.REDUCE) + m_ij = m_ij.after(m_ij.store(m_ij.after(rm2).maximum(S_reg[:, rm2])).end(rm2)) + ri_w = UOp.range(TM, 270) + m_ij = m_ij.after(m_ij[ri_w].store(warp_reduce(m_ij[ri_w], maximum=True)).end(ri_w)) + tile_max = m_ij.reshape(TM, 1).expand(TM, TN).maximum(-1e30) + S_reg = S_reg.after(S_reg.store(((S_reg - tile_max) * LOG2E).exp2())) + p_local, ri_ws = _reg((TM,), 8, 0, n_tile), UOp.range(TM, 295) + p_sum = p_local.after(p_local[ri_ws].store(sum((warp_reduce(S_reg[ri_ws, rn]) for rn in range(TN)), S_reg.const_like(0))).end(ri_ws)) + P_lds = QP_lds.flatten()[:WAVES_N * BLOCK_M * BLOCK_N].reshape(WAVES_N, BLOCK_M, BLOCK_N) + P_write = P_lds.reshape(WAVES_N, WAVES_M, TM, LANES_PER_WAVE_M, 1, TN, LANES_PER_WAVE_N, 1).permute((1, 0, 3, 6, 2, 4, 5, 7)) \ + .reshape(THREADS_PER_BLOCK, TM, TN) + P_store = P_write[tid].store(S_reg.cast(dtypes.half)) + beta_i, ri4, rj4 = UOp.placeholder((TM,), dtypes.float, slot=9, addrspace=AddrSpace.REG), UOp.range(TM, 330), UOp.range(TD, 331) + m_new = m_i[ri4].maximum(m_ij[ri4]) + alpha_val, beta_val = ((m_i[ri4] - m_new) * LOG2E).exp2(), ((m_ij[ri4] - m_new) * LOG2E).exp2() + correction = UOp.group(acc[ri4, rj4].store(alpha_val * acc[ri4, rj4]).end(rj4), + l_i[ri4].store(alpha_val * l_i[ri4] + beta_val * p_sum[ri4]), + m_i[ri4].store(m_new), beta_i[ri4].store(beta_val)).end(ri4) + acc, l_i, m_i, beta_i = acc.after(correction), l_i.after(correction), m_i.after(correction), beta_i.after(correction) + V_lds = UOp.placeholder((D, BLOCK_N + LDS_PAD), dtypes.half, slot=1, addrspace=AddrSpace.LOCAL)[:, :BLOCK_N] + V_copy, load_v = V_lds.after(qk_done).permute(1, 0), UOp.range(KV_ELEMS_PER_THREAD, 390) + vval = v.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_v].float() + V_store = V_copy.reshape(THREADS_PER_BLOCK, KV_ELEMS_PER_THREAD)[tid, load_v].store(vval).end(load_v) + pv_barrier = UOp.barrier(UOp.group(P_store, V_store)) + P_lds, V_lds = P_lds.after(pv_barrier), V_lds.after(pv_barrier) + pv_acc = _reg((TM, TD), 10, 0, n_tile).after(pv_barrier) + k_pv, tm2, tn2 = UOp.range(BLOCK_N//WMMA_K, 400, AxisType.REDUCE), UOp.range(TM//WMMA_ACC, 401), UOp.range(TD, 402) + pv_frag = pv_acc.reshape(TM // WMMA_ACC, WMMA_ACC, TD).permute(0, 2, 1)[tm2, tn2] + p_frag = P_lds[wave_n].reshape(WAVES_M, TM // WMMA_ACC, WMMA_M, BLOCK_N // WMMA_K, WMMA_K)[wave_m, tm2, lane_n, k_pv] + v_frag = V_lds.reshape(WAVES_N, TD, WMMA_N, BLOCK_N // WMMA_K, WMMA_K)[wave_n, tn2, lane_n, k_pv] + pv_done = pv_frag.store(UOp.wmma(p_frag, v_frag, pv_frag.after(k_pv), *WMMA_ARG)).end(tm2, tn2).end(k_pv) + pv_acc = pv_acc.after(pv_done) + ri5, rj5 = UOp.range(TM, 410), UOp.range(TD, 411) + n_tile_end = acc[ri5, rj5].store(acc[ri5, rj5] + beta_i[ri5] * pv_acc[ri5, rj5]).end(ri5, rj5).barrier().end(n_tile) + acc, l_i, m_i = acc.after(n_tile_end), l_i.after(n_tile_end), m_i.after(n_tile_end) + acc = acc.after(acc.store(acc * (1 / l_i).reshape(TM, 1).expand(TM, TD))) + o = o.reshape(WAVES_M, TM, LANES_PER_WAVE_M, 1, WAVES_N, TD, LANES_PER_WAVE_N, 1) \ + .permute((0, 4, 2, 6, 1, 3, 5, 7)).reshape(THREADS_PER_BLOCK, TM, TD) + return o[tid].store(acc).end(wave_m, wave_n, lane).end(block_m, block_bh).sink(arg=KernelInfo(opts_to_apply=())) + +def flash_attention(q:Tensor, assigned_kv:Tensor, valid_end:int|UOp) -> Tensor: + # cached flash attention on the half KV cache (already written through assigned_kv); valid_end stays bound at the graph level + T_real, q_start = q.shape[2], None + if resolve(T_real == 1): return amd_flash_attention_decode(q.half(), assigned_kv, valid_end, cast(int, assigned_kv.shape[3])) + if isinstance(T_real, UOp): + # symbolic chunk: pad the queries to the static tile; garbage rows are sliced off + T_pad = q.max_shape[2] + assert T_pad % BLOCK_M == 0, "chunk_size must be a multiple of 32" + q, q_start = q.pad_to((*q.shape[:2], T_pad, q.shape[3])), valid_end - T_real + B, H, T, D = q.shape + out = Tensor.empty(B*H, T, D, dtype="float32", device=q.device) + fxn = functools.partial(_amd_flash_attention, valid_kv_len=valid_end, q_start=q_start) + out = Tensor.custom_kernel(out, q.half().reshape(B*H, T, D), assigned_kv, fxn=fxn)[0].reshape(B, H, T, D) + return out if q_start is None else out[:, :, :T_real] + +# ******** gated delta net: fused recurrent scan ******** + +@functools.cache +def _gated_delta_prefill_kernel(core:UOp, q:UOp, k:UOp, v:UOp, beta:UOp, alpha:UOp, state:UOp, kq:UOp, start_pos:UOp|None=None) -> UOp: + batch, heads, tokens, value_dim, row_tile = *core.shape, 4 + key_dim, alpha_dim = q.shape[-1], alpha.shape[-1] if len(alpha.shape) == 4 else 1 + assert all(isinstance(x, int) for x in (batch, heads, tokens, value_dim, key_dim)) and key_dim % 32 == 0 and value_dim % row_tile == 0 + batch, heads, tokens, value_dim, key_dim = cast(tuple[int, int, int, int, int], (batch, heads, tokens, value_dim, key_dim)) + core, v = (x.reshape(batch*heads, tokens, value_dim) for x in (core, v)) + q, k = (x.reshape(batch*heads, tokens, key_dim) for x in (q, k)) + beta, kq = (x.reshape(batch*heads, tokens) for x in (beta, kq)) + alpha, state = alpha.reshape(batch*heads, tokens, alpha_dim), state.reshape(batch*heads, value_dim, key_dim) + bh_row, lane = UOp.range(batch*heads*value_dim//row_tile, 0), UOp.range(32, 1, axis_type=AxisType.LOCAL) + bh, row_base = bh_row // (value_dim//row_tile), (bh_row % (value_dim//row_tile))*row_tile + rows, cols = tuple(row_base+i for i in range(row_tile)), tuple(lane + i*32 for i in range(key_dim//32)) + current = UOp.placeholder((row_tile*key_dim//32,), dtypes.float32, slot=0, addrspace=AddrSpace.REG) + initial = None if start_pos is None else start_pos.eq(0) + current = current.after(current.store(UOp.stack(*(state[bh, row, col].float() if initial is None else + initial.where(0, state[bh, row, col].float()) for row in rows for col in cols)))) + token = UOp.range(tokens, 2, AxisType.REDUCE) + keys = tuple(k[bh, token, col].load() for col in cols) + queries = tuple(q[bh, token, col].load() for col in cols) + updates, stores = [], [] + for row_idx,row in enumerate(rows): + previous = tuple(current.after(token)[row_idx*key_dim//32+i].load() for i in range(key_dim//32)) + av, bv = alpha[bh, token, row if alpha_dim > 1 else 0].load(), beta[bh, token].load() + state_k = warp_reduce(sum((x*y for x,y in zip(previous, keys)), UOp.const(0, dtypes.float32)), full_wave=True) + state_q = warp_reduce(sum((x*y for x,y in zip(previous, queries)), UOp.const(0, dtypes.float32)), full_wave=True) + delta = (v[bh, token, row].load() - state_k*av) * bv + updates += [x*av + delta*y for x,y in zip(previous, keys)] + stores.append(core[bh, token, row.valid(lane.eq(0))].store(state_q*av + delta*kq[bh, token])) + step = UOp.group(*stores, current.store(UOp.stack(*updates))).end(token) + state_stores = (state[bh, row, col].store(current.after(step)[row_idx*key_dim//32+i].load().cast(state.dtype)) + for row_idx,row in enumerate(rows) for i,col in enumerate(cols)) + return UOp.group(*state_stores).end(lane, bh_row).sink(arg=KernelInfo(name="gated_delta_prefill", opts_to_apply=())) + +def gated_delta_prefill(q:Tensor, k:Tensor, v:Tensor, beta:Tensor, alpha:Tensor, state:Tensor, start_pos:Tensor|None=None) -> Tensor: + batch, heads, tokens, key_dim = q.shape + value_dim = v.shape[-1] + assert q.shape == k.shape and v.shape[:3] == beta.shape == (batch, heads, tokens) and state.shape == (batch, heads, value_dim, key_dim) + assert alpha.shape[:3] == (batch, heads, tokens) and (len(alpha.shape) == 3 or alpha.shape[-1] in (1, value_dim)) + assert key_dim % 32 == 0 and value_dim % 4 == 0 + core, kq = Tensor.empty_like(v), (q*k).sum(-1).contiguous() + srcs = (core, q.contiguous(), k.contiguous(), v.contiguous(), beta.contiguous(), alpha.contiguous(), state, kq) + if start_pos is None: return Tensor.custom_kernel(*srcs, fxn=_gated_delta_prefill_kernel)[0] + contig = tuple(x.uop if x.uop.op is Ops.AFTER else x.uop.contiguous() for x in srcs) + params = tuple(UOp.placeholder_like(x, slot=i) for i,x in enumerate(contig)) + assert start_pos.uop.is_bound_var + # the bound start_pos reaches the graph through the state AFTER chain, like the flash kernels' valid_end + call = _gated_delta_prefill_kernel(*params, kernel_var(start_pos.uop.src[0])).call(*contig) + return Tensor(contig[0].after(call)) diff --git a/tinygrad_repo/tinygrad/llm/model.py b/tinygrad_repo/tinygrad/llm/model.py index 1f4feb969..5231e8e2a 100644 --- a/tinygrad_repo/tinygrad/llm/model.py +++ b/tinygrad_repo/tinygrad/llm/model.py @@ -1,11 +1,17 @@ from __future__ import annotations -import functools, itertools, pathlib +import enum, functools, itertools, pathlib from dataclasses import dataclass, replace -from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function -from tinygrad.nn import Linear +from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function, dtypes +from tinygrad.llm.kernels.amd import Linear, gated_delta_prefill, flash_attention, amd_custom_kernels_supported from tinygrad.llm.gguf import gguf_load from tinygrad.uop.ops import resolve +class ExpertGating(enum.IntEnum): + SOFTMAX = 1 + SIGMOID = 2 + SOFTMAX_WEIGHT = 3 # softmax over the top-k selected logits + SQRT_SOFTPLUS = 4 + @functools.cache def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0, device:str|None=None) -> Tensor: freqs = 1.0 / (theta ** (Tensor.arange(0, dim, 2)[:(dim // 2)] / dim)) @@ -61,6 +67,7 @@ class TransformerConfig: num_experts: int = 0 num_experts_per_tok: int = 0 norm_topk_prob: bool = False + expert_gating_func: ExpertGating = ExpertGating.SOFTMAX q_lora_rank: int = 0 kv_lora_rank: int = 0 shared_expert_dim: int = 0 @@ -103,14 +110,21 @@ class FFNBlock: if hasattr(self, 'ffn_gate_exps'): h = x.unsqueeze(2) # (B, T, 1, D) - add expert dim for broadcasting logits = self.ffn_gate_inp(x) - if hasattr(self, 'exp_probs_b'): - probs = logits.sigmoid() - _, sel = pairwise_topk(probs + self.exp_probs_b["bias"], self.config.num_experts_per_tok) - probs = probs.gather(-1, sel) - if self.config.norm_topk_prob: probs = probs / probs.sum(axis=-1, keepdim=True) - else: - vals, sel = pairwise_topk(logits, self.config.num_experts_per_tok) - probs = vals.softmax(-1) if self.config.norm_topk_prob else logits.softmax(-1).gather(-1, sel) + bias = self.exp_probs_b["bias"] if hasattr(self, 'exp_probs_b') else None + gating, normalize_topk = self.config.expert_gating_func, self.config.norm_topk_prob + # fast path: without selection bias, normalized SOFTMAX is equivalent to SOFTMAX_WEIGHT + if gating == ExpertGating.SOFTMAX and bias is None and normalize_topk: + gating, normalize_topk = ExpertGating.SOFTMAX_WEIGHT, False + if gating == ExpertGating.SOFTMAX_WEIGHT: scores = logits + elif gating == ExpertGating.SOFTMAX: scores = logits.softmax(-1) + elif gating == ExpertGating.SIGMOID: scores = logits.sigmoid() + elif gating == ExpertGating.SQRT_SOFTPLUS: scores = logits.softplus().sqrt() + + _, sel = pairwise_topk(scores if bias is None else scores + bias, self.config.num_experts_per_tok) + probs = scores.gather(-1, sel) + # SOFTMAX_WEIGHT applies softmax after top-k selection + if gating == ExpertGating.SOFTMAX_WEIGHT: probs = probs.softmax(-1) + if normalize_topk: probs = probs / probs.sum(axis=-1, keepdim=True) probs = probs * self.config.routed_scaling_factor x_down = self.ffn_down_exps(sel, (self.ffn_gate_exps(sel, h).silu() * self.ffn_up_exps(sel, h)).contiguous()) # (B, T, k, D) out = (x_down * probs.unsqueeze(-1)).sum(axis=2) # (B, T, D) @@ -124,8 +138,6 @@ class FFNBlock: # given the token-prefix match, return how much cached state this block can still reuse def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return prefix_len - # return writes that reset this block's state after a cache mismatch - def _state_reset_ops(self) -> list[Tensor]: return [] def _init_state(self, x:Tensor): raise NotImplementedError def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: raise NotImplementedError @@ -169,7 +181,13 @@ class TransformerBlock(FFNBlock): k = apply_rope(k[..., :self.config.rope_dim], self.freqs_cis[start_pos:start_pos+T]).cat(k[..., self.config.rope_dim:], dim=-1) # NOTE: we don't want to change self.cache_kv, the function API doesn't support this well - assigned_kv = Tensor(self.cache_kv.uop.after(self.cache_kv[:, :, :, start_pos:start_pos+T, :].uop.store(Tensor.stack(k, v).uop))) + store = self.cache_kv[:, :, :, start_pos:start_pos+T, :].uop.store(Tensor.stack(k, v).cast(dtypes.half).uop) + assigned_kv = Tensor(self.cache_kv.uop.after(store)) + # on RDNA3, hybrid models use custom flash attention kernels on the KV cache + if amd_custom_kernels_supported(x.device) and self.config.ssm is not None: + attn = flash_attention(q, assigned_kv, start_pos+T) + attn = attn.transpose(1, 2).reshape(B, T, -1) # back to (B,T,D) + return self.attn_output(attn if not self.config.attn_output_gate else (attn * gate.sigmoid())) k = assigned_kv[0, :, :, 0:start_pos+T, :] v = assigned_kv[1, :, :, 0:start_pos+T, :] @@ -187,8 +205,9 @@ class TransformerBlock(FFNBlock): def _init_state(self, x:Tensor): if not hasattr(self, "cache_kv"): - # TODO: how is the dtype of this determined? - self.cache_kv = Tensor.empty(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, device=x.device) + # zeroed so the flash kernels can safely read whole tiles past the valid region (masked lanes multiply by 0) + self.cache_kv = Tensor.zeros(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, + dtype=dtypes.half, device=x.device) self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device) class MLATransformerBlock(FFNBlock): @@ -260,45 +279,72 @@ class GatedDeltaNetBlock(FFNBlock): def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: B, T, _ = x.shape - assert T == 1, "GatedDeltaNetBlock currently only supports T=1" + # bind ints to a variable so the reset flag stays a runtime value (it toggles when generation restarts at position 0) + start_pos = start_pos if isinstance(start_pos, UOp) else UOp.variable("start_pos", 0, self.config.max_context-1).bind(start_pos) + initial = Tensor(start_pos).eq(0) + is_kda = hasattr(self, "ssm_g_a") + symbolic = isinstance(T, UOp) + T_pad = x.max_shape[1] # symbolic chunks are padded to their max size: one graph serves every size # input processing x = x.half() - out_gate = self.ssm_g_b(self.ssm_g_a(x)) if hasattr(self, "ssm_g_a") else self.attn_gate(x) - out_gate = out_gate.reshape(B, 1, self.num_v_heads, self.head_v_dim) - beta = self.ssm_beta(x).sigmoid().reshape(B, self.num_v_heads, 1, 1) - alpha = self.ssm_f_b(self.ssm_f_a(x)) if hasattr(self, "ssm_f_a") else self.ssm_alpha(x) - alpha = ((alpha.float() + self.ssm_dt["bias"]).softplus().reshape(B, self.num_v_heads, -1) * - self.ssm_a.reshape(1, self.num_v_heads, -1)).exp().unsqueeze(-2) + out_gate = self.ssm_g_b(self.ssm_g_a(x)) if is_kda else self.attn_gate(x) + out_gate = out_gate.reshape(B, T, self.num_v_heads, self.head_v_dim) + beta = self.ssm_beta(x).sigmoid().reshape(B, T, self.num_v_heads) + alpha = self.ssm_f_b(self.ssm_f_a(x)) if is_kda else self.ssm_alpha(x) + log_alpha = ((alpha.float() + self.ssm_dt["bias"]).softplus().reshape(B, T, self.num_v_heads, -1) * + self.ssm_a.reshape(self.num_v_heads, -1)) - # qkv conv - conv_window = self.conv_state.cat(self.attn_qkv(x), dim=1) - conv_out = (conv_window * self.ssm_conv1d["weight"].T.unsqueeze(0)).sum(1).silu() + # qkv conv, conv_state is reset when starting from position 0 + conv_state = initial.where(0, self.conv_state) + # assemble the conv window in a static-size buffer: [conv_state | qkv rows | zero-pad]. + # padded steps are exact no-ops: beta=0 (delta rule off), log_alpha=0 (decay 1 after exp) + win = Tensor.zeros(B, self.ssm_conv_kernel-1 + T_pad, self.conv_channels).uop + win = win.after(win[:, :self.ssm_conv_kernel-1].store(conv_state.cast(win.dtype).uop)) + win = win.after(win[:, self.ssm_conv_kernel-1:self.ssm_conv_kernel-1+T].store(self.attn_qkv(x).cast(win.dtype).uop)) + conv_window = Tensor(win) + # the last conv_kernel-1 columns of the window become the next conv state + conv_state_store = self.conv_state.uop.store(conv_window[:, T:T+self.ssm_conv_kernel-1].cast(self.conv_state.dtype).uop) + + conv_out = functools.reduce(lambda a,b: a+b, + (conv_window[:, i:i+T_pad] * self.ssm_conv1d["weight"][:, i] for i in range(self.ssm_conv_kernel))).silu() + if symbolic: + out_gate = out_gate.pad_to((B, T_pad, self.num_v_heads, self.head_v_dim)) + beta, log_alpha = beta.pad_to((B, T_pad, self.num_v_heads)), log_alpha.pad_to((B, T_pad, *log_alpha.shape[2:])) q, k, v = conv_out.split([self.q_dim, self.q_dim, self.conv_channels - 2*self.q_dim], dim=-1) - q = q.reshape(B, self.num_k_heads, self.head_k_dim).normalize(dim=-1).repeat(1, self.num_v_heads//self.num_k_heads, 1) - k = k.reshape(B, self.num_k_heads, self.head_k_dim).normalize(dim=-1).repeat(1, self.num_v_heads//self.num_k_heads, 1) - v = v.reshape(B, self.num_v_heads, self.head_v_dim) - q, k, v = q.mul(self.head_k_dim**-0.5).unsqueeze(-1), k.unsqueeze(-1), v.unsqueeze(-1) + qk_eps = 1e-12 if is_kda else 1e-6 + q, k = (z.reshape(B, T_pad, self.num_k_heads, self.head_k_dim).normalize(dim=-1, eps=qk_eps) + .repeat(1, 1, self.num_v_heads//self.num_k_heads, 1) for z in (q, k)) + v = v.reshape(B, T_pad, self.num_v_heads, self.head_v_dim) + # layout the per-step operands to broadcast against the (B, H, V, K) state + q, k, v, beta = (z.transpose(1, 2).float() for z in (q, k, v, beta)) + q = q * self.head_k_dim**-0.5 + alpha = log_alpha.transpose(1, 2).exp() # per-channel decay for kda, per-head otherwise (B, H, T, V|1) - # recurrent - recurrent_state = self.recurrent_state * alpha - recurrent_state = recurrent_state + ((v - recurrent_state@k) * beta)@k.transpose(-1, -2) + # recurrent: scan over the (padded) tokens, updating the recurrent state. collect the per-step outputs + state = Tensor(self.recurrent_state.uop.after(conv_state_store)) # carry the conv write into this graph + if self.head_k_dim % 32 == 0 and self.head_v_dim % 4 == 0 and amd_custom_kernels_supported(x.device): + # one fused kernel for the whole scan; it resets and updates the recurrent state in place (RDNA3) + core = gated_delta_prefill(q, k, v, beta, alpha, state, Tensor(start_pos)).transpose(1, 2) + else: + q, k, v, beta = q.unsqueeze(-2), k.unsqueeze(-2), v.unsqueeze(-1), beta.unsqueeze(-1).unsqueeze(-1) + alpha = alpha.unsqueeze(-1) + state = initial.where(0, state.float()) + outs = [] + for t in range(T_pad): + s1 = state * alpha[:, :, t] # decay the state + delta = (v[:, :, t] - (s1*k[:, :, t]).sum(-1, keepdim=True)) * beta[:, :, t] # the delta rule update + state = s1 + delta * k[:, :, t] + outs.append((state * q[:, :, t]).sum(-1)) - # store the updated state - conv_state_store = self.conv_state.uop.store(conv_window[:, 1:, :].cast(self.conv_state.dtype).uop) - recurrent_state_store = self.recurrent_state.uop.store(recurrent_state.cast(self.recurrent_state.dtype).uop) - recurrent_state = Tensor(self.recurrent_state.uop.after(recurrent_state_store, conv_state_store)) + # store the updated recurrent state in place, then read the stacked outputs after the write + state_store = self.recurrent_state.uop.store(state.cast(self.recurrent_state.dtype).uop) + core = Tensor(outs[0].stack(*outs[1:], dim=1).contiguous().uop.after(state_store)) - # output - core_attn_out = self.ssm_norm((recurrent_state@q).squeeze(-1).reshape(B, 1, self.num_v_heads, self.head_v_dim)) - out_gate = out_gate.sigmoid() if hasattr(self, "ssm_g_a") else out_gate.silu() - return self.ssm_out((core_attn_out * out_gate).reshape(B, 1, -1).cast(x.dtype)) - - # recurrent state can't be partially reused after divergence, force a full rebuild - def _state_reset_ops(self): - return [self.conv_state.assign(self.conv_state.const_like(0)), - self.recurrent_state.assign(self.recurrent_state.const_like(0))] if hasattr(self, "conv_state") else [] - def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return 0 if prefix_len != cached_len else prefix_len + # output; undo the padding before the output projection + z = (self.ssm_norm(core) * (out_gate.sigmoid() if is_kda else out_gate.silu())).cast(x.dtype).contiguous() + if symbolic: z = z[:, :T] + return self.ssm_out(z.reshape(B, T, -1)) def _init_state(self, x): if not hasattr(self, "conv_state"): @@ -326,7 +372,8 @@ class Transformer: def forward(self, tokens:Tensor, start_pos:int|UOp, temperature:Tensor) -> Tensor: x = self.token_embd(tokens).float() # (B, T, D) for block in self.blk: x = block(x, start_pos) - logits = self.output(self.output_norm(x))[:, -1, :] + # only run the output projection on the last token + logits = self.output(self.output_norm(x[:, -1:]))[:, -1, :] # Gumbel-max trick: argmax(logits/temp - log(-log(uniform))) is equivalent to sampling from softmax(logits/temp) return (logits / temperature.maximum(1e-12) - (Tensor.rand_like(logits).maximum(1e-12).log().neg()).log()).argmax(-1, keepdim=True) @@ -397,6 +444,7 @@ class Transformer: qk_norm=int(state_dict['blk.0.attn_q_norm.weight'].shape[0]) if 'blk.0.attn_q_norm.weight' in state_dict else 0, num_experts=kv.get(f'{arch}.expert_count', 0), num_experts_per_tok=kv.get(f'{arch}.expert_used_count', 0), norm_topk_prob=kv.get(f'{arch}.expert_weights_norm', arch in ('qwen3moe', 'qwen35moe', 'kimi-linear')), + expert_gating_func=ExpertGating(kv.get(f'{arch}.expert_gating_func', ExpertGating.SOFTMAX)), kv_lora_rank=kv_lora_rank, q_lora_rank=kv.get(f'{arch}.attention.q_lora_rank', 0), leading_dense_blocks=kv.get(f'{arch}.leading_dense_block_count', 0), shared_expert_dim=kv.get( @@ -420,11 +468,15 @@ class Transformer: for _ in range(2): list(zip(range(2), self.generate([0]))) def get_start_pos(self, tokens:list[int]) -> int: + # recurrent state can't be partially reused after divergence: reuse it only when tokens extend the cached prefix + if self.has_recurrent_block: + return len(self._cached_tokens) if self._cached_tokens and len(self._cached_tokens) < len(tokens) \ + and tokens[:len(self._cached_tokens)] == self._cached_tokens else 0 prefix_len = sum(1 for _ in itertools.takewhile(lambda ab: ab[0] == ab[1], zip(tokens[:-1], self._cached_tokens))) return min(block._reusable_prefix_len(prefix_len, len(self._cached_tokens)) for block in self.blk) def generate(self, tokens:list[int], chunk_size:int=32, temperature:float=0.0): - if self.has_recurrent_block: chunk_size = 1 + if self.has_recurrent_block and not amd_custom_kernels_supported(self.token_embd.weight.device): chunk_size = 1 v_start_pos = UOp.variable("start_pos", 0, self.max_context-1) v_toks = UOp.variable("toks", 1, chunk_size) # TODO: use UOp.variable for temperature once float variables are supported @@ -433,7 +485,6 @@ class Transformer: t = Tensor(tokens + [0] * (self.max_context - len(tokens)), dtype="int32").reshape(1, self.max_context) # recompute start_pos from what's currently valid in the caches start_pos = self.get_start_pos(tokens) - if start_pos < len(self._cached_tokens) and (resets := [r for b in self.blk for r in b._state_reset_ops()]): Tensor.realize(*resets) out, prompt_len = None, len(tokens) while len(tokens) < self.max_context: n_toks = min(chunk_size, len(tokens) - start_pos) diff --git a/tinygrad_repo/tinygrad/mixin/creation.py b/tinygrad_repo/tinygrad/mixin/creation.py index c290bceb2..0015cc601 100644 --- a/tinygrad_repo/tinygrad/mixin/creation.py +++ b/tinygrad_repo/tinygrad/mixin/creation.py @@ -78,10 +78,9 @@ class CreationMixin(DTypeMixin, MovementMixin): from tinygrad.uop.ops import UOp new_shape = argfix(shape) dt = to_dtype(dtype) if dtype is not None else fill_value.dtype if isinstance(fill_value, UOp) else dtypes.from_py(fill_value) - val = cls.const(fill_value, dt) - val = val.reshape((1,)*len(new_shape)).expand(new_shape) + val = cls.const(fill_value, dt).expand(new_shape) if not buffer: return val - ret = val.empty_like(dt if dtype is not None else None, device) + ret = val.empty_like(None if dt in dtypes.weaks else dt, device) return cls._wrap_uop(ret._uop.after(ret._uop.store(val._uop))) def full_like(self, fill_value:ConstType, dtype:DTypeLike|None=None, device:str|tuple[str, ...]|None=None, buffer=True) -> Self: diff --git a/tinygrad_repo/tinygrad/mixin/dtype.py b/tinygrad_repo/tinygrad/mixin/dtype.py index b46bac675..688a612aa 100644 --- a/tinygrad_repo/tinygrad/mixin/dtype.py +++ b/tinygrad_repo/tinygrad/mixin/dtype.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING, Self -from tinygrad.dtype import DType, DTypeLike, dtypes, to_dtype +from tinygrad.dtype import DType, DTypeLike, dtypes, to_dtype, strong_dtype, commit_int from tinygrad.uop import Ops if TYPE_CHECKING: @@ -13,6 +13,9 @@ class DTypeMixin: @classmethod def _wrap_uop(cls, u:'UOp') -> Self: raise NotImplementedError + def commit_dtype(self, default_int:DType|None=None) -> DType: + return commit_int(self._uop.vmin, self._uop.vmax, default_int) if self.dtype is dtypes.weakint else strong_dtype(self.dtype) + def cast(self, dtype:DTypeLike) -> Self: """ Casts `self` to the given `dtype`. diff --git a/tinygrad_repo/tinygrad/mixin/elementwise.py b/tinygrad_repo/tinygrad/mixin/elementwise.py index f06fd5aa5..bed5e5dab 100644 --- a/tinygrad_repo/tinygrad/mixin/elementwise.py +++ b/tinygrad_repo/tinygrad/mixin/elementwise.py @@ -1,7 +1,7 @@ import math, functools, operator from typing import TYPE_CHECKING, Literal, Self from tinygrad.uop import Ops -from tinygrad.dtype import dtypes, ConstType, PyConst, least_upper_dtype, least_upper_float, weak_dtype +from tinygrad.dtype import dtypes, ConstType, DType, PyConst, least_upper_dtype, least_upper_float, weak_dtype from tinygrad.helpers import argfix, polyN from tinygrad.mixin.creation import CreationMixin @@ -9,6 +9,9 @@ if TYPE_CHECKING: from tinygrad.uop.ops import UOp, sint +def remint(u:'UOp', dt:DType) -> 'UOp': + return u.ccast(dt) if u.op is Ops.CONST else u.replace(src=(remint(u.src[0], dt),)+u.src[1:]) + class ElementwiseMixin(CreationMixin): # required to implement def alu(self, op: Ops, *src: Self) -> Self: @@ -25,7 +28,8 @@ class ElementwiseMixin(CreationMixin): # keep weak CONST weak, might lift weakint -> weakfloat def promote(t): if t._uop.base.is_invalid: return t # invalid bool is weak const - if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST: return t._wrap_uop(t._uop.const_like(t._uop.base.val, weak_dtype(out_dtype))) + if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST: + return t if t.dtype == (dt:=weak_dtype(out_dtype)) else t._wrap_uop(remint(t._uop, dt)) return t.cast(out_dtype) return promote(x), promote(y) @@ -115,7 +119,8 @@ class ElementwiseMixin(CreationMixin): ``` """ a, b = self._broadcasted(x, reverse) - return a + (-b) + # alu, not +: _broadcasted already promoted these, and a second promote would cast -b (only a weak CONST is kept weak) + return a.alu(Ops.ADD, -b) def mul(self, x: Self | ConstType, reverse: bool = False) -> Self: """ @@ -245,8 +250,9 @@ class ElementwiseMixin(CreationMixin): if dtypes.is_int(a.dtype) and dtypes.is_int(b.dtype): if rounding_mode == "trunc": return a.alu(Ops.CDIV, b) if rounding_mode == "floor": return a.alu(Ops.FLOORDIV, b) - a = a.cast(dtypes.default_float) - d = a * b.reciprocal() + if dtypes.is_int(a.dtype) or a.dtype == dtypes.bool: a = a.cast(dtypes.default_float) + # alu, not *: _broadcasted already promoted these, and a second promote would cast 1/b (only a weak CONST is kept weak) + d = a.alu(Ops.MUL, b.reciprocal()) if rounding_mode is None: return d if rounding_mode == "trunc": return d.trunc() if rounding_mode == "floor": return d.floor() @@ -414,7 +420,7 @@ class ElementwiseMixin(CreationMixin): Calculates (self.exp()+other.exp()).log(), elementwise. """ a, b = self._broadcasted(other) - m = a.maximum(b) + m = (mx:=a.maximum(b)).isfinite().where(mx, 0) return ((a-m).exp() + (b-m).exp()).log() + m def where(self, x: 'Self | ConstType | sint', y: 'Self | ConstType | sint') -> Self: @@ -868,7 +874,7 @@ class ElementwiseMixin(CreationMixin): print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).asinh().numpy()) ``` """ - return (self + (self.square() + 1).sqrt()).log() + return (sg:=(self<0).where(-1.0, 1.0)) * (self*sg + (self.square() + 1).sqrt()).log() def acosh(self) -> Self: """ @@ -930,10 +936,10 @@ class ElementwiseMixin(CreationMixin): print(Tensor([-0.9, -0.6, -0.3, 0., 0.3, 0.6, 0.9]).asin().numpy()) ``` """ - # https://personal.math.ubc.ca/~cbm/aands/page_81.htm 4.4.46 - coefficients = [-0.0012624911, 0.0066700901, -0.0170881256, 0.0308918810, -0.0501743046, 0.0889789874, -0.2145988016, 1.5707963050] - x = math.pi / 2 - (1.0 - self.abs()).sqrt() * polyN(self.abs(), coefficients) - return self.sign() * x + # https://personal.math.ubc.ca/~cbm/aands/page_81.htm 4.4.46, with a0 = pi/2 so asin(0) is exactly 0 + coefficients = [-0.0012624911, 0.0066700901, -0.0170881256, 0.0308918810, -0.0501743046, 0.0889789874, -0.2145988016, math.pi / 2] + a = (s:=(self >= 0).where(1.0, -1.0)) * self + return s * (math.pi / 2 - (1.0 - a).sqrt() * polyN(a, coefficients)) def acos(self) -> Self: """ @@ -965,7 +971,7 @@ class ElementwiseMixin(CreationMixin): print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).elu().numpy()) ``` """ - return self.relu() - alpha*(1-self.exp()).relu() + return (self > 0).where(self, alpha*((self - self.relu()).exp() - 1)) def celu(self, alpha=1.0) -> Self: """ @@ -977,7 +983,7 @@ class ElementwiseMixin(CreationMixin): print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).celu().numpy()) ``` """ - return self.maximum(0) + (alpha * ((self / alpha).exp() - 1)).minimum(0) + return alpha * (self / alpha).elu() def selu(self, alpha=1.67326, gamma=1.0507) -> Self: """ @@ -989,7 +995,7 @@ class ElementwiseMixin(CreationMixin): print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).selu().numpy()) ``` """ - return gamma * (self >= 0).where(self, alpha * (self.exp() - 1)) + return gamma * self.elu(alpha) def softplus(self, beta=1.0) -> Self: """ @@ -1060,8 +1066,8 @@ class ElementwiseMixin(CreationMixin): ``` """ # https://personal.math.ubc.ca/~cbm/aands/page_299.htm 7.1.26 - t = 1.0 / (1.0 + 0.3275911 * self.abs()) - return self.sign() * (1.0 - t * polyN(t, [1.061405429, -1.453152027, 1.421413741, -0.284496736, 0.254829592]) * (-self.square()).exp()) + t = 1.0 / (1.0 + 0.3275911 * (s:=(self >= 0).where(1.0, -1.0)) * self) + return s * (1.0 - t * polyN(t, [1.061405429, -1.453152027, 1.421413741, -0.284496736, 0.254829592]) * (-self.square()).exp()) def softsign(self) -> Self: """ diff --git a/tinygrad_repo/tinygrad/mixin/gradient.py b/tinygrad_repo/tinygrad/mixin/gradient.py index 14fa98c00..93e5d213b 100644 --- a/tinygrad_repo/tinygrad/mixin/gradient.py +++ b/tinygrad_repo/tinygrad/mixin/gradient.py @@ -3,6 +3,7 @@ import math, dataclasses from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, all_metadata, broadcast_axes from tinygrad.helpers import argsort from tinygrad.dtype import sum_acc_dtype +from tinygrad.function import renumber_invalid_outputs def reduce_gradient(ctx:UOp, ret:UOp, op:Ops): if op == Ops.ADD: return (ctx._broadcast_to(ret.src[0].shape),) @@ -15,7 +16,8 @@ def reduce_gradient(ctx:UOp, ret:UOp, op:Ops): def _compact_params(body:UOp, all_args:tuple[UOp, ...]) -> tuple[UOp, tuple[UOp, ...]]: """Remove unused PARAMs from body and return compacted (body, args).""" - used = sorted({p.arg.slot: p for p in body.toposort() if p.op is Ops.PARAM}.items()) + # NOTE: don't enter nested calls, their PARAMs are lexical params of the subprogram + used = sorted({p.arg.slot: p for p in body.toposort(enter_calls=False) if p.op is Ops.PARAM}.items()) body = body.substitute({p: p.replace(arg=dataclasses.replace(p.arg, slot=j)) for j,(_, p) in enumerate(used)}, walk=True) return body, tuple(all_args[i] for i,_ in used) @@ -23,27 +25,44 @@ def call_gradient(ctx:UOp, k:UOp, needed:set[int]) -> tuple[UOp|None, ...]: fxn, args = k.src[0], k.src[1:] if k.arg.grad_fxn is not None: # put const on a device, also TODO why do we still have NOOP... - def on_dev(g, i): return g.clone(device=args[i].device if k.op is Ops.CALL else k.device) if g.device is None else g - if ctx.op is Ops.TUPLE: + def on_dev(g, i): return g.clone(device=args[i].device) if g.device is None else g + # grads align with the call's src positions (None for the body and for RETURNED outputs, wherever they are) + def arg_grads(g): + git = iter(g) + return (None,) + tuple(next(git) if a.unsharded_base.op is not Ops.RETURNED else None for a in k.src[1:]) + if ctx.op is Ops.SINK: real = [on_dev(g, i) for i,g in enumerate(ctx.src) if g.op is not Ops.NOOP] - return (None,) + (k.arg.grad_fxn(*real, call=k) if len(real) > 1 else k.arg.grad_fxn(real[0], k)) - return (None,) + k.arg.grad_fxn(on_dev(ctx, 0), k) - assert fxn.op is Ops.TUPLE, f"expected TUPLE body for gradient, got {fxn.op}" + return arg_grads(k.arg.grad_fxn(*real, call=k) if len(real) > 1 else k.arg.grad_fxn(real[0], k)) + return arg_grads(k.arg.grad_fxn(on_dev(ctx, 0), k)) + # the RETURNED inputs are the call outputs: their positions in the args get the output gradients from the AFTER rule + assert fxn.op is Ops.SINK and k.num_returned, f"expected a CALL with RETURNED inputs or a grad_fxn, got {fxn.op}" + ret_pos = [i for i, a in enumerate(args) if a.unsharded_base.op is Ops.RETURNED] + # the body stores the outputs into output PARAMs: the values are the stored values in slot order + values = UOp.sink(*[st.src[1] for st in fxn.src if st.op is Ops.STORE]) params = {x.arg.slot:x for x in fxn.toposort(enter_calls=False) if x.op == Ops.PARAM} - grad_args = ctx.src - root_grad = UOp(Ops.TUPLE, src=tuple(UOp(Ops.NOOP) if g.op is Ops.NOOP else - g if g.base.op is Ops.CONST else g.param_like(len(args)+i) for i,g in enumerate(grad_args))) - grads = compute_gradient(fxn, root_grad, set(params.values())) + # grads are collected at the flat param storage: reshape to each arg's view (max view shrunk to symbolic) + def shaped_grad(grad:UOp, i:int) -> UOp: + a = args[i] + return grad.view_as(a.shard_shape, a.axis) if a.axis is not None and isinstance(a.device, tuple) else grad.view_as(a._shape) + grad_args = tuple(ctx.src[i] for i in ret_pos) + root_grad = UOp.sink(*[UOp(Ops.NOOP) if g.op is Ops.NOOP else + g if g.device is None else g.param_like(len(args)+i) for i,g in enumerate(grad_args)]) + grads = compute_gradient(values, root_grad, set(params.values())) # for precompiled calls, substitute forward outputs with params so intermediates aren't recomputed - fwd_subs = {src: src.param_like(len(args)+len(grad_args)+i) for i, src in enumerate(fxn.src)} if k.arg.precompile else {} - fwd_outs = tuple(k.gettuple(i) for i in range(len(fxn.src))) if k.arg.precompile else () + fwd_subs = {src: src.param_like(len(args)+len(grad_args)+i) for i, src in enumerate(values.src)} if k.arg.precompile else {} + fwd_outs = k.returned_outputs if k.arg.precompile else () # collect needed gradient bodies, compact unused params, create a single backward CALL - grad_bodies = [(i, grads[p]) for i in needed if (p:=params.get(i)) is not None and p in grads] - bwd_body = UOp.maketuple(*(gb for _, gb in grad_bodies)).substitute(fwd_subs, walk=True) + grad_bodies = [(i, shaped_grad(grads[p], i)) for i in needed if (p:=params.get(i)) is not None and p in grads] + bwd_body = UOp.sink(*[gb for _, gb in grad_bodies]).substitute(fwd_subs, walk=True) + bwd_body = renumber_invalid_outputs(bwd_body) + # NOTE: args includes the RETURNED inputs so the param slots above line up; they are unused and compacted away bwd_body, compact_args = _compact_params(bwd_body, (*args, *grad_args, *fwd_outs)) - bwd_call = bwd_body.call(*compact_args, name=(k.arg.name or "")+"_backward", precompile=k.arg.precompile_backward) + bwd_outs = UOp.call_outputs(bwd_body.src, *compact_args, name=(k.arg.name or "")+"_backward", + precompile=k.arg.precompile_backward).returned_outputs gb_map = {i: idx for idx, (i, _) in enumerate(grad_bodies)} - return (None,) + tuple(bwd_call.gettuple(gb_map[i]) if i in gb_map else None for i in range(len(args))) + # align gradients with the original source positions: None at RETURNED positions, gradients elsewhere + ret_set = set(ret_pos) + return (None,) + tuple(None if i in ret_set else (bwd_outs[gb_map[i]] if i in gb_map else None) for i in range(len(args))) # ctx is grad_output pm_gradient = PatternMatcher([ @@ -74,9 +93,9 @@ pm_gradient = PatternMatcher([ (UPat(Ops.STACK, name="ret"), lambda ctx, ret: tuple(ctx[i] for i in range(len(ret.src)))), (UPat(Ops.COPY, name="ret"), lambda ctx, ret: (ctx.copy_to_device(ret.src[0].device),)), (UPat(Ops.UNSHARD, name="ret"), lambda ctx, ret: ctx.shard(ret.device, ret.axis).src), - (UPat(Ops.TUPLE), lambda ctx: ctx.src), + (UPat(Ops.SINK), lambda ctx: ctx.src), (UPat(Ops.AFTER, src=(UPat.var("d"), UPat(Ops.CALL, name="k"))), lambda ctx, d, k: - (ctx, UOp.maketuple(*(ctx if i == k.src.index(d)-1 else UOp(Ops.NOOP) for i in range(len(k.src)-1))))), + (ctx, UOp.sink(*([ctx if i == k.src.index(d)-1 else UOp(Ops.NOOP) for i in range(len(k.src)-1)])))), # clone/assign gradient passes through to val (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE))), lambda ctx: (None, ctx)), (UPat(Ops.STORE, src=(UPat(), UPat())), lambda ctx: (None, ctx)), @@ -96,18 +115,9 @@ def compute_gradient(root:UOp, root_grad:UOp, targets:set[UOp]) -> dict[UOp, UOp grads: dict[UOp, UOp] = {root: root_grad} for t0 in reversed(walk): if t0 not in grads or grads[t0].op is Ops.NOOP: continue - # GETTUPLE: accumulate gradient into a TUPLE UOp on the FUNCTION, process when we hit the FUNCTION - if t0.op is Ops.GETTUPLE: - k = t0.src[0] # the FUNCTION - assert k.op is Ops.FUNCTION and k.src[0].op is Ops.TUPLE - n_outputs = len(k.src[0].src) - prev = grads[k].src if k in grads else tuple(UOp(Ops.NOOP) for _ in range(n_outputs)) - grads[k] = UOp.maketuple(*(prev[i] + grads[t0] if i == t0.arg and prev[i].op is not Ops.NOOP else - grads[t0] if i == t0.arg else prev[i] for i in range(n_outputs))) - continue - # FUNCTION/CALL: pass needed param set so backward only computes required gradients - # (FUNCTION uses implicit TUPLE gradient or grad_fxn; CALL requires an explicit grad_fxn) - if t0.op in {Ops.FUNCTION, Ops.CALL}: + # CALL: pass needed param set so backward only computes required gradients + # (calls with RETURNED inputs use the implicit body gradient or grad_fxn; opaque CALLs require an explicit grad_fxn) + if t0.op is Ops.CALL: needed = {i for i, arg in enumerate(t0.src[1:]) if arg in targets or in_target_path.get(arg, False)} lgrads:tuple[UOp|None, ...]|None = call_gradient(grads[t0], t0, needed) else: @@ -120,9 +130,9 @@ def compute_gradient(root:UOp, root_grad:UOp, targets:set[UOp]) -> dict[UOp, UOp if k._shape is not None and v._shape is not None and k._shape != v._shape: v = v.cast(sum_acc_dtype(v.dtype))._rop(Ops.ADD, broadcast_axes(k.shape, v.shape)).reshape(k.shape).cast(v.dtype) if k in grads and grads[k].op is not Ops.NOOP: - if v.op is Ops.TUPLE and grads[k].op is Ops.TUPLE: - grads[k] = UOp.maketuple(*(p + n if (p.op is not Ops.NOOP and n.op is not Ops.NOOP) else - n if p.op is Ops.NOOP else p for p, n in zip(grads[k].src, v.src))) + if v.op is Ops.SINK and grads[k].op is Ops.SINK: + grads[k] = UOp.sink(*[p + n if (p.op is not Ops.NOOP and n.op is not Ops.NOOP) else + n if p.op is Ops.NOOP else p for p, n in zip(grads[k].src, v.src)]) else: grads[k] = grads[k] + v else: grads[k] = v if len(forward_metadata:=all_metadata.get(t0, ())): diff --git a/tinygrad_repo/tinygrad/mixin/movement.py b/tinygrad_repo/tinygrad/mixin/movement.py index b916f8af9..44ec39155 100644 --- a/tinygrad_repo/tinygrad/mixin/movement.py +++ b/tinygrad_repo/tinygrad/mixin/movement.py @@ -46,6 +46,16 @@ class MovementMixin: """ return prod(self.shape) + @property + def max_shape(self) -> tuple[int, ...]: + """The shape with every symbolic dimension replaced by its maximum.""" + from tinygrad.uop.ops import to_max_shape # deferred: ops.py imports the mixins + return to_max_shape(self.shape) + + def max_numel(self) -> int: + """The number of elements in `max_shape`.""" + return prod(self.max_shape) + def size(self, dim:int|None=None) -> sint|tuple[sint, ...]: """ Returns the size of the tensor. If `dim` is specified, return the length along dimension `dim`. Otherwise return the shape of the tensor. @@ -540,6 +550,7 @@ class MovementMixin: if dims is None: return self.flatten().roll(shifts, 0).reshape(self.shape) dims, shifts = tuple(self._resolve_dim(d) for d in make_tuple(dims, 1)), make_tuple(shifts, 1) if len(dims) != len(shifts): raise RuntimeError(f"{len(dims)=} != {len(shifts)=}") + if 0 in self.shape: return self shrink_arg: list[tuple[sint, sint]|None] = [None] * self.ndim for d, s in zip(dims, shifts): shrink_arg[d] = (delta:=self.shape[d]-s%self.shape[d], delta+self.shape[d]) return self.repeat(*tuple(2 if i in dims else 1 for i in range(self.ndim))).shrink(tuple(shrink_arg)) diff --git a/tinygrad_repo/tinygrad/mixin/op.py b/tinygrad_repo/tinygrad/mixin/op.py index 15c9c0d1e..6aa3d097f 100644 --- a/tinygrad_repo/tinygrad/mixin/op.py +++ b/tinygrad_repo/tinygrad/mixin/op.py @@ -6,7 +6,7 @@ from tinygrad.mixin.movement import MovementMixin from tinygrad.mixin.reduce import ReduceMixin from tinygrad.uop import Ops from tinygrad.uop.ops import _broadcast_shape, resolve, smax, smin, identity_element -from tinygrad.dtype import ConstType, DType, DTypeLike, Invalid, PyConst, dtypes, least_upper_dtype, sum_acc_dtype, to_dtype +from tinygrad.dtype import ConstType, DType, DTypeLike, Invalid, PyConst, dtypes, least_upper_dtype, sum_acc_dtype, to_dtype, commit_int from tinygrad.helpers import all_int, argfix, argsort, ceildiv, flatten, flat_to_grouped, fully_flatten, get_shape, make_tuple, merge_dicts, prod from tinygrad.helpers import resolve_pool_pads, round_up, IMAGE, FLOAT16, WINO @@ -82,8 +82,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): parsed = {"size":size, "boundary":(0, size), "stride":1, "collapse_dim":False} if isinstance(index,(list,tuple)): flat = fully_flatten(index) - inferred = dtypes.bool if (flat and all(isinstance(s,bool) for s in flat)) else \ - (dtypes.default_int if flat and all_int(flat) else dtypes.default_float) + inferred = dtypes.from_py(flat) if not dtypes.is_int(inferred): raise IndexError(f"{index=} contains non-int element") index = self._wrap_uop(UOp._frompy([i+size if i<0 else i for i in flat], inferred, self.device)).reshape(get_shape(index)) elif is_adv(index): @@ -186,9 +185,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): if stop is None: stop, start = start, 0 lo, hi = (start, stop-step) if step > 0 else (stop-step, start) if dtype is None: - dtype = dtypes.default_float if any(isinstance(x, float) for x in (start, stop, step)) else dtypes.default_int - # an int range too large for default_int picks int64 - if dtype is dtypes.default_int and (lo < dtype.min or dtype.max < hi): dtype = dtypes.int64 + dtype = dtypes.default_float if any(isinstance(x, float) for x in (start, stop, step)) else commit_int(lo, hi) if lo < (dt:=to_dtype(dtype)).min or dt.max < hi: raise OverflowError(f"arange [{start}, {stop}) is not representable in dtype {dtype}") # NOTE: this matches numpy, torch raises RuntimeError if stop-start and step have different signs if (output_len:=ceildiv(stop-start, step)) <= 0: return cls.full((0,), 0, dtype=dtype, buffer=False) @@ -289,6 +286,12 @@ class OpMixin(ElementwiseMixin, ReduceMixin): if value == 0: return base return MovementMixin.pad(X.const_like(True, dtypes.bool), pads).where(base, value) + def pad_to(self, shape, *args, value:ConstType=0) -> Self: + # same mask trick as _pad_constant so the fill survives backends that realize PAD as 0-fill + ret = MovementMixin.pad_to(self, shape, *args) + if value == 0 or ret is self: return ret + return MovementMixin.pad_to(self.const_like(True, dtypes.bool), shape, *args).where(ret, value) + def _pad_circular(self, pX:tuple[tuple[sint, sint], ...]) -> Self: # shrink first for negative pads, then wrap the non-negative remainder X = self.shrink(tuple((-smin(pB,0), smin(pA+sh,sh)) for (pB,pA),sh in zip(pX, self.shape))) @@ -460,6 +463,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): """ assert gradient is not None or self.shape == tuple(), "when no gradient is provided, backward must be called on a scalar tensor" if not (self.is_floating_point() and all(t.is_floating_point() for t in targets)): raise RuntimeError("only float Tensors have gradient") + if any(t.dtype in dtypes.weaks for t in targets): raise RuntimeError("cannot take gradient wrt a weak Tensor") from tinygrad.mixin.gradient import compute_gradient if gradient is None: gradient = self.const_like(1.0) target_uops = [t._uop for t in targets] @@ -512,7 +516,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): ``` """ output_dtype = self.dtype if dtypes.is_float(self.dtype) else dtypes.float32 - numerator = self.cast(sum_acc_dtype(self.dtype)).sum(axis=axis, keepdim=keepdim) + numerator = self.cast(sum_acc_dtype(self.commit_dtype())).sum(axis=axis, keepdim=keepdim) denominator = prod([si for si, so in zip(self.shape, self.sum(axis=axis, keepdim=True).shape) if resolve(si != so)]) return numerator.div(denominator).cast(output_dtype) @@ -541,7 +545,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): output_dtype = self.dtype if dtypes.is_float(self.dtype) else dtypes.float32 squares = (self - self.mean(axis=axis, keepdim=True)).square() n = prod([si for si, so in zip(self.shape, squares.sum(axis=axis, keepdim=True).shape) if resolve(si != so)]) - numerator = squares.cast(sum_acc_dtype(self.dtype)).sum(axis=axis, keepdim=keepdim) + numerator = squares.cast(sum_acc_dtype(self.commit_dtype())).sum(axis=axis, keepdim=keepdim) return numerator.div(smax(n - correction, 0)).cast(output_dtype) def var_mean(self, axis:int|Sequence[int]|None=None, keepdim=False, correction=1) -> tuple[Self, Self]: @@ -647,7 +651,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): print(t.logsumexp(axis=1).numpy()) ``` """ - m = self.max(axis=axis, keepdim=True).detach() + m = (mx:=self.max(axis=axis, keepdim=True).detach()).isfinite().where(mx, 0) return (self - m).exp().sum(axis=axis, keepdim=keepdim).log() + (m if keepdim else m.squeeze(axis)) def _softmax(self, axis, dtype:DTypeLike|None=None) -> tuple[Self, Self, Self]: @@ -748,7 +752,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): def _cumalu(self, axis:int, op:Ops) -> Self: assert self.shape[axis] != 0 and op in (Ops.ADD, Ops.MAX, Ops.MUL) pads = (None,)*(self.ndim-1) + ((self.shape[axis]-1, 0),) - pooled = self.transpose(axis,-1)._pad_constant(pads, identity_element(op, self.dtype))._pool((self.shape[axis],)) + pooled = self.transpose(axis,-1)._pad_constant(pads, identity_element(op, self.commit_dtype()))._pool((self.shape[axis],)) return getattr(pooled, {Ops.ADD: "sum", Ops.MAX: "max", Ops.MUL: "prod"}[op])(-1).transpose(axis, -1) def _split_cumalu(self, axis:int, op:Ops) -> Self: @@ -757,7 +761,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): # TODO: someday the optimizer will find this on its own # for now this is a two stage cumsum SPLIT = 256 - value = identity_element(op, self.dtype) + value = identity_element(op, self.commit_dtype()) if not isinstance(s:=self.shape[axis], int) or s <= SPLIT*2: return self._cumalu(axis, op) chunks = self.transpose(axis,-1)._pad_constant((None,)*(self.ndim-1)+((round_up(s,SPLIT)-s,0),), value).unflatten(-1,(-1,SPLIT))._cumalu(-1, op) base = chunks[..., -1]._cumalu(-1, op)._pad_constant((None,)*(chunks.ndim-2) + ((1, -1),), value) @@ -805,7 +809,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): if self.ndim == 0: return self._split_cumalu(axis, Ops.MAX), type(self).zeros(self.shape, dtype=dtypes.int32, buffer=False) values, n = self._split_cumalu(axis, Ops.MAX), int(self.shape[axis]) x, values_t = self.transpose(axis, -1), values.transpose(axis, -1) - match = x.unsqueeze(-1).eq(values_t.unsqueeze(-2)) * type(self).ones(n, n, dtype=dtypes.bool, buffer=False).triu() + match = x.unsqueeze(-1).eq(values_t.unsqueeze(-2)) * self._tri(n, n) idx = (-(match * type(self).arange(n, 0, -1).reshape(n, 1)).max(-2) + n).cast(dtypes.int32) return values, idx.transpose(-1, axis) @@ -851,8 +855,8 @@ class OpMixin(ElementwiseMixin, ReduceMixin): x = self.transpose(axis, -1) last_dim_size = x.shape[-1] x_unsqueezed = x.unsqueeze(-2) - x_cummax = x.cummax(-1)[0].detach() - mask = type(self).ones(last_dim_size, last_dim_size, buffer=False, dtype=dtypes.bool).tril() + x_cummax = (mx:=x.cummax(-1)[0].detach()).isfinite().where(mx, 0) + mask = self._tri(last_dim_size, last_dim_size, 1).logical_not() ret = mask.where(x_unsqueezed - x_cummax.unsqueeze(-1), self.dtype.min).exp().sum(-1).log() + x_cummax return ret.transpose(-1, axis) @@ -949,7 +953,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): x = blue_box.cat(flipped_green_box.flip(flip_dims), dim=crossover_dim) x = x.flatten(dim, dim+n_stages-1).shrink_to(self.shape) # compute indices for sorted values - mask = type(self).ones(orig_len, orig_len, dtype=dtypes.bool, buffer=False).tril() + mask = self._tri(orig_len, orig_len, 1).logical_not() mask = mask.reshape((None, None) + (1,)*(self.ndim-dim-1)) def compute_counts(t:Self): return (mask & t.unsqueeze(dim).eq(t.unsqueeze(dim+1))).sum(dim+1) count_orig, count_sorted = compute_counts(self), compute_counts(x) @@ -1060,7 +1064,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): reshape[i] = expand[i] = size[i] if mode == "linear": arr = type(self).arange(size[i]) - num, den = (arr*(in_sz-1), size[i]-1) if align_corners else ((arr*2+1)*in_sz - size[i], size[i]*2) + num, den = (arr*(in_sz-1), max(size[i]-1, 1)) if align_corners else ((arr*2+1)*in_sz - size[i], size[i]*2) num = num.clip(0, (in_sz-1)*den) low, high, perc = [y.reshape(reshape).expand(expand) for y in (num//den, (num+den-1)//den, (num % den).cast(dtypes.float32)/den)] x = x.gather(i, low).lerp(x.gather(i, high), perc) @@ -1122,8 +1126,8 @@ class OpMixin(ElementwiseMixin, ReduceMixin): def _inv_mask(a:Self|PyConst, b:Self|PyConst) -> Self: return mask.any(-1).logical_not().where(a, b) if reduce == "sum": return mask.where(src, 0).sum(-1).add(self if include_self else _inv_mask(self, 0)) if reduce == "prod": return mask.where(src, 1).prod(-1).mul(self if include_self else _inv_mask(self, 1)) - if reduce == "amax": return mask.where(src, m := src.dtype.min).max(-1).maximum(self if include_self else _inv_mask(self, m)) - if reduce == "amin": return mask.where(src, m := src.dtype.max).min(-1).minimum(self if include_self else _inv_mask(self, m)) + if reduce == "amax": return mask.where(src, m := src.commit_dtype().min).max(-1).maximum(self if include_self else _inv_mask(self, m)) + if reduce == "amin": return mask.where(src, m := src.commit_dtype().max).min(-1).minimum(self if include_self else _inv_mask(self, m)) if reduce == "mean": count = mask.where(1, 0).sum(-1).add(1 if include_self else _inv_mask(1, 0)) return mask.where(src, 0).sum(-1).add(self if include_self else _inv_mask(self, 0)).div(count) @@ -1365,7 +1369,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): s_ = stride if stride is not None else k_ pads = resolve_pool_pads(padding, len(k_)) if ceil_mode: pads = self._apply_ceil_mode(pads, k_, s_, dilation) - pooled = self._pad_constant(((0,0),)*(self.ndim-len(k_)) + flat_to_grouped(pads), self.dtype.min)._pool(k_, s_, dilation) + pooled = self._pad_constant(((0,0),)*(self.ndim-len(k_)) + flat_to_grouped(pads), self.commit_dtype().min)._pool(k_, s_, dilation) if not return_indices: return pooled.max(axis) spatial_sz = int(prod(spatial_shape := self.shape[-len(k_):])) idx = type(self).arange(spatial_sz, 0, -1).reshape(spatial_shape) diff --git a/tinygrad_repo/tinygrad/mixin/reduce.py b/tinygrad_repo/tinygrad/mixin/reduce.py index 78695a40e..19931d7e6 100644 --- a/tinygrad_repo/tinygrad/mixin/reduce.py +++ b/tinygrad_repo/tinygrad/mixin/reduce.py @@ -1,6 +1,6 @@ from typing import Self, Sequence from tinygrad.uop import Ops -from tinygrad.dtype import DTypeLike, dtypes, strong_dtype, sum_acc_dtype, to_dtype +from tinygrad.dtype import DTypeLike, dtypes, sum_acc_dtype, to_dtype from tinygrad.helpers import make_tuple from tinygrad.mixin.dtype import DTypeMixin from tinygrad.mixin.movement import MovementMixin @@ -11,7 +11,7 @@ class ReduceMixin(DTypeMixin, MovementMixin): raise NotImplementedError def _reduce(self, op:Ops, axis:int|Sequence[int]|None=None, keepdim=False) -> Self: - self = self.cast(strong_dtype(self.dtype)) + self = self.cast(self.commit_dtype()) axis = tuple(self._resolve_dim(x) for x in (range(self.ndim) if axis is None else make_tuple(axis, 1))) if self.ndim == 0: axis = () ret = self._rop(op, axis) @@ -41,7 +41,7 @@ class ReduceMixin(DTypeMixin, MovementMixin): print(t.sum(axis=1).numpy()) ``` """ - ret = self.cast(sum_acc_dtype(self.dtype) if dtype is None else to_dtype(dtype))._reduce(Ops.ADD, axis, keepdim) + ret = self.cast(sum_acc_dtype(self.commit_dtype()) if dtype is None else to_dtype(dtype))._reduce(Ops.ADD, axis, keepdim) return ret.cast(self.dtype) if dtype is None and self.dtype in (dtypes.float16, dtypes.bfloat16, *dtypes.fp8s) else ret def prod(self, axis:int|Sequence[int]|None=None, keepdim=False, dtype:DTypeLike|None=None) -> Self: diff --git a/tinygrad_repo/tinygrad/nn/__init__.py b/tinygrad_repo/tinygrad/nn/__init__.py index e98186111..aae4600cc 100644 --- a/tinygrad_repo/tinygrad/nn/__init__.py +++ b/tinygrad_repo/tinygrad/nn/__init__.py @@ -99,7 +99,7 @@ class Conv2d: if isinstance(padding, str): if padding.lower() != 'same': raise ValueError(f"Invalid padding string {padding!r}, only 'same' is supported") if stride != 1: raise ValueError("padding='same' is not supported for strided convolutions") - pad = [(d*(k-1)//2, d*(k-1) - d*(k-1)//2) for d,k in zip(make_tuple(dilation, len(self.kernel_size)), self.kernel_size[::-1])] + pad = [(d*(k-1)//2, d*(k-1) - d*(k-1)//2) for d,k in zip(make_tuple(dilation, len(self.kernel_size))[::-1], self.kernel_size[::-1])] padding = tuple(flatten(pad)) self.stride, self.dilation, self.groups, self.padding = stride, dilation, groups, padding scale = 1 / math.sqrt(in_channels * prod(self.kernel_size)) @@ -305,7 +305,7 @@ class RMSNorm: from tinygrad.uop.ops import UOp, KernelInfo, Ops, AxisType def _embedding_bwd(grad_emb:UOp, call:UOp) -> tuple: - weight, idx = call.src[1:] + weight, idx = (a for a in call.src[1:] if a.unsharded_base.op is not Ops.RETURNED) is_vocab_sharded = isinstance(weight.device, tuple) and weight.axis == 0 # for multi-device: replicate grad_emb and idx on all devices if isinstance(weight.device, tuple): @@ -359,7 +359,7 @@ def _embedding_bwd(grad_emb:UOp, call:UOp) -> tuple: if device in ("CPU", "NULL"): atomic_arg = "__atomic_fetch_add({0}, {1}, __ATOMIC_RELAXED);" elif device == "AMD": atomic_arg = "__hip_atomic_fetch_add({0}, {1}, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);" else: raise NotImplementedError(f"no atomics for device {device}") - atomic = UOp(Ops.CUSTOM, src=(grad_weight.index(local_token_id, j_idx), grad_val), arg = atomic_arg) + atomic = UOp(Ops.CUSTOM, src=(grad_weight.index(local_token_id, j_idx), grad_val), arg=(atomic_arg, dtypes.void)) return atomic.end(i, j_outer, j_inner).sink(arg=KernelInfo(name="embedding_bwd", opts_to_apply=())) grad_weight_uop = grad_weight_uop.custom_kernel(grad_emb, idx, fxn=_embedding_bwd_kernel)[0] diff --git a/tinygrad_repo/tinygrad/nn/onnx.py b/tinygrad_repo/tinygrad/nn/onnx.py index a8a19f077..898dd2f68 100644 --- a/tinygrad_repo/tinygrad/nn/onnx.py +++ b/tinygrad_repo/tinygrad/nn/onnx.py @@ -46,6 +46,7 @@ class Domain(enum.Enum): MICROSOFT_NCHWC = "com.microsoft.nchwc" MICROSOFT_EXPERIMENTAL = "com.microsoft.experimental" PYTORCH_ATEN = "org.pytorch.aten" + TINYGRAD = "org.tinygrad" @classmethod def from_onnx(cls, domain: str | None) -> "Domain": return cls.ONNX if domain is None or domain == "" else cls(domain) @@ -537,6 +538,10 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT return ___wrapper return __decorator + # ***** Tinygrad Custom Ops ***** + def contiguous_1(x:Tensor): return x.contiguous() + Contiguous = {OpSetId(Domain.TINYGRAD, 1):contiguous_1} + # ***** Property/Graph Ops ***** def If(condition:Tensor, else_branch:OnnxRunner, then_branch:OnnxRunner, intermediate_tensors:dict[str, Tensor]): def run_branch(branch:OnnxRunner): @@ -556,8 +561,8 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT return tuple(t if cond else e for t,e in zip(then_out.values(), else_out.values())) def Identity(x:Tensor): return x - def Constant(sparse_value:Tensor|None=None, value:Tensor|None=None, value_float:float|None=None, value_floats:list[float]|None=None, - value_int:int|None=None, value_ints:list[int]|None=None, value_string:str|None=None, value_strings:list[str]|None=None): + def Constant(sparse_value:Tensor|None=None, value:Tensor|None=None, value_float:float|None=None, value_floats:tuple[float, ...]|None=None, + value_int:int|None=None, value_ints:tuple[int, ...]|None=None, value_string:str|None=None, value_strings:tuple[str, ...]|None=None): if value is not None: return value if value_float is not None: return Tensor(value_float, dtype=dtypes.float32) if value_floats is not None: return Tensor(list(value_floats), dtype=dtypes.float32) @@ -594,7 +599,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT # ***** Unary Ops (math) ***** def Not(x:Tensor): return x.logical_not() - def Clip(x: Tensor, min:Tensor|None=None, max:Tensor|None=None): return x if min is None and max is None else x.clip(min, max) # noqa: A002 # pylint: disable=redefined-builtin + def Clip(x: Tensor, min:Tensor|float|None=None, max:Tensor|float|None=None): return x if min is None and max is None else x.clip(min, max) # noqa: A002 # pylint: disable=redefined-builtin def IsInf(x:Tensor, detect_negative:int=1, detect_positive:int=1): return x.isinf(bool(detect_positive), bool(detect_negative)) # ***** Unary Ops (activation) ***** @@ -643,26 +648,26 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT def Min(*data_0:Tensor): return functools.reduce(Tensor.minimum, data_0) def Sum(*data_0:Tensor): return functools.reduce(Tensor.add, data_0) def Mean(*data_0:Tensor): return Sum(*data_0) / len(data_0) - def ReduceMax(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceMax(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.max(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceMin(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceMin(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.min(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceSum(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceSum(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.sum(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceMean(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceMean(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.mean(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceSumSquare(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceSumSquare(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data.square(), axes, keepdims, noop_with_empty_axes) - def ReduceProd(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceProd(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.prod(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceL1(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceL1(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data.abs(), axes, keepdims, noop_with_empty_axes) - def ReduceL2(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceL2(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): dtype = dtypes.float if data.dtype in (dtypes.float16, dtypes.bfloat16) else data.dtype return ReduceSum(data.cast(dtype).square(), axes, keepdims, noop_with_empty_axes).sqrt().cast(data.dtype) - def ReduceLogSum(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceLogSum(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data, axes, keepdims, noop_with_empty_axes).log() - def ReduceLogSumExp(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceLogSumExp(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data.exp(), axes, keepdims, noop_with_empty_axes).log() def ArgMax(x:Tensor, axis:int=0, keepdims:int=1, select_last_index:int=0): if select_last_index: return ((int(x.shape[axis])-1) - x.flip(axis).argmax(axis, keepdim=keepdims)).cast(dtypes.int64) @@ -671,32 +676,32 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT return ArgMax(-x, axis=axis, keepdims=keepdims, select_last_index=select_last_index) # ***** Movement Ops ***** - def Reshape(data:Tensor, shape:list[int], allowzero:int=0): + def Reshape(data:Tensor, shape:Sequence[int], allowzero:int=0): return data.reshape([x if x != 0 else (0 if allowzero else data.shape[i]) for i,x in enumerate(shape)]) def Flatten(x:Tensor, axis:int=1): return x.reshape(prod(x.shape[0:axis]), -1) def Expand(x:Tensor, shape:list[int]): return x.expand(_broadcast_shape(x.shape, tuple(shape))) def Shrink(x:Tensor, bias:float=0.0, lambd:float=0.5): return (x < -lambd)*(x+bias) + (x > lambd)*(x-bias) - def Transpose(x:Tensor, perm:list[int]|None=None): return x.permute(order=perm or list(range(x.ndim)[::-1])) + def Transpose(x:Tensor, perm:tuple[int, ...]|None=None): return x.permute(order=perm or list(range(x.ndim)[::-1])) - def Squeeze(data:Tensor, axes:list[int]|None=None): + def Squeeze(data:Tensor, axes:Sequence[int]|None=None): return data.squeeze() if axes is None else functools.reduce(lambda d, dim: d.squeeze(dim), sorted(axes, reverse=True), data) - def Unsqueeze(data:Tensor, axes:list[int]): return functools.reduce(lambda d, dim: d.unsqueeze(dim), sorted(axes), data) + def Unsqueeze(data:Tensor, axes:Sequence[int]): return functools.reduce(lambda d, dim: d.unsqueeze(dim), sorted(axes), data) def Tile(x:Tensor, repeats:list[int]): return x.repeat(repeats) def Concat(*xs:Tensor, axis:int): return Tensor.cat(*xs, dim=axis) - def Slice(data:Tensor, starts:list[int], ends:list[int], axes:list[int]|None=None, steps:list[int]|None=None): + def Slice(data:Tensor, starts:Sequence[int], ends:Sequence[int], axes:Sequence[int]|None=None, steps:list[int]|None=None): axes = axes or list(range(data.ndim)) steps = steps or [1] * data.ndim slices = [slice(None)] * data.ndim for i, axis in enumerate(axes): slices[axis] = slice(starts[i], ends[i], steps[i]) return data[tuple(slices)] - def Split(data:Tensor, split:list[int]|None=None, num_outputs:int=0, axis:int=0): + def Split(data:Tensor, split:Sequence[int]|None=None, num_outputs:int=0, axis:int=0): sz = int(data.shape[axis]) if split is None: split = [sz // num_outputs + (1 if i < sz % num_outputs else 0) for i in range(num_outputs)] return data.split(split, axis) - def Pad(x:Tensor, pads:list[int], constant_value:ConstType|None=None, axes:list[int]|None=None, + def Pad(x:Tensor, pads:Sequence[int], constant_value:ConstType|None=None, axes:list[int]|None=None, mode:Literal["constant", "reflect", "edge", "wrap"]="constant", value=0): value = _resolve_const(value if constant_value is None else constant_value) axes = axes or list(range(x.ndim)) @@ -704,7 +709,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT for i,axis in enumerate(axes): real_pads[axis%x.ndim], real_pads[axis%x.ndim+x.ndim] = pads[i], pads[i+len(axes)] return x.pad(padding=_onnx_pads_to_tiny_pads(real_pads), mode={"edge":"replicate", "wrap":"circular"}.get(mode, mode), value=value) - def CenterCropPad(t:Tensor, shape:list[int], axes:list[int]|None=None): + def CenterCropPad(t:Tensor, shape:list[int], axes:tuple[int, ...]|None=None): shrink_arg:list[None|tuple[sint,sint]] = [None] * t.ndim pad_arg:list[None|tuple[sint,sint]] = [None] * t.ndim for s, x in zip(shape, axes or range(t.ndim)): @@ -714,26 +719,26 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT return t.shrink(tuple(shrink_arg)).pad(tuple(pad_arg)) # ***** Processing Ops ***** - def AveragePool(X: Tensor, kernel_shape:list[int], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, count_include_pad:int=0, - dilations:list[int]|int=1, pads:list[int]|int=0, strides:list[int]|int=1): + def AveragePool(X: Tensor, kernel_shape:tuple[int, ...], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, count_include_pad:int=0, + dilations:tuple[int, ...]|int=1, pads:tuple[int, ...]|int=0, strides:tuple[int, ...]|int=1): pool_pads = _resolve_pool_pads(X, pads, kernel_shape, dilations, strides, auto_pad) return X.avg_pool2d(tuple(kernel_shape), strides, dilations, pool_pads, ceil_mode=ceil_mode, count_include_pad=count_include_pad) - def MaxPool(X: Tensor, kernel_shape:list[int], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, dilations:list[int]|int=1, pads:list[int]|int=0, - storage_order:int=0, strides:list[int]|int=1): + def MaxPool(X: Tensor, kernel_shape:tuple[int, ...], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, dilations:tuple[int, ...]|int=1, + pads:tuple[int, ...]|int=0, storage_order:int=0, strides:tuple[int, ...]|int=1): pool_pads = _resolve_pool_pads(X, pads, kernel_shape, dilations, strides, auto_pad) out = X.max_pool2d(tuple(kernel_shape), strides, dilations, pool_pads, ceil_mode=ceil_mode, return_indices=True) ret, idx = cast(tuple[Tensor, Tensor], out) return ret, idx.transpose(-2, -1).cast(dtypes.int64) if storage_order else idx.cast(dtypes.int64) - def Conv(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:list[int]|int=1, group:int=1, - kernel_shape:list[int]|None=None, pads:list[int]|int=0, strides:list[int]|int=1): + def Conv(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:tuple[int, ...]|int=1, group:int=1, + kernel_shape:tuple[int, ...]|None=None, pads:tuple[int, ...]|int=0, strides:tuple[int, ...]|int=1): return X.conv2d(W, B, stride=strides, groups=group, dilation=dilations, padding=_resolve_pool_pads(X, pads, kernel_shape or W.shape[2:], dilations, strides, auto_pad)) - def ConvTranspose(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:list[int]|int=1, group:int=1, - kernel_shape:list[int]|None=None, pads:list[int]|None=None, output_shape:list[int]|None=None, output_padding:list[int]|int=0, - strides:list[int]|int=1): + def ConvTranspose(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:tuple[int, ...]|int=1, group:int=1, + kernel_shape:tuple[int, ...]|None=None, pads:Sequence[int]|None=None, output_shape:Sequence[int]|None=None, + output_padding:tuple[int, ...]|int=0, strides:tuple[int, ...]|int=1): input_shape_, kernel_shape_ = X.shape[2:], (kernel_shape or W.shape[2:]) strides_, dilations_, output_padding_ = (make_tuple(x, len(input_shape_)) for x in (strides, dilations, output_padding)) if output_shape is not None: # we pad according to output_shape @@ -747,10 +752,10 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT pads = _onnx_pads_to_tiny_pads(pads) return X.conv_transpose2d(W, B, group, strides_, dilations_, pads, output_padding_) - def MaxUnpool(xT: Tensor, xI: Tensor, outshape: list[int]|None=None, kernel_shape:list[int]|None=None, pads:list[int]|int=0, - strides:list[int]|int=1): + def MaxUnpool(xT: Tensor, xI: Tensor, outshape: list[int]|None=None, kernel_shape:Sequence[int]|None=None, pads:tuple[int, ...]|int=0, + strides:tuple[int, ...]|int=1): if kernel_shape is None: kernel_shape = [] - pads_: int | tuple[int, ...] = tuple(pads) if isinstance(pads, list) else pads + pads_: int | tuple[int, ...] = pads if isinstance(pads, int) else _onnx_pads_to_tiny_pads(pads) return Tensor.max_unpool2d(xT, xI, tuple(kernel_shape), strides, 1, pads_, outshape if outshape is None else tuple(outshape)) def GlobalAveragePool(X:Tensor): return X.mean(axis=tuple(range(2, X.ndim)), keepdim=True) @@ -761,7 +766,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT if C is not None: ret = ret + beta * (C if broadcast == 0 else C.reshape([-1 if i < len(C.shape) else 1 for i in range(ret.ndim)][::-1])) return ret - def Einsum(*Inputs:list[Tensor], equation:str): return Tensor.einsum(equation, *Inputs) + def Einsum(*Inputs:Tensor, equation:str): return Tensor.einsum(equation, *Inputs) def CumSum(X:Tensor, axis:int|list[int], exclusive:int=0, reverse:int=0): axis = X._resolve_dim(_resolve_const(axis)) @@ -774,8 +779,8 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT k_ = _resolve_const(k) return x.triu(k_) if upper else x.tril(k_) - def Resize(X:Tensor, roi:list[float]|None=None, scales:list[float]|None=None, sizes:list[int]|None=None, antialias:int=0, - axes:list[int]|None=None, coordinate_transformation_mode:str='half_pixel', cubic_coeff_a:float=-0.75, exclude_outside:int=0, + def Resize(X:Tensor, roi:list[float]|None=None, scales:Sequence[float]|None=None, sizes:list[int]|None=None, antialias:int=0, + axes:Sequence[int]|None=None, coordinate_transformation_mode:str='half_pixel', cubic_coeff_a:float=-0.75, exclude_outside:int=0, extrapolation_value:float=0.0, keep_aspect_ratio_policy:str='stretch', mode:str='nearest', nearest_mode:str='round_prefer_floor'): def _apply_transformation(input_sz, output_sz, scale_dim, mode): index = Tensor.arange(output_sz) @@ -876,7 +881,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT gathered_values = [X.gather(i, idx) for idx in expanded_indices] X = sum(v * c for v, c in zip(gathered_values, expanded_coeffs)) return X.permute(*argsort(perm)) if perm else X - def Upsample(X, scales, mode): return Resize(X=X, scales=scales, mode=mode) # deprecated + def Upsample(X:Tensor, scales:Sequence[float], mode:str): return Resize(X=X, scales=scales, mode=mode) # deprecated def TopK(X:Tensor, K:int|list[int], axis:int=-1, largest:int=1, sorted:int=1): # noqa: A002 # pylint: disable=redefined-builtin val, idx = X.topk(_resolve_const(K), axis, bool(largest), bool(sorted)) @@ -937,9 +942,8 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT if segment_embedding is not None: embedding_sum = embedding_sum + embedding(segment_ids, segment_embedding.shape[0], segment_embedding) out = embedding_sum.layernorm(eps=epsilon) * gamma + beta return out, None, embedding_sum - def MeanVarianceNormalization(x:Tensor, axis:list[int]|None=None): - if axis is None: axis = [0,2,3] - return (x - x.mean(axis, keepdim=True)) / (x.std(axis, keepdim=True, correction=0) + 1e-9) + def MeanVarianceNormalization(x:Tensor, axes:Sequence[int]=(0,2,3)): + return (x - x.mean(axes, keepdim=True)) / (x.std(axes, keepdim=True, correction=0) + 1e-9) def LpNormalization(x:Tensor, axis:int=-1, p:int=2): return x / (x.abs().sum(axis, keepdim=True) if p == 1 else x.square().sum(axis, keepdim=True).sqrt()) @@ -1001,7 +1005,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT def attention_contrib(x:Tensor, weights:Tensor, bias:Tensor|None=None, mask_index:Tensor|None=None, past:Tensor|None=None, attention_bias:Tensor|None=None, past_sequence_length:Tensor|None=None, do_rotary:int=0, mask_filter_value:float=-10000.0, - num_heads:int|None=None, past_present_share_buffer:int|None=None, qkv_hidden_sizes:list[int]|None=None, + num_heads:int|None=None, past_present_share_buffer:int|None=None, qkv_hidden_sizes:Sequence[int]|None=None, rotary_embedding_dim:int|None=None, scale:float|None=None, unidirectional:int=0): assert not do_rotary and not attention_bias, "TODO" if qkv_hidden_sizes is None: qkv_hidden_sizes = [int(weights.shape[1] // 3)] * 3 @@ -1038,7 +1042,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT attn_scores = mask.where(attn_scores, mask_filter_value) if unidirectional: - causal_mask = Tensor.ones((seq_len, seq_len), dtype=dtypes.bool, buffer=False).tril() + causal_mask = Tensor._tri(seq_len, seq_len, 1).logical_not() attn_scores = causal_mask.where(attn_scores, mask_filter_value) output = attn_scores.softmax(-1) @ v @@ -1070,8 +1074,7 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT qk_matmul_return_val = scores if is_causal: - causal_mask = Tensor.ones(Q.shape[-2], K.shape[-2], dtype=dtypes.bool, buffer=False).tril(0) - scores = scores.masked_fill(causal_mask.logical_not(), -float("inf")) + scores = scores.masked_fill(Tensor._tri(Q.shape[-2], K.shape[-2], 1), -float("inf")) if attn_mask is not None: mask_to_add = attn_mask.where(0, -float("inf")) if attn_mask.dtype == dtypes.bool else attn_mask @@ -1246,8 +1249,8 @@ def get_onnx_ops() -> dict[str, types.FunctionType|dict[OpSetId, types.FunctionT ret = _qlinearop_float(GlobalAveragePool, [X], [x_zero_point], [x_scale], y_scale, y_zero_point) return ret.permute(0, *range(2, ret.ndim), 1) if channels_last else ret # NCHW -> NHWC - def ConvInteger(x: Tensor, w: Tensor, x_zero_point:Tensor = Tensor(0), w_zero_point:Tensor = Tensor(0), B: Tensor | None = None, **opts) -> Tensor: - return _op_integer(Conv, [x,w], [x_zero_point,w_zero_point], **{"B":B, **opts}) + def ConvInteger(x: Tensor, w: Tensor, x_zero_point:Tensor = Tensor(0), w_zero_point:Tensor = Tensor(0), **opts) -> Tensor: + return _op_integer(Conv, [x,w], [x_zero_point,w_zero_point], **opts) def MatMulInteger(A: Tensor, B: Tensor, a_zero_point: Tensor = Tensor(0), b_zero_point: Tensor = Tensor(0)) -> Tensor: return _op_integer(Tensor.matmul, [A,B], [a_zero_point,b_zero_point]) diff --git a/tinygrad_repo/tinygrad/nn/optim.py b/tinygrad_repo/tinygrad/nn/optim.py index b07fd75f1..2eca8b3e6 100644 --- a/tinygrad_repo/tinygrad/nn/optim.py +++ b/tinygrad_repo/tinygrad/nn/optim.py @@ -121,10 +121,10 @@ class LARS(Optimizer): self.b[i].assign(self.momentum * self.b[i] + g) # NOTE: self.b[i] is zero on the first run, no if required g = (g + self.momentum * self.b[i]) if self.nesterov else self.b[i] if self.ns_coefficients: g = g.reshape(g.shape[0], -1).newton_schulz(self.ns_steps, self.ns_coefficients).reshape(g.shape) - # muon does post momentum weight decay - if not self.pre_wd and self.wd > 0: t = t.detach() * (1.0 - self.wd * self.lr) # popular momentum does pre learning rate update if not self.classic: g = g * r * self.lr + # muon does post momentum weight decay + if not self.pre_wd and self.wd > 0: g = g + self.wd * self.lr * t.detach() ret.append(g.cast(t.dtype)) return ret, self.b diff --git a/tinygrad_repo/tinygrad/nn/state.py b/tinygrad_repo/tinygrad/nn/state.py index 17ffc538d..e6aa331d2 100644 --- a/tinygrad_repo/tinygrad/nn/state.py +++ b/tinygrad_repo/tinygrad/nn/state.py @@ -1,9 +1,9 @@ -import json, math, pathlib, zipfile, pickle, tarfile, struct, functools, io, zlib +import json, pathlib, struct, functools, io, zlib from collections import OrderedDict from typing import Any, Callable, BinaryIO, Iterable, cast from tinygrad.tensor import Tensor from tinygrad.dtype import dtypes -from tinygrad.helpers import prod, argsort, DEBUG, Timing, GlobalCounters, tqdm, round_up, T, strides_for_shape, CHUNK_SIZE +from tinygrad.helpers import prod, argsort, DEBUG, Timing, GlobalCounters, tqdm, round_up, T, strides_for_shape class TensorIO(io.RawIOBase, BinaryIO): def __init__(self, t: Tensor): @@ -84,59 +84,6 @@ def safe_save(tensors:dict[str, Tensor], fn:str, metadata:dict[str, Any]|None=No t[8:8+len(j)].assign(list(j.encode('utf-8'))) for k,v in safe_load(t).items(): v.assign(tensors[k]) -# tinyfs - -def fs_store(t:Tensor) -> Tensor: - """ - Store a tensor to storage. - """ - # TODO: this should work locally as well - data = t.contiguous().flatten().bitcast(dtypes.uint8) - - # pad to a multiple of 1mb - if (tsize := data.shape[0]) % CHUNK_SIZE != 0: data = data.pad((0, CHUNK_SIZE - tsize % CHUNK_SIZE)) - size = data.shape[0] - - base_chunks = math.ceil(size / CHUNK_SIZE) - tree_depth = math.ceil(math.log(base_chunks, CHUNK_SIZE // 16)) - - to_device = "CPU" if isinstance(t.device, str) and t.device.startswith("DISK") else t.device - - level_chunks = base_chunks - for _ in range(tree_depth + 1): - # assign data into tinyfs:store and read back hashes - data = Tensor.empty(data.shape[0], dtype=dtypes.uint8, device="tinyfs:store").assign(data)[:level_chunks * 16].to(to_device) - if (tsize := data.shape[0]) % CHUNK_SIZE != 0: data = data.pad((0, CHUNK_SIZE - tsize % CHUNK_SIZE)) - level_chunks = math.ceil(data.shape[0] / CHUNK_SIZE) - - return data[:16].contiguous() - -def fs_load(t:Tensor, size:int) -> Tensor: - """ - Load a tensor from storage. - - t should be a tensor of the hash to load - """ - # TODO: this should work locally as well - assert t.dtype == dtypes.uint8, "hash is expected to be uint8" - h = t.contiguous().flatten() - assert h.shape[0] == 16, "expected hash" - - base_chunks = math.ceil(size / CHUNK_SIZE) - tree_depth = math.ceil(math.log(base_chunks, CHUNK_SIZE // 16)) - data, level_chunks = h, 0 - for i in reversed(range(tree_depth + 1)): - # if not last level, its still hashes - if i > 0 or tree_depth == 0: - level_chunks = max(1, math.ceil(base_chunks / (CHUNK_SIZE // 16)**(i-1))) - out_sz = 16 * level_chunks - else: out_sz = CHUNK_SIZE * level_chunks - # assign hash into tinyfs:load and read back data - (load:=Tensor.empty(out_sz, dtype=dtypes.uint8, device="tinyfs:load"))[:data.shape[0]].assign(data) - data = load - - return data.to(t.device)[:size] - # state dict def get_state_dict(obj, prefix:str='', tensor_type=Tensor) -> dict[str, Tensor]: @@ -219,6 +166,7 @@ def load_state_dict(model, state_dict:dict[str, Tensor], strict=True, verbose=Tr @accept_filename def zip_extract(t: Tensor) -> dict[str, Tensor]: + import zipfile files: dict[str, Tensor] = {} with zipfile.ZipFile(TensorIO(t), "r") as myzip: # sadly, the extra length needs to be read from the local header of each file. @@ -249,6 +197,7 @@ def tar_extract(t: Tensor) -> dict[str, Tensor]: tensors = nn.state.tar_extract(Tensor(pathlib.Path("archive.tar"))) ``` """ + import tarfile with tarfile.open(fileobj=TensorIO(t), mode="r") as tar: return {member.name:t[member.offset_data:member.offset_data+member.size] for member in tar if member.type == tarfile.REGTYPE} @@ -303,6 +252,7 @@ def torch_load(t:Tensor) -> dict[str, Tensor]: "FloatTensor": None, "Parameter": Parameter} whitelist = {"torch", "collections", "numpy", "_codecs"} # NOTE: this is not for security, only speed class Dummy: pass + import pickle, zipfile, tarfile class TorchPickle(pickle.Unpickler): def find_class(self, module, name): module_root = module.split(".")[0] diff --git a/tinygrad_repo/tinygrad/renderer/__init__.py b/tinygrad_repo/tinygrad/renderer/__init__.py index 438af612b..3698c17d6 100644 --- a/tinygrad_repo/tinygrad/renderer/__init__.py +++ b/tinygrad_repo/tinygrad/renderer/__init__.py @@ -1,12 +1,17 @@ from __future__ import annotations from typing import Callable, cast -from dataclasses import dataclass +from dataclasses import dataclass, replace from tinygrad.helpers import prod, Target, EMULATED_DTYPES from tinygrad.uop.ops import Ops, UOp, sint, ssimplify, smin, GroupOp, PatternMatcher from tinygrad.dtype import AddrSpace, DType, dtypes -from tinygrad.codegen.opt.tc import TensorCore +from tinygrad.renderer.tc import TensorCore from tinygrad.device import Compiler +# an access takes its dtype from the buffer it indexes, so accessing at another dtype restates the storage on the buffer that owns it +def with_storage(x:UOp, dt:DType) -> UOp: + if x.op in {Ops.PARAM, Ops.BUFFER}: return x.replace(arg=replace(x.arg, dtype=dt)) + return x.replace(src=(with_storage(x.src[0], dt),)+x.src[1:]) + @dataclass(frozen=True) class Estimates: # number of FLOPS used in the Kernel @@ -35,8 +40,8 @@ class Estimates: while len(buf.src) and buf.op is not Ops.PARAM: buf = buf.src[0] if buf.op is Ops.PARAM: # u.src[0] is INDEX, cap at buffer size for re-reads (e.g. matmul) - accessed = mem.get((buf, u.op), 0) + u.src[0].max_numel() * u.src[0].dtype.scalar().itemsize * mults - mem[(buf, u.op)] = smin(accessed, buf.max_numel() * buf.dtype.scalar().itemsize) + accessed = mem.get((buf, u.op), 0) + u.src[0].max_numel() * u.src[0].dtype.itemsize * mults + mem[(buf, u.op)] = smin(accessed, buf.max_numel() * buf.dtype.itemsize) if u.op is Ops.RANGE: mult_stack.append(mults) if u.dtype is not dtypes.void: # unbounded loop, unknown trip count @@ -45,11 +50,10 @@ class Estimates: mults = mults.substitute({x:x.const_like(0) for x in mults.toposort() if x.op is Ops.SPECIAL}) if isinstance(mults, UOp) else mults elif u.op is Ops.END: mults = mult_stack.pop(-1) elif u.op is Ops.SPECIAL: mults *= cast(sint, u.src[0].ssimplify()) # NOTE: we don't push to the mult_stack here, you can't end these - elif u.op is Ops.PARAM and u.arg.addrspace == AddrSpace.ALU and u.expr == 'core_id': mults *= int(u.vmax) + 1 elif u.op is Ops.LOAD and u.src[0].addrspace != AddrSpace.REG: - lds += u.max_numel() * u.dtype.scalar().itemsize * mults + lds += u.max_numel() * u.dtype.itemsize * mults elif u.op is Ops.STORE and u.src[0].addrspace != AddrSpace.REG: - lds += u.max_numel() * u.src[1].dtype.scalar().itemsize * mults + lds += u.max_numel() * u.src[1].dtype.itemsize * mults elif u.op in GroupOp.ALU and u not in excluded: flops += (mults * (2 if u.op is Ops.MULACC else 1)) * u.max_numel() elif u.op is Ops.WMMA and u not in excluded: @@ -62,7 +66,6 @@ class Renderer: # TODO: make this generic with a list of supported types supports_float4: bool = True has_local: bool = True - has_threads: bool = False has_shared: bool = True # NOTE: these two should be in (x,y,z) order to match the max_sizes argument in get_grouped_dims global_max: tuple[int, ...]|None = (0x8FFFFFFF,) * (3) # TODO: Ops.SPECIAL int32 indexes right now diff --git a/tinygrad_repo/tinygrad/renderer/amd/elf.py b/tinygrad_repo/tinygrad/renderer/amd/elf.py index 718700cf8..8c918c063 100644 --- a/tinygrad_repo/tinygrad/renderer/amd/elf.py +++ b/tinygrad_repo/tinygrad/renderer/amd/elf.py @@ -13,7 +13,7 @@ from tinygrad.runtime.autogen.amd.cdna.ins import s_nop as s_nop_cdna _arch_map = {"gfx9": "cdna", "gfx10": "rdna3", "gfx11": "rdna3", "gfx12": "rdna4"} def assemble_linear(prg:UOp, lin:UOp, arch:str) -> bytes: - insts = [u.arg for u in lin.src] + insts = [u.arg[0] for u in lin.src] # ** scan for max vgpr/sgpr/accvgpr max_vgpr, max_sgpr, max_accvgpr = 0, 0, 0 diff --git a/tinygrad_repo/tinygrad/renderer/amd/sqtt.py b/tinygrad_repo/tinygrad/renderer/amd/sqtt.py index 08f565b85..f468f1445 100644 --- a/tinygrad_repo/tinygrad/renderer/amd/sqtt.py +++ b/tinygrad_repo/tinygrad/renderer/amd/sqtt.py @@ -664,7 +664,7 @@ def map_insts(data:bytes, lib:bytes, target:str) -> Iterator[tuple[PacketType, I if isinstance(p, (WAVESTART, WAVESTART_RDNA4, CDNA_WAVESTART)): assert p.wave not in wave_pc, "only one inflight wave per unit" wave_pc[p.wave] = next(iter(pc_map)) - elif isinstance(p, (WAVEEND, WAVEEND_RDNA4)): + elif isinstance(p, (WAVEEND, WAVEEND_RDNA4, CDNA_WAVEEND)): pc = wave_pc.pop(p.wave) yield (p, InstructionInfo(pc, p.wave, s_endpgm())) elif isinstance(p, IMMEDIATE_MASK): @@ -724,22 +724,17 @@ def format_packet(p) -> str: def print_packets(packets) -> None: skip = {"NOP", "TS_DELTA_SHORT", "TS_WAVE_STATE", "TS_DELTA_OR_MARK", "TS_DELTA_S5_W2", "TS_DELTA_S5_W3", "TS_DELTA_S8_W3", "REG", "EVENT"} if not getenv("NOSKIP") else {"NOP"} - for data in packets: - p, inst = data if isinstance(data, tuple) else (data, None) - if type(p).__name__.replace("_RDNA4", "") not in skip: print(format_packet(p), f"inst={inst.inst}" if inst is not None else '') + for p in packets: + if type(p).__name__.replace("_RDNA4", "") not in skip: print(format_packet(p)) if __name__ == "__main__": import sys, pickle from tinygrad.helpers import temp with open(temp("profile.pkl", append_user=True) if len(sys.argv) < 2 else sys.argv[1], "rb") as f: data = pickle.load(f) - prg_events = {e.tag: e for e in data if type(e).__name__ == "ProfileProgramEvent" and e.tag is not None} + prg_names = {e.tag: e.name for e in data if type(e).__name__ == "ProfileProgramEvent" and e.tag is not None} sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"] - dev_targets = {e.device:f"gfx{e.props['gfx_target_version']//1000}" for e in data if type(e).__name__ == "ProfileDeviceEvent" and e.props} evt_num = getenv("SQTT_EVENT", -1) for i, event in enumerate(sqtt_events): - prg = prg_events.get(event.kern) - print(f"=== event {i} {prg.name if prg is not None else ''} ===") - if evt_num == -1 or i == evt_num: - print_packets(map_insts(event.blob, prg.lib, dev_targets[prg.device]) if prg is not None else decode(event.blob)) - print("\n") + print(f"\n=== event {i} {prg_names.get(event.kern, '')} ===") + print_packets(decode(event.blob)) diff --git a/tinygrad_repo/tinygrad/renderer/cstyle.py b/tinygrad_repo/tinygrad/renderer/cstyle.py index e25ab0c60..a6ed23bcf 100644 --- a/tinygrad_repo/tinygrad/renderer/cstyle.py +++ b/tinygrad_repo/tinygrad/renderer/cstyle.py @@ -1,13 +1,13 @@ from typing import Literal, Callable import math, sys, struct from collections import defaultdict, Counter -from tinygrad.codegen.opt import tc +from tinygrad.renderer import tc from tinygrad.uop.ops import GroupOp, Ops, UOp, PatternMatcher, UPat, range_str, axis_letters -from tinygrad.helpers import strip_parens, getenv, prod, dedup, Target, NUM_CPU_THREADS, IMAGE, FLOAT16, is_image_shape +from tinygrad.uop.weak import commit_weak_consts +from tinygrad.helpers import strip_parens, getenv, prod, dedup, Target, IMAGE, FLOAT16, is_image_shape from tinygrad.dtype import dtypes, DType, AddrSpace, truncate, float_to_bf16 from tinygrad.renderer import Renderer - base_rewrite = PatternMatcher([ # local/reg buffers (UPat(Ops.BUFFER, name="x"), lambda ctx,x: ctx.render_buffer(x)), @@ -20,36 +20,36 @@ base_rewrite = PatternMatcher([ (UPat(Ops.IF, name="x"), lambda ctx,x: f"if ({ctx[x.src[0]]}) {{"), (UPat((Ops.ENDIF, Ops.END)), lambda ctx: "}"), + # const + (UPat.cvar("c").cast(dtypes.floats, name="x"), lambda ctx,x,c: None if math.isfinite(v:=c.val) else \ + f"({ctx.render_cast(x, ctx.nan if math.isnan(v) else ctx.infinity if v > 0 else f'-{ctx.infinity}')})"), + (UPat.cvar("c").cast(dtypes.float), lambda ctx,c: f"{c.val}f"), + (UPat.cvar("c").cast(dtypes.int64), lambda ctx,c: f"{c.val}l"), + (UPat.cvar("c").cast(dtypes.uint64, name="x"), lambda ctx,x,c: f"{truncate[x.dtype](c.val)}ul"), + (UPat.cvar("c").cast(dtypes.uint32, name="x"), lambda ctx,x,c: f"{truncate[x.dtype](c.val)}u"), + (UPat.cvar("c").cast(dtypes.bool), lambda ctx,c: "1" if c.val else "0"), + # consts are rendered to larger type and casted + (UPat.cvar("c").cast((*dtypes.fp8s, dtypes.bfloat16, dtypes.half), name="x"), lambda ctx,x,c: f"({ctx.render_cast(x, f'{c.val}f')})"), + (UPat.cvar("c").cast((dtypes.uint8, dtypes.uint16), name="x"), lambda ctx,x,c: f"({ctx.render_cast(x, f'{c.val}u')})"), + (UPat.cvar("c").cast((dtypes.int8, dtypes.int16), name="x"), lambda ctx,x,c: f"({ctx.render_cast(x, str(c.val))})"), + # default const render + (UPat.cvar("c").cast(), lambda ctx,c: str(c.val)), + # casting (UPat(Ops.CAST, name="x"), lambda ctx,x: f"__builtin_convertvector({ctx[x.src[0]]}, {ctx.render_type(x)})" \ if x.max_numel() > 1 and x.addrspace is AddrSpace.REG else None), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({ctx.render_cast(x, ctx[x.src[0]])})"), - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: ctx[x.src[0]] if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"(({ctx._render_dtype(x.dtype, addrspace=x.addrspace)})({ctx[x.src[0]]}))" + if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"__builtin_bit_cast({ctx.render_type(x)}, ({ctx.render_type(x.src[0])})({ctx[x.src[0]]}))"), # GPU stuff (UPat(Ops.BARRIER), lambda ctx: ctx.barrier), (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: f"{ctx.code_for_workitem[x.arg[0]](x.arg[-1])}; /* {(x.src[0]).render()} */"), - # const - (UPat(Ops.CONST, arg=math.inf, name="x"), lambda ctx, x: f"({ctx.render_cast(x, ctx.infinity)})"), - (UPat(Ops.CONST, arg=-math.inf, name="x"), lambda ctx, x: f"({ctx.render_cast(x, f'-{ctx.infinity}')})"), - (UPat(Ops.CONST, dtype=dtypes.floats, name="x"), lambda ctx,x: f"({ctx.render_cast(x, ctx.nan)})" if math.isnan(x.val) else None), - (UPat(Ops.CONST, dtype=dtypes.float, name="x"), lambda ctx,x: f"{x.val}f"), - (UPat(Ops.CONST, dtype=dtypes.int64, name="x"), lambda ctx,x: f"{x.val}l"), - (UPat(Ops.CONST, dtype=dtypes.uint64, name="x"), lambda ctx,x: f"{truncate[x.dtype](x.val)}ul"), - (UPat(Ops.CONST, dtype=dtypes.uint32, name="x"), lambda ctx,x: f"{truncate[x.dtype](x.val)}u"), - (UPat(Ops.CONST, dtype=dtypes.bool, name="x"), lambda ctx,x: "1" if x.val else "0"), - # consts are rendered to larger type and casted - (UPat(Ops.CONST, (*dtypes.fp8s, dtypes.bfloat16, dtypes.half), name="x"), lambda ctx,x: f"({ctx.render_cast(x, f'{x.val}f')})"), - (UPat(Ops.CONST, (dtypes.uint8, dtypes.uint16), name="x"), lambda ctx,x: f"({ctx.render_cast(x, f'{x.val}u')})"), - (UPat(Ops.CONST, (dtypes.int8, dtypes.int16), name="x"), lambda ctx,x: f"({ctx.render_cast(x, str(x.val))})"), - # default const render - (UPat(Ops.CONST, name="x"), lambda ctx,x: str(x.val)), - # SHRINK/INDEX (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var('idx')), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), - (UPat(Ops.SHRINK, src=(UPat.var("buf"), UPat.var('idx'), UPat.cvar()), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), + (UPat(Ops.SHRINK, src=(UPat.var("buf"), UPat.var('idx'), UPat.cvar().cast()), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), (UPat(Ops.STACK, name="x"), lambda ctx,x: f"{ctx.float4.replace('float4', ctx.render_type(x))}" + \ f"{ctx.float4_style[0]}{','.join([ctx[y] for y in x.src])}{ctx.float4_style[1]}"), @@ -71,11 +71,13 @@ base_rewrite = PatternMatcher([ f"({', '.join(f'({ctx.render_type(y)})({ctx[y]})' for y in x.src[1:])}))" + (";" if x.dtype is dtypes.void else "")), # custom passes through with format - (UPat((Ops.CUSTOM, Ops.CUSTOMI), name="x"), lambda ctx,x: x.arg.format(*[ctx[y] for y in x.src])), + (UPat((Ops.CUSTOM, Ops.CUSTOMI), name="x"), lambda ctx,x: x.arg[0].format(*[ctx[y] for y in x.src])), ]) def create_non_native_float_pats(dts:tuple[DType, ...], casting:bool=True): patterns = PatternMatcher([ + # a weak CONST states no width and cannot be restated: commit it at the emulated dtype a sibling src states + (UPat(GroupOp.ALU, name="x"), lambda x, dts=dts: commit_weak_consts(x, next((s.dtype for s in x.src if s.dtype in dts), None))), (UPat(Ops.WHERE, dtype=dts, src=(UPat.var("b"), UPat.var("x"), UPat.var("y")), name="w"), lambda w,b,x,y: b.where(x.cast(dtypes.float), y.cast(dtypes.float)).cast(w.dtype)), (UPat(GroupOp.ALU-{Ops.WHERE}, dtype=dts, name="x"), @@ -107,11 +109,11 @@ def uops_to_dtypes(uops:list[UOp]) -> list[tuple[DType, int]]: def _wmma_name(u:UOp) -> str: # sanitize spaces in DType.name (int8 = "signed char") - return f"WMMA_{'_'.join(map(str, u.arg[0]))}_{u.arg[1].name}_{u.dtype.scalar().name}".replace(" ", "_") + return f"WMMA_{'_'.join(map(str, u.arg[0]))}_{u.arg[1].name}_{u.dtype.name}".replace(" ", "_") # (name, dims, dtype_in, dtype_out, device, threads, upcast_sizes) def wmma_args(uops:list[UOp]): - return dedup((_wmma_name(uop), uop.arg[0], uop.arg[1], uop.dtype.scalar(), *(uop.arg[2:4]), + return dedup((_wmma_name(uop), uop.arg[0], uop.arg[1], uop.dtype, *(uop.arg[2:4]), tuple(uop.src[i].shape[-1] for i in range(3))) for uop in uops if uop.op is Ops.WMMA) @@ -126,7 +128,7 @@ class CStyleLanguage(Renderer): var_prefix: str = "const " var_suffix: str = "" barrier: str = "" - code_for_workitem: dict[Literal["g", "l", "i"], Callable] = {} + code_for_workitem: dict[Literal["g", "l"], Callable] = {} extra_args: list[str] = [] float4: str|None = None float4_style: tuple[str, str] = ('(', ')') @@ -163,8 +165,8 @@ class CStyleLanguage(Renderer): def render_index(self, x:UOp, buf:UOp, idx:UOp): if buf.addrspace == AddrSpace.ALU: # this is lane access in C - if idx.op is not Ops.CONST: return f"({self[buf]})[{self[idx]}]" - return self[buf]+(f"[{idx.val}]" if buf.max_numel() > self.gep_arr_threshold else f".{'xyzwabcd'[idx.val]}") + if not (idx.op is Ops.CAST and idx.src[0].op is Ops.CONST): return f"({self[buf]})[{self[idx]}]" + return self[buf]+(f"[{idx.src[0].val}]" if buf.max_numel() > self.gep_arr_threshold else f".{'xyzwabcd'[idx.src[0].val]}") return f"({self[buf]}+{strip_parens(self[idx]) if idx.arg == Ops.ADD else self[idx]})" def render_buffer(self, x:UOp): @@ -182,14 +184,16 @@ class CStyleLanguage(Renderer): if addrspace in (AddrSpace.LOCAL, AddrSpace.GLOBAL) or override_ptr: suffix = "*" if sz > 1: - return prefix + self.type_map.get(scalar:=dtype.scalar(), scalar.name).replace(" ", "_") + str(sz) + suffix - return prefix + self.type_map.get(scalar:=dtype.scalar(), scalar.name) + suffix + return prefix + self.type_map.get(dtype, dtype.name).replace(" ", "_") + str(sz) + suffix + return prefix + self.type_map.get(dtype, dtype.name) + suffix def render_type(self, u:UOp): return self._render_dtype(u.dtype, u.max_numel(), u.addrspace, shape=u._shape) - def render_access(self, u:UOp): + def render_ptr(self, u:UOp): + # the address of an access, vector-cast if the access reads/writes more lanes than the pointer's scalar type if u.max_numel() > 1 or u.dtype != u.src[0].dtype: - return f"*(({self._render_dtype(u.dtype, u.max_numel(), u.addrspace, override_ptr=True, shape=u._shape)})({self[u]}))" - else: return f"*{self[u]}" + return f"(({self._render_dtype(u.dtype, u.max_numel(), u.addrspace, override_ptr=True, shape=u._shape)})({self[u]}))" + else: return f"{self[u]}" + def render_access(self, u:UOp): return f"*{self.render_ptr(u)}" def render_cast(self, u:UOp, val:str) -> str: return f"({self.render_type(u)})({val})" # LEGACY @@ -210,7 +214,7 @@ class CStyleLanguage(Renderer): c: defaultdict[str, int] = defaultdict(int) name = "test" for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP}: continue + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST}: continue if u.op == Ops.STACK and len(u.src) == 0: continue if u.op is Ops.AFTER: r[u] = r[u.src[0]] @@ -228,7 +232,7 @@ class CStyleLanguage(Renderer): if u.op is Ops.SPECIAL: r[u] = u.arg elif u.op is Ops.RANGE: r[u] = f"{axis_letters[u.arg[-1]]}idx"+range_str(u) else: - prefix = {Ops.WMMA: "wmma", Ops.CONST: "const", Ops.BUFFER: "buf", Ops.CAST: "cast", Ops.BITCAST: "cast", Ops.STACK: "cast", + prefix = {Ops.WMMA: "wmma", Ops.BUFFER: "buf", Ops.CAST: "cast", Ops.BITCAST: "cast", Ops.STACK: "cast", Ops.INDEX: "bidx", Ops.LOAD: "val"}.get(u.op, "alu") r[u] = f"{prefix}{c[prefix]}" @@ -236,13 +240,14 @@ class CStyleLanguage(Renderer): assert l is not None, f"failed to render {u.op} {u.dtype} {[(x.op,x.dtype) for x in u.src]} {u.arg}" if u.op in {Ops.ENDIF, Ops.END}: depth -= 1 - if (u.op is not Ops.CAST or u.max_numel() == 1) and (u.op in {Ops.CONST, Ops.INDEX, Ops.SHRINK, Ops.CUSTOMI} or \ + if (u.op is not Ops.CAST or u.max_numel() == 1) and ((u.op is Ops.CAST and u.src[0].op is Ops.CONST) or \ + u.op in {Ops.INDEX, Ops.SHRINK, Ops.CUSTOMI} or \ (u.op is Ops.LOAD and u.src[0].addrspace == AddrSpace.REG and child_count[u] == 1) or \ - (u.op is Ops.CAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)) or \ + (u.op in {Ops.CAST, Ops.BITCAST} and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)) or \ (u.op in {Ops.STACK, *(GroupOp.ALU-{Ops.WHERE}), Ops.CAST, Ops.BITCAST} and child_count[u] == 1 and not getenv("EXPAND_SSA"))): r[u] = l else: - if u.op not in {Ops.RANGE, Ops.STORE, Ops.BUFFER} and u.dtype != dtypes.void: + if u.op not in {Ops.RANGE, Ops.BUFFER} and u.dtype != dtypes.void: l = f"{self.render_type(u)} {r[u]} = {l}" + (";" if u.op is not Ops.SPECIAL else "") kernel.append("\n".join(" "*depth + line for line in l.split("\n"))) if prefix: c[prefix] += 1 # if it was used, increment @@ -258,12 +263,12 @@ class ClangRenderer(CStyleLanguage): float4_style = ('{', '}') gep_arr_threshold = 0 has_local = False - has_threads = bool(getenv("THREADS", 1)) - global_max = (NUM_CPU_THREADS.value, 0, 0) + global_max = (1, 0, 0) infinity = "__builtin_inff()" nan = '__builtin_nanf("")' # language options + barrier = "__atomic_thread_fence(__ATOMIC_SEQ_CST);" buffer_suffix = " restrict" type_map = {dtypes.bool:"_Bool", dtypes.half:"__fp16"} code_for_op = {**({k:v for k,v in CStyleLanguage.code_for_op.items() if k not in [Ops.EXP2, Ops.SIN, Ops.LOG2, Ops.TRUNC, Ops.RECIPROCAL]}), @@ -311,16 +316,16 @@ class OpenCLRenderer(CStyleLanguage): smem_prefix = "__local " barrier = "barrier(CLK_LOCAL_MEM_FENCE);" float4 = "(float4)" - code_for_workitem = {"g": lambda x: f"get_group_id({x})", "l": lambda x: f"get_local_id({x})", "i": lambda x: f"get_global_id({x})"} + code_for_workitem = {"g": lambda x: f"get_group_id({x})", "l": lambda x: f"get_local_id({x})"} type_map = { dtypes.int8: "char", dtypes.uint8: "uchar", dtypes.uint32: "uint", dtypes.uint16: "ushort", dtypes.uint64: "ulong", dtypes.bfloat16: "ushort" } extra_matcher = create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast string_rewrite = PatternMatcher([ - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))" + if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), # bfloat16 constants need to be rendered as their bit pattern since bf16 is stored as ushort - (UPat(Ops.CONST, dtypes.bfloat16, name="x"), - lambda ctx,x: f"{(struct.unpack('I', struct.pack('f', float_to_bf16(x.val)))[0] >> 16)}u"), + (UPat.cvar("c").cast(dtypes.bfloat16), lambda ctx,c: f"{(struct.unpack('I', struct.pack('f', float_to_bf16(c.val)))[0] >> 16)}u"), # load/store image (OpenCL) (UPat.var('buf').index(UPat.var('idx_y'), UPat.var('idx_x')), lambda ctx,buf,idx_y,idx_x: f"IMAGE<{ctx[buf]}, {ctx[idx_y]}, {ctx[idx_x]}>"), (UPat(Ops.LOAD, dtype=dtypes.float, src=(UPat.var('buf').index(UPat.var('idx_y'), UPat.var('idx_x')), UPat.var("var"), UPat.var("gate"))), @@ -369,7 +374,8 @@ class MetalRenderer(CStyleLanguage): ]) + pm_manual_bf16_cast string_rewrite = PatternMatcher([ - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_type<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_type<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))" + if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), ]) + base_rewrite def render_kernel(self, function_name, kernel, bufs, uops, prefix=None): @@ -411,8 +417,7 @@ class CUDARenderer(CStyleLanguage): barrier = "__syncthreads();" float4 = "make_float4" gep_arr_threshold = 8 - code_for_workitem = {"g": lambda x: f"blockIdx.{chr(120+int(x))}", "l": lambda x: f"threadIdx.{chr(120+int(x))}", - "i": lambda x: f"(blockIdx.{chr(120+int(x))}*blockDim.{chr(120+int(x))}+threadIdx.{chr(120+int(x))})"} + code_for_workitem = {"g": lambda x: f"blockIdx.{chr(120+int(x))}", "l": lambda x: f"threadIdx.{chr(120+int(x))}"} code_for_op = { **CStyleLanguage.code_for_op, Ops.TRUNC: lambda x,dtype: f"htrunc({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"trunc({x})", Ops.SIN: lambda x,dtype: f"hsin({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"sin({x})", @@ -425,7 +430,8 @@ class CUDARenderer(CStyleLanguage): (UPat(Ops.CAST, dtypes.fp8s, UPat.var("x", dtypes.fp8s), name='y'), lambda x,y: x.cast(dtypes.float).cast(y.dtype) if x.dtype!=y.dtype else None), ]) string_rewrite = PatternMatcher([ - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"tg_bitcast<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"tg_bitcast<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))" + if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), ]) + base_rewrite def render_vector_prefix(self, dt:DType, count:int) -> str: @@ -471,7 +477,7 @@ class CUDARenderer(CStyleLanguage): class NVCCRenderer(CUDARenderer): def __init__(self, target:Target): super().__init__(target, use_nvcc=True) -def fp8_index(dtype: DType): return (dtypes.fp8e4m3, dtypes.fp8e5m2).index(dtype.scalar()) +def fp8_index(dtype: DType): return (dtypes.fp8e4m3, dtypes.fp8e5m2).index(dtype) def _ocml(op): return lambda x,dtype: f"__ocml_{op}_f{ {dtypes.half:16, dtypes.double:64}.get(dtype, 32)}({x})" class HIPRenderer(CStyleLanguage): @@ -494,21 +500,22 @@ class HIPRenderer(CStyleLanguage): (UPat(Ops.WMMA, name="x"), lambda ctx,x: f"__{_wmma_name(x)}({ctx[x.src[0]]}, {ctx[x.src[1]]}, {ctx[x.src[2]]}," f" {fp8_index(x.src[0].dtype)}, {fp8_index(x.src[0].dtype)}, 0, 0, 0, 0)" if x.arg[0][2] == 128 else None), (UPat(Ops.WMMA, name="x"), lambda ctx,x: f"__{_wmma_name(x)}({ctx[x.src[0]]}, {ctx[x.src[1]]}, {ctx[x.src[2]]}, 0, 0, 0)"), - (UPat(Ops.CONST, dtypes.fp8s, name="x"), lambda ctx,x: f"f32_to_fp8({ctx.nan}, {fp8_index(x.dtype)})" if math.isnan(x.val) else None), - (UPat(Ops.CONST, dtypes.fp8s, arg=math.inf, name="x"), lambda ctx,x: f"f32_to_fp8({ctx.infinity}, {fp8_index(x.dtype)})"), - (UPat(Ops.CONST, dtypes.fp8s, arg=-math.inf, name="x"), lambda ctx,x: f"f32_to_fp8(-{ctx.infinity}, {fp8_index(x.dtype)})"), - (UPat(Ops.CONST, dtypes.fp8s, name="x"), lambda ctx,x: f"f32_to_fp8({x.val}f, {fp8_index(x.dtype)})"), + (UPat.cvar("c").cast(dtypes.fp8s, name="x"), lambda ctx,x,c: + f"f32_to_fp8({ctx.nan if math.isnan(v:=c.val) else ctx.infinity if v == math.inf else f'-{ctx.infinity}' if v == -math.inf else f'{v}f'}," + f" {fp8_index(x.dtype)})"), (UPat(Ops.CAST, dtypes.fp8s, (UPat(dtype=dtypes.float),), name="x",), lambda ctx,x: f"f32_to_fp8({ctx[x.src[0]]}, {fp8_index(x.dtype)})"), (UPat(Ops.CAST, dtypes.float, (UPat.var("y", dtypes.fp8s),), name="x",), lambda ctx,x,y: f"__builtin_amdgcn_cvt_f32_{('fp8', 'bf8')[fp8_index(y.dtype)]}((unsigned int){ctx[x.src[0]]}, 0)"), ]) + base_rewrite + # a LOAD flagged nontemporal renders as the cache-bypassing builtin (only used on global loads) + self.string_rewrite = PatternMatcher([(UPat(Ops.LOAD, arg="nontemporal", src=(UPat.var("bidx"),)), + lambda ctx,bidx: f"__builtin_nontemporal_load({ctx.render_ptr(bidx)})")]) + self.string_rewrite # https://clang.llvm.org/docs/AttributeReference.html#amdgpu-flat-work-group-size # NOTE: this makes hlb_cifar10 twice as fast, there may be more gains in tweaking these parameters kernel_typedef = 'extern "C" __attribute__((global)) void __attribute__((amdgpu_flat_work_group_size(1, {launch_bounds})))' - code_for_workitem = {"g": lambda x: f"__ockl_get_group_id({x})", "l": lambda x: f"__ockl_get_local_id({x})", - "i": lambda x: f"(__ockl_get_group_id({x})*__ockl_get_local_size({x})+__ockl_get_local_id({x}))"} + code_for_workitem = {"g": lambda x: f"__ockl_get_group_id({x})", "l": lambda x: f"__ockl_get_local_id({x})"} code_for_op = {**CStyleLanguage.code_for_op, Ops.TRUNC: _ocml("trunc"), Ops.SIN: _ocml("sin"), Ops.LOG2: _ocml("log2"), Ops.EXP2: _ocml("exp2"), Ops.SQRT: _ocml("sqrt")} smem_prefix = "__attribute__((shared, aligned(16)))" @@ -521,8 +528,6 @@ class HIPRenderer(CStyleLanguage): (UPat(Ops.WMMA, name="x", dtype=dtypes.float), lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) if x.src[0].max_numel() == 8 and x.src[0].dtype in dtypes.fp8_ocp else None), - # bfloat16 constant casting - (UPat.cvar('x', dtypes.bfloat16), lambda x: cast_float_to_bf16(UOp.const(x.val, dtypes.float))), ]) def asm(self, prg:UOp, lin:UOp) -> bytes: @@ -538,21 +543,21 @@ class HIPRenderer(CStyleLanguage): prefix, ockl = [], [] type_map = { dtypes.bfloat16: "bf16", dtypes.float: "f32", dtypes.half: "f16", dtypes.fp8e4m3: "_fp8_fp8", dtypes.fp8e5m2: "_bf8_bf8" } used_dtypes = uops_to_dtypes(uops) - if any(u.op is Ops.CONST and not math.isfinite(u.val) for u in uops): + if any(u.op is Ops.CAST and u.src[0].op is Ops.CONST and not math.isfinite(u.src[0].val) for u in uops): prefix += ["#define INFINITY (__builtin_inff())", "#define NAN (__builtin_nanf(\"\"))"] if any(u.op is Ops.SPECIAL for u in uops): prefix.append("typedef long unsigned int size_t;") ockl = [(f"__ockl_get_{name}", "unsigned int", "size_t", "const") for name in ["local_id", "group_id", "local_size"]] ocml_ops = {Ops.EXP2: ("exp2", "pure"), Ops.LOG2: ("log2", "pure"), Ops.SQRT: ("sqrt", "const"), Ops.SIN: ("sin", ""), Ops.TRUNC: ("trunc", "")} ocml = [(f"__ocml_{ocml_ops[op][0]}_f{dt.bitsize}", dt.name, dt.name, ocml_ops[op][1]) - for op, dt in dedup((u.op, u.dtype.scalar()) for u in uops) if op in ocml_ops and dt in (dtypes.half, dtypes.float, dtypes.double)] + for op, dt in dedup((u.op, u.dtype) for u in uops) if op in ocml_ops and dt in (dtypes.half, dtypes.float, dtypes.double)] if any(dt == dtypes.bfloat16 for dt, _ in used_dtypes): prefix.append(f"typedef {'__bf16' if self.is_cdna4(self.target.arch) else 'unsigned short'} hip_bfloat16;") if any(dt == dtypes.half for dt, _ in used_dtypes): prefix.append("#define half _Float16") if any(dt in dtypes.fp8s for dt, _ in used_dtypes): prefix += ["typedef unsigned char hip_bf8;", "typedef unsigned char hip_fp8;"] if any((u.op is Ops.CAST and u.dtype in dtypes.fp8s and u.src[0].dtype == dtypes.float) or - (u.op is Ops.CONST and u.dtype in dtypes.fp8s) for u in uops): + (u.op is Ops.CAST and u.src[0].op is Ops.CONST and u.dtype in dtypes.fp8s) for u in uops): prefix.append("""static inline __attribute__((device)) unsigned char f32_to_fp8(float v, int is_bf8) { v = (((*(unsigned*)&v)&0x7F800000)!=0x7F800000)?__builtin_amdgcn_fmed3f(v,is_bf8?57344.0f:448.0f,is_bf8?-57344.0f:-448.0f) : v; return (unsigned char)(is_bf8?__builtin_amdgcn_cvt_pk_bf8_f32(v,v,0,false):__builtin_amdgcn_cvt_pk_fp8_f32(v,v,0,false));\n}""") diff --git a/tinygrad_repo/tinygrad/renderer/isa/__init__.py b/tinygrad_repo/tinygrad/renderer/isa/__init__.py index fb74f1569..a9d1b38ee 100644 --- a/tinygrad_repo/tinygrad/renderer/isa/__init__.py +++ b/tinygrad_repo/tinygrad/renderer/isa/__init__.py @@ -27,7 +27,7 @@ class IselContext: return Register(f"v{next(self.reg_n)}", 0, _cons=cons if isinstance(cons, tuple) else (cons,)) def greg(u:UOp): - if u.op in {Ops.NOOP, Ops.AFTER} and u.src: return greg(u.src[0]) + if u.op in {Ops.NOOP, Ops.AFTER, Ops.BITCAST} and u.src: return greg(u.src[0]) if isinstance(u.tag, tuple): return u.tag[0] return u.tag diff --git a/tinygrad_repo/tinygrad/renderer/isa/x86.py b/tinygrad_repo/tinygrad/renderer/isa/x86.py index 1346a0267..16fcff26c 100644 --- a/tinygrad_repo/tinygrad/renderer/isa/x86.py +++ b/tinygrad_repo/tinygrad/renderer/isa/x86.py @@ -2,11 +2,12 @@ # allow semicolons to put multiple ops on one line import sys, struct, functools from typing import cast +from dataclasses import replace from tinygrad.dtype import dtypes, DType, truncate, AddrSpace from tinygrad.uop import FastEnum, auto, Ops, GroupOp -from tinygrad.uop.ops import UOp, UPat, PatternMatcher +from tinygrad.uop.ops import UOp, UPat, PatternMatcher, promo_dtype from tinygrad.renderer.isa import ISARenderer, IselContext, Register, PreRegAllocContext, greg -from tinygrad.helpers import getenv, NUM_CPU_THREADS, unwrap, Target +from tinygrad.helpers import unwrap, Target # ***** X86 Ops ***** @@ -33,14 +34,11 @@ class X86Ops(FastEnum): # bitcasts VMOVD = auto(); VMOVQ = auto(); VMOVDm = auto(); VMOVQm = auto() # comparisons - VUCOMISS = auto(); VUCOMISD = auto() VCMPSS = auto(); VCMPSD = auto(); VCMPPS = auto(); VCMPPD = auto() - VPCMPGTB = auto(); VPCMPGTW = auto(); VPCMPGTD = auto(); VPCMPGTQ = auto() - VPCMPEQB = auto(); VPCMPEQW = auto(); VPCMPEQD = auto(); VPCMPEQQ = auto() SETNE = auto(); SETE = auto(); SETL = auto(); SETB = auto() # where CMOVNE = auto(); CMOVE = auto(); CMOVL = auto(); CMOVB = auto() - VPBLENDVB = auto(); VBLENDVPS = auto(); VBLENDVPD = auto() + VBLENDVPS = auto(); VBLENDVPD = auto() # jumps JNE = auto(); JE = auto(); JL = auto(); JB = auto(); JGE = auto(); JMP = auto() # vectorize / gep @@ -90,12 +88,11 @@ class X86GroupOp: X86Ops.VADDSS, X86Ops.VADDSD, X86Ops.VADDPS, X86Ops.VADDPD, X86Ops.VSUBSS, X86Ops.VSUBSD, X86Ops.VSUBPS, X86Ops.VSUBPD, X86Ops.VMULSS, X86Ops.VMULSD, X86Ops.VMULPS, X86Ops.VMULPD, X86Ops.VDIVSS, X86Ops.VDIVSD, X86Ops.VDIVPS, X86Ops.VDIVPD, X86Ops.VPADDB, X86Ops.VPADDW, X86Ops.VPADDD, X86Ops.VPADDQ, X86Ops.VPSUBB, X86Ops.VPSUBW, X86Ops.VPSUBD, X86Ops.VPSUBQ, - X86Ops.VPCMPEQB, X86Ops.VPCMPEQW, X86Ops.VPCMPEQD, X86Ops.VPCMPEQQ, X86Ops.VPBLENDVB, X86Ops.VBLENDVPS, X86Ops.VBLENDVPD, - X86Ops.VPCMPGTB, X86Ops.VPCMPGTW, X86Ops.VPCMPGTD, X86Ops.VPCMPGTQ, X86Ops.VCMPSS, X86Ops.VCMPSD, X86Ops.VCMPPS, X86Ops.VCMPPD, + X86Ops.VBLENDVPS, X86Ops.VBLENDVPD, X86Ops.VCMPSS, X86Ops.VCMPSD, X86Ops.VCMPPS, X86Ops.VCMPPD, X86Ops.VPMULLW, X86Ops.VPMULLD, X86Ops.VROUNDSS, X86Ops.VROUNDSD, X86Ops.VSQRTSS, X86Ops.VSQRTSD, X86Ops.VINSERTPS, X86Ops.VPINSRB, X86Ops.VPINSRW, X86Ops.VPINSRD, X86Ops.VPINSRQ, X86Ops.VPAND, X86Ops.VPOR, X86Ops.VPXOR, X86Ops.VPSLLVD, X86Ops.VPSLLVQ, X86Ops.VPSRLVD, X86Ops.VPSRLVQ, X86Ops.VPSRAVD, X86Ops.CMOVNE, X86Ops.CMOVE, X86Ops.CMOVL, X86Ops.CMOVB, - X86Ops.VCVTSI2SS, X86Ops.VCVTSI2SD, X86Ops.VCVTSS2SD, X86Ops.VCVTSD2SS, X86Ops.VUCOMISS, X86Ops.VUCOMISD, X86Ops.IDIV, X86Ops.DIV} + X86Ops.VCVTSI2SS, X86Ops.VCVTSI2SD, X86Ops.VCVTSS2SD, X86Ops.VCVTSD2SS, X86Ops.IDIV, X86Ops.DIV} # X86Ops that can write to memory WriteMem = {X86Ops.MOVm, X86Ops.MOVi, X86Ops.VMOVSSm, X86Ops.VMOVSDm, X86Ops.VMOVUPSm, X86Ops.VMOVDm, X86Ops.VMOVQm, @@ -110,7 +107,7 @@ class X86GroupOp: # X86Ops that write flags or can modify flags to undefined values WriteFlags = {X86Ops.CMP, X86Ops.CMPi, X86Ops.ADD, X86Ops.ADDi, X86Ops.SUB, X86Ops.SUBi, X86Ops.IMUL, X86Ops.IMULi, X86Ops.IDIV, X86Ops.DIV, X86Ops.SHL, X86Ops.SHLi, X86Ops.SHR, X86Ops.SHRi, X86Ops.SAR, X86Ops.SARi, X86Ops.AND, X86Ops.ANDi, X86Ops.XOR, X86Ops.XORi, - X86Ops.OR, X86Ops.ORi, X86Ops.VUCOMISS, X86Ops.VUCOMISD} + X86Ops.OR, X86Ops.ORi} # X86Ops whose first src is the rm field Rm1st = ReadMem1st | (ReadMem2nd & TwoAddress) | {X86Ops.VPSRLDQ} @@ -118,8 +115,6 @@ class X86GroupOp: # X86Ops whose second src is the rm field Rm2nd = ReadMem2nd - All = set(X86Ops) - # ***** X86 legalization ***** extra_matcher = PatternMatcher([ @@ -143,16 +138,16 @@ extra_matcher = PatternMatcher([ (UPat.var("m").where(UPat.var("a", (dtypes.bool,)+dtypes.int8s), UPat.var("b")), lambda m,a,b: m.where(a.cast(dtypes.int16), b.cast(dtypes.int16)).cast(a.dtype) if a.max_numel() == 1 else None), # float16 alus are done in float32 - (UPat(GroupOp.ALU, dtypes.float16, name="x"), lambda x: UOp(x.op, dtypes.float, - tuple(s.cast(dtypes.float) if s.dtype != dtypes.bool else s for s in x.src)).cast(x.dtype)), - (UPat(GroupOp.Comparison, src=(UPat.var("a", dtypes.float16), UPat.var("b")), name="x"), - lambda x,a,b: UOp(x.op, src=(a.cast(dtypes.float32), b.cast(dtypes.float32))).cast(x.dtype)), + (UPat(GroupOp.ALU, dtypes.float16, name="x"), lambda x: UOp(x.op, + src=tuple(s.cast(dtypes.float) if s.dtype != dtypes.bool else s for s in x.src)).cast(x.dtype)), + (UPat(GroupOp.Comparison, src=[UPat(dtype=dtypes.float16), UPat()], name="x"), + lambda x: UOp(x.op, src=tuple(s.cast(dtypes.float32) for s in x.src)).cast(x.dtype)), # no cmpne for packed ints, y != x => !(y==x) (UPat(Ops.CMPNE, src=(UPat.var("y", dtypes.ints), UPat.var("x")), name="cmp"), lambda y,x,cmp: UOp(Ops.CMPEQ, src=(y,x))^True if y.max_numel() > 1 else None), - # float where expects a mask - (UPat.var("m", dtypes.bool).where(UPat.var("a", dtypes.floats), UPat.var("b")), - lambda m,a,b: m.cast(a.dtype).ne(0).where(a, b) if m.src[0].dtype not in dtypes.floats else None), + # a float WHERE blends at the width of its value, so it needs a comparison at that width to make the mask + (UPat.var("m", dtypes.bool).where(UPat.var("a", dtypes.floats+(dtypes.weakfloat,)), UPat.var("b")).named("w"), + lambda m,a,b,w: m.cast(w.dtype).ne(0).where(a, b) if w.dtype in dtypes.floats and promo_dtype(m.src) is not w.dtype else None), # rewrite -x -> 0 - x (UPat(Ops.NEG, name="x"), lambda x: UOp(Ops.SUB, src=(x.const_like(0),) + x.src)), # TODO: add support for mod, requires support for accessing the 2nd+ reg of a multi output instruction @@ -165,37 +160,35 @@ def scratch_buffer(elem_dt:DType, count:int, slot:int) -> UOp: return UOp.placeholder((count,), elem_dt, slot, AddrSpace.LOCAL) def gated_load(ctx, addr:UOp, alt:UOp, gate:UOp, x:UOp): - local = scratch_buffer(addr.src[0].dtype.scalar(), x.max_numel(), next(ctx)) - local_idx = local.index(UOp.const(0, dtypes.int32), dtype=dtypes.uint64) - # the selected address is a 64bit value, the AFTER orders the load after the scratch store and carries the element dtype for the encoder - sel = gate.where(addr.replace(dtype=dtypes.uint64), local_idx) - ptr = UOp(Ops.AFTER, addr.dtype, (sel, (local_idx if x.max_numel() == 1 else local).store(alt))) - return ptr.load(dtype=x.dtype) + local = scratch_buffer(addr.src[0].dtype, x.max_numel(), next(ctx)) + local_idx = local.index(UOp.cconst(0, dtypes.int32)) + # the AFTER orders the load after the scratch store + sel = gate.where(addr, local_idx) + return UOp(Ops.AFTER, src=(sel, (local_idx if x.max_numel() == 1 else local).store(alt))).load() def gated_store(addr:UOp, gate:UOp, val:UOp): - local = scratch_buffer(addr.src[0].dtype.scalar(), val.max_numel(), -1) - sel = gate.where(addr.replace(dtype=dtypes.uint64), local.index(UOp.const(0, dtypes.int32), dtype=dtypes.uint64)) - return UOp(Ops.AFTER, addr.dtype, (sel,)).store(val) + local = scratch_buffer(addr.src[0].dtype, val.max_numel(), -1) + sel = gate.where(addr, local.index(UOp.cconst(0, dtypes.int32))) + return UOp(Ops.AFTER, src=(sel,)).store(val) + +# a gate the flags can be picked with, or the bool compared to zero that replaces one they can't: only an integer +# comparison sets the flags, see cmp. NOTE: the 0 is int so the bool zero-extends and compares as int (a byte compare renders +# different kernels) +def flag_gate(m:UOp) -> UOp|None: + return None if m.op in GroupOp.Comparison and m.src[0].dtype not in dtypes.floats else m.ne(UOp.cconst(0, dtypes.int)) # legalize the new style graph for isel. NOTE: this runs after the spec is verified, some of these rewrites violate it pre_isel_matcher = PatternMatcher([ - # noop casts: zero extending scalar 32bit int, same-width signed/unsigned, narrowing scalar int - (UPat.var("y", dtypes.uint32).cast(dtypes.int64s, name="x"), lambda y,x: x.replace(op=Ops.NOOP, arg=None) if y.max_numel() == 1 else None), + # widening a scalar uint32 is free, the 32bit write that produced it already zeroed the upper half + (UPat.var("y", dtypes.uint32).cast(dtypes.int64s, name="x"), lambda y,x: x.replace(op=Ops.BITCAST) if y.max_numel() == 1 else None), (UPat.var("y", dtypes.ints+(dtypes.bool,)).cast(dtypes.ints, name="x"), - lambda y,x: x.replace(op=Ops.NOOP, arg=None) if x.dtype.itemsize == y.dtype.itemsize and y.max_numel() == 1 else None), - (UPat.var("y", dtypes.ints).cast(dtypes.ints, name="x"), - lambda y,x: x.replace(op=Ops.NOOP, arg=None) if x.dtype.itemsize < y.dtype.itemsize and y.max_numel() == 1 else None), - # bitcasts between scalar floats and ints are real, rest are noops - (UPat.var("y").bitcast().named("x"), lambda y,x: None if y.dtype in dtypes.floats and x.dtype in dtypes.ints or \ - y.dtype in dtypes.ints and x.dtype in dtypes.floats else x.replace(op=Ops.NOOP, arg=None)), + lambda y,x: x.replace(op=Ops.BITCAST) if x.dtype.itemsize == y.dtype.itemsize else None), # gated load/store become a conditional move on the address, the load/store are unconditional (UPat((Ops.INDEX, Ops.SHRINK), name="addr").load(UPat.var("alt"), UPat.var("gate"), name="x"), gated_load), (UPat((Ops.INDEX, Ops.SHRINK), name="addr").store(UPat.var("val"), UPat.var("gate")), gated_store), - # TODO: remove this once we allow all flag producing ops in cmove - # if gate in scalar int cmove is not a comparison need to add one to set the flag - # NOTE: the 0 is int so the bool gate zero-extends and compares as int (a byte compare renders different kernels) - (UPat.var("m", dtypes.bool).where(UPat.var("a"), UPat.var("b")), - lambda m,a,b: m.ne(UOp.const(0, dtypes.int)).where(a,b) if m.op not in GroupOp.Comparison else None), + # a conditional backedge picks with the flags, and so does the cmove, which is legalized in isel + (UPat(Ops.END, src=(UPat(), UPat(), UPat.var("m", dtypes.bool)), name="x"), + lambda m,x: x.replace(src=x.src[:2]+(g,)) if (g:=flag_gate(m)) is not None else None), ]) # ***** X86 registers ***** @@ -217,28 +210,29 @@ CALLEE_SAVED = (RBX, RBP, GPR[12], GPR[13], GPR[14], GPR[15]) + ((RSI, RDI) + XM reg_strs = {"rax": {4:"eax", 2:"ax", 1:"al"}, "rcx": {4:"ecx", 2:"cx", 1:"cl"}, "rdx": {4:"edx", 2:"dx", 1:"dl"}, "rbx": {4:"ebx", 2:"bx", 1:"bl"}, "rsp": {4:"esp", 2:"sp", 1:"spl"}, "rbp": {4:"ebp", 2:"bp", 1:"bpl"}, "rsi": {4:"esi", 2:"si", 1:"sil"}, "rdi": {4:"edi", 2:"di", 1:"dil"}, - **{f"r{i}": {4:f"r{i}d", 2:f"r{i}w", 1:f"r{i}b"} for i in range(8, 16)}, **{f"xmm{i}": {64:f"zmm{i}", 32:f"ymm{i}"} for i in range(16)}} + **{f"r{i}": {4:f"r{i}d", 2:f"r{i}w", 1:f"r{i}b"} for i in range(8, 16)}} # ***** X86 instruction selection ***** def base(x:UOp, i:int) -> UOp: return s.src[0] if (s:=x.src[i]).op is Ops.INDEX else s -def lane(x:UOp, i:int) -> int: return s.src[1].val if (s:=x.src[i]).op is Ops.INDEX else 0 +def lane(x:UOp, i:int) -> int: return s.src[1].src[0].val if (s:=x.src[i]).op is Ops.INDEX else 0 def to_int(dt:DType): return {dtypes.float16: dtypes.int16, dtypes.float32: dtypes.int32, dtypes.float64: dtypes.int64}[dt] -def def_reg(dt:DType, reg:Register|None=None) -> UOp: return UOp(Ops.INS, dt, arg=X86Ops.DEFINE, tag=None if reg is None else (reg,)) -def imm(dt:DType, v:int) -> UOp: return UOp.const(truncate[dt](v), dt).rtag() +def def_reg(dt:DType, reg:Register|None=None) -> UOp: return UOp(Ops.INS, arg=(X86Ops.DEFINE, dt), tag=None if reg is None else (reg,)) +def imm(dt:DType, v:int) -> UOp: return UOp.cconst(truncate[dt](v), dt).rtag() def to_imm(c:UOp) -> UOp|None: - if c.op is not Ops.CONST: return None - if c.dtype is dtypes.int64: return imm(dtypes.int32, c.val) if not c.overflows(dtypes.int32) else None - if c.dtype is dtypes.uint64: return imm(dtypes.uint32, c.val) if not c.overflows(dtypes.uint32) else None - if c.dtype in dtypes.ints+(dtypes.bool,): return imm(c.dtype, c.val) + if not (c.op is Ops.CAST and (v:=c.src[0]).op is Ops.CONST): return None + if c.dtype in dtypes.int64s: return imm(dtypes.int32, v.val) if not v.overflows(dtypes.int32) else None + if c.dtype in dtypes.ints+(dtypes.bool,): return imm(c.dtype, v.val) return None +# the flag path, which only an integer comparison can take: an x86 float compare sets carry, zero and parity together when an +# operand is NaN, so a NaN reads as "below" and as "equal", and it clears sign and overflow, so nothing reads as "less" def cmp(x:UOp) -> UOp: - if x.src[0].dtype is dtypes.float32: return x.ins(X86Ops.VUCOMISS, dtype=dtypes.void) - if x.src[0].dtype is dtypes.float64: return x.ins(X86Ops.VUCOMISD, dtype=dtypes.void) + if x.src[0].dtype in dtypes.floats: raise RuntimeError(f"no flag compare for {x.src[0].dtype}, a float gate must be a mask") return x.ins(X86Ops.CMP, dtype=dtypes.void) if (i:=to_imm(x.src[1])) is None else x.ins(X86Ops.CMPi, dtype=dtypes.void, src=(x.src[0], i)) -def vcmp(x:UOp) -> UOp: - v = imm(dtypes.uint8, {Ops.CMPLT: 1, Ops.CMPNE: 4, Ops.CMPEQ: 0}[x.op]) - if x.dtype.scalar() is dtypes.float32: return x.ins(X86Ops.VCMPSS if x.max_numel() == 1 else X86Ops.VCMPPS, src=x.src + (v,)) - return x.ins(X86Ops.VCMPSD if x.max_numel() == 1 else X86Ops.VCMPPD, src=x.src + (v,)) +# comparisons that produce masks, the mask has the width of the operands +def mask(x:UOp) -> UOp: + dt, v = x.src[0].dtype, imm(dtypes.uint8, {Ops.CMPLT: 1, Ops.CMPNE: 4, Ops.CMPEQ: 0}[x.op]) + if dt is dtypes.float32: return x.ins(X86Ops.VCMPSS if x.max_numel() == 1 else X86Ops.VCMPPS, dtype=dt, src=x.src + (v,)) + return x.ins(X86Ops.VCMPSD if x.max_numel() == 1 else X86Ops.VCMPPD, dtype=dt, src=x.src + (v,)) # vinsertps xmm2, xmm0, xmm1, imm # inserts any 32 bit element in xmm1 into any position in xmm0 according to immm, result is written to xmm2 @@ -251,9 +245,9 @@ def vinsertps(x:UOp) -> UOp: # vpinsq xmm2, xmm0, rax, imm # inserts element in rax into any position in xmm0, result is written to xmm2 according to imm -def vpins(x:UOp) -> UOp: - op = {1: X86Ops.VPINSRB, 2: X86Ops.VPINSRW, 4: X86Ops.VPINSRD, 8: X86Ops.VPINSRQ}[x.dtype.scalar().itemsize] - return functools.reduce(lambda ret,i: x.ins(op, src=(ret, x.src[i], imm(dtypes.uint8, i))), range(len(x.src)), def_reg(x.dtype)) +def vpins(x:UOp, srcs:tuple[UOp, ...]) -> UOp: + op = {1: X86Ops.VPINSRB, 2: X86Ops.VPINSRW, 4: X86Ops.VPINSRD, 8: X86Ops.VPINSRQ}[x.dtype.itemsize] + return functools.reduce(lambda ret,i: x.ins(op, src=(ret, srcs[i], imm(dtypes.uint8, i))), range(len(srcs)), def_reg(x.dtype)) # we don't call ctx.vreg on the srcs to avoid duplicates, a rewrite will assign the tuple of valid registers to a vreg def idiv(ctx:IselContext, x:UOp) -> UOp: @@ -263,8 +257,8 @@ def idiv(ctx:IselContext, x:UOp) -> UOp: elif x.dtype in dtypes.uints: ext = [x.ins(X86Ops.MOVi, src=(imm(min(dtypes.uint32, x.dtype), 0),), tag=(RDX,))] else: ext = [x.ins(X86Ops.SARi, src=(x.src[0], imm(dtypes.uint8, x.dtype.itemsize * 8 - 1)), tag=(RDX,))] # for 8bit need to zero/sign extend al to ah - if x.dtype is dtypes.uint8: dividend = UOp(Ops.INS, arg=X86Ops.MOVZX, dtype=dtypes.int16, src=(x.src[0],), tag=(RAX,)) - elif x.dtype is dtypes.int8: dividend = UOp(Ops.INS, arg=X86Ops.MOVSX, dtype=dtypes.int16, src=(x.src[0],), tag=(RAX,)) + if x.dtype is dtypes.uint8: dividend = UOp(Ops.INS, arg=(X86Ops.MOVZX, dtypes.int16), src=(x.src[0],), tag=(RAX,)) + elif x.dtype is dtypes.int8: dividend = UOp(Ops.INS, arg=(X86Ops.MOVSX, dtypes.int16), src=(x.src[0],), tag=(RAX,)) else: dividend = x.ins(X86Ops.MOV, src=(x.src[0],), tag=(RAX,)) # divisor can't be in rax or rdx divisor = x.ins(X86Ops.MOV, src=(x.src[1],), tag=tuple(r for r in WGPR if r not in (RAX, RDX))) @@ -283,25 +277,30 @@ def shift(x:UOp, op:X86Ops) -> UOp: # it is materialized as an immediate so the address stays correct if the base register is ever spilled and refilled def fold_address(x:UOp) -> tuple[UOp, UOp, UOp, UOp]: def _disp(v:int) -> UOp: return imm(dtypes.int32 if abs(v) > dtypes.int8.max else dtypes.int8, v) - def _cast(v:UOp) -> UOp: return v.cast(dtypes.int64) if v.vmin < 0 else v + def _cast(v:UOp) -> UOp: return v.cast(dtypes.int64) if v.vmin < 0 else v.cast(dtypes.uint32) if v.dtype.itemsize < 4 else v if x.op not in {Ops.INDEX, Ops.SHRINK}: return (x, UOp(Ops.NOOP), _disp(0), imm(dtypes.uint8, x.dtype.itemsize)) base, idx = x.src[0], x.src[1] # buffers are indexed by element, everything else (the stack pointer) by byte scale = base.dtype.itemsize if base.op in {Ops.PARAM, Ops.BUFFER, Ops.AFTER} else 1 sz = imm(dtypes.uint8, base.dtype.itemsize) - if idx.op is Ops.ADD and idx.src[1].op is Ops.CONST: return (base, _cast(idx.src[0]), _disp(idx.src[1].val * scale), sz) - if idx.op is Ops.CONST: return (base, UOp(Ops.NOOP), _disp(idx.val * scale), sz) + if idx.op is Ops.ADD and (c:=idx.src[1]).op is Ops.CAST and c.src[0].op is Ops.CONST: + return (base, _cast(idx.src[0]), _disp(c.src[0].val * scale), sz) + if idx.op is Ops.CAST and idx.src[0].op is Ops.CONST: return (base, UOp(Ops.NOOP), _disp(idx.src[0].val * scale), sz) return (base, _cast(idx), _disp(0), sz) +# addresses are 64bit values +def lea(x:UOp) -> UOp: return x.ins(X86Ops.LEA, dtype=dtypes.uint64, src=fold_address(x)) + def abi(ctx:IselContext, x:UOp) -> UOp|None: if isinstance(x.tag, tuple): return None i = ctx.func_args.index(x) # buffer params hold addresses, their value moves as a 64bit int dt = dtypes.uint64 if x.op is Ops.PARAM and x.arg.addrspace is AddrSpace.GLOBAL else x.dtype + arg = replace(x.arg, dtype=dt) if x.op is Ops.PARAM else x.arg # the shape srcs of a PARAM are not values, tag them so they aren't materialized into registers - def _reg_arg(r:Register) -> tuple[UOp, ...]: return (x.replace(dtype=dt, src=tuple(s.rtag() for s in x.src), tag=(r,)),) + def _reg_arg(r:Register) -> tuple[UOp, ...]: return (x.replace(arg=arg, src=tuple(s.rtag() for s in x.src), tag=(r,)),) def _stack_arg(disp:int): - return (def_reg(dtypes.uint64, RSP), UOp(Ops.NOOP), UOp(Ops.INS, arg=X86Ops.FRAME_INDEX, dtype=dtypes.int32, tag=disp), imm(dtypes.uint8, 8)) + return (def_reg(dtypes.uint64, RSP), UOp(Ops.NOOP), UOp(Ops.INS, arg=(X86Ops.FRAME_INDEX, dtypes.int32), tag=disp), imm(dtypes.uint8, 8)) if sys.platform == "win32": src = _reg_arg((RCX, RDX, GPR[8], GPR[9])[i]) if i < 4 else _stack_arg((i-3)*8+32) else: src = _reg_arg((RDI, RSI, RDX, RCX, GPR[8], GPR[9])[i]) if i < 6 else _stack_arg((i-5)*8) # this move "cleanses" the abi register constraint @@ -312,7 +311,7 @@ GPR_DEST_OPS = {X86Ops.VPEXTRB, X86Ops.VPEXTRW, X86Ops.VPEXTRD, X86Ops.VPEXTRQ, XMM_OPS = {op for op in X86Ops if op.name.startswith('V')} - GPR_DEST_OPS def _is_vec_xmm(y: UOp) -> bool: - return (y.op is Ops.INS and y.arg in XMM_OPS) or (y.op not in (Ops.BUFFER, Ops.PARAM, Ops.AFTER, Ops.INS) and y.max_numel() > 1) + return (y.op is Ops.INS and y.arg[0] in XMM_OPS) or (y.op not in (Ops.BUFFER, Ops.PARAM, Ops.AFTER, Ops.INS) and y.max_numel() > 1) def _xmm_sz(x: UOp) -> X86Ops: bits = x.max_numel() * x.dtype.itemsize @@ -328,10 +327,10 @@ def _xmm_sz_m(x: UOp) -> X86Ops: def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: # register placeholders with real registers - if x.arg is X86Ops.DEFINE and x.tag is not None: return None - if x.arg is X86Ops.LOOP_CMP: return None + if x.op is Ops.INS and x.arg[0] is X86Ops.DEFINE and x.tag is not None: return None + if x.op is Ops.INS and x.arg[0] is X86Ops.LOOP_CMP: return None # this is an immediate - if x.arg is X86Ops.FRAME_INDEX: return None + if x.op is Ops.INS and x.arg[0] is X86Ops.FRAME_INDEX: return None # no register definition if x.dtype is dtypes.void: return None # already allocated vregs @@ -340,20 +339,18 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: defs = [] if isinstance(x.tag, tuple): defs = [ctx.vreg(x.tag)] elif x.op is Ops.BUFFER: defs = [ctx.vreg(WGPR)] - elif x.dtype in dtypes.floats or (x.op is Ops.INS and x.arg in XMM_OPS) or x.max_numel() > 1: defs = [ctx.vreg(XMM)] + elif x.dtype in dtypes.floats or (x.op is Ops.INS and x.arg[0] in XMM_OPS) or x.max_numel() > 1: defs = [ctx.vreg(XMM)] elif x.dtype in dtypes.ints+(dtypes.bool,): defs = [ctx.vreg(WGPR)] # TODO: add this once the scheduler can track register pressure - # if x.arg in X86GroupOp.WriteFlags: defs.append(ctx.vreg(RFLAGS)) + # if x.arg[0] in X86GroupOp.WriteFlags: defs.append(ctx.vreg(RFLAGS)) # the size src of a BUFFER is not a value, tag it so it isn't materialized into a register if x.op is Ops.BUFFER: return x.replace(src=tuple(s.rtag() for s in x.src), tag=tuple(defs)) return x.replace(tag=tuple(defs)) isel_matcher = PatternMatcher([ # **** Op -> Op **** - # cast of void is a noop - (UPat.var("y").cast(name="x"), lambda y,x: y if y.dtype == dtypes.void else None), # range is lowered to acc, cmp, jmp after regalloc - (UPat(Ops.RANGE, src=(UPat.cvar("c"),), allow_any_len=True, name="x"), lambda c,x: x.replace(src=(imm(c.dtype, c.val),) + x.src[1:])), + (UPat(Ops.RANGE, src=(UPat.cvar("c").cast(),), allow_any_len=True, name="x"), lambda c,x: x.replace(src=(imm(x.dtype, c.val),) + x.src[1:])), (UPat(Ops.RANGE, name="x"), lambda ctx,x: x.replace(tag=(ctx.vreg(WGPR),)) if not isinstance(x.tag, tuple) else None), # really all a backedge END is is an IF with a tag referencing the RANGE start label (UPat(Ops.END, src=(UPat(), UPat(), UPat(GroupOp.Comparison, name="cond")), name="x"), @@ -363,25 +360,29 @@ isel_matcher = PatternMatcher([ # so regalloc builds the prologue/epilogue naturally (UPat(Ops.SINK, name="x"), lambda x: x.replace(src=(x.ins(X86Ops.RET, src=x.src + tuple(def_reg(dtypes.uint64 if r in GPR else dtypes.float64, r) for r in CALLEE_SAVED)),)) \ - if not x.src or x.src[0].arg is not X86Ops.RET else None), + if not x.src or x.src[0].op is not Ops.INS or x.src[0].arg[0] is not X86Ops.RET else None), # function abi constraints (UPat((Ops.PARAM, Ops.SPECIAL), name="x"), abi), + # conditional moves between addresses, lea both srcs + (UPat.var("m").where(UPat((Ops.INDEX, Ops.SHRINK), name="a"), UPat((Ops.INDEX, Ops.SHRINK), name="b")), lambda m,a,b: + m.where(lea(a), lea(b)) if not _is_vec_xmm(a.src[0]) else None), # constants that can't be immediates, move them to registers - (UPat.cvar("x", dtypes.int64s), lambda x: x.ins(X86Ops.MOVABS, src=(imm(x.dtype, x.val),)) if not x.tag else None), - (UPat.cvar("x", dtypes.ints+(dtypes.bool,)), lambda x: x.ins(X86Ops.MOVi, src=(imm(x.dtype, x.val),)) if not x.tag else None), - (UPat.cvar("x", dtypes.floats), lambda x: - UOp.const(struct.unpack((dt:=to_int(x.dtype)).fmt, struct.pack(x.dtype.fmt, x.val))[0], dt).bitcast(x.dtype) if not x.tag else None), - # conditional moves that use masks NOTE: these currently assume a mask producing cmp exists - (UPat.var("m").where(UPat.var("a", dtypes.int8s+dtypes.int16s+dtypes.int32s+(dtypes.int64,)), UPat.var("b")), lambda m,a,b: - a.ins(X86Ops.VPBLENDVB, src=(b, a, m.replace(dtype=m.src[0].dtype))) if a.max_numel() > 1 else None), - (UPat.var("m").where(UPat.var("a", dtypes.float32), UPat.var("b")), lambda m,a,b: - a.ins(X86Ops.VBLENDVPS, src=(b, a, m.replace(dtype=m.src[0].dtype)))), - (UPat.var("m").where(UPat.var("a", dtypes.float64), UPat.var("b")), lambda m,a,b: - a.ins(X86Ops.VBLENDVPD, src=(b, a, m.replace(dtype=m.src[0].dtype)))), + (UPat.cvar("c").cast(dtypes.int64s, name="x"), lambda c,x: x.ins(X86Ops.MOVABS, src=(imm(x.dtype, c.val),)) if not x.tag else None), + (UPat.cvar("c").cast(dtypes.ints+(dtypes.bool,), name="x"), lambda c,x: x.ins(X86Ops.MOVi, src=(imm(x.dtype, c.val),)) if not x.tag else None), + (UPat.cvar("c").cast(dtypes.floats, name="x"), lambda c,x: + UOp.cconst(struct.unpack((dt:=to_int(x.dtype)).fmt, struct.pack(x.dtype.fmt, c.val))[0], dt).bitcast(x.dtype) if not x.tag else None), + # conditional moves that use masks, the mask has the width of the values + (UPat(GroupOp.Comparison, src=(UPat(dtype=dtypes.float32), UPat()), name="m").where(UPat.var("a", dtypes.float32), UPat.var("b")), lambda m,a,b: + a.ins(X86Ops.VBLENDVPS, src=(b, a, mask(m)))), + (UPat(GroupOp.Comparison, src=(UPat(dtype=dtypes.float64), UPat()), name="m").where(UPat.var("a", dtypes.float64), UPat.var("b")), lambda m,a,b: + a.ins(X86Ops.VBLENDVPD, src=(b, a, mask(m)))), # in this case we have a mask producing comparison whose user expects a bool, so we convert to bool - (UPat(GroupOp.Comparison, dtypes.bool, (UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: - UOp(Ops.AND, src=(x.replace(dtype=y.dtype).bitcast(dt:=to_int(y.dtype)), UOp.const(1, dt))).f(Ops.NOOP, dtype=dtypes.bool)), + (UPat(GroupOp.Comparison, src=(UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: + UOp(Ops.AND, src=(mask(x).bitcast(dt:=to_int(y.dtype)), UOp.cconst(1, dt))).bitcast(dtypes.bool)), # conditional moves that use flags + # TODO: remove this once we allow all flag producing ops in cmove + # the blends took every float gate a mask can serve, so a gate that is still not an integer comparison becomes one here + (UPat.var("m", dtypes.bool).where(UPat.var("a"), UPat.var("b")), lambda m,a,b: g.where(a, b) if (g:=flag_gate(m)) is not None else None), (UPat(Ops.CMPLT, src=(UPat(dtype=dtypes.sints), UPat()), name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVL, src=(b, a, cmp(m)))), (UPat(Ops.CMPLT, name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVB, src=(b, a, cmp(m)))), @@ -393,20 +394,10 @@ isel_matcher = PatternMatcher([ (UPat(Ops.IF, src=(UPat(Ops.CMPEQ, name="y"),), name="x"), lambda y,x: x.ins(X86Ops.JE, src=(cmp(y),))), (UPat(Ops.IF, src=(UPat(Ops.CMPNE, name="y"),), name="x"), lambda y,x: x.ins(X86Ops.JNE, src=(cmp(y),))), # comparisons whose user doesn't use the flag, move flag result to register - (UPat(Ops.CMPLT, dtypes.bool, (UPat(dtype=dtypes.uints), UPat()), name="x"), lambda x: x.ins(X86Ops.SETB, src=(cmp(x),))), - (UPat(Ops.CMPLT, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETL, src=(cmp(x),))), - (UPat(Ops.CMPEQ, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETE, src=(cmp(x),))), - (UPat(Ops.CMPNE, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETNE, src=(cmp(x),))), - # comparisons that produce masks (these aren't bool dtype) - (UPat(GroupOp.Comparison, src=(UPat(dtype=(dtypes.float32, dtypes.float64)), UPat()), name="x"), vcmp), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int8s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQB)), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int16s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQW)), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int32s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQD)), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int64s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQQ)), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int8s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTB, src=(b, a))), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int16s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTW, src=(b, a))), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int32s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTD, src=(b, a))), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int64s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTQ, src=(b, a))), + (UPat(Ops.CMPLT, src=(UPat(dtype=dtypes.uints), UPat()), name="x"), lambda x: x.ins(X86Ops.SETB, src=(cmp(x),))), + (UPat(Ops.CMPLT, name="x"), lambda x: x.ins(X86Ops.SETL, src=(cmp(x),))), + (UPat(Ops.CMPEQ, name="x"), lambda x: x.ins(X86Ops.SETE, src=(cmp(x),))), + (UPat(Ops.CMPNE, name="x"), lambda x: x.ins(X86Ops.SETNE, src=(cmp(x),))), # float unary (UPat.var("y", dtypes.float32).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSS, src=(y, y)) if x.max_numel() == 1 else x.ins(X86Ops.VSQRTPS)), (UPat.var("y", dtypes.float64).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSD, src=(y, y)) if x.max_numel() == 1 else x.ins(X86Ops.VSQRTPD)), @@ -415,20 +406,19 @@ isel_matcher = PatternMatcher([ (UPat.var("y", dtypes.float64).trunc().named("x"), lambda y,x: x.ins(X86Ops.VROUNDSD, src=(y, y, imm(dtypes.uint8, 3))) if x.max_numel() == 1 else x.ins(X86Ops.VROUNDPD, src=(y, imm(dtypes.uint8, 3)))), # for float16 we route the srcs through gprs, this is suboptimal for values in xmms, in that case we want vpunpcklwd - (UPat(Ops.STACK, dtypes.float16, name="x"), lambda x: - vpins(x.replace(src=tuple(s.bitcast(dtypes.int16) for s in x.src)))), + (UPat(Ops.STACK, dtypes.float16, name="x"), lambda x: vpins(x, tuple(s.bitcast(dtypes.int16) for s in x.src))), (UPat(Ops.STACK, dtypes.float32, name="x"), vinsertps), - (UPat(Ops.STACK, dtypes.ints+(dtypes.bool,), name="x"), vpins), + (UPat(Ops.STACK, dtypes.ints+(dtypes.bool,), name="x"), lambda x: vpins(x, x.src)), # INDEX on a vector register value extracts a single element - (UPat.var("y", dtypes.int8s+(dtypes.bool,)).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int8s+(dtypes.bool,)).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRB, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.int16s).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int16s).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRW, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.int32s).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int32s).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRD, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.int64s).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int64s).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRQ, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.floats).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.floats).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPSRLDQ, src=(y, imm(dtypes.uint8, c.val * x.dtype.itemsize))) if _is_vec_xmm(y) else None), # packed bitwise ((UPat() & UPat()).named("x"), lambda x: x.ins(X86Ops.VPAND) if x.max_numel() > 1 else None), @@ -453,15 +443,19 @@ isel_matcher = PatternMatcher([ # scalar int binary ((UPat(dtype=dtypes.ints).alu(Ops.CDIV, UPat())).named("x"), idiv), # scalar int binary with immediate - (UPat.var("a", dtypes.ints) << UPat.cvar("c"), lambda a,c: a.ins(X86Ops.SHLi, src=(a, imm(dtypes.uint8, c.val)))), - (UPat.var("a", dtypes.uints) >> UPat.cvar("c"), lambda a,c: a.ins(X86Ops.SHRi, src=(a, imm(dtypes.uint8, c.val)))), - (UPat.var("a", dtypes.sints) >> UPat.cvar("c"), lambda a,c: a.ins(X86Ops.SARi, src=(a, imm(dtypes.uint8, c.val)))), - (UPat.var("a", dtypes.ints) + UPat.cvar("c"), lambda a,c: a.ins(X86Ops.ADDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints) * UPat.cvar("c"), lambda a,c: a.ins(X86Ops.IMULi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints+(dtypes.bool,)) & UPat.cvar("c"), lambda a,c: a.ins(X86Ops.ANDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints+(dtypes.bool,)) | UPat.cvar("c"), lambda a,c: a.ins(X86Ops.ORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints+(dtypes.bool,)) ^ UPat.cvar("c"), lambda a,c: a.ins(X86Ops.XORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat(Ops.SUB, dtypes.ints, (UPat.var("a"), UPat.cvar("c"))), lambda a,c: a.ins(X86Ops.SUBi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints) << UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SHLi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.uints) >> UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SHRi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.sints) >> UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SARi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.ints) + UPat.cvar().cast(name="c"), lambda a,c: a.ins(X86Ops.ADDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints) * UPat.cvar().cast(name="c"), lambda a,c: a.ins(X86Ops.IMULi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) & UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.ANDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) | UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.ORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) ^ UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.XORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat(Ops.SUB, dtypes.ints, (UPat.var("a"), UPat.cvar().cast(name="c"))), + lambda a,c: a.ins(X86Ops.SUBi, src=(a, i)) if (i:=to_imm(c)) is not None else None), # scalar int binary with register ((UPat(dtype=dtypes.ints) << UPat()).named("x"), lambda x: shift(x, X86Ops.SHL)), ((UPat(dtype=dtypes.uints) >> UPat()).named("x"), lambda x: shift(x, X86Ops.SHR)), @@ -496,11 +490,12 @@ isel_matcher = PatternMatcher([ (UPat.var("y", dtypes.float64).cast(dtypes.float32, name="x"), lambda y,x: x.ins(X86Ops.VCVTSD2SS, src=(y, y))), (UPat.var("y", (dtypes.int32, dtypes.int64)).cast(dtypes.float32, name="x"), lambda y,x: x.ins(X86Ops.VCVTSI2SS, src=(def_reg(x.dtype), y))), (UPat.var("y", (dtypes.int32, dtypes.int64)).cast(dtypes.float64, name="x"), lambda y,x: x.ins(X86Ops.VCVTSI2SD, src=(def_reg(x.dtype), y))), - (UPat(dtype=dtypes.uints+(dtypes.bool,)).cast(dtypes.ints, name="x"), lambda x: + (UPat(dtype=(dtypes.uint8, dtypes.uint16, dtypes.bool)).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOVZX) if x.max_numel() == 1 and x.src[0].dtype.itemsize < x.dtype.itemsize else None), (UPat(dtype=dtypes.int32).cast(dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.MOVSXD) if x.max_numel() == 1 else None), (UPat(dtype=dtypes.sints).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOVSX) if x.max_numel() == 1 and x.src[0].dtype.itemsize < x.dtype.itemsize else None), + (UPat(dtype=dtypes.ints).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOV) if x.max_numel() == 1 else None), (UPat(dtype=(dtypes.uint8, dtypes.bool)).cast(dtypes.int16s, name="x"), lambda x: x.ins(X86Ops.VPMOVZXBW)), (UPat(dtype=(dtypes.uint8, dtypes.bool)).cast(dtypes.int32s, name="x"), lambda x: x.ins(X86Ops.VPMOVZXBD)), (UPat(dtype=(dtypes.uint8, dtypes.bool)).cast(dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.VPMOVZXBQ)), @@ -515,14 +510,13 @@ isel_matcher = PatternMatcher([ (UPat(dtype=dtypes.int32).cast(dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.VPMOVSXDQ)), # bitcasts between scalar floats and ints (UPat.var("y", dtypes.float16).bitcast(dtypes.int16s).named("x"), lambda y,x: x.ins(X86Ops.VPEXTRW, src=(y, imm(dtypes.uint8, 0)))), - (UPat(dtype=dtypes.int16s).bitcast(dtypes.float16).named("x"), vpins), + (UPat(dtype=dtypes.int16s).bitcast(dtypes.float16).named("x"), lambda x: vpins(x, x.src)), (UPat(dtype=dtypes.int32s).bitcast(dtypes.float32).named("x"), lambda x: x.ins(X86Ops.VMOVD)), (UPat(dtype=dtypes.int64s).bitcast(dtypes.float64).named("x"), lambda x: x.ins(X86Ops.VMOVQ)), (UPat(dtype=dtypes.float32).bitcast(dtypes.int32s).named("x"), lambda x: x.ins(X86Ops.VMOVDm)), (UPat(dtype=dtypes.float64).bitcast(dtypes.int64s).named("x"), lambda x: x.ins(X86Ops.VMOVQm)), # index on a buffer (or the stack pointer) computes an address, addresses are 64bit values - (UPat((Ops.INDEX, Ops.SHRINK), name="x"), - lambda x: x.ins(X86Ops.LEA, dtype=dtypes.uint64, src=fold_address(x)) if not _is_vec_xmm(x.src[0]) else None), + (UPat((Ops.INDEX, Ops.SHRINK), name="x"), lambda x: lea(x) if not _is_vec_xmm(x.src[0]) else None), # TODO: fuse stores, very few cases -- store cmp becomes setcc, store gep int becomes vpextr, store bitcast to int becomes vmovd/q # copy, load, store # NOTE: copy here violates the spec, it only happens post register allocation when a reg to reg move needs to be inserted @@ -551,7 +545,7 @@ isel_matcher = PatternMatcher([ # so we rematerialize. This is different from rematerialization you might want to do in regalloc because it is not optional, # regalloc shouldn't rematerialize if a src of the instruction is dead, but here you need to as there's no fallback load from stack def flag_rematerialize(ctx:PreRegAllocContext, x:UOp): - flag_def = x if x.arg in X86GroupOp.WriteFlags or x.op in (Ops.RANGE, Ops.END) else x.src[-1] if x.arg in X86GroupOp.ReadFlags else None + flag_def = x if x.op in (Ops.RANGE, Ops.END) or x.arg[0] in X86GroupOp.WriteFlags else x.src[-1] if x.arg[0] in X86GroupOp.ReadFlags else None if flag_def is None: return None if ctx.lock is not None and ctx.lock is not flag_def: ctx.clobbered.add(ctx.lock) ctx.lock = flag_def @@ -567,19 +561,19 @@ pre_regalloc_matcher = PatternMatcher([ # TODO: control flow should be overhauled so that this isn't necessary def lower_range(ctx, x:UOp) -> tuple[UOp, list[UOp]]: loop_label = "_".join(str(i) for i in x.arg[:-1]) - label = UOp(Ops.INS, arg=X86Ops.LABEL, tag=f".LOOP_{loop_label}") + label = UOp(Ops.INS, arg=(X86Ops.LABEL, dtypes.void), tag=f".LOOP_{loop_label}") # loop, cmp on backedge all we need is a jmp tag if x.dtype is dtypes.void: return (label, [label]) else: acc = x.ins(X86Ops.MOVi, src=(imm(x.dtype, 0),) + x.src[1:]) - cmp = UOp(Ops.INS, arg=X86Ops.CMPi if x.src[0].op is Ops.CONST else X86Ops.CMP, src=(acc, x.src[0])) - jump_out = UOp(Ops.INS, arg=X86Ops.JGE, src=(cmp,), tag=f".LOOP_OUT_{loop_label}") + cmp = UOp(Ops.INS, arg=(X86Ops.CMPi if x.src[0].op is Ops.CAST else X86Ops.CMP, dtypes.void), src=(acc, x.src[0])) + jump_out = UOp(Ops.INS, arg=(X86Ops.JGE, dtypes.void), src=(cmp,), tag=f".LOOP_OUT_{loop_label}") ctx.loop_label[acc] = loop_label return (acc, [acc, label, cmp, jump_out]) def lower_end(ctx, x:UOp) -> tuple[UOp, list[UOp]]: - end_label = UOp(Ops.INS, arg=X86Ops.LABEL, tag=f".LOOP_OUT_{ctx.loop_label[x.src[1]]}") - jmp = UOp(Ops.INS, arg=X86Ops.JMP, tag=f".LOOP_{ctx.loop_label[x.src[1]]}") + end_label = UOp(Ops.INS, arg=(X86Ops.LABEL, dtypes.void), tag=f".LOOP_OUT_{ctx.loop_label[x.src[1]]}") + jmp = UOp(Ops.INS, arg=(X86Ops.JMP, dtypes.void), tag=f".LOOP_{ctx.loop_label[x.src[1]]}") inc = x.src[1].ins(X86Ops.ADDi, src=(imm(x.src[1].dtype, 1),)) return (inc, [inc, jmp, end_label]) @@ -591,16 +585,16 @@ def lower_loop(ctx, x:UOp) -> tuple[UOp, list[UOp]]: # final rewrite to match the isa spec post_regalloc_matcher = PatternMatcher([ # rewrite FRAME_INDEX to IMM now that the stack size is known - (UPat(Ops.INS, arg=X86Ops.FRAME_INDEX, name="x"), lambda ctx,x: (nx:=x.const_like(ctx.stack_size + x.tag), [nx])), + (UPat(Ops.INS, name="x"), lambda ctx,x: (nx:=UOp.cconst(ctx.stack_size + x.tag, x.dtype), [nx]) if x.arg[0] is X86Ops.FRAME_INDEX else None), # expand the cmp here so we can preserve rng src edge to get label from ctx - (UPat(Ops.INS, arg=X86Ops.LOOP_CMP, name="x"), lower_loop), + (UPat(Ops.INS, name="x"), lambda ctx,x: lower_loop(ctx, x) if x.arg[0] is X86Ops.LOOP_CMP else None), # rewrite RANGE to ACC = 0 -> LABEL -> JUMP if ACC >= loop bound (UPat(Ops.RANGE, name="x"), lower_range), # rewrite END to ACC + 1 -> JUMP -> LABEL, also add the out of loop JUMP to the src so this becomes the jump target (UPat(Ops.END, name="x"), lower_end), # rewrite two address instructions to two address form, if reused src wasn't coalesced insert a move (UPat(Ops.INS, name="x"), lambda ctx,x: (nx:=x.replace(src=x.src[1:]), - [ctx.ren.copy(x.src[0], greg(x)), nx] if greg(x) != greg(x.src[0]) else [nx]) if x.arg in X86GroupOp.TwoAddress else None), + [ctx.ren.copy(x.src[0], greg(x)), nx] if greg(x) != greg(x.src[0]) else [nx]) if x.arg[0] in X86GroupOp.TwoAddress else None), ]) # ***** X86 instruction encoding ***** @@ -614,7 +608,7 @@ def encode(x:UOp, opc:int, reg:int|None=None, pp:int=0, sel:int=0, we:int=0) -> rm = cast(Register, greg(rm_uop)).index idx = cast(Register, greg(idx_uop)).index if idx_uop is not None and greg(idx_uop) is not None else 4 # for a memory operand the rm size is the element size from the address, otherwise it's the size of the value in the register - rm_sz = sz_uop.val if sz_uop is not None else rm_uop.dtype.itemsize + rm_sz = sz_uop.src[0].val if sz_uop is not None else rm_uop.dtype.itemsize reg_sz = reg_uop.dtype.itemsize if reg_uop is not None else 0 sz = reg_sz or rm_sz @@ -625,18 +619,19 @@ def encode(x:UOp, opc:int, reg:int|None=None, pp:int=0, sel:int=0, we:int=0) -> r, _x, b = reg >> 3, idx >> 3, rm >> 3 if sel: # VEX bytes vvvv = cast(Register, greg(vvvv_uop)).index if vvvv_uop is not None else 0 - l = (max(reg_sz, rm_sz) > 16) & 0b1 - if sel == 1 and _x == b == we == 0: inst += bytes([0xC5, (~r & 0b1) << 7 | (~vvvv & 0b1111) << 3 | l << 2 | pp]) - else: inst += bytes([0xC4, (~r & 0b1) << 7 | (~_x & 0b1) << 6 | (~b & 0b1) << 5 | sel, we << 7 | (~vvvv & 0b1111) << 3 | l << 2 | pp]) + if sel == 1 and _x == b == we == 0: inst += bytes([0xC5, (~r & 0b1) << 7 | (~vvvv & 0b1111) << 3 | pp]) + else: inst += bytes([0xC4, (~r & 0b1) << 7 | (~_x & 0b1) << 6 | (~b & 0b1) << 5 | sel, we << 7 | (~vvvv & 0b1111) << 3 | pp]) else: # optional PREFIX and REX bytes # PREFIX byte signaling 16 bit variant of instruction if sz == 2: inst += bytes([0x66]) # bit signaling 64 bit variant of instruction w = sz == 8 - # REX byte is required when 64 bit or an extended reg is used (index 8 - 15) or lower 8 bits of (rsp, rbp, rsi, rdi) are accessed - if w | r | _x | b | (reg_sz == 1 & reg >> 2) | (rm_sz == 1 & rm >> 2): inst += bytes([0b0100 << 4 | w << 3 | r << 2 | _x << 1 | b]) # legacy 8bit opcode is 1 less than 16-64bit variants - if (rm_sz == 1 or reg_sz == 1) and x.arg not in X86GroupOp.ReadFlags | {X86Ops.LEA}: opc -= 1 + demote = (rm_sz == 1 or reg_sz == 1) and x.arg[0] not in X86GroupOp.ReadFlags | {X86Ops.LEA} + # REX byte is required when 64 bit or an extended reg is used (index 8 - 15) or lower 8 bits of (rsp, rbp, rsi, rdi) are accessed + if w | r | _x | b | (reg_sz == 1 & reg >> 2) | (rm_sz == 1 & rm >> 2) | (demote and disp_uop is None and rm >= 4): + inst += bytes([0b0100 << 4 | w << 3 | r << 2 | _x << 1 | b]) + if demote: opc -= 1 # OPCODE byte inst += opc.to_bytes((opc.bit_length() + 7) // 8, 'big') # MODRM byte @@ -647,10 +642,10 @@ def encode(x:UOp, opc:int, reg:int|None=None, pp:int=0, sel:int=0, we:int=0) -> # 0b10 -- signals memory access with 32bit displacement # 0b11 -- signals no memory access if disp_uop is not None: - assert disp_uop.op is Ops.CONST, "displacement must be a constant" + assert disp_uop.op is Ops.CAST, "displacement must be a const" assert disp_uop.dtype in (dtypes.int8, dtypes.int32), "displacement can only be 1 or 4 byte signed int" # rbp/r13 always require a displacement - if disp_uop.val != 0 or rm == 0b101: mod = 0b01 if disp_uop.dtype.itemsize == 1 else 0b10 + if disp_uop.src[0].val != 0 or rm == 0b101: mod = 0b01 if disp_uop.dtype.itemsize == 1 else 0b10 else: mod = 0b00 else: mod = 0b11 # x 0b0 and idx 0b100 means rsp which means no index exists @@ -664,32 +659,32 @@ def encode(x:UOp, opc:int, reg:int|None=None, pp:int=0, sel:int=0, we:int=0) -> # DISP byte if mod == 0b01 or mod == 0b10: assert disp_uop is not None - inst += struct.pack(unwrap(disp_uop.dtype.fmt), disp_uop.val) + inst += struct.pack(unwrap(disp_uop.dtype.fmt), disp_uop.src[0].val) # IMM byte if imm_uop is not None: - if imm_uop.op is Ops.CONST: inst += struct.pack(unwrap(imm_uop.dtype.fmt), imm_uop.val) + if imm_uop.op is Ops.CAST: inst += struct.pack(unwrap(imm_uop.dtype.fmt), imm_uop.src[0].val) elif isinstance(greg(imm_uop), Register): inst += bytes([(greg(imm_uop).index & 0b1111) << 4 | 0b0000]) return inst # get the encoding structure of the uop # when a uop writes to memory it takes the form of a store, dtype is void, no definition address:tuple[UOp|None, ...] - if x.arg in X86GroupOp.WriteMem: + if x.arg[0] in X86GroupOp.WriteMem: if len(x.src) > 4: address, rest = x.src[:4], x.src[4:] else: address, rest = (x, None, None, None), x.src - imm_uop = rest[:1] if rest and rest[0].op is Ops.CONST else (None,) + imm_uop = rest[:1] if rest and rest[0].op is Ops.CAST else (None,) return _encode(rest[0], *address, *(None, *rest[1:])) if reg is None else _encode(None, *address, *(None, *imm_uop)) - if x.arg in X86GroupOp.Rm1st: + if x.arg[0] in X86GroupOp.Rm1st: if len(x.src) > 3: address, rest = x.src[:4], x.src[4:] else: address, rest = (x.src[0], None, None, None), x.src[1:] - imm_uop = rest[:1] if rest and rest[0].op is Ops.CONST else (None,) + imm_uop = rest[:1] if rest and rest[0].op is Ops.CAST else (None,) return _encode(x, *address, *(None, *imm_uop)) if reg is None else _encode(None, *address, *(x if sel else None, *imm_uop)) - if x.arg in X86GroupOp.Rm2nd: + if x.arg[0] in X86GroupOp.Rm2nd: if len(x.src) > 4: address, rest = x.src[1:5], x.src[:1] + x.src[5:] else: address, rest = (x.src[1], None, None, None), x.src[:1] + x.src[2:] - # cmp/vucomiss reg, rm don't define a new register + # cmp reg, rm doesn't define a new register return _encode(x, *address, *rest) if x.dtype is not dtypes.void else _encode(rest[0], *address) return None @@ -701,7 +696,7 @@ def encode(x:UOp, opc:int, reg:int|None=None, pp:int=0, sel:int=0, we:int=0) -> encodings = { # moves X86Ops.MOVABS: lambda x: - bytes([0b0100 << 4 | 0b1 << 3 | 0b00 << 2 | greg(x).index >> 3, 0xB8 + (greg(x).index & 0b111)]) + struct.pack(x.dtype.fmt, x.src[0].val), + bytes([0b0100 << 4 | 0b1 << 3 | 0b00 << 2 | greg(x).index >> 3, 0xB8 + (greg(x).index & 0b111)]) + struct.pack(x.dtype.fmt, x.src[0].src[0].val), X86Ops.MOV: lambda x: encode(x, 0x8B), X86Ops.MOVi: lambda x: encode(x, 0xC7, reg=0), X86Ops.MOVm: lambda x: encode(x, 0x89), X86Ops.LEA: lambda x: encode(x, 0x8D), X86Ops.VMOVSS: lambda x: encode(x, 0x10, pp=2, sel=1), X86Ops.VMOVSSm: lambda x: encode(x, 0x11, pp=2, sel=1), @@ -724,8 +719,8 @@ encodings = { X86Ops.VCVTPS2PD: lambda x: encode(x, 0x5A, pp=0, sel=1), X86Ops.VCVTPD2PS: lambda x: encode(x, 0x5A, pp=1, sel=1), X86Ops.VCVTTPS2DQ: lambda x: encode(x, 0x5B, pp=2, sel=1), X86Ops.VCVTTPD2DQ: lambda x: encode(x, 0xE6, pp=1, sel=1), # the int src is the 2nd src (the rm field), if it was folded into a memory operand its width is the element size of the address - X86Ops.VCVTSI2SS: lambda x: encode(x, 0x2A, pp=2, sel=1, we=(x.src[4].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), - X86Ops.VCVTSI2SD: lambda x: encode(x, 0x2A, pp=3, sel=1, we=(x.src[4].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), + X86Ops.VCVTSI2SS: lambda x: encode(x, 0x2A, pp=2, sel=1, we=(x.src[4].src[0].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), + X86Ops.VCVTSI2SD: lambda x: encode(x, 0x2A, pp=3, sel=1, we=(x.src[4].src[0].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), X86Ops.VCVTTSS2SI: lambda x: encode(x, 0x2C, pp=2, sel=1, we=x.dtype.itemsize == 8), X86Ops.VCVTTSD2SI: lambda x: encode(x, 0x2C, pp=3, sel=1, we=x.dtype.itemsize == 8), # int division @@ -754,18 +749,12 @@ encodings = { # packed int binary X86Ops.VPSLLVD: lambda x: encode(x, 0x47, pp=1, sel=2), X86Ops.VPSLLVQ: lambda x: encode(x, 0x47, pp=1, sel=2, we=1), X86Ops.VPSRLVD: lambda x: encode(x, 0x45, pp=1, sel=2), X86Ops.VPSRLVQ: lambda x: encode(x, 0x45, pp=1, sel=2, we=1), - X86Ops.VPCMPGTB: lambda x: encode(x, 0x64, pp=1, sel=1), X86Ops.VPCMPGTW: lambda x: encode(x, 0x65, pp=1, sel=1), - X86Ops.VPCMPGTD: lambda x: encode(x, 0x66, pp=1, sel=1), X86Ops.VPCMPGTQ: lambda x: encode(x, 0x37, pp=1, sel=2), - X86Ops.VPCMPEQB: lambda x: encode(x, 0x74, pp=1, sel=1), X86Ops.VPCMPEQW: lambda x: encode(x, 0x75, pp=1, sel=1), - X86Ops.VPCMPEQD: lambda x: encode(x, 0x76, pp=1, sel=1), X86Ops.VPCMPEQQ: lambda x: encode(x, 0x29, pp=1, sel=2), X86Ops.VPMULLW: lambda x: encode(x, 0xD5, pp=1, sel=1), X86Ops.VPMULLD: lambda x: encode(x, 0x40, pp=1, sel=2), X86Ops.VPADDB: lambda x: encode(x, 0xFC, pp=1, sel=1), X86Ops.VPADDW: lambda x: encode(x, 0xFD, pp=1, sel=1), X86Ops.VPADDD: lambda x: encode(x, 0xFE, pp=1, sel=1), X86Ops.VPADDQ: lambda x: encode(x, 0xD4, pp=1, sel=1), X86Ops.VPSUBB: lambda x: encode(x, 0xF8, pp=1, sel=1), X86Ops.VPSUBW: lambda x: encode(x, 0xF9, pp=1, sel=1), X86Ops.VPSUBD: lambda x: encode(x, 0xFA, pp=1, sel=1), X86Ops.VPSUBQ: lambda x: encode(x, 0xFB, pp=1, sel=1), X86Ops.VPSRAVD: lambda x: encode(x, 0x46, pp=1, sel=2), - # float cmp - X86Ops.VUCOMISS: lambda x: encode(x, 0x2E, pp=0, sel=1), X86Ops.VUCOMISD: lambda x: encode(x, 0x2E, pp=1, sel=1), # scalar / packed float binary X86Ops.VADDSS: lambda x: encode(x, 0x58, pp=2, sel=1), X86Ops.VADDPS: lambda x: encode(x, 0x58, pp=0, sel=1), X86Ops.VADDSD: lambda x: encode(x, 0x58, pp=3, sel=1), X86Ops.VADDPD: lambda x: encode(x, 0x58, pp=1, sel=1), @@ -781,7 +770,6 @@ encodings = { X86Ops.CMOVB: lambda x: encode(x, 0x0F42), X86Ops.CMOVL: lambda x: encode(x, 0x0F4C), X86Ops.CMOVE: lambda x: encode(x, 0x0F44), X86Ops.CMOVNE: lambda x: encode(x, 0x0F45), X86Ops.VBLENDVPS: lambda x: encode(x, 0x4A, pp=1, sel=3), X86Ops.VBLENDVPD: lambda x: encode(x, 0x4B, pp=1, sel=3), - X86Ops.VPBLENDVB: lambda x: encode(x, 0x4C, pp=1, sel=3), # shuffles X86Ops.VPSRLDQ: lambda x: encode(x, 0x73, reg=3, pp=1, sel=1), X86Ops.VPINSRB: lambda x: encode(x, 0x20, pp=1, sel=3), X86Ops.VPINSRW: lambda x: encode(x, 0xC4, pp=1, sel=1), @@ -803,8 +791,7 @@ encodings = { class X86Renderer(ISARenderer): device = "CPU" has_local = False - has_threads = bool(getenv("THREADS", 1)) - global_max = (NUM_CPU_THREADS.value, 0, 0) + global_max = (1, 0, 0) extra_matcher = extra_matcher pre_isel_matcher = pre_isel_matcher isel_matcher = isel_matcher @@ -816,46 +803,46 @@ class X86Renderer(ISARenderer): super().__init__(target) from tinygrad.runtime.support.compiler_cpu import X86Compiler self.compiler = X86Compiler() - def is_two_address(self, x:UOp) -> bool: return x.arg in X86GroupOp.TwoAddress + def is_two_address(self, x:UOp) -> bool: return x.op is Ops.INS and x.arg[0] in X86GroupOp.TwoAddress def stack_pointer(self) -> UOp: return def_reg(dtypes.uint64, RSP) # the value of a BUFFER is its address, it moves through registers and the stack as a 64bit int def copy(self, x:UOp, reg:Register): - dt = dtypes.uint64 if x.op is Ops.BUFFER else x.dtype - ret = isel_matcher.rewrite(UOp(Ops.COPY, dt, (x,), tag=reg)) + if x.op is Ops.BUFFER: x = x.replace(arg=replace(x.arg, dtype=dtypes.uint64)) + ret = isel_matcher.rewrite(UOp(Ops.COPY, src=(x,), tag=reg)) assert ret is not None, f"failed to copy {x}" return ret def spill(self, disp:UOp, x:UOp) -> UOp: - if x.op is Ops.BUFFER: x = x.replace(dtype=dtypes.uint64) + if x.op is Ops.BUFFER: x = x.replace(arg=replace(x.arg, dtype=dtypes.uint64)) is_xmm = isinstance(x.tag, tuple) and x.tag[0].cons[0].size == 16 op = X86Ops.VMOVUPSm if is_xmm else X86Ops.MOVm - return UOp(Ops.INS, dtypes.void, fold_address(self.stack_pointer().index(disp)) + (x,), op, x.tag) + return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)) + (x,), arg=(op, dtypes.void), tag=x.tag) def fill(self, disp:UOp, x:UOp, reg:Register) -> UOp: is_xmm = reg.cons[0].size == 16 dt = dtypes.uint64 if x.op is Ops.BUFFER else x.dtype - return UOp(Ops.INS, dt, fold_address(self.stack_pointer().index(disp)), X86Ops.VMOVUPS if is_xmm else X86Ops.MOV, (reg,)) + return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)), arg=(X86Ops.VMOVUPS if is_xmm else X86Ops.MOV, dt), tag=(reg,)) def asm_str(self, uops:list[UOp], function_name:str) -> str: - def _format_op(x:UOp) -> str: return f" {(o[7:-1] if (o:=str(x.arg))[-1] in ('i', 'm') else o[7:]).lower():7s}" + def _format_op(x:UOp) -> str: return f" {(o[7:-1] if (o:=str(x.arg[0]))[-1] in ('i', 'm') else o[7:]).lower():7s}" def _format_operands(x:UOp) -> str: def _format(src:tuple[UOp, ...]) -> list[str]: - return [str(s.val) if s.op is Ops.CONST else reg_strs[o].get(s.dtype.itemsize, o) if \ + return [str(s.src[0].val) if s.op is Ops.CAST else reg_strs[o].get(s.dtype.itemsize, o) if \ (o:=str(greg(s))) in reg_strs else o for s in src if greg(s) is not None] def _mem_adress(base:UOp, idx:UOp, disp:UOp, sz:UOp) -> list[str]: - return [f"[{greg(base)}" + (f" + {greg(idx)}*{sz.val}" if greg(idx) else "") + (f" + {disp.val}" if disp.val else "") + "]"] + return [f"[{greg(base)}" + (f" + {greg(idx)}*{sz.src[0].val}" if greg(idx) else "") + (f" + {d}" if (d:=disp.src[0].val) else "") + "]"] - if len(x.src) > 4 and x.arg in X86GroupOp.WriteMem: ret = _mem_adress(*x.src[:4]) + _format(x.src[4:]) - elif len(x.src) > 3 and x.arg in X86GroupOp.Rm1st: ret = _format((x,)) + _mem_adress(*x.src[:4]) + _format(x.src[4:]) - elif len(x.src) > 4 and x.arg in X86GroupOp.Rm2nd: ret = _format((x, x.src[0])) + _mem_adress(*x.src[1:5]) + _format(x.src[5:]) + if len(x.src) > 4 and x.arg[0] in X86GroupOp.WriteMem: ret = _mem_adress(*x.src[:4]) + _format(x.src[4:]) + elif len(x.src) > 3 and x.arg[0] in X86GroupOp.Rm1st: ret = _format((x,)) + _mem_adress(*x.src[:4]) + _format(x.src[4:]) + elif len(x.src) > 4 and x.arg[0] in X86GroupOp.Rm2nd: ret = _format((x, x.src[0])) + _mem_adress(*x.src[1:5]) + _format(x.src[5:]) else: ret = _format((x,) + x.src) return ", ".join(ret) asm = [f".{function_name}:"] for u in uops: - if u.op is not Ops.INS or u.arg is X86Ops.DEFINE: continue - if u.arg is X86Ops.LABEL: asm.append(f"{str(u.tag)}:") - elif u.arg is X86Ops.RET: asm.append(_format_op(u)) + if u.op is not Ops.INS or u.arg[0] is X86Ops.DEFINE: continue + if u.arg[0] is X86Ops.LABEL: asm.append(f"{str(u.tag)}:") + elif u.arg[0] is X86Ops.RET: asm.append(_format_op(u)) else: asm.append(_format_op(u) + " " + _format_operands(u)) return "\n".join(asm) @@ -864,15 +851,15 @@ class X86Renderer(ISARenderer): jumps: dict[UOp, int] = {} binary = bytearray() for u in uops: - if u.op is not Ops.INS or u.arg is X86Ops.DEFINE: continue - if u.arg is X86Ops.LOOP_CMP: continue - if u.arg is X86Ops.LABEL: + if u.op is not Ops.INS or u.arg[0] is X86Ops.DEFINE: continue + if u.arg[0] is X86Ops.LOOP_CMP: continue + if u.arg[0] is X86Ops.LABEL: targets[u.tag] = len(binary) continue - if u.arg not in encodings or (l:=encodings[u.arg](u)) is None: - raise RuntimeError(f"failed to encode {u.arg} with {u.dtype} srcs {[x.dtype for x in u.src]}") + if u.arg[0] not in encodings or (l:=encodings[u.arg[0]](u)) is None: + raise RuntimeError(f"failed to encode {u.arg[0]} with {u.dtype} srcs {[x.dtype for x in u.src]}") binary.extend(l) - if u.arg in (X86Ops.JL, X86Ops.JB, X86Ops.JE, X86Ops.JNE, X86Ops.JGE, X86Ops.JMP): jumps[u] = len(binary) + if u.arg[0] in (X86Ops.JL, X86Ops.JB, X86Ops.JE, X86Ops.JNE, X86Ops.JGE, X86Ops.JMP): jumps[u] = len(binary) # fixup jump targets now that encoding size is known for u in uops: if (t:=jumps.get(u)) is not None: binary[t-4:t] = (targets[u.tag] - t).to_bytes(4, 'little', signed=True) diff --git a/tinygrad_repo/tinygrad/renderer/llvmir.py b/tinygrad_repo/tinygrad/renderer/llvmir.py index 99da4d134..d25e1d1ef 100644 --- a/tinygrad_repo/tinygrad/renderer/llvmir.py +++ b/tinygrad_repo/tinygrad/renderer/llvmir.py @@ -1,11 +1,11 @@ import math, struct, sys -from tinygrad.codegen.opt import tc +from tinygrad.renderer import tc from tinygrad.renderer import Renderer from tinygrad.renderer.cstyle import HIPRenderer, create_non_native_float_pats, pm_manual_bf16_cast from tinygrad.codegen.decomp.transcendental import xexp2, xlog2 from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, GroupOp, range_str from tinygrad.dtype import dtypes, float_to_fp8, DType, truncate, AddrSpace -from tinygrad.helpers import prod, Target, NUM_CPU_THREADS, getenv, OSX +from tinygrad.helpers import prod, Target, OSX def is_volatile(u:UOp) -> bool: return (buf:=u.buf_uop).op is Ops.PARAM and buf.arg.volatile @@ -81,8 +81,8 @@ base_rewrite = PatternMatcher([ (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat((Ops.BUFFER, Ops.PARAM, Ops.AFTER)),), allow_any_len=True, name="x"), lambda ctx,x: f" {ctx[x]} = getelementptr inbounds {ldt(x.dtype)}, {ldt(x.dtype, ptr=True)} {ctx[x.src[0]]}, {ldt(x.src[1].dtype)} {ctx[x.src[1]]}"), # register index - (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.cvar("idx")), name="x"), lambda ctx,buf,idx,x: - f" {ctx[x]} = extractelement {ldt(buf.dtype, buf.max_numel())} {ctx[buf]}, i32 {idx.val}" if buf.addrspace == AddrSpace.ALU else None), + (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.cvar("c").cast()), name="x"), lambda ctx,buf,c,x: + f" {ctx[x]} = extractelement {ldt(buf.dtype, buf.max_numel())} {ctx[buf]}, i32 {c.val}" if buf.addrspace == AddrSpace.ALU else None), # load/store (UPat(Ops.LOAD, src=(UPat.var("idx"), UPat.var("alt"), UPat.var("mask")), name="x"), @@ -142,7 +142,7 @@ base_rewrite = PatternMatcher([ (UPat(Ops.IF, name="x"), lambda ctx,x: f" br i1 {ctx[x.src[0]]}, label %ifbody_{ctx[x][1:]}, label %ifskip_{ctx[x][1:]}\nifbody_{ctx[x][1:]}:"), (UPat(Ops.ENDIF, name="x"), lambda ctx,x: f" br label %ifskip_{ctx[x.src[0]][1:]}\nifskip_{ctx[x.src[0]][1:]}:"), - (UPat(Ops.BARRIER), lambda ctx: "") + (UPat(Ops.BARRIER), lambda ctx: " fence seq_cst") ]) class LLVMRenderer(Renderer): @@ -165,7 +165,7 @@ class LLVMRenderer(Renderer): local_args: list[str] = [] name = "test" for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP}: continue + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST}: continue if u.op is Ops.AFTER: r[u] = r[u.src[0]] continue @@ -185,7 +185,7 @@ class LLVMRenderer(Renderer): kernel.append(f" {r[u]} = addrspacecast [{size} x {ldt(u.dtype)}] addrspace(3)* @{r[u][1:]} to [{size} x {ldt(u.dtype)}]*") else: kernel.append(f" {r[u]} = alloca [{size} x {ldt(u.dtype)}], align 16") - elif u.op is Ops.CONST: r[u] = lconst(u.val, u.dtype) + elif u.op is Ops.CAST and u.src[0].op is Ops.CONST: r[u] = lconst(u.src[0].val, u.dtype) elif u.op is Ops.CAST and ldt(u.dtype) == ldt(u.src[0].dtype): r[u] = r[u.src[0]] # cast from signed to unsigned of the same size is a noop, or pointer cast else: @@ -203,8 +203,7 @@ class LLVMRenderer(Renderer): class CPULLVMRenderer(LLVMRenderer): has_local = False - has_threads = bool(getenv("THREADS", 1)) - global_max = (NUM_CPU_THREADS.value, 0, 0) + global_max = (1, 0, 0) abi = 'win64cc' if sys.platform == 'win32' else None string_rewrite = base_rewrite def render(self, uops: list[UOp]) -> str: return "\n".join((k:=self._render_kernel(uops))[0] + (k[1], self._render_footer(uops))) @@ -238,8 +237,8 @@ class AMDLLVMRenderer(LLVMRenderer): (UPat(Ops.CAST, dtypes.fp8s, (UPat(dtype=dtypes.float),), name="x",), lambda ctx,x: f" {ctx[x]} = call i8 @f32_to_fp8({ldt(x.src[0].dtype)} {ctx[x.src[0]]}, i1 {'1' if x.dtype == dtypes.fp8e5m2 else '0'})"), (UPat(Ops.CAST, dtypes.float, (UPat.var("y", dtypes.fp8s),), name="x",), lambda ctx,x,y: - f" {ctx[x.src[0]]}_i32 = zext i8 {ctx[x.src[0]]} to i32\n" - f" {ctx[x]} = call float @llvm.amdgcn.cvt.f32.{'bf8' if y.dtype == dtypes.fp8e5m2 else 'fp8'}(i32 {ctx[x.src[0]]}_i32, i32 0)"), + f" {ctx[x]}_i32 = zext i8 {ctx[x.src[0]]} to i32\n" + f" {ctx[x]} = call float @llvm.amdgcn.cvt.f32.{'bf8' if y.dtype == dtypes.fp8e5m2 else 'fp8'}(i32 {ctx[x]}_i32, i32 0)"), ]) + base_rewrite extra_matcher = LLVMRenderer.extra_matcher + create_non_native_float_pats(dtypes.fp8s) + PatternMatcher([ # amd llvm intrinsics llvm.log2/llvm.exp2 don't support double @@ -279,43 +278,9 @@ exit: %packed = phi i32 [%packed_bf8, %do_bf8], [%packed_fp8, %do_fp8]\n %trunc (UPat(Ops.WMMA, name="wmma"), lambda ctx, wmma, rdna4=AMDLLVMRenderer.is_rdna4(target.arch), cdna=self.is_cdna: render_wmma_amd(ctx, wmma, cdna, rdna4)) ]) - if self.is_cdna: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) - if x.arg[0][2] == 128 and x.src[0].dtype.itemsize <= 8 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.max_numel() == 4 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 4 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) - if x.max_numel() == 4 and x.src[0].dtype in dtypes.fp8_ocp and x.src[0].max_numel() == 8 else None), - ]) - if target.arch in {"gfx1100", "gfx1151"}: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.int32), lambda x: x.replace( - src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) - if x.src[0].dtype == dtypes.int8 and x.src[0].max_numel() == 16 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.half), lambda x: UOp(Ops.STACK, src=tuple(x.replace( - src=(x.src[0], x.src[1], UOp(Ops.STACK, src=tuple(x.src[2].index(UOp.const(j//2, dtypes.int16)) - if j%2 == 0 else UOp.const(0.0, x.src[2].dtype) - for j in range(x.max_numel()*2)))), - arg=(*x.arg[:4], None)).index(UOp.const(i*2, dtypes.int16)) - for i in range(x.max_numel()))) if x.max_numel() == 8 else None), - (UPat(Ops.WMMA, name="x"), lambda x: x.replace( - src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 16 else None), - ]) - if target.arch in {"gfx1200", "gfx1201"}: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( - dtype=dtypes.uint16, - src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) - .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None) - ]) + if self.is_cdna: self.extra_matcher += tc.pm_validate_wmma_cdna + if target.arch in {"gfx1100", "gfx1151"}: self.extra_matcher += tc.pm_validate_wmma_rdna3 + if target.arch in {"gfx1200", "gfx1201"}: self.extra_matcher += tc.pm_validate_wmma_rdna4 def supported_dtypes(self): return {d for d in super().supported_dtypes() if (d not in dtypes.fp8_ocp or self.target.arch == "gfx950") and d not in dtypes.fp8_fnuz} diff --git a/tinygrad_repo/tinygrad/renderer/nir.py b/tinygrad_repo/tinygrad/renderer/nir.py index b0d3a38f2..2917da319 100644 --- a/tinygrad_repo/tinygrad/renderer/nir.py +++ b/tinygrad_repo/tinygrad/renderer/nir.py @@ -1,7 +1,7 @@ from typing import Callable, Any from tinygrad.dtype import AddrSpace, DType, dtypes, truncate from tinygrad.helpers import DEBUG, OSX, unwrap, fromimport, Target, is_image_shape, round_up -from tinygrad.renderer import Renderer +from tinygrad.renderer import Renderer, with_storage from tinygrad.renderer.cstyle import CUDARenderer from tinygrad.uop.ops import GroupOp, Ops, UOp, PatternMatcher, UPat, range_str from tinygrad.runtime.autogen import mesa, libc @@ -92,8 +92,6 @@ nload = nir_instr(nc=lambda u:u.max_numel(), bs=lambda u:u.dtype.bitsize, num_co ngid = nir_instr(nc=3, bs=32)(lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_load_workgroup_id)) nlid = nir_instr(nc=3, bs=32)(lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_load_local_invocation_id)) -ngsz = nir_instr(nc=3, bs=32)(lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_load_workgroup_size)) -def nid(b): return nalu(b, "iadd", nalu(b, "imul", ngid(b), ngsz(b)), nlid(b)) nbarrier = nir_instr(has_def=False, intrins={"EXECUTION_SCOPE":mesa.SCOPE_WORKGROUP})( lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_barrier)) @@ -121,32 +119,33 @@ class NIRRenderer(Renderer): code_for_op = {**{k:lambda:None for k in u_aop.keys()}, **{k:lambda:None for k in s_aop.keys()}, **{k:lambda:None for k in f_aop.keys()}} extra_matcher = PatternMatcher([ - # handle negative unsigned CONST - (UPat.cvar("x", dtypes.uints), lambda x: UOp.const(x.dtype.max+x.val+1, x.dtype) if x.val < 0 else None), # from ptx (UPat.var('x', dtype=dtypes.bool) uint8 + # a bool is one bit in NIR but a byte in memory, so every access to a bool buffer goes through a uint8 view of it (UPat(Ops.LOAD, dtypes.bool, name="x"), - lambda x: x.replace(dtype=dtypes.uint8, src=x.src[0:1]+((x.src[1].cast(dtypes.uint8),) if len(x.src)>=2 else ())+x.src[2:]).cast(dtypes.bool)), - (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), - lambda x: x.replace(src=(x.src[0], x.src[1].cast(dtypes.uint8))+x.src[2:])), + lambda x: x.replace(src=(with_storage(x.src[0], dtypes.uint8),)+((x.src[1].cast(dtypes.uint8),) if len(x.src)>=2 else ()) + +x.src[2:]).cast(dtypes.bool)), + (UPat(Ops.STORE, src=(UPat(name="idx"), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), + lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8), x.src[1].cast(dtypes.uint8))+x.src[2:])), # NIR requires shift amount to be 32 bit: https://docs.mesa3d.org/nir/alu.html#nir-alu-op-ishl (UPat((Ops.SHL, Ops.SHR), name="x"), lambda x: x.replace(src=(x.src[0], x.src[1].cast(dtypes.uint))) if x.src[1].dtype.bitsize != 32 else None), # OpConvertFToU is undefined if Result Type is not wide enough, cast through int32 # ref: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpConvertFToU (UPat(Ops.CAST, (dtypes.uchar, dtypes.ushort), src=(UPat.var("x", dtypes.floats),), name="c"), lambda x,c: x.cast(dtypes.int32).cast(c.dtype)), # load/store use pointer arithmetic, and the cast does nothing. NOTE: this doesn't apply to image indexing cause it's 1-D - (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True, name="x"), lambda x,buf,off: x.replace( - src=(buf,off.cast(dtypes.long))+x.src[2:]) if buf.addrspace != AddrSpace.REG and not is_image_shape(buf._shape) else None), + # nor to REG/ALU register picks, which keep their own index dtype + (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True, name="x"), + lambda x,buf,off: x.replace(src=(buf,off.ccast(dtypes.long))+x.src[2:]) + if buf.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) and not is_image_shape(buf._shape) else None), # images need index to be int for nir (coordinates only: the INDEX keeps its access dtype) (UPat.var("buf").index(UPat.var("idx_y"), UPat.var("idx_x"), name="x"), lambda x,buf,idx_y,idx_x: x.replace(src=(buf, idx_y.cast(dtypes.int), idx_x.cast(dtypes.int)))), ]) def_rewrite = PatternMatcher([ - (UPat(Ops.CONST, name="x"), lambda ctx,x: nimm(ctx.b, x.val, x.dtype)), + (UPat.cvar("c").cast(name="x"), lambda ctx,x,c: nimm(ctx.b, c.val, x.dtype)), (UPat(Ops.PARAM, name="x"), lambda ctx,x: ctx.param(ctx.b, x, x.dtype.itemsize if x.addrspace is AddrSpace.ALU else 8)), - (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: nchannel(ctx.b, {'g':ngid, 'l':nlid, 'i': nid}[x.arg[0]](ctx.b), int(x.arg[-1]))), + (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: nchannel(ctx.b, {'g':ngid, 'l':nlid}[x.arg[0]](ctx.b), int(x.arg[-1]))), (UPat(Ops.STORE, src=(UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"),UPat.var("off")), allow_any_len=True), UPat.var("val"))), lambda ctx,buf,off,val: nstore(ctx.b, buf.addrspace, nidx(ctx.b, ctx.r[buf], ctx.r[off], buf.addrspace, buf.dtype.itemsize), ctx.r[val])), (UPat(Ops.LOAD, src=(UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True), UPat.var("alt"), @@ -180,21 +179,21 @@ class NIRRenderer(Renderer): def param(self, b:mesa.nir_builder, x, sz:int) -> mesa.nir_def: raise NotImplementedError("needs param") def prerender(self, uops:list[UOp]): self.b = mesa.nir_builder_init_simple_shader(mesa.MESA_SHADER_COMPUTE, mesa.nir_shader_compiler_options.from_buffer_copy(self.nir_options), None) - self.b.shader.contents.info.workgroup_size_variable = any([u.op == Ops.SPECIAL and u.arg[0] == 'i' for u in uops]) def postrender(self, uops:list[UOp]): pass def render(self, uops:list[UOp]): self.prerender(uops) - for u in [u for u in uops if u.op is Ops.SPECIAL and u.arg[0] == "l"]: self.b.shader.contents.info.workgroup_size[int(u.arg[-1])] = u.src[0].val + for u in [u for u in uops if u.op is Ops.SPECIAL and u.arg[0] == "l"]: + self.b.shader.contents.info.workgroup_size[int(u.arg[-1])] = u.src[0].src[0].val self.r: dict[UOp, Any] = {} self.param_idx = 0 ranges: list[mesa.nir_def|None] = [] for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP} or (u.op is Ops.STACK and len(u.src) == 0): pass + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST} or (u.op is Ops.STACK and len(u.src) == 0): pass elif u.op in {Ops.INDEX, Ops.SHRINK}: # INDEX on a register value picks the element, memory INDEX is handled in the LOAD/STORE patterns - if u.src[0].op not in {Ops.PARAM, Ops.BUFFER, Ops.AFTER}: self.r[u] = nchannel(self.b, self.r[u.src[0]], u.src[1].val) + if u.src[0].op not in {Ops.PARAM, Ops.BUFFER, Ops.AFTER}: self.r[u] = nchannel(self.b, self.r[u.src[0]], u.src[1].src[0].val) elif u.op is Ops.AFTER: self.r[u] = self.r[u.src[0]] elif u.op == Ops.SINK: diff --git a/tinygrad_repo/tinygrad/renderer/ptx.py b/tinygrad_repo/tinygrad/renderer/ptx.py index 1b6b77859..c173bd104 100644 --- a/tinygrad_repo/tinygrad/renderer/ptx.py +++ b/tinygrad_repo/tinygrad/renderer/ptx.py @@ -1,10 +1,10 @@ from typing import cast, Callable import struct from collections import defaultdict -from tinygrad.codegen.opt import tc +from tinygrad.renderer import tc from tinygrad.uop.ops import Ops, UOp, PatternMatcher, UPat, GroupOp from tinygrad.dtype import dtypes, DType, AddrSpace -from tinygrad.renderer import Renderer +from tinygrad.renderer import Renderer, with_storage from tinygrad.renderer.cstyle import CUDARenderer from tinygrad.helpers import flatten, prod, unwrap, Target @@ -45,15 +45,15 @@ ptx_matcher = PatternMatcher([ # upcast to float32 all the ops that don't support half (UPat(doesnt_support_half, dtype=dtypes.half, name="x"), lambda x: (UOp(x.op, src=tuple(vv.cast(dtypes.float32) for vv in x.src), arg=x.arg).cast(dtypes.half))), - # load/store bool -> uint8 (only for memory, not registers) + # a bool is a predicate register in PTX but a byte in memory, so a bool buffer is accessed through a uint8 view of it (UPat(Ops.LOAD, dtypes.bool, src=(UPat(name="idx"),), name="x", allow_any_len=True), - lambda x,idx: UOp(x.op, dtypes.uint8, x.src[0:1] + ((x.src[1].cast(dtypes.uint8),) if len(x.src) >= 2 else ()) + x.src[2:]).cast(dtypes.bool) \ - if idx.addrspace != AddrSpace.REG else None), + lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8),) + ((x.src[1].cast(dtypes.uint8),) if len(x.src) >= 2 else ()) + + x.src[2:]).cast(dtypes.bool) if idx.addrspace != AddrSpace.REG else None), (UPat(Ops.STORE, src=(UPat(name="idx"), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), - lambda x,idx: UOp(x.op, src=(x.src[0], x.src[1].cast(dtypes.uint8))+x.src[2:]) if idx.addrspace != AddrSpace.REG else None), + lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8), x.src[1].cast(dtypes.uint8))+x.src[2:]) if idx.addrspace != AddrSpace.REG else None), # ptx shr and shl instructions require y to be uint - (UPat.var("x") << UPat.var("y"), lambda x,y: UOp(Ops.SHL, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), - (UPat.var("x") >> UPat.var("y"), lambda x,y: UOp(Ops.SHR, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), + (UPat.var("x") << UPat.var("y"), lambda x,y: UOp(Ops.SHL, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), + (UPat.var("x") >> UPat.var("y"), lambda x,y: UOp(Ops.SHR, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), ]) def mem_type(x:UOp) -> str: return 'shared' if x.addrspace == AddrSpace.LOCAL else 'global' @@ -64,7 +64,7 @@ def render_wmma(ctx: "PTXRenderer", wmma: UOp): for src, regs in zip(wmma.src, ctx.wmma_r): for i, reg in enumerate(regs): # pack input and acc registers - if (elems_per_reg := 4 // src.dtype.scalar().itemsize) == 1: yield f"mov.b32 {reg}, {ctx.r[src][i]};" + if (elems_per_reg := 4 // src.dtype.itemsize) == 1: yield f"mov.b32 {reg}, {ctx.r[src][i]};" else: yield f"mov.b32 {reg}, {{{', '.join(ctx.r[src][i * elems_per_reg : (i+1) * elems_per_reg])}}};" dt_map_in, dt_map_out = {dtypes.float: "tf32", dtypes.half: "f16"}, {dtypes.float: "f32", dtypes.half: "f16"} @@ -79,8 +79,8 @@ def modifier(a: DType, b: DType): return '.rzi' if dtypes.is_int(a) and dtypes.i (a.itemsize < b.itemsize or dtypes.is_int(b) or b == dtypes.bool) else '' string_rewrite = PatternMatcher([ - (UPat.cvar("x", dtypes.bool), lambda ctx, x: f"setp.ne.s16 {ctx.r[x]}, {render_val(x.val, x.dtype)}, 0;"), - (UPat.cvar("x"), lambda ctx, x: f"mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {render_val(x.val, x.dtype)};"), + (UPat.cvar("c").cast(dtypes.bool, name="x"), lambda ctx, x, c: f"setp.ne.s16 {ctx.r[x]}, {render_val(c.val, x.dtype)}, 0;"), + (UPat.cvar("c").cast(name="x"), lambda ctx, x, c: f"mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {render_val(c.val, x.dtype)};"), (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: f"mov.u32 %{x.arg}, %{'ctaid' if x.arg[0] == 'g' else 'tid'}.{chr(120+int(x.arg[-1]))};"), (UPat(Ops.PARAM, name="x"), lambda ctx, x: f"ld.param.{ctx.types[dtypes.ulong] if x.addrspace is AddrSpace.GLOBAL else ctx.mem_types[x.dtype]} {ctx.r[x]}, [data{x.arg.slot}+0];"), @@ -101,17 +101,17 @@ string_rewrite = PatternMatcher([ if loc.addrspace == AddrSpace.REG else None), (UPat(Ops.STORE, src=(UPat((Ops.INDEX, Ops.SHRINK), name="loc"), UPat.var("var"))), lambda ctx, loc, var: f"st.{mem_type(loc)}" + \ - f"{f'.v{cnt}' if ((cnt:=var.max_numel())>1) else ''}.{ctx.mem_types[var.dtype.scalar()]} " + \ + f"{f'.v{cnt}' if ((cnt:=var.max_numel())>1) else ''}.{ctx.mem_types[var.dtype]} " + \ f"[{ctx.r[loc]}+0], {('{' + ', '.join(ctx.r[var]) + '}') if var.max_numel() > 1 else ctx.r[var]};"), (UPat(Ops.LOAD, name="x", src=(UPat((Ops.INDEX, Ops.SHRINK), name="loc"), UPat.var("alt"), UPat.var("gate"))), lambda ctx, x, loc, alt, gate: flatten([ - [f"mov.{ctx.mem_types[x.dtype.scalar()]} {v}, {render_val(0, x.dtype.scalar())};" for v in ctx.r[x]], - [f"@{ctx.r[gate]} ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype.scalar()]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];"] + [f"mov.{ctx.mem_types[x.dtype]} {v}, {render_val(0, x.dtype)};" for v in ctx.r[x]], + [f"@{ctx.r[gate]} ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];"] ]) if alt.max_numel() > 1 else [ - f"@{ctx.r[gate]} ld.{mem_type(loc)}.{ctx.mem_types[x.dtype.scalar()]} {ctx.r[x]}, [{ctx.r[loc]}+0];", - f"@!{ctx.r[gate]} mov.b{ctx.types[x.dtype.scalar()][1:]} {ctx.r[x]}, {ctx.r[alt]};"]), + f"@{ctx.r[gate]} ld.{mem_type(loc)}.{ctx.mem_types[x.dtype]} {ctx.r[x]}, [{ctx.r[loc]}+0];", + f"@!{ctx.r[gate]} mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {ctx.r[alt]};"]), (UPat(Ops.LOAD, name="x", src=(UPat((Ops.INDEX, Ops.SHRINK), name="loc"),)), - lambda ctx, x, loc: f"ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype.scalar()]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];" \ + lambda ctx, x, loc: f"ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];" \ if x.max_numel() > 1 else f"ld.{mem_type(loc)}.{ctx.mem_types[x.dtype]} {ctx.r[x]}, [{ctx.r[loc]}+0];"), # simple (UPat(Ops.BUFFER, name="x"), lambda ctx, x: [] if x.addrspace == AddrSpace.REG else [ @@ -186,7 +186,7 @@ class PTXRenderer(Renderer): name = "test" for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP}: continue + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST}: continue if u.op is Ops.AFTER: self.r[u] = self.r[u.src[0]] continue @@ -197,26 +197,26 @@ class PTXRenderer(Renderer): r[u] = [cast(str,r[x]) for x in u.src] continue if u.op is Ops.BUFFER and u.addrspace == AddrSpace.REG: - r[u] = [ssa("reg", u, self.types[u.dtype.scalar()]) for _ in range(u.max_numel())] + r[u] = [ssa("reg", u, self.types[u.dtype]) for _ in range(u.max_numel())] continue if u.op in {Ops.INDEX, Ops.SHRINK, Ops.LOAD} and u.src[0].addrspace in (AddrSpace.REG, AddrSpace.ALU): # on REG, INDEX/SHRINK pick the register (must be CONST) and LOAD is a noop - if u.op is not Ops.LOAD and u.src[1].op is not Ops.CONST: + if u.op is not Ops.LOAD and not (u.src[1].op is Ops.CAST and u.src[1].src[0].op is Ops.CONST): raise RuntimeError(f"PTX does not support dynamic register indexing: {u}") - r[u] = r[u.src[0]] if u.op is Ops.LOAD else r[u.src[0]][u.src[1].val] + r[u] = r[u.src[0]] if u.op is Ops.LOAD else r[u.src[0]][u.src[1].src[0].val] continue if u.op is Ops.SPECIAL: r[u] = "%" + u.arg elif u.op is Ops.LOAD: - r[u] = [ssa('val', dtype=self.types[u.dtype.scalar()]) for _ in range(u.max_numel())] if u.max_numel() > 1 else ssa('val', u) + r[u] = [ssa('val', dtype=self.types[u.dtype]) for _ in range(u.max_numel())] if u.max_numel() > 1 else ssa('val', u) elif u.op is Ops.PARAM: bufs.append((f"data{u.arg.slot}", u)) elif u.op is Ops.WMMA: # registers for packing/unpacking input and acc - self.wmma_r = [[ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[0]]), 4 // u.src[0].dtype.scalar().itemsize)], - [ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[1]]), 4 // u.src[0].dtype.scalar().itemsize)], - [ssa("wmma_acc", dtype="b32") for _ in range(0, len(r[u.src[2]]), 4 // u.dtype.scalar().itemsize)]] - r[u] = [ssa("wmma", dtype=self.types[u.dtype.scalar()]) for _ in range(u.max_numel())] + self.wmma_r = [[ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[0]]), 4 // u.src[0].dtype.itemsize)], + [ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[1]]), 4 // u.src[0].dtype.itemsize)], + [ssa("wmma_acc", dtype="b32") for _ in range(0, len(r[u.src[2]]), 4 // u.dtype.itemsize)]] + r[u] = [ssa("wmma", dtype=self.types[u.dtype]) for _ in range(u.max_numel())] prefix, dtype = {Ops.CAST: ("cast", None), Ops.BITCAST: ("cast", None), Ops.END: ("pred", "pred"), Ops.RANGE: ("ridx", None), - Ops.CONST: ("const", None), Ops.BUFFER: ("local", "u64"), Ops.INDEX: ("bidx", "u64"), Ops.SHRINK: ("bidx", "u64"), + Ops.BUFFER: ("local", "u64"), Ops.INDEX: ("bidx", "u64"), Ops.SHRINK: ("bidx", "u64"), Ops.PARAM: ("dat", "u64" if u.addrspace is AddrSpace.GLOBAL else None), **{op: ("alu", None) for op in GroupOp.ALU}}.get(u.op, (None, None)) if u.op is Ops.RANGE and u.dtype == dtypes.void: prefix = None # loop headers don't have a register if prefix: r[u] = ssa(prefix, u, dtype) diff --git a/tinygrad_repo/tinygrad/codegen/opt/tc.py b/tinygrad_repo/tinygrad/renderer/tc.py similarity index 81% rename from tinygrad_repo/tinygrad/codegen/opt/tc.py rename to tinygrad_repo/tinygrad/renderer/tc.py index 52df9ce56..507a619a3 100644 --- a/tinygrad_repo/tinygrad/codegen/opt/tc.py +++ b/tinygrad_repo/tinygrad/renderer/tc.py @@ -1,6 +1,7 @@ import math, functools from dataclasses import dataclass from tinygrad.dtype import DType, dtypes +from tinygrad.uop.ops import PatternMatcher, UOp, UPat, Ops @dataclass(frozen=True) class TensorCore: # D = A * B + C, A is (M x K), B is (K x N), C and D are (M x N) @@ -135,6 +136,41 @@ amd_cdna4 = amd_cdna_1616128 + amd_cdna_161632 + amd_cdna_161616 def get_amd(arch): return {"gfx942": amd_cdna3, "gfx950": amd_cdna4, "gfx1200": amd_rdna4, "gfx1201": amd_rdna4}.get(arch, amd_rdna3) +pm_validate_wmma_rdna3 = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.int32), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) + if x.src[0].dtype == dtypes.int8 and x.src[0].max_numel() == 16 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.half), lambda x: UOp(Ops.STACK, src=tuple(x.replace( + src=(x.src[0], x.src[1], UOp(Ops.STACK, src=tuple(x.src[2].index(UOp.const(j//2, dtypes.int16)) + if j%2 == 0 else UOp.const(0.0, x.src[2].dtype) + for j in range(x.max_numel()*2)))), + arg=(*x.arg[:4], None)).index(UOp.const(i*2, dtypes.int16)) + for i in range(x.max_numel()))) if x.max_numel() == 8 else None), + (UPat(Ops.WMMA, name="x"), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 16 else None), +]) + +pm_validate_wmma_rdna4 = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) + .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None) +]) + +pm_validate_wmma_cdna = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) + if x.arg[0][2] == 128 and x.src[0].dtype.itemsize <= 8 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.max_numel() == 4 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 4 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) + if x.max_numel() == 4 and x.src[0].dtype in dtypes.fp8_ocp and x.src[0].max_numel() == 8 else None), +]) # ***** Apple Metal ***** metal = [TensorCore(dims=(8,8,8), threads=32, elements_per_thread=(2,2,2), dtype_in=di, dtype_out=do, diff --git a/tinygrad_repo/tinygrad/renderer/wgsl.py b/tinygrad_repo/tinygrad/renderer/wgsl.py index 4cecf8116..6f0128891 100644 --- a/tinygrad_repo/tinygrad/renderer/wgsl.py +++ b/tinygrad_repo/tinygrad/renderer/wgsl.py @@ -1,57 +1,55 @@ from tinygrad.dtype import DType, dtypes, truncate, AddrSpace from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat from tinygrad.renderer.cstyle import CStyleLanguage, base_rewrite -from tinygrad.helpers import strip_parens +from tinygrad.helpers import strip_parens, ceildiv -def _mask(dt:DType): return 0xFF if dt.itemsize == 1 else 0xFFFF +# a field of `width` bits sitting in the low bits of val: shift it up to the sign bit, then let the arithmetic shift fill +def sign_extend(val:UOp, width:int): return (val << (32-width)).bitcast(dtypes.int) >> (32-width) -def sign_extend(val:UOp, sext_am:int): - return (UOp.where((val >> (sext_am - 1)) > 0, UOp.const(0xffffffff << sext_am, dtypes.uint32), UOp.const(0, dtypes.uint32)) \ - | val.bitcast(dtypes.uint32)).bitcast(dtypes.int) +# a packed field of dt: the word it lives in, its offset in that word, and its mask. width is 8*itemsize, bool is one bit in a byte +def packed_field(bidx:UOp, dt:DType) -> tuple[UOp, UOp, int]: + elems, width = 4//dt.itemsize, 8*dt.itemsize + return bidx.src[0].index(bidx.src[1] // elems), (bidx.src[1].cast(dtypes.uint32) % elems) * width, (1 << width)-1 # store for char: buf[idx/4] <- (var << (idx%4)*8)) -def packed_store(bidx:UOp, var:UOp, gate:UOp|None=None): - elems, mask = 4//var.dtype.itemsize, _mask(var.dtype) - shift_am, div_idx = (bidx.src[1].cast(dtypes.uint32) % elems) * (8*var.dtype.itemsize), bidx.src[1] // elems +def packed_store(s:UOp): + bidx, var, *gate = s.src + idx, shift_am, mask = packed_field(bidx, var.dtype) # bool does its mask math at int32: renderer rewrites run after weak dtypes are lowered, and bool & 0xFF would create a weakint const if var.dtype == dtypes.bool: var = var.cast(dtypes.int32) new_v, wmask = (var & mask).cast(dtypes.uint32) << shift_am, ((mask << shift_am) ^ 0xFFFFFFFF).cast(dtypes.uint32) - idx = UOp(Ops.INDEX, src=(bidx.src[0], div_idx)) - buf = UOp.load(idx, *((UOp.const(0, dtypes.uint32), gate) if gate is not None else ()), dtype=dtypes.uint32) - return UOp.store(idx, (buf & wmask) | new_v, *((gate,) if gate is not None else ())) + buf = idx.cast(dtypes.uint32).load(*((UOp.const(0, dtypes.uint32), *gate) if gate else ())) + return idx.store((buf & wmask) | new_v, *gate) # load for char: sign_extend(buf[idx/4] >> ((idx%4)*8)) -def packed_load(root:UOp, bidx:UOp, dtype:DType, var:UOp|None=None, gate:UOp|None=None): - elems, mask = 4//dtype.itemsize, _mask(dtype) - shift_am, div_idx = (bidx.src[1].cast(dtypes.uint32) % elems) * (8*dtype.itemsize), bidx.src[1] // elems - idx = UOp(Ops.INDEX, src=(bidx.src[0], div_idx)) - load = UOp.load(idx, *((var, gate) if var is not None and gate is not None else root.src[1:]), dtype=dtypes.uint32, arg=root.arg) - val = (load.cast(dtypes.uint32) >> shift_am) & mask +def packed_load(root:UOp): + bidx, *alt = root.src + idx, shift_am, mask = packed_field(bidx, dtype:=root.dtype) + load = idx.cast(dtypes.uint32).load(*((alt[0].cast(dtypes.uint32), *alt[1:]) if alt else ()), arg=root.arg) + val = (load >> shift_am) & mask return sign_extend(val, 8*dtype.itemsize).cast(dtype) if dtype in [dtypes.char, dtypes.short] else val.cast(dtype) def is_packed(x:UOp): - if x.op is Ops.LOAD: dt, addrspace = x.dtype, x.src[0].addrspace - elif x.op is Ops.STORE: dt, addrspace = x.src[1].dtype, x.src[0].addrspace - else: dt, addrspace = x.dtype, x.addrspace - return dt.itemsize < 4 and dt != dtypes.half and addrspace != AddrSpace.REG -def _packed_size(u:UOp): return u.max_numel() // (4//u.dtype.itemsize) if is_packed(u) else u.max_numel() + dt = x.src[1].dtype if x.op is Ops.STORE else x.buf_uop.dtype + return dt.itemsize < 4 and dt != dtypes.half and x.buf_uop.addrspace != AddrSpace.REG +def _packed_size(u:UOp): return ceildiv(u.max_numel(), 4//u.dtype.itemsize) if is_packed(u) else u.max_numel() def is_nan(a): bs, (exp, mant) = a.dtype.bitsize, dtypes.finfo(a.dtype) return (a.bitcast(getattr(dtypes, f"uint{bs}")) & ((1 << (bs - 1)) - 1)) > (((1 << exp) - 1) << mant) +# the read-modify-write packed_store emits: a load of the very index being stored to, masked (a gated store loads with 3 srcs) +packed_rmw = UPat(Ops.LOAD, src=(UPat(Ops.CAST, dtype=dtypes.uint32, src=(UPat.var("b"),)),), allow_any_len=True) & UPat.var("wmask") + wgsl_matcher = PatternMatcher([ (UPat((Ops.CMPLT, Ops.XOR), src=(UPat(name="a", dtype=dtypes.bool), UPat.var("b")), name="c"), lambda a,b,c: a.cast(dtypes.int).alu(c.op, b.cast(dtypes.int)).cast(dtypes.bool)), - (UPat.load(UPat.var("b"), UPat.var("c"), UPat.var("gate"), name="l"), - lambda l,b,c,gate: packed_load(l,b,l.dtype,c.cast(dtypes.uint32),gate) if is_packed(l) else None), - (UPat.load(UPat.var("b"), name='l'), lambda l,b: packed_load(l,b,l.dtype) if is_packed(l) else None), - (UPat.store(UPat.var("b"), UPat.var("var"), UPat.var("gate"), name="s"), - lambda b,var,gate,s: packed_store(b,var,gate) if is_packed(s) else None), - (UPat.store(UPat.var("b"), UPat.var("var"), name="s"), lambda b,var,s: packed_store(b,var) if is_packed(s) else None), + (UPat(Ops.LOAD, src=(UPat(Ops.INDEX),), allow_any_len=True, name="l"), lambda l: packed_load(l) if is_packed(l) else None), + (UPat(Ops.STORE, name="s"), lambda s: packed_store(s) if is_packed(s) else None), (UPat.var("a") << UPat.var("b"),lambda a,b:(a.bitcast(dtypes.uint32)<> UPat.var("y"), lambda x,y: UOp(Ops.SHR, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), - # fix nan check: 'a != a -> is_nan()' - (UPat.var("a") != UPat.var("a"), is_nan), + (UPat.var("x") >> UPat.var("y"), lambda x,y: UOp(Ops.SHR, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), + # fix nan check: 'a != a -> is_nan()'. the decomp rewrites (a != a).logical_not() to CMPEQ, so match both forms + (UPat.var("a", dtypes.floats) != UPat.var("a"), is_nan), + (UPat.var("a", dtypes.floats).alu(Ops.CMPEQ, UPat.var("a")), lambda a: is_nan(a).ne(True)), ]) class WGSLRenderer(CStyleLanguage): @@ -67,11 +65,13 @@ class WGSLRenderer(CStyleLanguage): dtypes.char: "i32", dtypes.int32: "i32", dtypes.uint32: "u32", dtypes.bool: "bool", dtypes.half: "f16" } string_rewrite = PatternMatcher([ + (UPat(Ops.CAST, dtype=dtypes.uint32, src=(UPat(Ops.INDEX, name="x"),)), lambda ctx,x: ctx[x] if is_packed(x) else None), (UPat(Ops.NEG, dtypes.uints, src=(UPat.var('x'))), lambda ctx,x: f"(0-{ctx[x]})"), - (UPat.cvar("x", dtype=dtypes.bool), lambda x: "true" if x.val else "false"), - (UPat(Ops.CONST, dtype=(dtypes.uchar, dtypes.ushort, dtypes.uint32), name="x"), - lambda x: f"bitcast({x.val})" if x.val < 0 else f"{x.val&0xFFFFFFFF}u"), - (UPat(Ops.CONST, dtype=dtypes.int32, name="x"), lambda ctx,x: f"{truncate[x.dtype](x.val)}"), + (UPat.cvar("c").cast(dtypes.bool), lambda c: "true" if c.val else "false"), + (UPat.cvar("c").cast((dtypes.uchar, dtypes.ushort, dtypes.uint32)), + lambda c: f"bitcast({c.val})" if c.val < 0 else f"{c.val&0xFFFFFFFF}u"), + # a negative const must state its type: contextual conversion of a bare abstract int rejects it in a u32 position + (UPat.cvar("c").cast(dtypes.int32, name="x"), lambda ctx,x,c: f"i32({v})" if (v:=truncate[x.dtype](c.val)) < 0 else f"{v}"), (UPat(Ops.BUFFER, name="x"), lambda ctx,x: f"var{'' if x.addrspace == AddrSpace.LOCAL else ''} {ctx[x]}: array<{ctx.buf_map(x)},{_packed_size(x)}>;"), (UPat(Ops.BITCAST, dtype=dtypes.half, name="x", src=(UPat(dtype=(dtypes.short, dtypes.ushort, dtypes.uint32),),)), @@ -86,10 +86,10 @@ class WGSLRenderer(CStyleLanguage): (UPat.load(UPat.var("b"), UPat.var("v"), UPat.var("gate")), lambda ctx,b,v,gate: f"select({ctx[v]}, {ctx.render_load(ctx[b], b.src[0])}, {ctx[gate]})"), (UPat.load(UPat.var("b")), lambda ctx, b: ctx.render_load(ctx[b], b)), - (UPat.store(UPat.var("b"), UPat.var("v")), lambda ctx,b,v:\ - # (load & mask) | var -> mask = v.src[0].src[1], var = v.src[1] - f"atomicAnd(&{ctx[b]},{ctx[v.src[0].src[1]]});\n atomicAdd(&{ctx[b]},{ctx[v.src[1]]});" if is_packed(b) \ - else f"{ctx[b]} = {ctx[v]};"), + # packed_store writes (load & wmask) | new_v: atomicAnd clears the field, atomicAdd sets it. new_v is gone when it is 0 + (UPat.store(UPat.var("b"), UPat.any(packed_rmw, packed_rmw | UPat.var("nv"))), lambda ctx,b,wmask,nv=None: + f"atomicAnd(&{ctx[b]},{ctx[wmask]});"+(f"\n atomicAdd(&{ctx[b]},{ctx[nv]});" if nv is not None else "") if is_packed(b) else None), + (UPat.store(UPat.var("b"), UPat.var("v")), lambda ctx,b,v: f"{ctx[b]} = {ctx[v]};"), (UPat(Ops.INDEX, src=(UPat.var("b"), UPat.var("idx"))), lambda ctx,b,idx: f"{ctx[b]}[{strip_parens(ctx[idx]) if idx.arg is Ops.ADD else ctx[idx]}]"), ]) + base_rewrite diff --git a/tinygrad_repo/tinygrad/runtime/autogen/__init__.py b/tinygrad_repo/tinygrad/runtime/autogen/__init__.py index 3a1920662..299f979a0 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/__init__.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/__init__.py @@ -1,4 +1,4 @@ -import glob, importlib, os, pathlib, shutil, subprocess, tarfile, tempfile +import glob, importlib, os, pathlib, subprocess from tinygrad.helpers import fetch, flatten, system, getenv root = (here:=pathlib.Path(__file__).parent).parents[2] @@ -10,10 +10,14 @@ rocr_src = "https://github.com/ROCm/rocm-systems/archive/refs/tags/rocm-7.1.1.ta linux_headers_deb = "https://snapshot.debian.org/archive/debian/20260207T145350Z/pool/main/l/linux/linux-libc-dev_6.18.9-1_all.deb" linux_headers_kern_deb = "https://snapshot.debian.org/archive/debian/20260207T145350Z/pool/main/l/linux/linux-headers-6.18.9+deb14-common_6.18.9-1_all.deb" liburing_src = "https://raw.githubusercontent.com/axboe/liburing/refs/tags/liburing-2.14/src/include/liburing.h" +bnxt_src = ["https://raw.githubusercontent.com/torvalds/linux/v6.18/drivers/" + s for s in + ("infiniband/hw/bnxt_re/roce_hsi.h", "infiniband/hw/bnxt_re/qplib_rcfw.h", "infiniband/hw/bnxt_re/qplib_res.h", + "net/ethernet/broadcom/bnxt/bnxt_hwrm.h")] ggml_common_src = "https://raw.githubusercontent.com/ggml-org/ggml/d4fcfe88a8bcf5c9840be14be6c2fbf1f5b3b2db/src/ggml-common.h" cudart_src = "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-12.0.146-archive.tar.xz" nvrtc_src = "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/linux-x86_64/cuda_nvrtc-linux-x86_64-12.0.140-archive.tar.xz" opencl_src = "https://github.com/KhronosGroup/OpenCL-Headers/archive/2e30669d48718fd460f085b4b35b160dad51ce9d.tar.gz" +comgr_2_src = "https://repo.radeon.com/rocm/apt/6.2/pool/main/c/comgr/comgr_2.8.0.60200-66~24.04_amd64.deb" macossdk = "/var/db/xcode_select_link/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" llvm_lib = ( @@ -31,6 +35,7 @@ def load(name, files, **kwargs): if not (f:=(root/(path:=kwargs.pop("path", __name__)).replace('.','/')/f"{name}.py")).exists() or getenv('REGEN'): files, kwargs['args'] = files() if callable(files) else files, args() if callable(args:=kwargs.get('args', [])) else args if (srcs:=kwargs.pop('srcs', None)): + import tempfile, tarfile srcpath = (td:=tempfile.TemporaryDirectory(f"autogen-src-{name.replace('/','-')}")).name + "/" for src in (srcs if isinstance(srcs, list) else [srcs]): if 'tar' in src: @@ -49,6 +54,8 @@ def load(name, files, **kwargs): if srcs: td.cleanup() return importlib.import_module(f"{path}.{name.replace('/', '.')}") +def _extract_deb(path:str): subprocess.run("ar x *.deb && tar xf data.tar.*", cwd=path, shell=True, check=True) + def __getattr__(nm): match nm: case "libc": @@ -98,14 +105,11 @@ def __getattr__(nm): # this defines all syscall numbers. should probably unify linux autogen? case "io_uring": return load("io_uring", ["{}/liburing.h", "{}/usr/include/linux/io_uring.h", "{}/usr/include/asm-generic/unistd.h"], - args=["-I{}/usr/include"], srcs=[linux_headers_deb, liburing_src], rules=[('__NR', 'NR')], - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_deb.split('/')[-1]} && tar xf data.tar.xz", cwd=path, shell=True, check=True)) + args=["-I{}/usr/include"], srcs=[linux_headers_deb, liburing_src], rules=[('__NR', 'NR')], preprocess=_extract_deb) case "llvm": return load("llvm", lambda: [system("llvm-config-20 --includedir")+"/llvm-c/**/*.h"], dll=llvm_lib, args=lambda: system("llvm-config-20 --cflags").split(), recsym=True, prolog=["from tinygrad.helpers import WIN, OSX"]) - case "pci": return load("pci", ["{}/usr/include/linux/pci_regs.h"], srcs=linux_headers_deb, - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_deb.split('/')[-1]} && tar xf data.tar.xz", cwd=path, shell=True, check=True)) - case "vfio": return load("vfio", ["{}/usr/include/linux/vfio.h"], args=["-I{}/usr/include"], srcs=linux_headers_deb, - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_deb.split('/')[-1]} && tar xf data.tar.xz", cwd=path, shell=True, check=True)) + case "pci": return load("pci", ["{}/usr/include/linux/pci_regs.h"], srcs=linux_headers_deb, preprocess=_extract_deb) + case "vfio": return load("vfio", ["{}/usr/include/linux/vfio.h"], args=["-I{}/usr/include"], srcs=linux_headers_deb, preprocess=_extract_deb) # could add rule: WGPU_COMMA -> ',' case "webgpu": return load("webgpu", [root/"extra/webgpu/webgpu.h"], dll=webgpu_lib, prolog=["from tinygrad.helpers import WIN, OSX", "import sysconfig, os"]) @@ -115,9 +119,10 @@ def __getattr__(nm): dll="os.getenv('ROCM_PATH', '/opt/rocm')+'/lib/libamdhip64.so'", args=["-D__HIP_PLATFORM_AMD__", "-I/opt/rocm/include", "-x", "c++"], prolog=["import os"]) case "comgr" | "comgr_3": - return load("comgr_3" if nm == "comgr_3" else "comgr", ["/opt/rocm/include/amd_comgr/amd_comgr.h"], - dll= "[os.getenv('ROCM_PATH', '/opt/rocm')+'/lib/libamd_comgr.so', 'amd_comgr']", - args=["-D__HIP_PLATFORM_AMD__", "-I/opt/rocm/include", "-x", "c++"], prolog=["import os"]) + prefix = "{}/opt/rocm-6.2.0" if nm == "comgr" else "/opt/rocm" + return load(nm, [f"{prefix}/include/amd_comgr/amd_comgr.h"], dll="[os.getenv('ROCM_PATH', '/opt/rocm')+'/lib/libamd_comgr.so', 'amd_comgr']", + args=["-D__HIP_PLATFORM_AMD__", f"-I{prefix}/include", "-x", "c++"], prolog=["import os"], srcs=comgr_2_src if nm == "comgr" else None, + **({'preprocess':_extract_deb} if nm == "comgr" else {})) case "hsa": return load("hsa", [*[f"{{}}/projects/rocr-runtime/runtime/hsa-runtime/core/inc/{s}.h" for s in ["registers"]], *[f"{{}}/projects/rocr-runtime/runtime/hsa-runtime/inc/{s}.h" for s in [ "hsa", "hsa_ext_amd", "amd_hsa_signal", "amd_hsa_queue", "amd_hsa_kernel_code", @@ -157,7 +162,7 @@ def __getattr__(nm): *[f"python3 src/compiler/nir/nir_{s}_h.py --outdir gen" for s in ["intrinsics", "intrinsics_indices"]]]), cwd=path, shell=True, check=True), srcs="https://gitlab.freedesktop.org/mesa/mesa/-/archive/mesa-25.2.7/mesa-25.2.7.tar.gz", dll=f"'tinymesa_cpu' if DEV.renderer == 'LVP' else 'tinymesa', {tinymesa_path}, emsg='pip install tinymesa==25.2.7.2'", - prolog=["from tinygrad.helpers import DEV", "import gzip, base64, platform, sysconfig, os"], + prolog=["from tinygrad.helpers import DEV", "import gzip, base64, sysconfig, os"], epilog=lambda path: [system(f"{root}/extra/mesa/lvp_nir_options.sh {path}")]) case "libclang": return load("libclang", @@ -180,6 +185,17 @@ def __getattr__(nm): return load("mlx5", [root/"extra/mlx_driver/mlx5.h", f"{kh}/mlx5_ifc.h"], srcs=linux_headers_kern_deb, args=["-Du8=unsigned char", "-Du16=unsigned short", "-Du32=unsigned int", "-Du64=unsigned long long", "-D__be16=unsigned short", "-D__be32=unsigned int", "-D__be64=unsigned long long", f"-I{kh}"], - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_kern_deb.split('/')[-1]} && tar xf data.tar.xz", - cwd=path, shell=True, check=True)) + preprocess=_extract_deb) + case "bnxt": + kh = "{}/usr/src/linux-headers-6.18.9+deb14-common/include" + return load("bnxt", [f"{kh}/linux/bnxt/hsi.h", *[f"{{}}/{s.split('/')[-1]}" for s in bnxt_src]], + srcs=[linux_headers_kern_deb, *bnxt_src], + args=["-Du8=unsigned char", "-Du32=unsigned int", "-Du64=unsigned long long", "-D__le16=unsigned short", + "-D__le32=unsigned int", "-D__le64=unsigned long long", "-D__be16=unsigned short", "-D__be32=unsigned int", f"-I{kh}"], + patterns=[r"hwrm_((ver_get|func_(qcaps|qcfg|reset|drv_rgtr|backing_store_(qcaps|cfg)_v2)|stat_ctx_alloc|ring_alloc" + r"|vnic_(alloc|cfg)|cfa_l2_filter_alloc|port_phy_cfg)_(input|output)|(cmd|resp)_hdr)$", + r"((cmdq|creq)_(base|init|add_gid|create_(cq|qp)|initialize_fw|modify_qp|query_version|register_mr)(_resp)?" + r"|cq_(base|req)|sq_(rdma_hdr|sge))$", + r"(BNXT|CMDQ|CREQ|CQ|SQ|DBC|PTU|RCFW|HWRM|VNIC|RING_ALLOC|STAT_CTX|CFA_L2_FILTER|PORT_PHY_CFG|FIRMWARE_FIRST" + r"|FUNC_(QCAPS|QCFG|RESET|DRV_RGTR|BACKING_STORE))_"], preprocess=_extract_deb) case _: raise AttributeError(f"no such autogen: {nm}") diff --git a/tinygrad_repo/tinygrad/runtime/autogen/am/__init__.py b/tinygrad_repo/tinygrad/runtime/autogen/am/__init__.py index 54c5f2cc9..6d6507fb6 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/am/__init__.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/am/__init__.py @@ -8,7 +8,7 @@ am_src="https://github.com/ROCm/ROCK-Kernel-Driver/archive/33970e1351f5e51102960 rocm_src="https://github.com/ROCm/rocm-systems/archive/cccc350dc620e61ae2554978b62ab3532dc10bd9.tar.gz" AMD, AMDINC = "{}/drivers/gpu/drm/amd", "{}/drivers/gpu/drm/amd/include" inc, kern_rules = ["-include", "stdint.h"], [(r'le32_to_cpu', ''),] -fw_src="https://gitlab.com/kernel-firmware/linux-firmware/-/archive/1e2c15348485939baf1b6d1f5a7a3b799d80703d/1e2c15348485939baf1b6d1f5a7a3b799d80703d.tar.gz" +fw_src="https://gitlab.com/kernel-firmware/linux-firmware/-/archive/0a6871b19abf5d6e024b5d208b101ae53e7fa0de/0a6871b19abf5d6e024b5d208b101ae53e7fa0de.tar.gz" pmc_src="https://raw.githubusercontent.com/ROCm/rocm-systems/cccc350dc620e61ae2554978b62ab3532dc10bd9/projects/rocprofiler-compute/src/rocprof_compute_soc/profile_configs/counter_defs.yaml" reg_files = { @@ -22,11 +22,12 @@ reg_files = { reg_patterns = { "gc": ["GCVM", "GCMC_VM", "CP_(HQD|MQD|MEC|ME_CNTL|PERFMON|RB_WPTR_POLL_CNTL|INT_CNTL|STAT|PFP_PRGRM|ME_PRGRM|COHER_START)", "COMPUTE_", "(SQ|GL2C|TCC)_PERFCOUNTER", "SQ_THREAD_TRACE", "SPI_(CONFIG_CNTL|COMPUTE_QUEUE_RESET)", "GRBM", "SH_MEM", "RLC", "TCP", "GB_ADDR_CONFIG", - "SDMA[01]_(WATCHDOG_CNTL|UTCL1_(CNTL|PAGE)|MCU_CNTL|F32_CNTL|CNTL|QUEUE0_|RLC_CGCG_CTRL)", "SCRATCH_REG[67]"], + "SDMA[01]_(WATCHDOG_CNTL|UTCL1_(CNTL|PAGE)|MCU_CNTL|F32_CNTL|CNTL|QUEUE0_|RLC_CGCG_CTRL)", "SCRATCH_REG[0-367]"], "mmhub": ["MMVM", "MMMC_VM", "MM_ATC_L2_MISC_CG"], "nbio": (nbio:=["BIF_BX_PF[01]_GPU_HDP_FLUSH", "BIF_BX_PF0_RSMU", "BIF_BX0_(REMAP_HDP_MEM_FLUSH_CNTL|BIF_DOORBELL_INT_CNTL|PCIE_INDEX2|PCIE_DATA2)", "BIFC_(DOORBELL_ACCESS_EN_PF|GFX_INT_MONITOR_MASK)", "XCC_DOORBELL_FENCE", "DOORBELL0_CTRL_ENTRY", "GDC_S2A0_S2A_DOORBELL_ENTRY", - "S2A_DOORBELL_ENTRY", "RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN", "RCC_DEV0_EPF2_STRAP2"]), + "S2A_DOORBELL_ENTRY", "RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN", + "BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL", "RCC_DEV0_EPF2_STRAP2"]), "nbif": nbio, "mp": ["MP([01]|ASP)_SMN_C2PMSG"], "hdp": ["HDP_MEM_POWER_CTRL"], "oss": ["IH_"], "sdma": ["SDMA_GFX", "SDMA_CNTL"] } @@ -38,7 +39,7 @@ def __getattr__(nm): case "am": return load("am/am", [root/f"extra/amdpci/headers/{s}.h" for s in ["v11_structs", "v12_structs", "amdgpu_vm", "discovery", "amdgpu_ucode", "psp_gfx_if", "amdgpu_psp", "amdgpu_irq", "amdgpu_doorbell"]] + [f"{AMD}/amdkfd/soc15_int.h"] + \ [f"{AMDINC}/ivsrcid/{s}.h" for s in [f"gfx/irqsrcs_gfx_{x}_0" for x in ('9','11_0','12_0')] + [f"sdma0/irqsrcs_sdma0_{x}_0" for x in (4,5)]] + \ - [f"{AMDINC}/{s}.h" for s in ["v9_structs", "soc15_ih_clientid"]], args=inc, srcs=am_src, rules=kern_rules) + [f"{AMDINC}/{s}.h" for s in ["v9_structs", "soc15_ih_clientid"]] + [f"{AMD}/amdgpu/{s}.h" for s in ["mxgpu_nv", "amdgpu_virt"]], args=inc, srcs=am_src, rules=kern_rules) case "pm4_soc15": return load("am/pm4_soc15", [f"{AMD}/amdkfd/kfd_pm4_headers_ai.h", f"{AMD}/amdgpu/soc15d.h"], srcs=am_src) case "pm4_nv": return load("am/pm4_nv", [f"{AMD}/amdkfd/kfd_pm4_headers_ai.h", f"{AMD}/amdgpu/nvd.h"], srcs=am_src) case "sdma_4_0_0": return load("am/sdma_4_0_0", [root/"extra/hip_gpu_driver/sdma_registers.h", f"{AMD}/amdgpu/vega10_sdma_pkt_open.h"], diff --git a/tinygrad_repo/tinygrad/runtime/autogen/am/am.py b/tinygrad_repo/tinygrad/runtime/autogen/am/am.py index 15bc876f4..bbe5ee5a1 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/am/am.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/am/am.py @@ -4110,6 +4110,110 @@ class struct_v9_gfx_meta_data(c.Struct): struct_v9_gfx_meta_data.register_fields([('ce_payload', struct_v9_ce_ib_state, 0), ('reserved1', c.Array[uint32_t, Literal[54]], 40), ('de_payload', struct_v9_de_ib_state, 256), ('DeIbBaseAddrLo', uint32_t, 364), ('DeIbBaseAddrHi', uint32_t, 368), ('reserved2', c.Array[uint32_t, Literal[931]], 372)]) enum_soc15_ih_clientid: dict[int, str] = {(SOC15_IH_CLIENTID_IH:=0): 'SOC15_IH_CLIENTID_IH', (SOC15_IH_CLIENTID_ACP:=1): 'SOC15_IH_CLIENTID_ACP', (SOC15_IH_CLIENTID_ATHUB:=2): 'SOC15_IH_CLIENTID_ATHUB', (SOC15_IH_CLIENTID_BIF:=3): 'SOC15_IH_CLIENTID_BIF', (SOC15_IH_CLIENTID_DCE:=4): 'SOC15_IH_CLIENTID_DCE', (SOC15_IH_CLIENTID_ISP:=5): 'SOC15_IH_CLIENTID_ISP', (SOC15_IH_CLIENTID_PCIE0:=6): 'SOC15_IH_CLIENTID_PCIE0', (SOC15_IH_CLIENTID_RLC:=7): 'SOC15_IH_CLIENTID_RLC', (SOC15_IH_CLIENTID_SDMA0:=8): 'SOC15_IH_CLIENTID_SDMA0', (SOC15_IH_CLIENTID_SDMA1:=9): 'SOC15_IH_CLIENTID_SDMA1', (SOC15_IH_CLIENTID_SE0SH:=10): 'SOC15_IH_CLIENTID_SE0SH', (SOC15_IH_CLIENTID_SE1SH:=11): 'SOC15_IH_CLIENTID_SE1SH', (SOC15_IH_CLIENTID_SE2SH:=12): 'SOC15_IH_CLIENTID_SE2SH', (SOC15_IH_CLIENTID_SE3SH:=13): 'SOC15_IH_CLIENTID_SE3SH', (SOC15_IH_CLIENTID_UVD1:=14): 'SOC15_IH_CLIENTID_UVD1', (SOC15_IH_CLIENTID_THM:=15): 'SOC15_IH_CLIENTID_THM', (SOC15_IH_CLIENTID_UVD:=16): 'SOC15_IH_CLIENTID_UVD', (SOC15_IH_CLIENTID_VCE0:=17): 'SOC15_IH_CLIENTID_VCE0', (SOC15_IH_CLIENTID_VMC:=18): 'SOC15_IH_CLIENTID_VMC', (SOC15_IH_CLIENTID_XDMA:=19): 'SOC15_IH_CLIENTID_XDMA', (SOC15_IH_CLIENTID_GRBM_CP:=20): 'SOC15_IH_CLIENTID_GRBM_CP', (SOC15_IH_CLIENTID_ATS:=21): 'SOC15_IH_CLIENTID_ATS', (SOC15_IH_CLIENTID_ROM_SMUIO:=22): 'SOC15_IH_CLIENTID_ROM_SMUIO', (SOC15_IH_CLIENTID_DF:=23): 'SOC15_IH_CLIENTID_DF', (SOC15_IH_CLIENTID_VCE1:=24): 'SOC15_IH_CLIENTID_VCE1', (SOC15_IH_CLIENTID_PWR:=25): 'SOC15_IH_CLIENTID_PWR', (SOC15_IH_CLIENTID_RESERVED:=26): 'SOC15_IH_CLIENTID_RESERVED', (SOC15_IH_CLIENTID_UTCL2:=27): 'SOC15_IH_CLIENTID_UTCL2', (SOC15_IH_CLIENTID_EA:=28): 'SOC15_IH_CLIENTID_EA', (SOC15_IH_CLIENTID_UTCL2LOG:=29): 'SOC15_IH_CLIENTID_UTCL2LOG', (SOC15_IH_CLIENTID_MP0:=30): 'SOC15_IH_CLIENTID_MP0', (SOC15_IH_CLIENTID_MP1:=31): 'SOC15_IH_CLIENTID_MP1', (SOC15_IH_CLIENTID_MAX:=32): 'SOC15_IH_CLIENTID_MAX', (SOC15_IH_CLIENTID_VCN:=16): 'SOC15_IH_CLIENTID_VCN', (SOC15_IH_CLIENTID_VCN1:=14): 'SOC15_IH_CLIENTID_VCN1', (SOC15_IH_CLIENTID_SDMA2:=1): 'SOC15_IH_CLIENTID_SDMA2', (SOC15_IH_CLIENTID_SDMA3:=4): 'SOC15_IH_CLIENTID_SDMA3', (SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid:=5): 'SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid', (SOC15_IH_CLIENTID_SDMA4:=5): 'SOC15_IH_CLIENTID_SDMA4', (SOC15_IH_CLIENTID_SDMA5:=17): 'SOC15_IH_CLIENTID_SDMA5', (SOC15_IH_CLIENTID_SDMA6:=19): 'SOC15_IH_CLIENTID_SDMA6', (SOC15_IH_CLIENTID_SDMA7:=24): 'SOC15_IH_CLIENTID_SDMA7', (SOC15_IH_CLIENTID_VMC1:=6): 'SOC15_IH_CLIENTID_VMC1'} enum_soc21_ih_clientid: dict[int, str] = {(SOC21_IH_CLIENTID_IH:=0): 'SOC21_IH_CLIENTID_IH', (SOC21_IH_CLIENTID_ATHUB:=2): 'SOC21_IH_CLIENTID_ATHUB', (SOC21_IH_CLIENTID_BIF:=3): 'SOC21_IH_CLIENTID_BIF', (SOC21_IH_CLIENTID_DCN:=4): 'SOC21_IH_CLIENTID_DCN', (SOC21_IH_CLIENTID_ISP:=5): 'SOC21_IH_CLIENTID_ISP', (SOC21_IH_CLIENTID_MP3:=6): 'SOC21_IH_CLIENTID_MP3', (SOC21_IH_CLIENTID_RLC:=7): 'SOC21_IH_CLIENTID_RLC', (SOC21_IH_CLIENTID_GFX:=10): 'SOC21_IH_CLIENTID_GFX', (SOC21_IH_CLIENTID_IMU:=11): 'SOC21_IH_CLIENTID_IMU', (SOC21_IH_CLIENTID_VCN1:=14): 'SOC21_IH_CLIENTID_VCN1', (SOC21_IH_CLIENTID_THM:=15): 'SOC21_IH_CLIENTID_THM', (SOC21_IH_CLIENTID_VCN:=16): 'SOC21_IH_CLIENTID_VCN', (SOC21_IH_CLIENTID_VPE1:=17): 'SOC21_IH_CLIENTID_VPE1', (SOC21_IH_CLIENTID_VMC:=18): 'SOC21_IH_CLIENTID_VMC', (SOC21_IH_CLIENTID_GRBM_CP:=20): 'SOC21_IH_CLIENTID_GRBM_CP', (SOC21_IH_CLIENTID_ROM_SMUIO:=22): 'SOC21_IH_CLIENTID_ROM_SMUIO', (SOC21_IH_CLIENTID_DF:=23): 'SOC21_IH_CLIENTID_DF', (SOC21_IH_CLIENTID_VPE:=24): 'SOC21_IH_CLIENTID_VPE', (SOC21_IH_CLIENTID_PWR:=25): 'SOC21_IH_CLIENTID_PWR', (SOC21_IH_CLIENTID_LSDMA:=26): 'SOC21_IH_CLIENTID_LSDMA', (SOC21_IH_CLIENTID_MP0:=30): 'SOC21_IH_CLIENTID_MP0', (SOC21_IH_CLIENTID_MP1:=31): 'SOC21_IH_CLIENTID_MP1', (SOC21_IH_CLIENTID_MAX:=32): 'SOC21_IH_CLIENTID_MAX'} +enum_idh_request: dict[int, str] = {(IDH_REQ_GPU_INIT_ACCESS:=1): 'IDH_REQ_GPU_INIT_ACCESS', (IDH_REL_GPU_INIT_ACCESS:=2): 'IDH_REL_GPU_INIT_ACCESS', (IDH_REQ_GPU_FINI_ACCESS:=3): 'IDH_REQ_GPU_FINI_ACCESS', (IDH_REL_GPU_FINI_ACCESS:=4): 'IDH_REL_GPU_FINI_ACCESS', (IDH_REQ_GPU_RESET_ACCESS:=5): 'IDH_REQ_GPU_RESET_ACCESS', (IDH_REQ_GPU_INIT_DATA:=6): 'IDH_REQ_GPU_INIT_DATA', (IDH_LOG_VF_ERROR:=200): 'IDH_LOG_VF_ERROR', (IDH_READY_TO_RESET:=201): 'IDH_READY_TO_RESET', (IDH_RAS_POISON:=202): 'IDH_RAS_POISON', (IDH_REQ_RAS_ERROR_COUNT:=203): 'IDH_REQ_RAS_ERROR_COUNT', (IDH_REQ_RAS_CPER_DUMP:=204): 'IDH_REQ_RAS_CPER_DUMP', (IDH_REQ_RAS_BAD_PAGES:=205): 'IDH_REQ_RAS_BAD_PAGES', (IDH_REQ_RAS_CHK_CRITI:=206): 'IDH_REQ_RAS_CHK_CRITI'} +enum_idh_event: dict[int, str] = {(IDH_CLR_MSG_BUF:=0): 'IDH_CLR_MSG_BUF', (IDH_READY_TO_ACCESS_GPU:=1): 'IDH_READY_TO_ACCESS_GPU', (IDH_FLR_NOTIFICATION:=2): 'IDH_FLR_NOTIFICATION', (IDH_FLR_NOTIFICATION_CMPL:=3): 'IDH_FLR_NOTIFICATION_CMPL', (IDH_SUCCESS:=4): 'IDH_SUCCESS', (IDH_FAIL:=5): 'IDH_FAIL', (IDH_QUERY_ALIVE:=6): 'IDH_QUERY_ALIVE', (IDH_REQ_GPU_INIT_DATA_READY:=7): 'IDH_REQ_GPU_INIT_DATA_READY', (IDH_RAS_POISON_READY:=8): 'IDH_RAS_POISON_READY', (IDH_PF_SOFT_FLR_NOTIFICATION:=9): 'IDH_PF_SOFT_FLR_NOTIFICATION', (IDH_RAS_ERROR_DETECTED:=10): 'IDH_RAS_ERROR_DETECTED', (IDH_RAS_ERROR_COUNT_READY:=11): 'IDH_RAS_ERROR_COUNT_READY', (IDH_RAS_CPER_DUMP_READY:=14): 'IDH_RAS_CPER_DUMP_READY', (IDH_RAS_BAD_PAGES_READY:=15): 'IDH_RAS_BAD_PAGES_READY', (IDH_RAS_BAD_PAGES_NOTIFICATION:=16): 'IDH_RAS_BAD_PAGES_NOTIFICATION', (IDH_UNRECOV_ERR_NOTIFICATION:=17): 'IDH_UNRECOV_ERR_NOTIFICATION', (IDH_REQ_RAS_CHK_CRITI_READY:=18): 'IDH_REQ_RAS_CHK_CRITI_READY', (IDH_TEXT_MESSAGE:=255): 'IDH_TEXT_MESSAGE'} +class struct_amdgpu_virt_ops(c.Struct): pass +enum_amdgpu_sriov_vf_mode: dict[int, str] = {(SRIOV_VF_MODE_BARE_METAL:=0): 'SRIOV_VF_MODE_BARE_METAL', (SRIOV_VF_MODE_ONE_VF:=1): 'SRIOV_VF_MODE_ONE_VF', (SRIOV_VF_MODE_MULTI_VF:=2): 'SRIOV_VF_MODE_MULTI_VF'} +@c.record +class struct_amdgpu_mm_table(c.Struct): + SIZE = 24 + bo: c.POINTER[struct_amdgpu_bo] + cpu_addr: c.POINTER[ctypes.c_uint32] + gpu_addr: int +class struct_amdgpu_bo(c.Struct): pass +struct_amdgpu_mm_table.register_fields([('bo', c.POINTER[struct_amdgpu_bo], 0), ('cpu_addr', c.POINTER[uint32_t], 8), ('gpu_addr', uint64_t, 16)]) +class struct_amdgpu_vf_error_buffer(c.Struct): pass +@c.record +class struct_amdgpu_virt_fw_reserve(c.Struct): + SIZE = 32 + p_pf2vf: c.POINTER[struct_amd_sriov_msg_pf2vf_info_header] + p_vf2pf: c.POINTER[struct_amd_sriov_msg_vf2pf_info_header] + ras_telemetry: ctypes.c_void_p + checksum_key: int +@c.record +class struct_amd_sriov_msg_pf2vf_info_header(c.Struct): + SIZE = 16 + size: int + version: int + reserved: c.Array[ctypes.c_uint32, Literal[2]] +struct_amd_sriov_msg_pf2vf_info_header.register_fields([('size', uint32_t, 0), ('version', uint32_t, 4), ('reserved', c.Array[uint32_t, Literal[2]], 8)]) +@c.record +class struct_amd_sriov_msg_vf2pf_info_header(c.Struct): + SIZE = 16 + size: int + version: int + reserved: c.Array[ctypes.c_uint32, Literal[2]] +struct_amd_sriov_msg_vf2pf_info_header.register_fields([('size', uint32_t, 0), ('version', uint32_t, 4), ('reserved', c.Array[uint32_t, Literal[2]], 8)]) +struct_amdgpu_virt_fw_reserve.register_fields([('p_pf2vf', c.POINTER[struct_amd_sriov_msg_pf2vf_info_header], 0), ('p_vf2pf', c.POINTER[struct_amd_sriov_msg_vf2pf_info_header], 8), ('ras_telemetry', ctypes.c_void_p, 16), ('checksum_key', ctypes.c_uint32, 24)]) +enum_AMDGIM_FEATURE_FLAG: dict[int, str] = {(AMDGIM_FEATURE_ERROR_LOG_COLLECT:=1): 'AMDGIM_FEATURE_ERROR_LOG_COLLECT', (AMDGIM_FEATURE_GIM_LOAD_UCODES:=2): 'AMDGIM_FEATURE_GIM_LOAD_UCODES', (AMDGIM_FEATURE_GIM_FLR_VRAMLOST:=4): 'AMDGIM_FEATURE_GIM_FLR_VRAMLOST', (AMDGIM_FEATURE_GIM_MM_BW_MGR:=8): 'AMDGIM_FEATURE_GIM_MM_BW_MGR', (AMDGIM_FEATURE_PP_ONE_VF:=16): 'AMDGIM_FEATURE_PP_ONE_VF', (AMDGIM_FEATURE_INDIRECT_REG_ACCESS:=32): 'AMDGIM_FEATURE_INDIRECT_REG_ACCESS', (AMDGIM_FEATURE_AV1_SUPPORT:=64): 'AMDGIM_FEATURE_AV1_SUPPORT', (AMDGIM_FEATURE_VCN_RB_DECOUPLE:=128): 'AMDGIM_FEATURE_VCN_RB_DECOUPLE', (AMDGIM_FEATURE_MES_INFO_ENABLE:=256): 'AMDGIM_FEATURE_MES_INFO_ENABLE', (AMDGIM_FEATURE_RAS_CAPS:=512): 'AMDGIM_FEATURE_RAS_CAPS', (AMDGIM_FEATURE_RAS_TELEMETRY:=1024): 'AMDGIM_FEATURE_RAS_TELEMETRY', (AMDGIM_FEATURE_RAS_CPER:=2048): 'AMDGIM_FEATURE_RAS_CPER', (AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK:=4096): 'AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK'} +enum_AMDGIM_REG_ACCESS_FLAG: dict[int, str] = {(AMDGIM_FEATURE_IH_REG_PSP_EN:=1): 'AMDGIM_FEATURE_IH_REG_PSP_EN', (AMDGIM_FEATURE_MMHUB_REG_RLC_EN:=2): 'AMDGIM_FEATURE_MMHUB_REG_RLC_EN', (AMDGIM_FEATURE_GC_REG_RLC_EN:=4): 'AMDGIM_FEATURE_GC_REG_RLC_EN', (AMDGIM_FEATURE_L1_TLB_CNTL_PSP_EN:=8): 'AMDGIM_FEATURE_L1_TLB_CNTL_PSP_EN', (AMDGIM_FEATURE_REG_ACCESS_SQ_CONFIG:=16): 'AMDGIM_FEATURE_REG_ACCESS_SQ_CONFIG'} +@c.record +class struct_amdgim_pf2vf_info_v1(c.Struct): + SIZE = 44 + header: struct_amd_sriov_msg_pf2vf_info_header + uvd_enc_max_pixels_count: int + uvd_enc_max_bandwidth: int + vce_enc_max_pixels_count: int + vce_enc_max_bandwidth: int + mecfw_kboffset: int + feature_flags: int + checksum: int +struct_amdgim_pf2vf_info_v1.register_fields([('header', struct_amd_sriov_msg_pf2vf_info_header, 0), ('uvd_enc_max_pixels_count', ctypes.c_uint32, 16), ('uvd_enc_max_bandwidth', ctypes.c_uint32, 20), ('vce_enc_max_pixels_count', ctypes.c_uint32, 24), ('vce_enc_max_bandwidth', ctypes.c_uint32, 28), ('mecfw_kboffset', ctypes.c_uint32, 32), ('feature_flags', ctypes.c_uint32, 36), ('checksum', ctypes.c_uint32, 40)]) +@c.record +class struct_amdgim_vf2pf_info_v1(c.Struct): + SIZE = 128 + header: struct_amd_sriov_msg_vf2pf_info_header + driver_version: c.Array[ctypes.c_char, Literal[64]] + driver_cert: int + os_info: int + fb_usage: int + gfx_usage: int + gfx_health: int + compute_usage: int + compute_health: int + vce_enc_usage: int + vce_enc_health: int + uvd_enc_usage: int + uvd_enc_health: int + checksum: int +struct_amdgim_vf2pf_info_v1.register_fields([('header', struct_amd_sriov_msg_vf2pf_info_header, 0), ('driver_version', c.Array[ctypes.c_char, Literal[64]], 16), ('driver_cert', ctypes.c_uint32, 80), ('os_info', ctypes.c_uint32, 84), ('fb_usage', ctypes.c_uint32, 88), ('gfx_usage', ctypes.c_uint32, 92), ('gfx_health', ctypes.c_uint32, 96), ('compute_usage', ctypes.c_uint32, 100), ('compute_health', ctypes.c_uint32, 104), ('vce_enc_usage', ctypes.c_uint32, 108), ('vce_enc_health', ctypes.c_uint32, 112), ('uvd_enc_usage', ctypes.c_uint32, 116), ('uvd_enc_health', ctypes.c_uint32, 120), ('checksum', ctypes.c_uint32, 124)]) +@c.record +class struct_amdgim_vf2pf_info_v2(c.Struct): + SIZE = 1024 + header: struct_amd_sriov_msg_vf2pf_info_header + checksum: int + driver_version: c.Array[ctypes.c_ubyte, Literal[64]] + driver_cert: int + os_info: int + fb_usage: int + gfx_usage: int + gfx_health: int + compute_usage: int + compute_health: int + vce_enc_usage: int + vce_enc_health: int + uvd_enc_usage: int + uvd_enc_health: int + reserved: c.Array[ctypes.c_uint32, Literal[224]] +struct_amdgim_vf2pf_info_v2.register_fields([('header', struct_amd_sriov_msg_vf2pf_info_header, 0), ('checksum', uint32_t, 16), ('driver_version', c.Array[uint8_t, Literal[64]], 20), ('driver_cert', uint32_t, 84), ('os_info', uint32_t, 88), ('fb_usage', uint32_t, 92), ('gfx_usage', uint32_t, 96), ('gfx_health', uint32_t, 100), ('compute_usage', uint32_t, 104), ('compute_health', uint32_t, 108), ('vce_enc_usage', uint32_t, 112), ('vce_enc_health', uint32_t, 116), ('uvd_enc_usage', uint32_t, 120), ('uvd_enc_health', uint32_t, 124), ('reserved', c.Array[uint32_t, Literal[224]], 128)]) +@c.record +class struct_amdgpu_virt_ras_err_handler_data(c.Struct): + SIZE = 24 + bps: c.POINTER[struct_eeprom_table_record] + bps_bo: c.POINTER[c.POINTER[struct_amdgpu_bo]] + count: int + last_reserved: int +class struct_eeprom_table_record(c.Struct): pass +struct_amdgpu_virt_ras_err_handler_data.register_fields([('bps', c.POINTER[struct_eeprom_table_record], 0), ('bps_bo', c.POINTER[c.POINTER[struct_amdgpu_bo]], 8), ('count', ctypes.c_int32, 16), ('last_reserved', ctypes.c_int32, 20)]) +class struct_amdgpu_virt_ras(c.Struct): pass +@c.record +class struct_amdgpu_virt_region(c.Struct): + SIZE = 8 + offset: int + size_kb: int +struct_amdgpu_virt_region.register_fields([('offset', uint32_t, 0), ('size_kb', uint32_t, 4)]) +class struct_amdgpu_virt(c.Struct): pass +class struct_amdgpu_video_codec_info(c.Struct): pass AMDGPU_VM_MAX_UPDATE_SIZE = 0x3FFFF AMDGPU_PTE_VALID = (1 << 0) AMDGPU_PTE_SYSTEM = (1 << 1) @@ -4465,4 +4569,47 @@ SDMA0_5_0__SRCID__SDMA_CTXEMPTY = 243 SDMA0_5_0__SRCID__SDMA_DOORBELL_INVALID = 244 SDMA0_5_0__SRCID__SDMA_FROZEN = 245 SDMA0_5_0__SRCID__SDMA_POLL_TIMEOUT = 246 -SDMA0_5_0__SRCID__SDMA_SRBMWRITE = 247 \ No newline at end of file +SDMA0_5_0__SRCID__SDMA_SRBMWRITE = 247 +NV_MAILBOX_POLL_ACK_TIMEDOUT = 500 +NV_MAILBOX_POLL_MSG_TIMEDOUT = 15000 +NV_MAILBOX_POLL_FLR_TIMEDOUT = 10000 +NV_MAILBOX_POLL_MSG_REP_MAX = 11 +mmMAILBOX_CONTROL = 0xE5E +NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE = (mmMAILBOX_CONTROL * 4) +NV_MAIBOX_CONTROL_RCV_OFFSET_BYTE = (NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE + 1) +mmMAILBOX_MSGBUF_TRN_DW0 = 0xE56 +mmMAILBOX_MSGBUF_TRN_DW1 = 0xE57 +mmMAILBOX_MSGBUF_TRN_DW2 = 0xE58 +mmMAILBOX_MSGBUF_TRN_DW3 = 0xE59 +mmMAILBOX_MSGBUF_RCV_DW0 = 0xE5A +mmMAILBOX_MSGBUF_RCV_DW1 = 0xE5B +mmMAILBOX_MSGBUF_RCV_DW2 = 0xE5C +mmMAILBOX_MSGBUF_RCV_DW3 = 0xE5D +mmMAILBOX_INT_CNTL = 0xE5F +AMDGPU_SRIOV_CAPS_SRIOV_VBIOS = (1 << 0) +AMDGPU_SRIOV_CAPS_ENABLE_IOV = (1 << 1) +AMDGPU_SRIOV_CAPS_IS_VF = (1 << 2) +AMDGPU_PASSTHROUGH_MODE = (1 << 3) +AMDGPU_SRIOV_CAPS_RUNTIME = (1 << 4) +AMDGPU_VF_MMIO_ACCESS_PROTECT = (1 << 5) +AMDGPU_RLCG_GC_WRITE_LEGACY = (0x8 << 28) +AMDGPU_RLCG_GC_WRITE = (0x0 << 28) +AMDGPU_RLCG_GC_READ = (0x1 << 28) +AMDGPU_RLCG_MMHUB_WRITE = (0x2 << 28) +AMDGPU_RLCG_VFGATE_DISABLED = 0x4000000 +AMDGPU_RLCG_WRONG_OPERATION_TYPE = 0x2000000 +AMDGPU_RLCG_REG_NOT_IN_RANGE = 0x1000000 +AMDGPU_RLCG_SCRATCH1_ADDRESS_MASK = 0xFFFFF +AMDGPU_RLCG_SCRATCH1_ERROR_MASK = 0xF000000 +mmRCC_IOV_FUNC_IDENTIFIER = 0xDE5 +mmBIF_IOV_FUNC_IDENTIFIER = 0x1503 +AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT = 2 +AMDGPU_SRIOV_CRIT_DATA_SIGNATURE = "INDA" +AMDGPU_SRIOV_CRIT_DATA_SIG_LEN = 4 +AMDGPU_VF_ERROR_ENTRY_SIZE = 16 +AMDGIM_DATAEXCHANGE_OFFSET = (64 * 1024) +AMDGIM_GET_STRUCTURE_RESERVED_SIZE = lambda total,u8,u16,u32,u64: (total - (((u8)+3) / 4 + ((u16)+1) / 2 + (u32) + (u64)*2)) # type: ignore +AMDGPU_VIRT_CAPS_LIST = lambda X: X(AMDGPU_VIRT_CAP_POWER_LIMIT) # type: ignore +amdgpu_sriov_fullaccess = lambda adev: (amdgpu_sriov_vf((adev)) and not amdgpu_sriov_runtime((adev))) # type: ignore +amdgpu_sriov_rlcg_error_report_enabled = lambda adev: (amdgpu_sriov_reg_indirect_mmhub(adev) or amdgpu_sriov_reg_indirect_gc(adev)) # type: ignore +amdgpu_sriov_multi_vf_mode = lambda adev: (amdgpu_sriov_vf(adev) and not amdgpu_sriov_is_pp_one_vf(adev)) # type: ignore \ No newline at end of file diff --git a/tinygrad_repo/tinygrad/runtime/autogen/am/fw.py b/tinygrad_repo/tinygrad/runtime/autogen/am/fw.py index 4b6d62e14..b4fa3880a 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/am/fw.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/am/fw.py @@ -1,81 +1,82 @@ hashes = { - 'psp_13_0_0_sos.bin': 'b5592f46885585b935e013f46c949db8ff2f15c0b346caf70e7fcd2776623d13', + 'psp_13_0_0_sos.bin': '4a51299f6d0a15bbba9694419f7891e6accc01dbd2dd67c06add7bfd75a45ac6', 'psp_13_0_10_sos.bin': '0bcaaad9cd8578d3841ae69155a6bd4fc3ceae8f4fb5a6ba4f576e7ace94d1d9', - 'psp_13_0_12_sos.bin': '89da90bf4286b38678b1fd175c78462a426afa3d258d15872cd14072d7098b9b', - 'psp_13_0_14_sos.bin': 'a4f0d5f76d27b77409ec0b71d7cc6a848ddfd29f8c84f3003edf74ad3999fb7d', - 'psp_13_0_6_sos.bin': '27657daa0f91ad8095d3610224a7de748b8b348a4cb211ecb5fccabe47369716', - 'psp_13_0_7_sos.bin': 'ef1af0ecea38abbac6f85cce71789f19848c498d0cb8ef13748dab2d65b23c31', + 'psp_13_0_12_sos.bin': '7113a165c75c232d4cb7193a920b503e0bf082689adde3b45fdc38f58bfd18b3', + 'psp_13_0_14_sos.bin': 'db863768cb25e806b68033e9237e0869f9f3603119df4d369ff4d80418d585d0', + 'psp_13_0_15_sos.bin': '3b28d53e75a88131155e3931378ac8434eca4880ada9211d3b4e8915b6289583', + 'psp_13_0_6_sos.bin': '36cce3a9441a0dcde81badd8fcf0416de8e4c39a7707865eff4d9d75e6bb0466', + 'psp_13_0_7_sos.bin': '94db505fa6482f258c33a0a8d412050f6d843ab4ada368252e988f82f8a26fa8', 'psp_14_0_2_sos.bin': '7b538448b57d4f9dd06b2eea90d4f86a16e65e3027cdecee8db71c2c5f1fa243', - 'psp_14_0_3_sos.bin': '23bea01a0c6f36d00759d0765d46cb4cb4aa87398b2fbccacbf547a890c0bf51', - 'smu_13_0_0.bin': '2ffac37fd8534965eeba19755db0e5ec80278213487dc4af0fbc8453befb64b1', - 'smu_13_0_0_kicker.bin': '7f83656a2a89b7fce1c8a85e96d91cd8265a91fe883a7027f1a0ed18ced501de', - 'smu_13_0_10.bin': 'daedb9cbdf48942be7ffe00d31b7c16bb36e11ff5a9d7495f218e95c07717b71', + 'psp_14_0_3_sos.bin': '28469a0857c813c54a0492423cdf0b0caf757428400036377e19c47e5af62478', + 'smu_13_0_0.bin': '93e46a5526f19dcc3d13bfd9e23f88bc8eee52138bfe9caf0951b4eef5e49914', + 'smu_13_0_0_kicker.bin': 'd0ef51d9ed06d0c17e06667302be21e7aedd86ed7a72be6e2f55b102214131cc', + 'smu_13_0_10.bin': '9376ae64149e6b0b684898ffbc12c2230f8c50a2e9447dc7dafc95c0c16b5227', 'smu_13_0_14.bin': 'a4f36de75fdcecd8000246762e027b4be489b6787afea57675225b0b39d35625', 'smu_13_0_6.bin': 'ad7232264e8c57c2094244fbdd5a55d7a4575ffe9b44d229884bc0b6a44fb0b1', - 'smu_13_0_7.bin': 'ccecc0fd0196b9613c920a51c2fd9436e739ff19dda5bdf74d97562387231732', - 'smu_14_0_2.bin': '6951995d1d606f4dc60c895f19d34ed18aa40e62129f83d8510c45e8aa9ae2fc', - 'smu_14_0_3.bin': 'df230947ddb7bbfd6e77d1280001db886e69adf2b2a448b47fa668a48bc0009f', - 'smu_14_0_3_kicker.bin': '8ddc1da5b4e1619796c2cc81f19f388a35bf7d78bfe476cee559625589cb4dc7', - 'sdma_4_4_2.bin': '456061b814268425843537da6f2191c8861d4e1a18d4c5d90c44ea6be18c78ff', - 'sdma_4_4_4.bin': 'af47a2940e72b932d3e3a7e8f34f7a182624e5e433f7c56dff939ca5549cd33a', - 'sdma_4_4_5.bin': '6127baabea3de7b18db3868c983b02c0fbf2cd75997f7f11241a5b1be27e5134', + 'smu_13_0_7.bin': '68ec18bd605e680085c927ff72c609f8c771aff0718d0cfab58a3744dff8e5b7', + 'smu_14_0_2.bin': '1b2054e3f710d1ab8dbdf6ff35914ad376b51caa6337831260c955add874b2ee', + 'smu_14_0_3.bin': '4e1522d3c96c1028be2961dfcfc5f1ff783fb77724b260a99c4c8b4a901ef3fe', + 'smu_14_0_3_kicker.bin': '9ff142656ae5f57be1b5ecc134e9da8f76650e793fbc5c499acd75094ff24453', + 'sdma_4_4_2.bin': 'ff885711d2d5d75ceed51cf239e93c882584b918cd5d5d1ff58ee5aecc0c50ae', + 'sdma_4_4_4.bin': '06a9d4d02c187844313a78469321d6091e59a334f0ce3b61d770d810c984d70b', + 'sdma_4_4_5.bin': 'e2a30faa9403933fbfba7ce8e9feba460fff6ecdb15304818d24c9f3eeaad0a6', 'sdma_5_2_6.bin': '3a163db00eb7e4752be8adbd61cf7dd8f08d924e59a6f798ced7dfcd89f340ed', 'sdma_5_2_7.bin': '16fe80dc866b323e15a06f51646ef0f036878ad34da66921fcdb8167207d6b2b', - 'sdma_6_0_0.bin': '0f3da6b211f376356335b41be07149f650c10cfa4e23f7e25d53836006ed11f5', - 'sdma_6_0_1.bin': 'ff565d3c215a30737560d4e3df6fc2c637738407e91d212fb200fdfb185b6744', - 'sdma_6_0_2.bin': '398380184bb69113ef4c8964a3b55f6184deb0c1ffd96c9683490a3eec3ba8f3', - 'sdma_6_0_3.bin': '0e8a83513087db865ba926f8b65cfb003fd41098f707e178d7a7ae2941fed0b1', - 'sdma_6_1_0.bin': '22e55d0ad5f0247a7f0fffc67cfd3161b39f24ad6062ff3c91ec7ff38bd7e1e1', - 'sdma_6_1_1.bin': '74533a581b8e3e2743b3c9c803d0666405e80898c4a630acefed82cb6b516ba2', - 'sdma_6_1_2.bin': '4fe04b0286ec739b0414e8aee17e62e85e691f0246d1d9b56bc18a1219072314', - 'sdma_6_1_3.bin': '35c9ed7e3a237c0d4a83b4975c63b62488f72aeafbb648342f384618e103f66b', + 'sdma_6_0_0.bin': '82cd01a24171af12de6c7ac4ee7471aa2dfcf51f9677e7bae9cd4c75e07761ee', + 'sdma_6_0_1.bin': '708c2c2c45262c98ebe8e34e32c3f1ce8eb5b007bab560c9ea9b576a8e4d6768', + 'sdma_6_0_2.bin': '16c374344d2894da751f7028f9ec1f7520035fa9548d8c750d99a00a4afa86c7', + 'sdma_6_0_3.bin': 'd47ac4db523aa35d77b27d57c35d4c04f431229dec6c0d667c36d98b985a6933', + 'sdma_6_1_0.bin': '85f9f3714de68eee74cdf4852d709bc36a5c73a00e943b707bc2ce10d6b7bab4', + 'sdma_6_1_1.bin': 'e7b7a23923ab691665e6ad16bbc8431a92f7c049da4b0b19a82c45fba03d4979', + 'sdma_6_1_2.bin': '5947d78eb308a3f6a62d772c5a6493b21439c73eac139f9c22f080f660b4f4f3', + 'sdma_6_1_3.bin': '8c651f32cbf030b6239ecc44f01bc9f5d5a193f333e21f2103736aff33227361', 'sdma_7_0_0.bin': 'beaafb53993a106edd392392d5896245ae2a957c6d0f495d0002eec72ad8ad38', 'sdma_7_0_1.bin': '73c29e1c1714ebc95d2221ba56e187910902891593010653bf9518937e414a59', - 'gc_10_3_6_pfp.bin': '793d678427887a0e724c79e356440aec33e6d1301f2a4e63543500249ebec064', + 'gc_10_3_6_pfp.bin': '042f5d2d223aac6a62b500a47d0d0bf33984200110da0ffca4fe5df9a96571c0', 'gc_10_3_7_pfp.bin': '3ae29aac3f424f7de97f82ce7158beba69509afb2dcbf1a428dc315df474a524', - 'gc_11_0_0_pfp.bin': 'e175cb0f580a38c961a6f7366142c08e413995f57f78f39795368b15442df8a3', - 'gc_11_0_1_pfp.bin': 'f5bf21dfbd9e72a30b4caf4704282c27854710c1b7c4affbb2a19530466b12a8', - 'gc_11_0_2_pfp.bin': '001c4dec1119e29314d725cc1280fc4f0cd9cabdf61ea5ee2260cfd4e62ec141', - 'gc_11_0_3_pfp.bin': '0488034c85be97125e39e860308d33c3f76a01df8250092a32d4d55acb2526fd', - 'gc_11_0_4_pfp.bin': '5ae8b7bb6316f87ae8b978354c088e3bd8c890959382d72886377cda25b1ffd1', - 'gc_11_5_0_pfp.bin': '0124f540871a7759fa8aaae046d458dfb34aeea12a1183ff962c3f1a33067d5a', - 'gc_11_5_1_pfp.bin': '7794ea46d0d3cf9cb3f7938affbdf09dd7a9970340da5cd02b774cb393436d24', - 'gc_11_5_2_pfp.bin': '55e64741de28c506524959f7f696713a72aafe46f49ccd827781d67a9475b386', - 'gc_11_5_3_pfp.bin': 'ce805040fb347fddbc89b2715e66b446865dda9e2056a9b233269b72bc09c387', - 'gc_12_0_0_pfp.bin': '16bfd64c10fe73b5e760055069a60e5841dba16c0ed4edb56c20d675e23901f6', - 'gc_12_0_1_pfp.bin': '49efb319305c5fffd90ac1eef7d7a0bdec72998ecb5cf4526996311788a53dc3', - 'gc_10_3_6_me.bin': '141b59faad3f2f1be16a2178833b7ca8e97519e1e844c8fda6689572c3767902', + 'gc_11_0_0_pfp.bin': 'b360393c8629144b194f69a3cd961ed509331feff7a5cc1e4eb21c901da2710a', + 'gc_11_0_1_pfp.bin': 'fb1ee527c05c55679c80a8bcf60fbb533724891baeb0eabc2917fc44e63a45dc', + 'gc_11_0_2_pfp.bin': '9020f53788ad881fa01aa656fc082f9f8d3cdfc81f70aaac0bed6e6001491128', + 'gc_11_0_3_pfp.bin': '362db904fa16c1fea2af7ad1295532434df7f85662b4a69332f51ae6c7290b61', + 'gc_11_0_4_pfp.bin': 'aad22ca342c47d857bc1107a9aa9127e5e4ba7f7fd42d432213b1850bda1f4e1', + 'gc_11_5_0_pfp.bin': '82ccf0265d841351183b011a79422799431f0c11f6d11165d64d7dfe404bda31', + 'gc_11_5_1_pfp.bin': '633404d8db1dc03fe997f7d0d0e15ef908069727abaf9de55841be3f3c97348b', + 'gc_11_5_2_pfp.bin': 'baee1456dd1800cdaedd4998c2dd7d76cdc0cf0ec928679fe67b02485905ea2c', + 'gc_11_5_3_pfp.bin': 'fee840b049b5e082215df72a93fad80a64f07ef6f638408a2d56fae97449a2cb', + 'gc_12_0_0_pfp.bin': 'd1b043c60920e509e5c8f9677221fb78ff7985f68b605e8f39a04a57333a9366', + 'gc_12_0_1_pfp.bin': '9d8d6188efeca5ef05482d9299c4f102fab7db3dae51a23e59de9baa34997123', + 'gc_10_3_6_me.bin': '776d2299bc4f3abffd4a7999f5a21a4e38aced8b6b4c199a83610dbabf08176d', 'gc_10_3_7_me.bin': '9eb0b56e9bcc9dad5d53437b162226fcb37e5df102832260f1232832f3658edf', - 'gc_11_0_0_me.bin': 'f8fba8a63dd4293b8fc1e4aab78b6fac630e575d1d62838c7996d9210f82aea1', - 'gc_11_0_1_me.bin': '5030040b00955de94876341ec64ea43b96640413d7a03dc460a83c8386bf76e0', - 'gc_11_0_2_me.bin': '0f21fd43f1dfbc6ccced9a2b3774de25c993c61a689aabab8b45333937b7945e', - 'gc_11_0_3_me.bin': '3acb5061dba342ade81d329d1932f19ec01f0c5bf44e6e3568008a951a351bac', - 'gc_11_0_4_me.bin': 'e4f1f6abcd213d54ad9e885d9f550083b0e2f67d983566015e8a53981e1cb155', - 'gc_11_5_0_me.bin': '8f906b64d0a29503daa662c93ec44d076fcac11b78f70cd50ce0af2b500a05a6', - 'gc_11_5_1_me.bin': '7e42602bcbaf1e511f8b4f6ed2246844ad1f6e351ce2b663d89062a7be263663', - 'gc_11_5_2_me.bin': 'aae26255d8efff81e0e3bbcb727efb8b837d8e25fe85c708545f5328f1077b50', - 'gc_11_5_3_me.bin': '93cd588348b16fe432609fe8da6e6b5da0a52da5c5884882aecf7b1001f72700', - 'gc_12_0_0_me.bin': 'd7eba5197f2580f32b8256b1d9cb68e723e9e644293a34446a7913e3c093cba5', - 'gc_12_0_1_me.bin': '365e7f193b39cbb10d3af44905fefaca0e9844721801755276baebac7b19c1ea', - 'gc_10_3_6_mec.bin': '247943415658159704a21f670dd7b3e7cb2d2fc0c17b000a5098715979c8d95e', + 'gc_11_0_0_me.bin': 'f2f5a793d811c6abad1a18af0fcf7694c443478f224176da86650c22aa71ca7a', + 'gc_11_0_1_me.bin': '476db2ec7e33d1e126b1736649208443e3ccc68aa60e4978574cdbced2b26543', + 'gc_11_0_2_me.bin': 'f5fe48f97acbd3ce13b35929290bfbac01ce522631cc91dcef1fdeb3ff35c8ed', + 'gc_11_0_3_me.bin': 'd02c25070e5bdf0ec0146f5c9d6d2f8b86de43bd2a318a0b67eb5201963bafdc', + 'gc_11_0_4_me.bin': 'f075220f75ffe43eacc5986ff8448946c27405e632764fda83323e7ec8d55566', + 'gc_11_5_0_me.bin': '338019a1fcdab39729e3f492ffc9f5970c2c81b12c8a4f431494ca28cfdadedf', + 'gc_11_5_1_me.bin': '4c4dd30c22d4f7f2c5d3a19c645f505e30cdac115a91c65791e2651b22932175', + 'gc_11_5_2_me.bin': 'cab2999186d26c0e9a3d46b5a43d2854d88be880cb764c096ad2b43038566384', + 'gc_11_5_3_me.bin': '94e2d74e834725b3d51e03e830160e95c56f3a31e93f5d61c852afd8fe8cc779', + 'gc_12_0_0_me.bin': 'fb10cb3535ae4a6a8fb3e78166cf30c5b717341b1f20cde73065c62b642adfed', + 'gc_12_0_1_me.bin': '56a1ae0031aa938f6b61348a56404ab2cee92f1f45630fc82a801aa4d908f98a', + 'gc_10_3_6_mec.bin': '7003c4a77537e9edaf67064104cd9371fac38a84f71f948349140b28d3c210e8', 'gc_10_3_7_mec.bin': 'ee58a523375bcf5b89400b32b801f95e182b632a26bce4f2bed5c07928d486dc', - 'gc_11_0_0_mec.bin': '801a09c9bf06188260db9b51ad8f978f15d84c72ca91b90643a2ef8af4074776', - 'gc_11_0_1_mec.bin': '6afadcb7504bb11bcc9d4a205cdf73f7934a615e28f178fcf7285971df2ccd05', - 'gc_11_0_2_mec.bin': '0da0edee28c73a6fa1191f77853d380ec2503cbf43e0aaae4617f32f1f8a48fa', - 'gc_11_0_3_mec.bin': '323cfa6658b6b5169830f852e2ff0552acae8dfb9e44b42c63de7b2900d3fd9e', - 'gc_11_0_4_mec.bin': '5d89cf6b60354f3746c2cbd1ff0cb1a741556ca20d72745242cb69b553d0985c', - 'gc_11_5_0_mec.bin': 'a01c324ab14ec89792449a621a541829b9af26865019027a411a14b910145dfa', - 'gc_11_5_1_mec.bin': 'eab05719371caa68df09d4f7574e3958a3c4f5044ab3c7b0d2b214add0c6d1c4', - 'gc_11_5_2_mec.bin': 'a374b2335802e24f8b9a3ce40000a1d37a52a14eb87099bebcc6680c27cc93e5', - 'gc_11_5_3_mec.bin': '165025437cba80dd32c19ebbc83b756fa7adac7053ff7780ba4aa2f8089c6a3f', - 'gc_12_0_0_mec.bin': '1931593440b8f9423580d9e2cdc5b34e7c682cdffe1ca4b74b0c2f6a0420236d', - 'gc_12_0_1_mec.bin': 'f57541688a5108730bf210663f1137ffc2121f3acfe614a6de09ec1982c69a2f', - 'gc_9_4_3_mec.bin': '3159176e72301fb88dc416721fb3d0ab82ece484cf93a43c3f37430c7e6673a1', - 'gc_9_4_3_sjt_mec.bin': 'd19468dbb47849640bd0e6cdc8d7e25a3c8442c7ca2ca81357702e0d6baab50f', - 'gc_9_4_4_mec.bin': '5004f73e43db2dd45e77d65942e33d4a69e7157618cfd23944c30f801c77a0f3', - 'gc_9_4_4_sjt_mec.bin': '627a9e98102e70fe3bf0947eb764187f29f5e775d1130c7310e0ba5fc0502dbe', - 'gc_9_5_0_mec.bin': 'c5eca4311a6f6e8f81cf41c2c46941d5dcf90789ee8326901da2dfc86ac14c31', - 'gc_9_5_0_sjt_mec.bin': 'f162e509379288e3f3b1eead541b315c2262d625d433287ecd34ca185614d312', + 'gc_11_0_0_mec.bin': '1dd1de8ecf5455ea4719c502b64b32ac18763d5601128c01b4a4a36211a122c2', + 'gc_11_0_1_mec.bin': '505ae64eccb2e4b4751fe18ec1b584e1f6b4c81d0f5ec089afbcf378cad59711', + 'gc_11_0_2_mec.bin': '19bf080d6e672de5ed3fb86e3fdbdda4d700d8e3bda2dbdcc923101484ad645b', + 'gc_11_0_3_mec.bin': 'a37bc1a4e245300a5c3e26da34ea213842447d7df6c5c81e9fc78887a2fde26f', + 'gc_11_0_4_mec.bin': '850d5302b4fee6022f42f706c2de103531b45b7794a45f2d6dce6015767a1ad6', + 'gc_11_5_0_mec.bin': '5e022bae6638967d82e2b1077e3024f52bc83b3cb850aa31fba51469c7517c4c', + 'gc_11_5_1_mec.bin': 'e49964d5e58686c53e66d98d4e3b9fab70e98fad3b28379c6e60aed03c83ee80', + 'gc_11_5_2_mec.bin': '9691d7bff5d2c933d8eecb7d171635612a76a2dd1441cffcd65a8a02bdb5a2c5', + 'gc_11_5_3_mec.bin': 'd368f3886b9245dd0d21d57fccfd8aa7e872c2564e23f292abe735348121277e', + 'gc_12_0_0_mec.bin': '9c7602d6ebf1f7e6ec7a5d1ceefded18f35fa1c08fbea1e3e1a0d78d519db8e8', + 'gc_12_0_1_mec.bin': 'caf1dbaf72b0ef0c4c973947414033aeec002994f63967bb53e9165195a3c2c3', + 'gc_9_4_3_mec.bin': '99bc12230f00b930cf286105a35cc6110d87461cd48cb4fdf3cb6caff73ac1e7', + 'gc_9_4_3_sjt_mec.bin': '2945dbd098c4158870df7dc4ccb33d40031fd1cce37cdbe5df291d8941d03567', + 'gc_9_4_4_mec.bin': '7f14258f8301d2717e0a707ccfad7b3091af478b0df6d5134adfd56caa7429d8', + 'gc_9_4_4_sjt_mec.bin': '0bbef279bbc07c502098b80765b876f69fcda9834e5ed269a7d8236c85e89e19', + 'gc_9_5_0_mec.bin': '0c39078c53e10e99538901df5fc14e7f1b1f3639ea825b1b3126ae87a28b2464', + 'gc_9_5_0_sjt_mec.bin': 'a769745367567fc6f389695aa5f48c154c07560e21a93052185e19f950205240', 'gc_11_0_0_imu.bin': 'b4f8fc056b45709a6abf48e7885fb1b4ab8d3cc092cbfa2c554a78564a6403bc', 'gc_11_0_1_imu.bin': 'ac71f4eec713fc35b4a1fe27531e3eb04edd81eeac2cef64df01ac50d8510805', 'gc_11_0_2_imu.bin': '9befca62b0b0cfd252c3df4a9edca295526f4d43821cd99a6326454995a6ca2d', @@ -90,17 +91,17 @@ hashes = { 'gc_10_3_6_rlc.bin': 'acfbac75c0dcfbfe40e222640ef17eb3dc8d206d30bc3863f275f2dd1cb132a5', 'gc_10_3_7_rlc.bin': 'a02585ebe3b36d942e883057119572d9497600c52fc65b8a523487eb65d874f2', 'gc_11_0_0_rlc.bin': 'dabd49039772d02f5fd5e48dc21d35ad52a6b1283b470dabca86ca159c4c7c8e', - 'gc_11_0_1_rlc.bin': '86145719a58e9428562930c6b5ee3b6ced4701d34a80d0b4d84d6026c93134f2', + 'gc_11_0_1_rlc.bin': '5f07dc1f0a75ecd9cb56d805ea869184a50ed9e43d811ebf833b8906534650ef', 'gc_11_0_2_rlc.bin': 'b43eb2fd0600f50a1a5796bc9983d6b39b5c20960234920f5e89cb362193e0b8', - 'gc_11_0_3_rlc.bin': '29b0b456f5b53076ddffa6f09de3bb697219e8e7b33504bf6c197e8b858426dc', - 'gc_11_0_4_rlc.bin': '823573078b608108fbe4dd8176c396ec582632913db9c59a512d82b068f8eba0', - 'gc_11_5_0_rlc.bin': '68cd85567f4f2f8d6b80db294988806d956bf826979c3597daccb71c7ee6aadd', + 'gc_11_0_3_rlc.bin': '890d8e0123efb40c0179dd8ac3e9af073a0b87cbbccfec1db54e5ed2315a8d39', + 'gc_11_0_4_rlc.bin': '257ced82d7bec41249b06592ee0c44fb8f9262de2c6af9c52dc6f6a8a702063e', + 'gc_11_5_0_rlc.bin': '0dc8b6ef5530a4a53938c8baa0d49cd458607d95233237859fa98d44feb3e985', 'gc_11_5_1_rlc.bin': '92731ecabbeb77865fb71787b4268dc738a58779f1190bdc2056482cb88a08f6', - 'gc_11_5_2_rlc.bin': 'ef3a9209d3eccfbe18fce9e972c146ac283719798bb788096c176b796dc9aee5', + 'gc_11_5_2_rlc.bin': 'c9ad70b8ac309257cb8929bb6b4efa6b551ec1e5229d7a419332a9797f31fc9e', 'gc_11_5_3_rlc.bin': '10a68940c6258d5818d9c05fd98eb0ccc8d5aee99b2769fbad30e5abd0d9327e', 'gc_12_0_0_rlc.bin': '6436b582734a413456fff3d3c7195e71cc9e78a7ed31ee21c83ffd6fae1ad186', 'gc_12_0_1_rlc.bin': '6ba4459532246a5c415d3cb33c9b1248294e48f67b827e2accb292a8d1a5c0ec', - 'gc_9_4_3_rlc.bin': '5345d388712d547b0ae16f199ad5ccadb65643584b3efa7817049ddeb3fdcd12', + 'gc_9_4_3_rlc.bin': '54cbd0de3a0ec35d2e58e992babeee2a237f870ccdf37e734652e4daeeba59d5', 'gc_9_4_4_rlc.bin': 'e0c3585c72f8136670ca63e607fba32c1ae4948f493f13e33fc4d466bd6318a8', 'gc_9_5_0_rlc.bin': '9b1268f5751153fe57f527c9acb417bfa53ed42c9bc083c9d3da2ba61fe5fdc4', } \ No newline at end of file diff --git a/tinygrad_repo/tinygrad/runtime/autogen/am/regs.py b/tinygrad_repo/tinygrad/runtime/autogen/am/regs.py index 6cb6f94e0..e5b9f556e 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/am/regs.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/am/regs.py @@ -510,6 +510,10 @@ gc_9_4_3 = { 'regTCP_PERFCOUNTER_FILTER': (4793, 0, {'buffer': (0, 0), 'flat': (1, 1), 'dim': (2, 4), 'data_format': (5, 10), 'num_format': (11, 14), 'sw_mode': (15, 19), 'num_samples': (20, 21), 'opcode_type': (22, 24), 'glc': (25, 25), 'slc': (26, 26), 'compression_enable': (27, 27), 'addr_mode': (28, 30)}), 'regTCP_PERFCOUNTER_FILTER_EN': (4794, 0, {'buffer': (0, 0), 'flat': (1, 1), 'dim': (2, 2), 'data_format': (3, 3), 'num_format': (4, 4), 'sw_mode': (5, 5), 'num_samples': (6, 6), 'opcode_type': (7, 7), 'glc': (8, 8), 'slc': (9, 9), 'compression_enable': (10, 10), 'addr_mode': (11, 11)}), 'regCP_PERFMON_CNTX_CNTL': (216, 1, {'perfmon_enable': (31, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regCP_COHER_START_DELAY': (8315, 1, {'start_delay_count': (0, 5)}), @@ -1791,6 +1795,10 @@ gc_11_0_0 = { 'regTCP_CNTL2': (6563, 1, {'ls_disable_clocks': (0, 7), 'tcp_fmt_mgcg_disable': (8, 8), 'tcpf_latency_bypass_disable': (9, 9), 'tcp_write_data_mgcg_disable': (10, 10), 'tcp_inner_block_mgcg_disable': (11, 11), 'tcp_adrs_img_calc_mgcg_disable': (12, 12), 'v64_combine_enable': (13, 13), 'tagram_addr_swizzle_disable': (14, 14), 'return_order_override': (15, 15), 'power_opt_disable': (16, 16), 'gcr_rsp_fgcg_disable': (17, 17), 'perf_en_override': (18, 19), 'tc_td_ram_clken_disable': (20, 20), 'tc_td_data_clken_disable': (21, 21), 'tcp_gl1_req_clken_disable': (22, 22), 'tcp_gl1r_src_clken_disable': (23, 23), 'spare_bit': (26, 26), 'tagram_xy_bias_override': (27, 28), 'tcp_req_mgcg_disable': (29, 29), 'tcp_miss_mgcg_disable': (30, 30), 'disable_mipmap_param_calc_self_gating': (31, 31)}), 'regTCP_DEBUG_INDEX': (6565, 1, {'index': (0, 4)}), 'regTCP_DEBUG_DATA': (6566, 1, {'data': (0, 17)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -3366,6 +3374,10 @@ gc_11_0_3 = { 'regTCP_CNTL': (6562, 1, {'force_hit': (0, 0), 'force_miss': (1, 1), 'flat_buf_cache_swizzle': (5, 5), 'td_data_en_override': (6, 6), 'enable_128b_dcc_comp_read_for_indep64': (7, 7), 'disable_write_combining': (9, 9), 'force_eow_total_cnt': (15, 20), 'force_eow_set_cnt': (22, 26), 'disable_z_map': (28, 28), 'force_order_between_read_write_to_same_address': (29, 29), 'astc_ve_msb_tolerant': (31, 31)}), 'regTCP_CNTL2': (6563, 1, {'ls_disable_clocks': (0, 7), 'tcp_fmt_mgcg_disable': (8, 8), 'tcpf_latency_bypass_disable': (9, 9), 'tcp_write_data_mgcg_disable': (10, 10), 'tcp_inner_block_mgcg_disable': (11, 11), 'tcp_adrs_img_calc_mgcg_disable': (12, 12), 'v64_combine_enable': (13, 13), 'tagram_addr_swizzle_disable': (14, 14), 'return_order_override': (15, 15), 'power_opt_disable': (16, 16), 'gcr_rsp_fgcg_disable': (17, 17), 'perf_en_override': (18, 19), 'tc_td_ram_clken_disable': (20, 20), 'tc_td_data_clken_disable': (21, 21), 'tcp_gl1_req_clken_disable': (22, 22), 'tcp_gl1r_src_clken_disable': (23, 23), 'spare_bit': (26, 26), 'tagram_xy_bias_override': (27, 28), 'tcp_req_mgcg_disable': (29, 29), 'tcp_miss_mgcg_disable': (30, 30), 'disable_mipmap_param_calc_self_gating': (31, 31)}), 'regTCP_CREDIT': (6564, 1, {'lfifo_ram_depth': (0, 9), 'gl1_req_credit': (10, 15), 'req_fifo_credit': (16, 22), 'td_ram_credit': (23, 27), 'td_data_credit': (29, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -4791,6 +4803,10 @@ gc_11_5_0 = { 'regTCP_STATUS': (6561, 1, {'tcp_busy': (0, 0), 'input_busy': (1, 1), 'adrs_busy': (2, 2), 'tagrams_busy': (3, 3), 'cntrl_busy': (4, 4), 'lfifo_busy': (5, 5), 'read_busy': (6, 6), 'format_busy': (7, 7), 'vm_busy': (8, 8), 'memif_busy': (9, 9), 'gcr_busy': (10, 10), 'ofifo_busy': (11, 11), 'ofifo_queue_busy': (12, 13), 'xnack_prt': (15, 15)}), 'regTCP_CNTL': (6562, 1, {'force_hit': (0, 0), 'force_miss': (1, 1), 'flat_buf_cache_swizzle': (5, 5), 'td_data_en_override': (6, 6), 'enable_128b_dcc_comp_read_for_indep64': (7, 7), 'disable_decompression_power_opt': (8, 8), 'disable_write_combining': (9, 9), 'force_eow_total_cnt': (15, 20), 'force_eow_set_cnt': (22, 26), 'disable_z_map': (28, 28), 'force_order_between_read_write_to_same_address': (29, 29), 'astc_ve_msb_tolerant': (31, 31)}), 'regTCP_CNTL2': (6563, 1, {'ls_disable_clocks': (0, 7), 'tcp_fmt_mgcg_disable': (8, 8), 'tcpf_latency_bypass_disable': (9, 9), 'tcp_write_data_mgcg_disable': (10, 10), 'tcp_inner_block_mgcg_disable': (11, 11), 'tcp_adrs_img_calc_mgcg_disable': (12, 12), 'v64_combine_enable': (13, 13), 'tagram_addr_swizzle_disable': (14, 14), 'return_order_override': (15, 15), 'power_opt_disable': (16, 16), 'gcr_rsp_fgcg_disable': (17, 17), 'perf_en_override': (18, 19), 'tcp_gl1_req_clken_disable': (22, 22), 'tcp_gl1r_src_clken_disable': (23, 23), 'tcp_force_2x_to_load': (24, 24), 'spare_bit': (26, 26), 'tagram_xy_bias_override': (27, 28), 'tcp_req_mgcg_disable': (29, 29), 'tcp_miss_mgcg_disable': (30, 30), 'disable_mipmap_param_calc_self_gating': (31, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -6048,6 +6064,10 @@ gc_12_0_0 = { 'regCP_ME_CNTL': (2051, 1, {'ce_invalidate_icache': (4, 4), 'pfp_invalidate_icache': (6, 6), 'me_invalidate_icache': (8, 8), 'pfp_pipe0_disable': (12, 12), 'pfp_pipe1_disable': (13, 13), 'me_pipe0_disable': (14, 14), 'me_pipe1_disable': (15, 15), 'ce_pipe0_reset': (16, 16), 'ce_pipe1_reset': (17, 17), 'pfp_pipe0_reset': (18, 18), 'pfp_pipe1_reset': (19, 19), 'me_pipe0_reset': (20, 20), 'me_pipe1_reset': (21, 21), 'ce_halt': (24, 24), 'ce_step': (25, 25), 'pfp_halt': (26, 26), 'pfp_step': (27, 27), 'me_halt': (28, 28), 'me_step': (29, 29)}), 'regGRBM_GFX_CNTL': (2304, 1, {'pipeid': (0, 1), 'meid': (2, 3), 'vmid': (4, 7), 'queueid': (8, 10), 'ctxid': (11, 13)}), 'regGRBM_NOWHERE': (2305, 1, {'data': (0, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -8977,6 +8997,7 @@ nbio_4_3_0 = { 'regBIF_BX_PF0_GPU_HDP_FLUSH_REQ': (262, 2, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), 'regBIF_BX_PF0_GPU_HDP_FLUSH_DONE': (263, 2, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), 'regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN': (192, 2, {'bif_doorbell_aper_en': (0, 0)}), + 'regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL': (247, 2, {'hdp_mem_flush_addr': (0, 0)}), 'regRCC_DEV0_EPF2_STRAP2': (53506, 5, {'strap_no_soft_reset_dev0_f2': (7, 7), 'strap_resize_bar_en_dev0_f2': (8, 8), 'strap_max_pasid_width_dev0_f2': (9, 13), 'strap_msi_pervector_mask_cap_dev0_f2': (14, 14), 'strap_aer_en_dev0_f2': (16, 16), 'strap_acs_en_dev0_f2': (17, 17), 'strap_cpl_abort_err_en_dev0_f2': (20, 20), 'strap_dpa_en_dev0_f2': (21, 21), 'strap_vc_en_dev0_f2': (23, 23), 'strap_msi_multi_cap_dev0_f2': (24, 26), 'strap_pasid_en_dev0_f2': (28, 28), 'strap_pasid_exe_permission_supported_dev0_f2': (29, 29), 'strap_pasid_global_invalidate_supported_dev0_f2': (30, 30), 'strap_pasid_priv_mode_supported_dev0_f2': (31, 31)}), 'regRCC_DEV0_EPF2_STRAP20': (53524, 5, {}), 'regBIF_BX_PF1_GPU_HDP_FLUSH_REQ': (36390, 5, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), @@ -9084,6 +9105,7 @@ nbio_7_9_0 = { 'regS2A_DOORBELL_ENTRY_14_CTRL': (31374, 5, {'s2a_doorbell_port14_enable': (0, 0), 's2a_doorbell_port14_awid': (1, 5), 's2a_doorbell_port14_fence_enable': (6, 6), 's2a_doorbell_port14_range_offset': (7, 16), 's2a_doorbell_port14_range_size': (17, 24), 's2a_doorbell_port14_64bit_support_dis': (25, 25), 's2a_doorbell_port14_need_deduct_range_offset': (26, 26), 's2a_doorbell_port14_awaddr_31_28_value': (28, 31)}), 'regS2A_DOORBELL_ENTRY_15_CTRL': (31375, 5, {'s2a_doorbell_port15_enable': (0, 0), 's2a_doorbell_port15_awid': (1, 5), 's2a_doorbell_port15_fence_enable': (6, 6), 's2a_doorbell_port15_range_offset': (7, 16), 's2a_doorbell_port15_range_size': (17, 24), 's2a_doorbell_port15_64bit_support_dis': (25, 25), 's2a_doorbell_port15_need_deduct_range_offset': (26, 26), 's2a_doorbell_port15_awaddr_31_28_value': (28, 31)}), 'regXCC_DOORBELL_FENCE': (29708, 5, {'xcc_0_doorbell_fence': (0, 0), 'xcc_1_doorbell_fence': (1, 1), 'xcc_2_doorbell_fence': (2, 2), 'xcc_3_doorbell_fence': (3, 3), 'xcc_4_doorbell_fence': (4, 4), 'xcc_5_doorbell_fence': (5, 5), 'xcc_6_doorbell_fence': (6, 6), 'xcc_7_doorbell_fence': (7, 7), 'shub_slv_mode': (16, 16), 'rmote_cp_sent': (17, 17), 'cp_0_sent': (18, 18), 'cp_1_sent': (19, 19), 'cp_2_sent': (20, 20), 'cp_3_sent': (21, 21), 'cp_4_sent': (22, 22), 'cp_5_sent': (23, 23), 'cp_6_sent': (24, 24), 'cp_7_sent': (25, 25), 'remote_client_sent': (26, 26), 'remote_client_clr_pending': (27, 27)}), + 'regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL': (247, 2, {'hdp_mem_flush_addr': (0, 0)}), } nbio_7_11_0 = { 'regBIF_BX0_PCIE_INDEX2': (32782, 0, {'pcie_index2': (0, 31)}), @@ -9131,6 +9153,7 @@ nbif_6_3_1 = { 'regRCC_DEV0_EPF2_STRAP20': (53524, 5, {}), 'regBIF_BX_PF1_GPU_HDP_FLUSH_REQ': (36390, 5, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), 'regBIF_BX_PF1_GPU_HDP_FLUSH_DONE': (36391, 5, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), + 'regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL': (247, 2, {'hdp_mem_flush_addr': (0, 0)}), } mp_11_0_0 = { 'mmMP0_SMN_C2PMSG_32': (96, 0, {'content': (0, 31)}), diff --git a/tinygrad_repo/tinygrad/runtime/autogen/bnxt.py b/tinygrad_repo/tinygrad/runtime/autogen/bnxt.py new file mode 100644 index 000000000..7d8227ec8 --- /dev/null +++ b/tinygrad_repo/tinygrad/runtime/autogen/bnxt.py @@ -0,0 +1,5407 @@ +# mypy: disable-error-code="empty-body" +from __future__ import annotations +import ctypes +from typing import Literal, TypeAlias +from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR +from tinygrad.runtime.support import c +@c.record +class struct_hwrm_cmd_hdr(c.Struct): + SIZE = 16 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int +struct_hwrm_cmd_hdr.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8)]) +@c.record +class struct_hwrm_resp_hdr(c.Struct): + SIZE = 8 + error_code: int + req_type: int + seq_id: int + resp_len: int +struct_hwrm_resp_hdr.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6)]) +@c.record +class struct_hwrm_ver_get_input(c.Struct): + SIZE = 24 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + hwrm_intf_maj: int + hwrm_intf_min: int + hwrm_intf_upd: int + unused_0: c.Array[ctypes.c_ubyte, Literal[5]] +struct_hwrm_ver_get_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('hwrm_intf_maj', ctypes.c_ubyte, 16), ('hwrm_intf_min', ctypes.c_ubyte, 17), ('hwrm_intf_upd', ctypes.c_ubyte, 18), ('unused_0', c.Array[ctypes.c_ubyte, Literal[5]], 19)]) +@c.record +class struct_hwrm_ver_get_output(c.Struct): + SIZE = 176 + error_code: int + req_type: int + seq_id: int + resp_len: int + hwrm_intf_maj_8b: int + hwrm_intf_min_8b: int + hwrm_intf_upd_8b: int + hwrm_intf_rsvd_8b: int + hwrm_fw_maj_8b: int + hwrm_fw_min_8b: int + hwrm_fw_bld_8b: int + hwrm_fw_rsvd_8b: int + mgmt_fw_maj_8b: int + mgmt_fw_min_8b: int + mgmt_fw_bld_8b: int + mgmt_fw_rsvd_8b: int + netctrl_fw_maj_8b: int + netctrl_fw_min_8b: int + netctrl_fw_bld_8b: int + netctrl_fw_rsvd_8b: int + dev_caps_cfg: int + roce_fw_maj_8b: int + roce_fw_min_8b: int + roce_fw_bld_8b: int + roce_fw_rsvd_8b: int + hwrm_fw_name: c.Array[ctypes.c_char, Literal[16]] + mgmt_fw_name: c.Array[ctypes.c_char, Literal[16]] + netctrl_fw_name: c.Array[ctypes.c_char, Literal[16]] + active_pkg_name: c.Array[ctypes.c_char, Literal[16]] + roce_fw_name: c.Array[ctypes.c_char, Literal[16]] + chip_num: int + chip_rev: int + chip_metal: int + chip_bond_id: int + chip_platform_type: int + max_req_win_len: int + max_resp_len: int + def_req_timeout: int + flags: int + unused_0: c.Array[ctypes.c_ubyte, Literal[2]] + always_1: int + hwrm_intf_major: int + hwrm_intf_minor: int + hwrm_intf_build: int + hwrm_intf_patch: int + hwrm_fw_major: int + hwrm_fw_minor: int + hwrm_fw_build: int + hwrm_fw_patch: int + mgmt_fw_major: int + mgmt_fw_minor: int + mgmt_fw_build: int + mgmt_fw_patch: int + netctrl_fw_major: int + netctrl_fw_minor: int + netctrl_fw_build: int + netctrl_fw_patch: int + roce_fw_major: int + roce_fw_minor: int + roce_fw_build: int + roce_fw_patch: int + max_ext_req_len: int + max_req_timeout: int + unused_1: c.Array[ctypes.c_ubyte, Literal[3]] + valid: int +struct_hwrm_ver_get_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('hwrm_intf_maj_8b', ctypes.c_ubyte, 8), ('hwrm_intf_min_8b', ctypes.c_ubyte, 9), ('hwrm_intf_upd_8b', ctypes.c_ubyte, 10), ('hwrm_intf_rsvd_8b', ctypes.c_ubyte, 11), ('hwrm_fw_maj_8b', ctypes.c_ubyte, 12), ('hwrm_fw_min_8b', ctypes.c_ubyte, 13), ('hwrm_fw_bld_8b', ctypes.c_ubyte, 14), ('hwrm_fw_rsvd_8b', ctypes.c_ubyte, 15), ('mgmt_fw_maj_8b', ctypes.c_ubyte, 16), ('mgmt_fw_min_8b', ctypes.c_ubyte, 17), ('mgmt_fw_bld_8b', ctypes.c_ubyte, 18), ('mgmt_fw_rsvd_8b', ctypes.c_ubyte, 19), ('netctrl_fw_maj_8b', ctypes.c_ubyte, 20), ('netctrl_fw_min_8b', ctypes.c_ubyte, 21), ('netctrl_fw_bld_8b', ctypes.c_ubyte, 22), ('netctrl_fw_rsvd_8b', ctypes.c_ubyte, 23), ('dev_caps_cfg', ctypes.c_uint32, 24), ('roce_fw_maj_8b', ctypes.c_ubyte, 28), ('roce_fw_min_8b', ctypes.c_ubyte, 29), ('roce_fw_bld_8b', ctypes.c_ubyte, 30), ('roce_fw_rsvd_8b', ctypes.c_ubyte, 31), ('hwrm_fw_name', c.Array[ctypes.c_char, Literal[16]], 32), ('mgmt_fw_name', c.Array[ctypes.c_char, Literal[16]], 48), ('netctrl_fw_name', c.Array[ctypes.c_char, Literal[16]], 64), ('active_pkg_name', c.Array[ctypes.c_char, Literal[16]], 80), ('roce_fw_name', c.Array[ctypes.c_char, Literal[16]], 96), ('chip_num', ctypes.c_uint16, 112), ('chip_rev', ctypes.c_ubyte, 114), ('chip_metal', ctypes.c_ubyte, 115), ('chip_bond_id', ctypes.c_ubyte, 116), ('chip_platform_type', ctypes.c_ubyte, 117), ('max_req_win_len', ctypes.c_uint16, 118), ('max_resp_len', ctypes.c_uint16, 120), ('def_req_timeout', ctypes.c_uint16, 122), ('flags', ctypes.c_ubyte, 124), ('unused_0', c.Array[ctypes.c_ubyte, Literal[2]], 125), ('always_1', ctypes.c_ubyte, 127), ('hwrm_intf_major', ctypes.c_uint16, 128), ('hwrm_intf_minor', ctypes.c_uint16, 130), ('hwrm_intf_build', ctypes.c_uint16, 132), ('hwrm_intf_patch', ctypes.c_uint16, 134), ('hwrm_fw_major', ctypes.c_uint16, 136), ('hwrm_fw_minor', ctypes.c_uint16, 138), ('hwrm_fw_build', ctypes.c_uint16, 140), ('hwrm_fw_patch', ctypes.c_uint16, 142), ('mgmt_fw_major', ctypes.c_uint16, 144), ('mgmt_fw_minor', ctypes.c_uint16, 146), ('mgmt_fw_build', ctypes.c_uint16, 148), ('mgmt_fw_patch', ctypes.c_uint16, 150), ('netctrl_fw_major', ctypes.c_uint16, 152), ('netctrl_fw_minor', ctypes.c_uint16, 154), ('netctrl_fw_build', ctypes.c_uint16, 156), ('netctrl_fw_patch', ctypes.c_uint16, 158), ('roce_fw_major', ctypes.c_uint16, 160), ('roce_fw_minor', ctypes.c_uint16, 162), ('roce_fw_build', ctypes.c_uint16, 164), ('roce_fw_patch', ctypes.c_uint16, 166), ('max_ext_req_len', ctypes.c_uint16, 168), ('max_req_timeout', ctypes.c_uint16, 170), ('unused_1', c.Array[ctypes.c_ubyte, Literal[3]], 172), ('valid', ctypes.c_ubyte, 175)]) +@c.record +class struct_hwrm_func_reset_input(c.Struct): + SIZE = 24 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + enables: int + vf_id: int + func_reset_level: int + unused_0: int +struct_hwrm_func_reset_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('enables', ctypes.c_uint32, 16), ('vf_id', ctypes.c_uint16, 20), ('func_reset_level', ctypes.c_ubyte, 22), ('unused_0', ctypes.c_ubyte, 23)]) +@c.record +class struct_hwrm_func_reset_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + unused_0: c.Array[ctypes.c_ubyte, Literal[7]] + valid: int +struct_hwrm_func_reset_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('unused_0', c.Array[ctypes.c_ubyte, Literal[7]], 8), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_hwrm_func_qcaps_input(c.Struct): + SIZE = 24 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + fid: int + unused_0: c.Array[ctypes.c_ubyte, Literal[6]] +struct_hwrm_func_qcaps_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('fid', ctypes.c_uint16, 16), ('unused_0', c.Array[ctypes.c_ubyte, Literal[6]], 18)]) +@c.record +class struct_hwrm_func_qcaps_output(c.Struct): + SIZE = 144 + error_code: int + req_type: int + seq_id: int + resp_len: int + fid: int + port_id: int + flags: int + mac_address: c.Array[ctypes.c_ubyte, Literal[6]] + max_rsscos_ctx: int + max_cmpl_rings: int + max_tx_rings: int + max_rx_rings: int + max_l2_ctxs: int + max_vnics: int + first_vf_id: int + max_vfs: int + max_stat_ctx: int + max_encap_records: int + max_decap_records: int + max_tx_em_flows: int + max_tx_wm_flows: int + max_rx_em_flows: int + max_rx_wm_flows: int + max_mcast_filters: int + max_flow_id: int + max_hw_ring_grps: int + max_sp_tx_rings: int + max_msix_vfs: int + flags_ext: int + max_schqs: int + mpc_chnls_cap: int + max_key_ctxs_alloc: int + flags_ext2: int + tunnel_disable_flag: int + xid_partition_cap: int + device_serial_number: c.Array[ctypes.c_ubyte, Literal[8]] + ctxs_per_partition: int + max_tso_segs: int + roce_vf_max_av: int + roce_vf_max_cq: int + roce_vf_max_mrw: int + roce_vf_max_qp: int + roce_vf_max_srq: int + roce_vf_max_gid: int + flags_ext3: int + max_roce_vfs: int + max_crypto_rx_flow_filters: int + unused_3: c.Array[ctypes.c_ubyte, Literal[3]] + valid: int +struct_hwrm_func_qcaps_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('fid', ctypes.c_uint16, 8), ('port_id', ctypes.c_uint16, 10), ('flags', ctypes.c_uint32, 12), ('mac_address', c.Array[ctypes.c_ubyte, Literal[6]], 16), ('max_rsscos_ctx', ctypes.c_uint16, 22), ('max_cmpl_rings', ctypes.c_uint16, 24), ('max_tx_rings', ctypes.c_uint16, 26), ('max_rx_rings', ctypes.c_uint16, 28), ('max_l2_ctxs', ctypes.c_uint16, 30), ('max_vnics', ctypes.c_uint16, 32), ('first_vf_id', ctypes.c_uint16, 34), ('max_vfs', ctypes.c_uint16, 36), ('max_stat_ctx', ctypes.c_uint16, 38), ('max_encap_records', ctypes.c_uint32, 40), ('max_decap_records', ctypes.c_uint32, 44), ('max_tx_em_flows', ctypes.c_uint32, 48), ('max_tx_wm_flows', ctypes.c_uint32, 52), ('max_rx_em_flows', ctypes.c_uint32, 56), ('max_rx_wm_flows', ctypes.c_uint32, 60), ('max_mcast_filters', ctypes.c_uint32, 64), ('max_flow_id', ctypes.c_uint32, 68), ('max_hw_ring_grps', ctypes.c_uint32, 72), ('max_sp_tx_rings', ctypes.c_uint16, 76), ('max_msix_vfs', ctypes.c_uint16, 78), ('flags_ext', ctypes.c_uint32, 80), ('max_schqs', ctypes.c_ubyte, 84), ('mpc_chnls_cap', ctypes.c_ubyte, 85), ('max_key_ctxs_alloc', ctypes.c_uint16, 86), ('flags_ext2', ctypes.c_uint32, 88), ('tunnel_disable_flag', ctypes.c_uint16, 92), ('xid_partition_cap', ctypes.c_uint16, 94), ('device_serial_number', c.Array[ctypes.c_ubyte, Literal[8]], 96), ('ctxs_per_partition', ctypes.c_uint16, 104), ('max_tso_segs', ctypes.c_uint16, 106), ('roce_vf_max_av', ctypes.c_uint32, 108), ('roce_vf_max_cq', ctypes.c_uint32, 112), ('roce_vf_max_mrw', ctypes.c_uint32, 116), ('roce_vf_max_qp', ctypes.c_uint32, 120), ('roce_vf_max_srq', ctypes.c_uint32, 124), ('roce_vf_max_gid', ctypes.c_uint32, 128), ('flags_ext3', ctypes.c_uint32, 132), ('max_roce_vfs', ctypes.c_uint16, 136), ('max_crypto_rx_flow_filters', ctypes.c_uint16, 138), ('unused_3', c.Array[ctypes.c_ubyte, Literal[3]], 140), ('valid', ctypes.c_ubyte, 143)]) +@c.record +class struct_hwrm_func_qcfg_input(c.Struct): + SIZE = 24 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + fid: int + unused_0: c.Array[ctypes.c_ubyte, Literal[6]] +struct_hwrm_func_qcfg_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('fid', ctypes.c_uint16, 16), ('unused_0', c.Array[ctypes.c_ubyte, Literal[6]], 18)]) +@c.record +class struct_hwrm_func_qcfg_output(c.Struct): + SIZE = 176 + error_code: int + req_type: int + seq_id: int + resp_len: int + fid: int + port_id: int + vlan: int + flags: int + mac_address: c.Array[ctypes.c_ubyte, Literal[6]] + pci_id: int + alloc_rsscos_ctx: int + alloc_cmpl_rings: int + alloc_tx_rings: int + alloc_rx_rings: int + alloc_l2_ctx: int + alloc_vnics: int + admin_mtu: int + mru: int + stat_ctx_id: int + port_partition_type: int + port_pf_cnt: int + dflt_vnic_id: int + max_mtu_configured: int + min_bw: int + max_bw: int + evb_mode: int + options: int + alloc_vfs: int + alloc_mcast_filters: int + alloc_hw_ring_grps: int + alloc_sp_tx_rings: int + alloc_stat_ctx: int + alloc_msix: int + registered_vfs: int + l2_doorbell_bar_size_kb: int + active_endpoints: int + always_1: int + reset_addr_poll: int + legacy_l2_db_size_kb: int + svif_info: int + mpc_chnls: int + db_page_size: int + roce_vnic_id: int + partition_min_bw: int + partition_max_bw: int + host_mtu: int + flags2: int + stag_vid: int + port_kdnet_mode: int + kdnet_pcie_function: int + port_kdnet_fid: int + unused_5: int + roce_bidi_opt_mode: int + num_ktls_tx_key_ctxs: int + num_ktls_rx_key_ctxs: int + lag_id: int + parif: int + fw_lag_id: int + unused_6: int + num_quic_tx_key_ctxs: int + num_quic_rx_key_ctxs: int + roce_max_av_per_vf: int + roce_max_cq_per_vf: int + roce_max_mrw_per_vf: int + roce_max_qp_per_vf: int + roce_max_srq_per_vf: int + roce_max_gid_per_vf: int + xid_partition_cfg: int + mirror_vnic_id: int + max_link_width: int + max_link_speed: int + negotiated_link_width: int + negotiated_link_speed: int + unused_7: c.Array[ctypes.c_ubyte, Literal[2]] + pcie_compliance: int + unused_8: int + l2_db_multi_page_size_kb: int + unused_9: c.Array[ctypes.c_ubyte, Literal[5]] + valid: int +struct_hwrm_func_qcfg_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('fid', ctypes.c_uint16, 8), ('port_id', ctypes.c_uint16, 10), ('vlan', ctypes.c_uint16, 12), ('flags', ctypes.c_uint16, 14), ('mac_address', c.Array[ctypes.c_ubyte, Literal[6]], 16), ('pci_id', ctypes.c_uint16, 22), ('alloc_rsscos_ctx', ctypes.c_uint16, 24), ('alloc_cmpl_rings', ctypes.c_uint16, 26), ('alloc_tx_rings', ctypes.c_uint16, 28), ('alloc_rx_rings', ctypes.c_uint16, 30), ('alloc_l2_ctx', ctypes.c_uint16, 32), ('alloc_vnics', ctypes.c_uint16, 34), ('admin_mtu', ctypes.c_uint16, 36), ('mru', ctypes.c_uint16, 38), ('stat_ctx_id', ctypes.c_uint16, 40), ('port_partition_type', ctypes.c_ubyte, 42), ('port_pf_cnt', ctypes.c_ubyte, 43), ('dflt_vnic_id', ctypes.c_uint16, 44), ('max_mtu_configured', ctypes.c_uint16, 46), ('min_bw', ctypes.c_uint32, 48), ('max_bw', ctypes.c_uint32, 52), ('evb_mode', ctypes.c_ubyte, 56), ('options', ctypes.c_ubyte, 57), ('alloc_vfs', ctypes.c_uint16, 58), ('alloc_mcast_filters', ctypes.c_uint32, 60), ('alloc_hw_ring_grps', ctypes.c_uint32, 64), ('alloc_sp_tx_rings', ctypes.c_uint16, 68), ('alloc_stat_ctx', ctypes.c_uint16, 70), ('alloc_msix', ctypes.c_uint16, 72), ('registered_vfs', ctypes.c_uint16, 74), ('l2_doorbell_bar_size_kb', ctypes.c_uint16, 76), ('active_endpoints', ctypes.c_ubyte, 78), ('always_1', ctypes.c_ubyte, 79), ('reset_addr_poll', ctypes.c_uint32, 80), ('legacy_l2_db_size_kb', ctypes.c_uint16, 84), ('svif_info', ctypes.c_uint16, 86), ('mpc_chnls', ctypes.c_ubyte, 88), ('db_page_size', ctypes.c_ubyte, 89), ('roce_vnic_id', ctypes.c_uint16, 90), ('partition_min_bw', ctypes.c_uint32, 92), ('partition_max_bw', ctypes.c_uint32, 96), ('host_mtu', ctypes.c_uint16, 100), ('flags2', ctypes.c_uint16, 102), ('stag_vid', ctypes.c_uint16, 104), ('port_kdnet_mode', ctypes.c_ubyte, 106), ('kdnet_pcie_function', ctypes.c_ubyte, 107), ('port_kdnet_fid', ctypes.c_uint16, 108), ('unused_5', ctypes.c_ubyte, 110), ('roce_bidi_opt_mode', ctypes.c_ubyte, 111), ('num_ktls_tx_key_ctxs', ctypes.c_uint32, 112), ('num_ktls_rx_key_ctxs', ctypes.c_uint32, 116), ('lag_id', ctypes.c_ubyte, 120), ('parif', ctypes.c_ubyte, 121), ('fw_lag_id', ctypes.c_ubyte, 122), ('unused_6', ctypes.c_ubyte, 123), ('num_quic_tx_key_ctxs', ctypes.c_uint32, 124), ('num_quic_rx_key_ctxs', ctypes.c_uint32, 128), ('roce_max_av_per_vf', ctypes.c_uint32, 132), ('roce_max_cq_per_vf', ctypes.c_uint32, 136), ('roce_max_mrw_per_vf', ctypes.c_uint32, 140), ('roce_max_qp_per_vf', ctypes.c_uint32, 144), ('roce_max_srq_per_vf', ctypes.c_uint32, 148), ('roce_max_gid_per_vf', ctypes.c_uint32, 152), ('xid_partition_cfg', ctypes.c_uint16, 156), ('mirror_vnic_id', ctypes.c_uint16, 158), ('max_link_width', ctypes.c_ubyte, 160), ('max_link_speed', ctypes.c_ubyte, 161), ('negotiated_link_width', ctypes.c_ubyte, 162), ('negotiated_link_speed', ctypes.c_ubyte, 163), ('unused_7', c.Array[ctypes.c_ubyte, Literal[2]], 164), ('pcie_compliance', ctypes.c_ubyte, 166), ('unused_8', ctypes.c_ubyte, 167), ('l2_db_multi_page_size_kb', ctypes.c_uint16, 168), ('unused_9', c.Array[ctypes.c_ubyte, Literal[5]], 170), ('valid', ctypes.c_ubyte, 175)]) +@c.record +class struct_hwrm_func_drv_rgtr_input(c.Struct): + SIZE = 112 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + flags: int + enables: int + os_type: int + ver_maj_8b: int + ver_min_8b: int + ver_upd_8b: int + unused_0: c.Array[ctypes.c_ubyte, Literal[3]] + timestamp: int + unused_1: c.Array[ctypes.c_ubyte, Literal[4]] + vf_req_fwd: c.Array[ctypes.c_uint32, Literal[8]] + async_event_fwd: c.Array[ctypes.c_uint32, Literal[8]] + ver_maj: int + ver_min: int + ver_upd: int + ver_patch: int +struct_hwrm_func_drv_rgtr_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('flags', ctypes.c_uint32, 16), ('enables', ctypes.c_uint32, 20), ('os_type', ctypes.c_uint16, 24), ('ver_maj_8b', ctypes.c_ubyte, 26), ('ver_min_8b', ctypes.c_ubyte, 27), ('ver_upd_8b', ctypes.c_ubyte, 28), ('unused_0', c.Array[ctypes.c_ubyte, Literal[3]], 29), ('timestamp', ctypes.c_uint32, 32), ('unused_1', c.Array[ctypes.c_ubyte, Literal[4]], 36), ('vf_req_fwd', c.Array[ctypes.c_uint32, Literal[8]], 40), ('async_event_fwd', c.Array[ctypes.c_uint32, Literal[8]], 72), ('ver_maj', ctypes.c_uint16, 104), ('ver_min', ctypes.c_uint16, 106), ('ver_upd', ctypes.c_uint16, 108), ('ver_patch', ctypes.c_uint16, 110)]) +@c.record +class struct_hwrm_func_drv_rgtr_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + flags: int + unused_0: c.Array[ctypes.c_ubyte, Literal[3]] + valid: int +struct_hwrm_func_drv_rgtr_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('flags', ctypes.c_uint32, 8), ('unused_0', c.Array[ctypes.c_ubyte, Literal[3]], 12), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_hwrm_func_backing_store_cfg_v2_input(c.Struct): + SIZE = 64 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + type: int + instance: int + flags: int + page_dir: int + num_entries: int + entry_size: int + page_size_pbl_level: int + subtype_valid_cnt: int + split_entry_0: int + split_entry_1: int + split_entry_2: int + split_entry_3: int + enables: int + next_bs_offset: int +struct_hwrm_func_backing_store_cfg_v2_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('type', ctypes.c_uint16, 16), ('instance', ctypes.c_uint16, 18), ('flags', ctypes.c_uint32, 20), ('page_dir', ctypes.c_uint64, 24), ('num_entries', ctypes.c_uint32, 32), ('entry_size', ctypes.c_uint16, 36), ('page_size_pbl_level', ctypes.c_ubyte, 38), ('subtype_valid_cnt', ctypes.c_ubyte, 39), ('split_entry_0', ctypes.c_uint32, 40), ('split_entry_1', ctypes.c_uint32, 44), ('split_entry_2', ctypes.c_uint32, 48), ('split_entry_3', ctypes.c_uint32, 52), ('enables', ctypes.c_uint32, 56), ('next_bs_offset', ctypes.c_uint32, 60)]) +@c.record +class struct_hwrm_func_backing_store_cfg_v2_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + rsvd0: c.Array[ctypes.c_ubyte, Literal[7]] + valid: int +struct_hwrm_func_backing_store_cfg_v2_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('rsvd0', c.Array[ctypes.c_ubyte, Literal[7]], 8), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_hwrm_func_backing_store_qcaps_v2_input(c.Struct): + SIZE = 24 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + type: int + rsvd: c.Array[ctypes.c_ubyte, Literal[6]] +struct_hwrm_func_backing_store_qcaps_v2_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('type', ctypes.c_uint16, 16), ('rsvd', c.Array[ctypes.c_ubyte, Literal[6]], 18)]) +@c.record +class struct_hwrm_func_backing_store_qcaps_v2_output(c.Struct): + SIZE = 56 + error_code: int + req_type: int + seq_id: int + resp_len: int + type: int + entry_size: int + flags: int + instance_bit_map: int + ctx_init_value: int + ctx_init_offset: int + entry_multiple: int + rsvd: int + max_num_entries: int + min_num_entries: int + next_valid_type: int + subtype_valid_cnt: int + exact_cnt_bit_map: int + split_entry_0: int + split_entry_1: int + split_entry_2: int + split_entry_3: int + max_instance_count: int + rsvd3: int + valid: int +struct_hwrm_func_backing_store_qcaps_v2_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('type', ctypes.c_uint16, 8), ('entry_size', ctypes.c_uint16, 10), ('flags', ctypes.c_uint32, 12), ('instance_bit_map', ctypes.c_uint32, 16), ('ctx_init_value', ctypes.c_ubyte, 20), ('ctx_init_offset', ctypes.c_ubyte, 21), ('entry_multiple', ctypes.c_ubyte, 22), ('rsvd', ctypes.c_ubyte, 23), ('max_num_entries', ctypes.c_uint32, 24), ('min_num_entries', ctypes.c_uint32, 28), ('next_valid_type', ctypes.c_uint16, 32), ('subtype_valid_cnt', ctypes.c_ubyte, 34), ('exact_cnt_bit_map', ctypes.c_ubyte, 35), ('split_entry_0', ctypes.c_uint32, 36), ('split_entry_1', ctypes.c_uint32, 40), ('split_entry_2', ctypes.c_uint32, 44), ('split_entry_3', ctypes.c_uint32, 48), ('max_instance_count', ctypes.c_uint16, 52), ('rsvd3', ctypes.c_ubyte, 54), ('valid', ctypes.c_ubyte, 55)]) +@c.record +class struct_hwrm_port_phy_cfg_input(c.Struct): + SIZE = 64 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + flags: int + enables: int + port_id: int + force_link_speed: int + auto_mode: int + auto_duplex: int + auto_pause: int + mgmt_flag: int + auto_link_speed: int + auto_link_speed_mask: int + wirespeed: int + lpbk: int + force_pause: int + unused_1: int + preemphasis: int + eee_link_speed_mask: int + force_pam4_link_speed: int + tx_lpi_timer: int + auto_link_pam4_speed_mask: int + force_link_speeds2: int + auto_link_speeds2_mask: int + unused_2: c.Array[ctypes.c_ubyte, Literal[6]] +struct_hwrm_port_phy_cfg_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('flags', ctypes.c_uint32, 16), ('enables', ctypes.c_uint32, 20), ('port_id', ctypes.c_uint16, 24), ('force_link_speed', ctypes.c_uint16, 26), ('auto_mode', ctypes.c_ubyte, 28), ('auto_duplex', ctypes.c_ubyte, 29), ('auto_pause', ctypes.c_ubyte, 30), ('mgmt_flag', ctypes.c_ubyte, 31), ('auto_link_speed', ctypes.c_uint16, 32), ('auto_link_speed_mask', ctypes.c_uint16, 34), ('wirespeed', ctypes.c_ubyte, 36), ('lpbk', ctypes.c_ubyte, 37), ('force_pause', ctypes.c_ubyte, 38), ('unused_1', ctypes.c_ubyte, 39), ('preemphasis', ctypes.c_uint32, 40), ('eee_link_speed_mask', ctypes.c_uint16, 44), ('force_pam4_link_speed', ctypes.c_uint16, 46), ('tx_lpi_timer', ctypes.c_uint32, 48), ('auto_link_pam4_speed_mask', ctypes.c_uint16, 52), ('force_link_speeds2', ctypes.c_uint16, 54), ('auto_link_speeds2_mask', ctypes.c_uint16, 56), ('unused_2', c.Array[ctypes.c_ubyte, Literal[6]], 58)]) +@c.record +class struct_hwrm_port_phy_cfg_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + unused_0: c.Array[ctypes.c_ubyte, Literal[7]] + valid: int +struct_hwrm_port_phy_cfg_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('unused_0', c.Array[ctypes.c_ubyte, Literal[7]], 8), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_hwrm_vnic_alloc_input(c.Struct): + SIZE = 24 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + flags: int + virtio_net_fid: int + vnic_id: int +struct_hwrm_vnic_alloc_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('flags', ctypes.c_uint32, 16), ('virtio_net_fid', ctypes.c_uint16, 20), ('vnic_id', ctypes.c_uint16, 22)]) +@c.record +class struct_hwrm_vnic_alloc_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + vnic_id: int + unused_0: c.Array[ctypes.c_ubyte, Literal[3]] + valid: int +struct_hwrm_vnic_alloc_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('vnic_id', ctypes.c_uint32, 8), ('unused_0', c.Array[ctypes.c_ubyte, Literal[3]], 12), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_hwrm_vnic_cfg_input(c.Struct): + SIZE = 48 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + flags: int + enables: int + vnic_id: int + dflt_ring_grp: int + rss_rule: int + cos_rule: int + lb_rule: int + mru: int + default_rx_ring_id: int + default_cmpl_ring_id: int + queue_id: int + rx_csum_v2_mode: int + l2_cqe_mode: int + raw_qp_id: int +struct_hwrm_vnic_cfg_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('flags', ctypes.c_uint32, 16), ('enables', ctypes.c_uint32, 20), ('vnic_id', ctypes.c_uint16, 24), ('dflt_ring_grp', ctypes.c_uint16, 26), ('rss_rule', ctypes.c_uint16, 28), ('cos_rule', ctypes.c_uint16, 30), ('lb_rule', ctypes.c_uint16, 32), ('mru', ctypes.c_uint16, 34), ('default_rx_ring_id', ctypes.c_uint16, 36), ('default_cmpl_ring_id', ctypes.c_uint16, 38), ('queue_id', ctypes.c_uint16, 40), ('rx_csum_v2_mode', ctypes.c_ubyte, 42), ('l2_cqe_mode', ctypes.c_ubyte, 43), ('raw_qp_id', ctypes.c_uint32, 44)]) +@c.record +class struct_hwrm_vnic_cfg_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + unused_0: c.Array[ctypes.c_ubyte, Literal[7]] + valid: int +struct_hwrm_vnic_cfg_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('unused_0', c.Array[ctypes.c_ubyte, Literal[7]], 8), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_hwrm_ring_alloc_input(c.Struct): + SIZE = 96 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + enables: int + ring_type: int + cmpl_coal_cnt: int + flags: int + page_tbl_addr: int + fbo: int + page_size: int + page_tbl_depth: int + schq_id: int + length: int + logical_id: int + cmpl_ring_id: int + queue_id: int + rx_buf_size: int + rx_ring_id: int + nq_ring_id: int + ring_arb_cfg: int + steering_tag: int + reserved3: int + stat_ctx_id: int + reserved4: int + max_bw: int + int_mode: int + mpc_chnls_type: int + rx_rate_profile_sel: int + unused_4: int + cq_handle: int + dpi: int + unused_5: c.Array[ctypes.c_uint16, Literal[3]] +struct_hwrm_ring_alloc_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('enables', ctypes.c_uint32, 16), ('ring_type', ctypes.c_ubyte, 20), ('cmpl_coal_cnt', ctypes.c_ubyte, 21), ('flags', ctypes.c_uint16, 22), ('page_tbl_addr', ctypes.c_uint64, 24), ('fbo', ctypes.c_uint32, 32), ('page_size', ctypes.c_ubyte, 36), ('page_tbl_depth', ctypes.c_ubyte, 37), ('schq_id', ctypes.c_uint16, 38), ('length', ctypes.c_uint32, 40), ('logical_id', ctypes.c_uint16, 44), ('cmpl_ring_id', ctypes.c_uint16, 46), ('queue_id', ctypes.c_uint16, 48), ('rx_buf_size', ctypes.c_uint16, 50), ('rx_ring_id', ctypes.c_uint16, 52), ('nq_ring_id', ctypes.c_uint16, 54), ('ring_arb_cfg', ctypes.c_uint16, 56), ('steering_tag', ctypes.c_uint16, 58), ('reserved3', ctypes.c_uint32, 60), ('stat_ctx_id', ctypes.c_uint32, 64), ('reserved4', ctypes.c_uint32, 68), ('max_bw', ctypes.c_uint32, 72), ('int_mode', ctypes.c_ubyte, 76), ('mpc_chnls_type', ctypes.c_ubyte, 77), ('rx_rate_profile_sel', ctypes.c_ubyte, 78), ('unused_4', ctypes.c_ubyte, 79), ('cq_handle', ctypes.c_uint64, 80), ('dpi', ctypes.c_uint16, 88), ('unused_5', c.Array[ctypes.c_uint16, Literal[3]], 90)]) +@c.record +class struct_hwrm_ring_alloc_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + ring_id: int + logical_ring_id: int + push_buffer_index: int + unused_0: c.Array[ctypes.c_ubyte, Literal[2]] + valid: int +struct_hwrm_ring_alloc_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('ring_id', ctypes.c_uint16, 8), ('logical_ring_id', ctypes.c_uint16, 10), ('push_buffer_index', ctypes.c_ubyte, 12), ('unused_0', c.Array[ctypes.c_ubyte, Literal[2]], 13), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_hwrm_cfa_l2_filter_alloc_input(c.Struct): + SIZE = 96 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + flags: int + enables: int + l2_addr: c.Array[ctypes.c_ubyte, Literal[6]] + num_vlans: int + t_num_vlans: int + l2_addr_mask: c.Array[ctypes.c_ubyte, Literal[6]] + l2_ovlan: int + l2_ovlan_mask: int + l2_ivlan: int + l2_ivlan_mask: int + unused_1: c.Array[ctypes.c_ubyte, Literal[2]] + t_l2_addr: c.Array[ctypes.c_ubyte, Literal[6]] + unused_2: c.Array[ctypes.c_ubyte, Literal[2]] + t_l2_addr_mask: c.Array[ctypes.c_ubyte, Literal[6]] + t_l2_ovlan: int + t_l2_ovlan_mask: int + t_l2_ivlan: int + t_l2_ivlan_mask: int + src_type: int + unused_3: int + src_id: int + tunnel_type: int + unused_4: int + dst_id: int + mirror_vnic_id: int + pri_hint: int + unused_5: int + unused_6: int + l2_filter_id_hint: int +struct_hwrm_cfa_l2_filter_alloc_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('flags', ctypes.c_uint32, 16), ('enables', ctypes.c_uint32, 20), ('l2_addr', c.Array[ctypes.c_ubyte, Literal[6]], 24), ('num_vlans', ctypes.c_ubyte, 30), ('t_num_vlans', ctypes.c_ubyte, 31), ('l2_addr_mask', c.Array[ctypes.c_ubyte, Literal[6]], 32), ('l2_ovlan', ctypes.c_uint16, 38), ('l2_ovlan_mask', ctypes.c_uint16, 40), ('l2_ivlan', ctypes.c_uint16, 42), ('l2_ivlan_mask', ctypes.c_uint16, 44), ('unused_1', c.Array[ctypes.c_ubyte, Literal[2]], 46), ('t_l2_addr', c.Array[ctypes.c_ubyte, Literal[6]], 48), ('unused_2', c.Array[ctypes.c_ubyte, Literal[2]], 54), ('t_l2_addr_mask', c.Array[ctypes.c_ubyte, Literal[6]], 56), ('t_l2_ovlan', ctypes.c_uint16, 62), ('t_l2_ovlan_mask', ctypes.c_uint16, 64), ('t_l2_ivlan', ctypes.c_uint16, 66), ('t_l2_ivlan_mask', ctypes.c_uint16, 68), ('src_type', ctypes.c_ubyte, 70), ('unused_3', ctypes.c_ubyte, 71), ('src_id', ctypes.c_uint32, 72), ('tunnel_type', ctypes.c_ubyte, 76), ('unused_4', ctypes.c_ubyte, 77), ('dst_id', ctypes.c_uint16, 78), ('mirror_vnic_id', ctypes.c_uint16, 80), ('pri_hint', ctypes.c_ubyte, 82), ('unused_5', ctypes.c_ubyte, 83), ('unused_6', ctypes.c_uint32, 84), ('l2_filter_id_hint', ctypes.c_uint64, 88)]) +@c.record +class struct_hwrm_cfa_l2_filter_alloc_output(c.Struct): + SIZE = 24 + error_code: int + req_type: int + seq_id: int + resp_len: int + l2_filter_id: int + flow_id: int + unused_0: c.Array[ctypes.c_ubyte, Literal[3]] + valid: int +struct_hwrm_cfa_l2_filter_alloc_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('l2_filter_id', ctypes.c_uint64, 8), ('flow_id', ctypes.c_uint32, 16), ('unused_0', c.Array[ctypes.c_ubyte, Literal[3]], 20), ('valid', ctypes.c_ubyte, 23)]) +@c.record +class struct_hwrm_stat_ctx_alloc_input(c.Struct): + SIZE = 48 + req_type: int + cmpl_ring: int + seq_id: int + target_id: int + resp_addr: int + stats_dma_addr: int + update_period_ms: int + stat_ctx_flags: int + unused_0: int + stats_dma_length: int + flags: int + steering_tag: int + stat_ctx_id: int + alloc_seq_id: int + unused_1: c.Array[ctypes.c_ubyte, Literal[6]] +struct_hwrm_stat_ctx_alloc_input.register_fields([('req_type', ctypes.c_uint16, 0), ('cmpl_ring', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('target_id', ctypes.c_uint16, 6), ('resp_addr', ctypes.c_uint64, 8), ('stats_dma_addr', ctypes.c_uint64, 16), ('update_period_ms', ctypes.c_uint32, 24), ('stat_ctx_flags', ctypes.c_ubyte, 28), ('unused_0', ctypes.c_ubyte, 29), ('stats_dma_length', ctypes.c_uint16, 30), ('flags', ctypes.c_uint16, 32), ('steering_tag', ctypes.c_uint16, 34), ('stat_ctx_id', ctypes.c_uint32, 36), ('alloc_seq_id', ctypes.c_uint16, 40), ('unused_1', c.Array[ctypes.c_ubyte, Literal[6]], 42)]) +@c.record +class struct_hwrm_stat_ctx_alloc_output(c.Struct): + SIZE = 16 + error_code: int + req_type: int + seq_id: int + resp_len: int + stat_ctx_id: int + unused_0: c.Array[ctypes.c_ubyte, Literal[3]] + valid: int +struct_hwrm_stat_ctx_alloc_output.register_fields([('error_code', ctypes.c_uint16, 0), ('req_type', ctypes.c_uint16, 2), ('seq_id', ctypes.c_uint16, 4), ('resp_len', ctypes.c_uint16, 6), ('stat_ctx_id', ctypes.c_uint32, 8), ('unused_0', c.Array[ctypes.c_ubyte, Literal[3]], 12), ('valid', ctypes.c_ubyte, 15)]) +@c.record +class struct_cmdq_init(c.Struct): + SIZE = 16 + cmdq_pbl: int + cmdq_size_cmdq_lvl: int + creq_ring_id: int + prod_idx: int +struct_cmdq_init.register_fields([('cmdq_pbl', ctypes.c_uint64, 0), ('cmdq_size_cmdq_lvl', ctypes.c_uint16, 8), ('creq_ring_id', ctypes.c_uint16, 10), ('prod_idx', ctypes.c_uint32, 12)]) +@c.record +class struct_cmdq_base(c.Struct): + SIZE = 16 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + reserved8: int + resp_addr: int +struct_cmdq_base.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('reserved8', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8)]) +@c.record +class struct_creq_base(c.Struct): + SIZE = 16 + type: int + reserved56: c.Array[ctypes.c_ubyte, Literal[7]] + v: int + event: int + reserved48: c.Array[ctypes.c_ubyte, Literal[6]] +struct_creq_base.register_fields([('type', ctypes.c_ubyte, 0), ('reserved56', c.Array[ctypes.c_ubyte, Literal[7]], 1), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('reserved48', c.Array[ctypes.c_ubyte, Literal[6]], 10)]) +@c.record +class struct_cmdq_query_version(c.Struct): + SIZE = 16 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + reserved8: int + resp_addr: int +struct_cmdq_query_version.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('reserved8', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8)]) +@c.record +class struct_creq_query_version_resp(c.Struct): + SIZE = 16 + type: int + status: int + cookie: int + fw_maj: int + fw_minor: int + fw_bld: int + fw_rsvd: int + v: int + event: int + reserved16: int + intf_maj: int + intf_minor: int + intf_bld: int + intf_rsvd: int +struct_creq_query_version_resp.register_fields([('type', ctypes.c_ubyte, 0), ('status', ctypes.c_ubyte, 1), ('cookie', ctypes.c_uint16, 2), ('fw_maj', ctypes.c_ubyte, 4), ('fw_minor', ctypes.c_ubyte, 5), ('fw_bld', ctypes.c_ubyte, 6), ('fw_rsvd', ctypes.c_ubyte, 7), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('reserved16', ctypes.c_uint16, 10), ('intf_maj', ctypes.c_ubyte, 12), ('intf_minor', ctypes.c_ubyte, 13), ('intf_bld', ctypes.c_ubyte, 14), ('intf_rsvd', ctypes.c_ubyte, 15)]) +@c.record +class struct_cmdq_initialize_fw(c.Struct): + SIZE = 112 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + reserved8: int + resp_addr: int + qpc_pg_size_qpc_lvl: int + mrw_pg_size_mrw_lvl: int + srq_pg_size_srq_lvl: int + cq_pg_size_cq_lvl: int + tqm_pg_size_tqm_lvl: int + tim_pg_size_tim_lvl: int + log2_dbr_pg_size: int + qpc_page_dir: int + mrw_page_dir: int + srq_page_dir: int + cq_page_dir: int + tqm_page_dir: int + tim_page_dir: int + number_of_qp: int + number_of_mrw: int + number_of_srq: int + number_of_cq: int + max_qp_per_vf: int + max_mrw_per_vf: int + max_srq_per_vf: int + max_cq_per_vf: int + max_gid_per_vf: int + stat_ctx_id: int +struct_cmdq_initialize_fw.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('reserved8', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8), ('qpc_pg_size_qpc_lvl', ctypes.c_ubyte, 16), ('mrw_pg_size_mrw_lvl', ctypes.c_ubyte, 17), ('srq_pg_size_srq_lvl', ctypes.c_ubyte, 18), ('cq_pg_size_cq_lvl', ctypes.c_ubyte, 19), ('tqm_pg_size_tqm_lvl', ctypes.c_ubyte, 20), ('tim_pg_size_tim_lvl', ctypes.c_ubyte, 21), ('log2_dbr_pg_size', ctypes.c_uint16, 22), ('qpc_page_dir', ctypes.c_uint64, 24), ('mrw_page_dir', ctypes.c_uint64, 32), ('srq_page_dir', ctypes.c_uint64, 40), ('cq_page_dir', ctypes.c_uint64, 48), ('tqm_page_dir', ctypes.c_uint64, 56), ('tim_page_dir', ctypes.c_uint64, 64), ('number_of_qp', ctypes.c_uint32, 72), ('number_of_mrw', ctypes.c_uint32, 76), ('number_of_srq', ctypes.c_uint32, 80), ('number_of_cq', ctypes.c_uint32, 84), ('max_qp_per_vf', ctypes.c_uint32, 88), ('max_mrw_per_vf', ctypes.c_uint32, 92), ('max_srq_per_vf', ctypes.c_uint32, 96), ('max_cq_per_vf', ctypes.c_uint32, 100), ('max_gid_per_vf', ctypes.c_uint32, 104), ('stat_ctx_id', ctypes.c_uint32, 108)]) +@c.record +class struct_creq_initialize_fw_resp(c.Struct): + SIZE = 16 + type: int + status: int + cookie: int + reserved32: int + v: int + event: int + reserved48: c.Array[ctypes.c_ubyte, Literal[6]] +struct_creq_initialize_fw_resp.register_fields([('type', ctypes.c_ubyte, 0), ('status', ctypes.c_ubyte, 1), ('cookie', ctypes.c_uint16, 2), ('reserved32', ctypes.c_uint32, 4), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('reserved48', c.Array[ctypes.c_ubyte, Literal[6]], 10)]) +@c.record +class struct_cmdq_create_qp(c.Struct): + SIZE = 104 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + reserved8: int + resp_addr: int + qp_handle: int + qp_flags: int + type: int + sq_pg_size_sq_lvl: int + rq_pg_size_rq_lvl: int + unused_0: int + dpi: int + sq_size: int + rq_size: int + sq_fwo_sq_sge: int + rq_fwo_rq_sge: int + scq_cid: int + rcq_cid: int + srq_cid: int + pd_id: int + sq_pbl: int + rq_pbl: int + irrq_addr: int + orrq_addr: int + request_xid: int + steering_tag: int + reserved16: int +struct_cmdq_create_qp.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('reserved8', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8), ('qp_handle', ctypes.c_uint64, 16), ('qp_flags', ctypes.c_uint32, 24), ('type', ctypes.c_ubyte, 28), ('sq_pg_size_sq_lvl', ctypes.c_ubyte, 29), ('rq_pg_size_rq_lvl', ctypes.c_ubyte, 30), ('unused_0', ctypes.c_ubyte, 31), ('dpi', ctypes.c_uint32, 32), ('sq_size', ctypes.c_uint32, 36), ('rq_size', ctypes.c_uint32, 40), ('sq_fwo_sq_sge', ctypes.c_uint16, 44), ('rq_fwo_rq_sge', ctypes.c_uint16, 46), ('scq_cid', ctypes.c_uint32, 48), ('rcq_cid', ctypes.c_uint32, 52), ('srq_cid', ctypes.c_uint32, 56), ('pd_id', ctypes.c_uint32, 60), ('sq_pbl', ctypes.c_uint64, 64), ('rq_pbl', ctypes.c_uint64, 72), ('irrq_addr', ctypes.c_uint64, 80), ('orrq_addr', ctypes.c_uint64, 88), ('request_xid', ctypes.c_uint32, 96), ('steering_tag', ctypes.c_uint16, 100), ('reserved16', ctypes.c_uint16, 102)]) +@c.record +class struct_creq_create_qp_resp(c.Struct): + SIZE = 16 + type: int + status: int + cookie: int + xid: int + v: int + event: int + optimized_transmit_enabled: int + reserved48: c.Array[ctypes.c_ubyte, Literal[5]] +struct_creq_create_qp_resp.register_fields([('type', ctypes.c_ubyte, 0), ('status', ctypes.c_ubyte, 1), ('cookie', ctypes.c_uint16, 2), ('xid', ctypes.c_uint32, 4), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('optimized_transmit_enabled', ctypes.c_ubyte, 10), ('reserved48', c.Array[ctypes.c_ubyte, Literal[5]], 11)]) +@c.record +class struct_cmdq_modify_qp(c.Struct): + SIZE = 144 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + qp_type: int + resp_addr: int + modify_mask: int + qp_cid: int + network_type_en_sqd_async_notify_new_state: int + access: int + pkey: int + qkey: int + dgid: c.Array[ctypes.c_uint32, Literal[4]] + flow_label: int + sgid_index: int + hop_limit: int + traffic_class: int + dest_mac: c.Array[ctypes.c_uint16, Literal[3]] + tos_dscp_tos_ecn: int + path_mtu_pingpong_push_enable: int + timeout: int + retry_cnt: int + rnr_retry: int + min_rnr_timer: int + rq_psn: int + sq_psn: int + max_rd_atomic: int + max_dest_rd_atomic: int + enable_cc: int + sq_size: int + rq_size: int + sq_sge: int + rq_sge: int + max_inline_data: int + dest_qp_id: int + pingpong_push_dpi: int + src_mac: c.Array[ctypes.c_uint16, Literal[3]] + vlan_pcp_vlan_dei_vlan_id: int + irrq_addr: int + orrq_addr: int + ext_modify_mask: int + ext_stats_ctx_id: int + schq_id: int + unused_0: int + reserved32: int +struct_cmdq_modify_qp.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('qp_type', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8), ('modify_mask', ctypes.c_uint32, 16), ('qp_cid', ctypes.c_uint32, 20), ('network_type_en_sqd_async_notify_new_state', ctypes.c_ubyte, 24), ('access', ctypes.c_ubyte, 25), ('pkey', ctypes.c_uint16, 26), ('qkey', ctypes.c_uint32, 28), ('dgid', c.Array[ctypes.c_uint32, Literal[4]], 32), ('flow_label', ctypes.c_uint32, 48), ('sgid_index', ctypes.c_uint16, 52), ('hop_limit', ctypes.c_ubyte, 54), ('traffic_class', ctypes.c_ubyte, 55), ('dest_mac', c.Array[ctypes.c_uint16, Literal[3]], 56), ('tos_dscp_tos_ecn', ctypes.c_ubyte, 62), ('path_mtu_pingpong_push_enable', ctypes.c_ubyte, 63), ('timeout', ctypes.c_ubyte, 64), ('retry_cnt', ctypes.c_ubyte, 65), ('rnr_retry', ctypes.c_ubyte, 66), ('min_rnr_timer', ctypes.c_ubyte, 67), ('rq_psn', ctypes.c_uint32, 68), ('sq_psn', ctypes.c_uint32, 72), ('max_rd_atomic', ctypes.c_ubyte, 76), ('max_dest_rd_atomic', ctypes.c_ubyte, 77), ('enable_cc', ctypes.c_uint16, 78), ('sq_size', ctypes.c_uint32, 80), ('rq_size', ctypes.c_uint32, 84), ('sq_sge', ctypes.c_uint16, 88), ('rq_sge', ctypes.c_uint16, 90), ('max_inline_data', ctypes.c_uint32, 92), ('dest_qp_id', ctypes.c_uint32, 96), ('pingpong_push_dpi', ctypes.c_uint32, 100), ('src_mac', c.Array[ctypes.c_uint16, Literal[3]], 104), ('vlan_pcp_vlan_dei_vlan_id', ctypes.c_uint16, 110), ('irrq_addr', ctypes.c_uint64, 112), ('orrq_addr', ctypes.c_uint64, 120), ('ext_modify_mask', ctypes.c_uint32, 128), ('ext_stats_ctx_id', ctypes.c_uint32, 132), ('schq_id', ctypes.c_uint16, 136), ('unused_0', ctypes.c_uint16, 138), ('reserved32', ctypes.c_uint32, 140)]) +@c.record +class struct_creq_modify_qp_resp(c.Struct): + SIZE = 16 + type: int + status: int + cookie: int + xid: int + v: int + event: int + pingpong_push_state_index_enabled: int + reserved8: int + lag_src_mac: int +struct_creq_modify_qp_resp.register_fields([('type', ctypes.c_ubyte, 0), ('status', ctypes.c_ubyte, 1), ('cookie', ctypes.c_uint16, 2), ('xid', ctypes.c_uint32, 4), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('pingpong_push_state_index_enabled', ctypes.c_ubyte, 10), ('reserved8', ctypes.c_ubyte, 11), ('lag_src_mac', ctypes.c_uint32, 12)]) +@c.record +class struct_cmdq_create_cq(c.Struct): + SIZE = 64 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + reserved8: int + resp_addr: int + cq_handle: int + pg_size_lvl: int + cq_fco_cnq_id: int + dpi: int + cq_size: int + pbl: int + steering_tag: int + reserved48: c.Array[ctypes.c_ubyte, Literal[2]] + coalescing: int + reserved64: int +struct_cmdq_create_cq.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('reserved8', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8), ('cq_handle', ctypes.c_uint64, 16), ('pg_size_lvl', ctypes.c_uint32, 24), ('cq_fco_cnq_id', ctypes.c_uint32, 28), ('dpi', ctypes.c_uint32, 32), ('cq_size', ctypes.c_uint32, 36), ('pbl', ctypes.c_uint64, 40), ('steering_tag', ctypes.c_uint16, 48), ('reserved48', c.Array[ctypes.c_ubyte, Literal[2]], 50), ('coalescing', ctypes.c_uint32, 52), ('reserved64', ctypes.c_uint64, 56)]) +@c.record +class struct_creq_create_cq_resp(c.Struct): + SIZE = 16 + type: int + status: int + cookie: int + xid: int + v: int + event: int + reserved48: c.Array[ctypes.c_ubyte, Literal[6]] +struct_creq_create_cq_resp.register_fields([('type', ctypes.c_ubyte, 0), ('status', ctypes.c_ubyte, 1), ('cookie', ctypes.c_uint16, 2), ('xid', ctypes.c_uint32, 4), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('reserved48', c.Array[ctypes.c_ubyte, Literal[6]], 10)]) +@c.record +class struct_cmdq_register_mr(c.Struct): + SIZE = 56 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + reserved8: int + resp_addr: int + log2_pg_size_lvl: int + access: int + log2_pbl_pg_size: int + key: int + pbl: int + va: int + mr_size: int + steering_tag: int + reserved48: c.Array[ctypes.c_ubyte, Literal[6]] +struct_cmdq_register_mr.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('reserved8', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8), ('log2_pg_size_lvl', ctypes.c_ubyte, 16), ('access', ctypes.c_ubyte, 17), ('log2_pbl_pg_size', ctypes.c_uint16, 18), ('key', ctypes.c_uint32, 20), ('pbl', ctypes.c_uint64, 24), ('va', ctypes.c_uint64, 32), ('mr_size', ctypes.c_uint64, 40), ('steering_tag', ctypes.c_uint16, 48), ('reserved48', c.Array[ctypes.c_ubyte, Literal[6]], 50)]) +@c.record +class struct_creq_register_mr_resp(c.Struct): + SIZE = 16 + type: int + status: int + cookie: int + xid: int + v: int + event: int + reserved48: c.Array[ctypes.c_ubyte, Literal[6]] +struct_creq_register_mr_resp.register_fields([('type', ctypes.c_ubyte, 0), ('status', ctypes.c_ubyte, 1), ('cookie', ctypes.c_uint16, 2), ('xid', ctypes.c_uint32, 4), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('reserved48', c.Array[ctypes.c_ubyte, Literal[6]], 10)]) +@c.record +class struct_cmdq_add_gid(c.Struct): + SIZE = 48 + opcode: int + cmd_size: int + flags: int + cookie: int + resp_size: int + reserved8: int + resp_addr: int + gid: c.Array[ctypes.c_uint32, Literal[4]] + src_mac: c.Array[ctypes.c_uint16, Literal[3]] + vlan: int + ipid: int + stats_ctx: int + unused_0: int +struct_cmdq_add_gid.register_fields([('opcode', ctypes.c_ubyte, 0), ('cmd_size', ctypes.c_ubyte, 1), ('flags', ctypes.c_uint16, 2), ('cookie', ctypes.c_uint16, 4), ('resp_size', ctypes.c_ubyte, 6), ('reserved8', ctypes.c_ubyte, 7), ('resp_addr', ctypes.c_uint64, 8), ('gid', c.Array[ctypes.c_uint32, Literal[4]], 16), ('src_mac', c.Array[ctypes.c_uint16, Literal[3]], 32), ('vlan', ctypes.c_uint16, 38), ('ipid', ctypes.c_uint16, 40), ('stats_ctx', ctypes.c_uint16, 42), ('unused_0', ctypes.c_uint32, 44)]) +@c.record +class struct_creq_add_gid_resp(c.Struct): + SIZE = 16 + type: int + status: int + cookie: int + xid: int + v: int + event: int + reserved48: c.Array[ctypes.c_ubyte, Literal[6]] +struct_creq_add_gid_resp.register_fields([('type', ctypes.c_ubyte, 0), ('status', ctypes.c_ubyte, 1), ('cookie', ctypes.c_uint16, 2), ('xid', ctypes.c_uint32, 4), ('v', ctypes.c_ubyte, 8), ('event', ctypes.c_ubyte, 9), ('reserved48', c.Array[ctypes.c_ubyte, Literal[6]], 10)]) +@c.record +class struct_sq_sge(c.Struct): + SIZE = 16 + va_or_pa: int + l_key: int + size: int +struct_sq_sge.register_fields([('va_or_pa', ctypes.c_uint64, 0), ('l_key', ctypes.c_uint32, 8), ('size', ctypes.c_uint32, 12)]) +@c.record +class struct_sq_rdma_hdr(c.Struct): + SIZE = 32 + wqe_type: int + flags: int + wqe_size: int + reserved8: int + imm_data: int + length: int + reserved32_1: int + remote_va: int + remote_key: int + timestamp: int +struct_sq_rdma_hdr.register_fields([('wqe_type', ctypes.c_ubyte, 0), ('flags', ctypes.c_ubyte, 1), ('wqe_size', ctypes.c_ubyte, 2), ('reserved8', ctypes.c_ubyte, 3), ('imm_data', ctypes.c_uint32, 4), ('length', ctypes.c_uint32, 8), ('reserved32_1', ctypes.c_uint32, 12), ('remote_va', ctypes.c_uint64, 16), ('remote_key', ctypes.c_uint32, 24), ('timestamp', ctypes.c_uint32, 28)]) +@c.record +class struct_cq_base(c.Struct): + SIZE = 32 + reserved64_1: int + reserved64_2: int + reserved64_3: int + cqe_type_toggle: int + status: int + reserved16: int + opaque: int +struct_cq_base.register_fields([('reserved64_1', ctypes.c_uint64, 0), ('reserved64_2', ctypes.c_uint64, 8), ('reserved64_3', ctypes.c_uint64, 16), ('cqe_type_toggle', ctypes.c_ubyte, 24), ('status', ctypes.c_ubyte, 25), ('reserved16', ctypes.c_uint16, 26), ('opaque', ctypes.c_uint32, 28)]) +@c.record +class struct_cq_req(c.Struct): + SIZE = 32 + qp_handle: int + sq_cons_idx: int + reserved16_1: int + reserved32_2: int + reserved64: int + cqe_type_toggle: int + status: int + reserved16_2: int + reserved32_1: int +struct_cq_req.register_fields([('qp_handle', ctypes.c_uint64, 0), ('sq_cons_idx', ctypes.c_uint16, 8), ('reserved16_1', ctypes.c_uint16, 10), ('reserved32_2', ctypes.c_uint32, 12), ('reserved64', ctypes.c_uint64, 16), ('cqe_type_toggle', ctypes.c_ubyte, 24), ('status', ctypes.c_ubyte, 25), ('reserved16_2', ctypes.c_uint16, 26), ('reserved32_1', ctypes.c_uint32, 28)]) +HWRM_VER_GET = 0x0 +HWRM_FUNC_ECHO_RESPONSE = 0xb +HWRM_ERROR_RECOVERY_QCFG = 0xc +HWRM_FUNC_DRV_IF_CHANGE = 0xd +HWRM_FUNC_BUF_UNRGTR = 0xe +HWRM_FUNC_VF_CFG = 0xf +HWRM_RESERVED1 = 0x10 +HWRM_FUNC_RESET = 0x11 +HWRM_FUNC_GETFID = 0x12 +HWRM_FUNC_VF_ALLOC = 0x13 +HWRM_FUNC_VF_FREE = 0x14 +HWRM_FUNC_QCAPS = 0x15 +HWRM_FUNC_QCFG = 0x16 +HWRM_FUNC_CFG = 0x17 +HWRM_FUNC_QSTATS = 0x18 +HWRM_FUNC_CLR_STATS = 0x19 +HWRM_FUNC_DRV_UNRGTR = 0x1a +HWRM_FUNC_VF_RESC_FREE = 0x1b +HWRM_FUNC_VF_VNIC_IDS_QUERY = 0x1c +HWRM_FUNC_DRV_RGTR = 0x1d +HWRM_FUNC_DRV_QVER = 0x1e +HWRM_FUNC_BUF_RGTR = 0x1f +HWRM_PORT_PHY_CFG = 0x20 +HWRM_PORT_MAC_CFG = 0x21 +HWRM_PORT_TS_QUERY = 0x22 +HWRM_PORT_QSTATS = 0x23 +HWRM_PORT_LPBK_QSTATS = 0x24 +HWRM_PORT_CLR_STATS = 0x25 +HWRM_PORT_LPBK_CLR_STATS = 0x26 +HWRM_PORT_PHY_QCFG = 0x27 +HWRM_PORT_MAC_QCFG = 0x28 +HWRM_PORT_MAC_PTP_QCFG = 0x29 +HWRM_PORT_PHY_QCAPS = 0x2a +HWRM_PORT_PHY_I2C_WRITE = 0x2b +HWRM_PORT_PHY_I2C_READ = 0x2c +HWRM_PORT_LED_CFG = 0x2d +HWRM_PORT_LED_QCFG = 0x2e +HWRM_PORT_LED_QCAPS = 0x2f +HWRM_QUEUE_QPORTCFG = 0x30 +HWRM_QUEUE_QCFG = 0x31 +HWRM_QUEUE_CFG = 0x32 +HWRM_FUNC_VLAN_CFG = 0x33 +HWRM_FUNC_VLAN_QCFG = 0x34 +HWRM_QUEUE_PFCENABLE_QCFG = 0x35 +HWRM_QUEUE_PFCENABLE_CFG = 0x36 +HWRM_QUEUE_PRI2COS_QCFG = 0x37 +HWRM_QUEUE_PRI2COS_CFG = 0x38 +HWRM_QUEUE_COS2BW_QCFG = 0x39 +HWRM_QUEUE_COS2BW_CFG = 0x3a +HWRM_QUEUE_DSCP_QCAPS = 0x3b +HWRM_QUEUE_DSCP2PRI_QCFG = 0x3c +HWRM_QUEUE_DSCP2PRI_CFG = 0x3d +HWRM_VNIC_ALLOC = 0x40 +HWRM_VNIC_FREE = 0x41 +HWRM_VNIC_CFG = 0x42 +HWRM_VNIC_QCFG = 0x43 +HWRM_VNIC_TPA_CFG = 0x44 +HWRM_VNIC_TPA_QCFG = 0x45 +HWRM_VNIC_RSS_CFG = 0x46 +HWRM_VNIC_RSS_QCFG = 0x47 +HWRM_VNIC_PLCMODES_CFG = 0x48 +HWRM_VNIC_PLCMODES_QCFG = 0x49 +HWRM_VNIC_QCAPS = 0x4a +HWRM_VNIC_UPDATE = 0x4b +HWRM_RING_ALLOC = 0x50 +HWRM_RING_FREE = 0x51 +HWRM_RING_CMPL_RING_QAGGINT_PARAMS = 0x52 +HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS = 0x53 +HWRM_RING_AGGINT_QCAPS = 0x54 +HWRM_RING_SCHQ_ALLOC = 0x55 +HWRM_RING_SCHQ_CFG = 0x56 +HWRM_RING_SCHQ_FREE = 0x57 +HWRM_RING_RESET = 0x5e +HWRM_RING_GRP_ALLOC = 0x60 +HWRM_RING_GRP_FREE = 0x61 +HWRM_RING_CFG = 0x62 +HWRM_RING_QCFG = 0x63 +HWRM_RESERVED5 = 0x64 +HWRM_RESERVED6 = 0x65 +HWRM_VNIC_RSS_COS_LB_CTX_ALLOC = 0x70 +HWRM_VNIC_RSS_COS_LB_CTX_FREE = 0x71 +HWRM_QUEUE_MPLS_QCAPS = 0x80 +HWRM_QUEUE_MPLSTC2PRI_QCFG = 0x81 +HWRM_QUEUE_MPLSTC2PRI_CFG = 0x82 +HWRM_QUEUE_VLANPRI_QCAPS = 0x83 +HWRM_QUEUE_VLANPRI2PRI_QCFG = 0x84 +HWRM_QUEUE_VLANPRI2PRI_CFG = 0x85 +HWRM_QUEUE_GLOBAL_CFG = 0x86 +HWRM_QUEUE_GLOBAL_QCFG = 0x87 +HWRM_QUEUE_ADPTV_QOS_RX_FEATURE_QCFG = 0x88 +HWRM_QUEUE_ADPTV_QOS_RX_FEATURE_CFG = 0x89 +HWRM_QUEUE_ADPTV_QOS_TX_FEATURE_QCFG = 0x8a +HWRM_QUEUE_ADPTV_QOS_TX_FEATURE_CFG = 0x8b +HWRM_QUEUE_QCAPS = 0x8c +HWRM_QUEUE_ADPTV_QOS_RX_TUNING_QCFG = 0x8d +HWRM_QUEUE_ADPTV_QOS_RX_TUNING_CFG = 0x8e +HWRM_QUEUE_ADPTV_QOS_TX_TUNING_QCFG = 0x8f +HWRM_CFA_L2_FILTER_ALLOC = 0x90 +HWRM_CFA_L2_FILTER_FREE = 0x91 +HWRM_CFA_L2_FILTER_CFG = 0x92 +HWRM_CFA_L2_SET_RX_MASK = 0x93 +HWRM_CFA_VLAN_ANTISPOOF_CFG = 0x94 +HWRM_CFA_TUNNEL_FILTER_ALLOC = 0x95 +HWRM_CFA_TUNNEL_FILTER_FREE = 0x96 +HWRM_CFA_ENCAP_RECORD_ALLOC = 0x97 +HWRM_CFA_ENCAP_RECORD_FREE = 0x98 +HWRM_CFA_NTUPLE_FILTER_ALLOC = 0x99 +HWRM_CFA_NTUPLE_FILTER_FREE = 0x9a +HWRM_CFA_NTUPLE_FILTER_CFG = 0x9b +HWRM_CFA_EM_FLOW_ALLOC = 0x9c +HWRM_CFA_EM_FLOW_FREE = 0x9d +HWRM_CFA_EM_FLOW_CFG = 0x9e +HWRM_TUNNEL_DST_PORT_QUERY = 0xa0 +HWRM_TUNNEL_DST_PORT_ALLOC = 0xa1 +HWRM_TUNNEL_DST_PORT_FREE = 0xa2 +HWRM_QUEUE_ADPTV_QOS_TX_TUNING_CFG = 0xa3 +HWRM_STAT_CTX_ENG_QUERY = 0xaf +HWRM_STAT_CTX_ALLOC = 0xb0 +HWRM_STAT_CTX_FREE = 0xb1 +HWRM_STAT_CTX_QUERY = 0xb2 +HWRM_STAT_CTX_CLR_STATS = 0xb3 +HWRM_PORT_QSTATS_EXT = 0xb4 +HWRM_PORT_PHY_MDIO_WRITE = 0xb5 +HWRM_PORT_PHY_MDIO_READ = 0xb6 +HWRM_PORT_PHY_MDIO_BUS_ACQUIRE = 0xb7 +HWRM_PORT_PHY_MDIO_BUS_RELEASE = 0xb8 +HWRM_PORT_QSTATS_EXT_PFC_WD = 0xb9 +HWRM_RESERVED7 = 0xba +HWRM_PORT_TX_FIR_CFG = 0xbb +HWRM_PORT_TX_FIR_QCFG = 0xbc +HWRM_PORT_ECN_QSTATS = 0xbd +HWRM_FW_LIVEPATCH_QUERY = 0xbe +HWRM_FW_LIVEPATCH = 0xbf +HWRM_FW_RESET = 0xc0 +HWRM_FW_QSTATUS = 0xc1 +HWRM_FW_HEALTH_CHECK = 0xc2 +HWRM_FW_SYNC = 0xc3 +HWRM_FW_STATE_QCAPS = 0xc4 +HWRM_FW_STATE_QUIESCE = 0xc5 +HWRM_FW_STATE_BACKUP = 0xc6 +HWRM_FW_STATE_RESTORE = 0xc7 +HWRM_FW_SET_TIME = 0xc8 +HWRM_FW_GET_TIME = 0xc9 +HWRM_FW_SET_STRUCTURED_DATA = 0xca +HWRM_FW_GET_STRUCTURED_DATA = 0xcb +HWRM_FW_IPC_MAILBOX = 0xcc +HWRM_FW_ECN_CFG = 0xcd +HWRM_FW_ECN_QCFG = 0xce +HWRM_FW_SECURE_CFG = 0xcf +HWRM_EXEC_FWD_RESP = 0xd0 +HWRM_REJECT_FWD_RESP = 0xd1 +HWRM_FWD_RESP = 0xd2 +HWRM_FWD_ASYNC_EVENT_CMPL = 0xd3 +HWRM_OEM_CMD = 0xd4 +HWRM_PORT_PRBS_TEST = 0xd5 +HWRM_PORT_SFP_SIDEBAND_CFG = 0xd6 +HWRM_PORT_SFP_SIDEBAND_QCFG = 0xd7 +HWRM_FW_STATE_UNQUIESCE = 0xd8 +HWRM_PORT_DSC_DUMP = 0xd9 +HWRM_PORT_EP_TX_QCFG = 0xda +HWRM_PORT_EP_TX_CFG = 0xdb +HWRM_PORT_CFG = 0xdc +HWRM_PORT_QCFG = 0xdd +HWRM_PORT_MAC_QCAPS = 0xdf +HWRM_TEMP_MONITOR_QUERY = 0xe0 +HWRM_REG_POWER_QUERY = 0xe1 +HWRM_CORE_FREQUENCY_QUERY = 0xe2 +HWRM_REG_POWER_HISTOGRAM = 0xe3 +HWRM_MONITOR_PAX_HISTOGRAM_START = 0xe4 +HWRM_MONITOR_PAX_HISTOGRAM_COLLECT = 0xe5 +HWRM_STAT_QUERY_ROCE_STATS = 0xe6 +HWRM_STAT_QUERY_ROCE_STATS_EXT = 0xe7 +HWRM_WOL_FILTER_ALLOC = 0xf0 +HWRM_WOL_FILTER_FREE = 0xf1 +HWRM_WOL_FILTER_QCFG = 0xf2 +HWRM_WOL_REASON_QCFG = 0xf3 +HWRM_CFA_METER_QCAPS = 0xf4 +HWRM_CFA_METER_PROFILE_ALLOC = 0xf5 +HWRM_CFA_METER_PROFILE_FREE = 0xf6 +HWRM_CFA_METER_PROFILE_CFG = 0xf7 +HWRM_CFA_METER_INSTANCE_ALLOC = 0xf8 +HWRM_CFA_METER_INSTANCE_FREE = 0xf9 +HWRM_CFA_METER_INSTANCE_CFG = 0xfa +HWRM_CFA_VFR_ALLOC = 0xfd +HWRM_CFA_VFR_FREE = 0xfe +HWRM_CFA_VF_PAIR_ALLOC = 0x100 +HWRM_CFA_VF_PAIR_FREE = 0x101 +HWRM_CFA_VF_PAIR_INFO = 0x102 +HWRM_CFA_FLOW_ALLOC = 0x103 +HWRM_CFA_FLOW_FREE = 0x104 +HWRM_CFA_FLOW_FLUSH = 0x105 +HWRM_CFA_FLOW_STATS = 0x106 +HWRM_CFA_FLOW_INFO = 0x107 +HWRM_CFA_DECAP_FILTER_ALLOC = 0x108 +HWRM_CFA_DECAP_FILTER_FREE = 0x109 +HWRM_CFA_VLAN_ANTISPOOF_QCFG = 0x10a +HWRM_CFA_REDIRECT_TUNNEL_TYPE_ALLOC = 0x10b +HWRM_CFA_REDIRECT_TUNNEL_TYPE_FREE = 0x10c +HWRM_CFA_PAIR_ALLOC = 0x10d +HWRM_CFA_PAIR_FREE = 0x10e +HWRM_CFA_PAIR_INFO = 0x10f +HWRM_FW_IPC_MSG = 0x110 +HWRM_CFA_REDIRECT_TUNNEL_TYPE_INFO = 0x111 +HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE = 0x112 +HWRM_CFA_FLOW_AGING_TIMER_RESET = 0x113 +HWRM_CFA_FLOW_AGING_CFG = 0x114 +HWRM_CFA_FLOW_AGING_QCFG = 0x115 +HWRM_CFA_FLOW_AGING_QCAPS = 0x116 +HWRM_CFA_CTX_MEM_RGTR = 0x117 +HWRM_CFA_CTX_MEM_UNRGTR = 0x118 +HWRM_CFA_CTX_MEM_QCTX = 0x119 +HWRM_CFA_CTX_MEM_QCAPS = 0x11a +HWRM_CFA_COUNTER_QCAPS = 0x11b +HWRM_CFA_COUNTER_CFG = 0x11c +HWRM_CFA_COUNTER_QCFG = 0x11d +HWRM_CFA_COUNTER_QSTATS = 0x11e +HWRM_CFA_TCP_FLAG_PROCESS_QCFG = 0x11f +HWRM_CFA_EEM_QCAPS = 0x120 +HWRM_CFA_EEM_CFG = 0x121 +HWRM_CFA_EEM_QCFG = 0x122 +HWRM_CFA_EEM_OP = 0x123 +HWRM_CFA_ADV_FLOW_MGNT_QCAPS = 0x124 +HWRM_CFA_TFLIB = 0x125 +HWRM_CFA_LAG_GROUP_MEMBER_RGTR = 0x126 +HWRM_CFA_LAG_GROUP_MEMBER_UNRGTR = 0x127 +HWRM_CFA_TLS_FILTER_ALLOC = 0x128 +HWRM_CFA_TLS_FILTER_FREE = 0x129 +HWRM_CFA_RELEASE_AFM_FUNC = 0x12a +HWRM_ENGINE_CKV_STATUS = 0x12e +HWRM_ENGINE_CKV_CKEK_ADD = 0x12f +HWRM_ENGINE_CKV_CKEK_DELETE = 0x130 +HWRM_ENGINE_CKV_KEY_ADD = 0x131 +HWRM_ENGINE_CKV_KEY_DELETE = 0x132 +HWRM_ENGINE_CKV_FLUSH = 0x133 +HWRM_ENGINE_CKV_RNG_GET = 0x134 +HWRM_ENGINE_CKV_KEY_GEN = 0x135 +HWRM_ENGINE_CKV_KEY_LABEL_CFG = 0x136 +HWRM_ENGINE_CKV_KEY_LABEL_QCFG = 0x137 +HWRM_ENGINE_QG_CONFIG_QUERY = 0x13c +HWRM_ENGINE_QG_QUERY = 0x13d +HWRM_ENGINE_QG_METER_PROFILE_CONFIG_QUERY = 0x13e +HWRM_ENGINE_QG_METER_PROFILE_QUERY = 0x13f +HWRM_ENGINE_QG_METER_PROFILE_ALLOC = 0x140 +HWRM_ENGINE_QG_METER_PROFILE_FREE = 0x141 +HWRM_ENGINE_QG_METER_QUERY = 0x142 +HWRM_ENGINE_QG_METER_BIND = 0x143 +HWRM_ENGINE_QG_METER_UNBIND = 0x144 +HWRM_ENGINE_QG_FUNC_BIND = 0x145 +HWRM_ENGINE_SG_CONFIG_QUERY = 0x146 +HWRM_ENGINE_SG_QUERY = 0x147 +HWRM_ENGINE_SG_METER_QUERY = 0x148 +HWRM_ENGINE_SG_METER_CONFIG = 0x149 +HWRM_ENGINE_SG_QG_BIND = 0x14a +HWRM_ENGINE_QG_SG_UNBIND = 0x14b +HWRM_ENGINE_CONFIG_QUERY = 0x154 +HWRM_ENGINE_STATS_CONFIG = 0x155 +HWRM_ENGINE_STATS_CLEAR = 0x156 +HWRM_ENGINE_STATS_QUERY = 0x157 +HWRM_ENGINE_STATS_QUERY_CONTINUOUS_ERROR = 0x158 +HWRM_ENGINE_RQ_ALLOC = 0x15e +HWRM_ENGINE_RQ_FREE = 0x15f +HWRM_ENGINE_CQ_ALLOC = 0x160 +HWRM_ENGINE_CQ_FREE = 0x161 +HWRM_ENGINE_NQ_ALLOC = 0x162 +HWRM_ENGINE_NQ_FREE = 0x163 +HWRM_ENGINE_ON_DIE_RQE_CREDITS = 0x164 +HWRM_ENGINE_FUNC_QCFG = 0x165 +HWRM_FUNC_RESOURCE_QCAPS = 0x190 +HWRM_FUNC_VF_RESOURCE_CFG = 0x191 +HWRM_FUNC_BACKING_STORE_QCAPS = 0x192 +HWRM_FUNC_BACKING_STORE_CFG = 0x193 +HWRM_FUNC_BACKING_STORE_QCFG = 0x194 +HWRM_FUNC_VF_BW_CFG = 0x195 +HWRM_FUNC_VF_BW_QCFG = 0x196 +HWRM_FUNC_HOST_PF_IDS_QUERY = 0x197 +HWRM_FUNC_QSTATS_EXT = 0x198 +HWRM_STAT_EXT_CTX_QUERY = 0x199 +HWRM_FUNC_SPD_CFG = 0x19a +HWRM_FUNC_SPD_QCFG = 0x19b +HWRM_FUNC_PTP_PIN_QCFG = 0x19c +HWRM_FUNC_PTP_PIN_CFG = 0x19d +HWRM_FUNC_PTP_CFG = 0x19e +HWRM_FUNC_PTP_TS_QUERY = 0x19f +HWRM_FUNC_PTP_EXT_CFG = 0x1a0 +HWRM_FUNC_PTP_EXT_QCFG = 0x1a1 +HWRM_FUNC_KEY_CTX_ALLOC = 0x1a2 +HWRM_FUNC_BACKING_STORE_CFG_V2 = 0x1a3 +HWRM_FUNC_BACKING_STORE_QCFG_V2 = 0x1a4 +HWRM_FUNC_DBR_PACING_CFG = 0x1a5 +HWRM_FUNC_DBR_PACING_QCFG = 0x1a6 +HWRM_FUNC_DBR_PACING_BROADCAST_EVENT = 0x1a7 +HWRM_FUNC_BACKING_STORE_QCAPS_V2 = 0x1a8 +HWRM_FUNC_DBR_PACING_NQLIST_QUERY = 0x1a9 +HWRM_FUNC_DBR_RECOVERY_COMPLETED = 0x1aa +HWRM_FUNC_SYNCE_CFG = 0x1ab +HWRM_FUNC_SYNCE_QCFG = 0x1ac +HWRM_FUNC_KEY_CTX_FREE = 0x1ad +HWRM_FUNC_LAG_MODE_CFG = 0x1ae +HWRM_FUNC_LAG_MODE_QCFG = 0x1af +HWRM_FUNC_LAG_CREATE = 0x1b0 +HWRM_FUNC_LAG_UPDATE = 0x1b1 +HWRM_FUNC_LAG_FREE = 0x1b2 +HWRM_FUNC_LAG_QCFG = 0x1b3 +HWRM_FUNC_TTX_PACING_RATE_PROF_QUERY = 0x1c3 +HWRM_FUNC_TTX_PACING_RATE_QUERY = 0x1c4 +HWRM_SELFTEST_QLIST = 0x200 +HWRM_SELFTEST_EXEC = 0x201 +HWRM_SELFTEST_IRQ = 0x202 +HWRM_SELFTEST_RETRIEVE_SERDES_DATA = 0x203 +HWRM_PCIE_QSTATS = 0x204 +HWRM_MFG_FRU_WRITE_CONTROL = 0x205 +HWRM_MFG_TIMERS_QUERY = 0x206 +HWRM_MFG_OTP_CFG = 0x207 +HWRM_MFG_OTP_QCFG = 0x208 +HWRM_MFG_HDMA_TEST = 0x209 +HWRM_MFG_FRU_EEPROM_WRITE = 0x20a +HWRM_MFG_FRU_EEPROM_READ = 0x20b +HWRM_MFG_SOC_IMAGE = 0x20c +HWRM_MFG_SOC_QSTATUS = 0x20d +HWRM_MFG_PARAM_CRITICAL_DATA_FINALIZE = 0x20e +HWRM_MFG_PARAM_CRITICAL_DATA_READ = 0x20f +HWRM_MFG_PARAM_CRITICAL_DATA_HEALTH = 0x210 +HWRM_MFG_PRVSN_EXPORT_CSR = 0x211 +HWRM_MFG_PRVSN_IMPORT_CERT = 0x212 +HWRM_MFG_PRVSN_GET_STATE = 0x213 +HWRM_MFG_GET_NVM_MEASUREMENT = 0x214 +HWRM_MFG_PSOC_QSTATUS = 0x215 +HWRM_MFG_SELFTEST_QLIST = 0x216 +HWRM_MFG_SELFTEST_EXEC = 0x217 +HWRM_STAT_GENERIC_QSTATS = 0x218 +HWRM_MFG_PRVSN_EXPORT_CERT = 0x219 +HWRM_STAT_DB_ERROR_QSTATS = 0x21a +HWRM_MFG_TESTS = 0x21b +HWRM_MFG_WRITE_CERT_NVM = 0x21c +HWRM_PORT_POE_CFG = 0x230 +HWRM_PORT_POE_QCFG = 0x231 +HWRM_PORT_PHY_FDRSTAT = 0x232 +HWRM_UDCC_QCAPS = 0x258 +HWRM_UDCC_CFG = 0x259 +HWRM_UDCC_QCFG = 0x25a +HWRM_UDCC_SESSION_CFG = 0x25b +HWRM_UDCC_SESSION_QCFG = 0x25c +HWRM_UDCC_SESSION_QUERY = 0x25d +HWRM_UDCC_COMP_CFG = 0x25e +HWRM_UDCC_COMP_QCFG = 0x25f +HWRM_UDCC_COMP_QUERY = 0x260 +HWRM_QUEUE_PFCWD_TIMEOUT_QCAPS = 0x261 +HWRM_QUEUE_PFCWD_TIMEOUT_CFG = 0x262 +HWRM_QUEUE_PFCWD_TIMEOUT_QCFG = 0x263 +HWRM_QUEUE_ADPTV_QOS_RX_QCFG = 0x264 +HWRM_QUEUE_ADPTV_QOS_TX_QCFG = 0x265 +HWRM_TF = 0x2bc +HWRM_TF_VERSION_GET = 0x2bd +HWRM_TF_SESSION_OPEN = 0x2c6 +HWRM_TF_SESSION_REGISTER = 0x2c8 +HWRM_TF_SESSION_UNREGISTER = 0x2c9 +HWRM_TF_SESSION_CLOSE = 0x2ca +HWRM_TF_SESSION_QCFG = 0x2cb +HWRM_TF_SESSION_RESC_QCAPS = 0x2cc +HWRM_TF_SESSION_RESC_ALLOC = 0x2cd +HWRM_TF_SESSION_RESC_FREE = 0x2ce +HWRM_TF_SESSION_RESC_FLUSH = 0x2cf +HWRM_TF_SESSION_RESC_INFO = 0x2d0 +HWRM_TF_SESSION_HOTUP_STATE_SET = 0x2d1 +HWRM_TF_SESSION_HOTUP_STATE_GET = 0x2d2 +HWRM_TF_TBL_TYPE_GET = 0x2da +HWRM_TF_TBL_TYPE_SET = 0x2db +HWRM_TF_TBL_TYPE_BULK_GET = 0x2dc +HWRM_TF_EM_INSERT = 0x2ea +HWRM_TF_EM_DELETE = 0x2eb +HWRM_TF_EM_HASH_INSERT = 0x2ec +HWRM_TF_EM_MOVE = 0x2ed +HWRM_TF_TCAM_SET = 0x2f8 +HWRM_TF_TCAM_GET = 0x2f9 +HWRM_TF_TCAM_MOVE = 0x2fa +HWRM_TF_TCAM_FREE = 0x2fb +HWRM_TF_GLOBAL_CFG_SET = 0x2fc +HWRM_TF_GLOBAL_CFG_GET = 0x2fd +HWRM_TF_IF_TBL_SET = 0x2fe +HWRM_TF_IF_TBL_GET = 0x2ff +HWRM_TF_RESC_USAGE_SET = 0x300 +HWRM_TF_RESC_USAGE_QUERY = 0x301 +HWRM_TF_TBL_TYPE_ALLOC = 0x302 +HWRM_TF_TBL_TYPE_FREE = 0x303 +HWRM_TFC_TBL_SCOPE_QCAPS = 0x380 +HWRM_TFC_TBL_SCOPE_ID_ALLOC = 0x381 +HWRM_TFC_TBL_SCOPE_CONFIG = 0x382 +HWRM_TFC_TBL_SCOPE_DECONFIG = 0x383 +HWRM_TFC_TBL_SCOPE_FID_ADD = 0x384 +HWRM_TFC_TBL_SCOPE_FID_REM = 0x385 +HWRM_TFC_TBL_SCOPE_POOL_ALLOC = 0x386 +HWRM_TFC_TBL_SCOPE_POOL_FREE = 0x387 +HWRM_TFC_SESSION_ID_ALLOC = 0x388 +HWRM_TFC_SESSION_FID_ADD = 0x389 +HWRM_TFC_SESSION_FID_REM = 0x38a +HWRM_TFC_IDENT_ALLOC = 0x38b +HWRM_TFC_IDENT_FREE = 0x38c +HWRM_TFC_IDX_TBL_ALLOC = 0x38d +HWRM_TFC_IDX_TBL_ALLOC_SET = 0x38e +HWRM_TFC_IDX_TBL_SET = 0x38f +HWRM_TFC_IDX_TBL_GET = 0x390 +HWRM_TFC_IDX_TBL_FREE = 0x391 +HWRM_TFC_GLOBAL_ID_ALLOC = 0x392 +HWRM_TFC_TCAM_SET = 0x393 +HWRM_TFC_TCAM_GET = 0x394 +HWRM_TFC_TCAM_ALLOC = 0x395 +HWRM_TFC_TCAM_ALLOC_SET = 0x396 +HWRM_TFC_TCAM_FREE = 0x397 +HWRM_TFC_IF_TBL_SET = 0x398 +HWRM_TFC_IF_TBL_GET = 0x399 +HWRM_TFC_TBL_SCOPE_CONFIG_GET = 0x39a +HWRM_TFC_RESC_USAGE_QUERY = 0x39b +HWRM_TFC_GLOBAL_ID_FREE = 0x39c +HWRM_TFC_TCAM_PRI_UPDATE = 0x39d +HWRM_TFC_HOT_UPGRADE_PROCESS = 0x3a0 +HWRM_SV = 0x400 +HWRM_DBG_SERDES_TEST = 0xff0e +HWRM_DBG_LOG_BUFFER_FLUSH = 0xff0f +HWRM_DBG_READ_DIRECT = 0xff10 +HWRM_DBG_READ_INDIRECT = 0xff11 +HWRM_DBG_WRITE_DIRECT = 0xff12 +HWRM_DBG_WRITE_INDIRECT = 0xff13 +HWRM_DBG_DUMP = 0xff14 +HWRM_DBG_ERASE_NVM = 0xff15 +HWRM_DBG_CFG = 0xff16 +HWRM_DBG_COREDUMP_LIST = 0xff17 +HWRM_DBG_COREDUMP_INITIATE = 0xff18 +HWRM_DBG_COREDUMP_RETRIEVE = 0xff19 +HWRM_DBG_FW_CLI = 0xff1a +HWRM_DBG_I2C_CMD = 0xff1b +HWRM_DBG_RING_INFO_GET = 0xff1c +HWRM_DBG_CRASHDUMP_HEADER = 0xff1d +HWRM_DBG_CRASHDUMP_ERASE = 0xff1e +HWRM_DBG_DRV_TRACE = 0xff1f +HWRM_DBG_QCAPS = 0xff20 +HWRM_DBG_QCFG = 0xff21 +HWRM_DBG_CRASHDUMP_MEDIUM_CFG = 0xff22 +HWRM_DBG_USEQ_ALLOC = 0xff23 +HWRM_DBG_USEQ_FREE = 0xff24 +HWRM_DBG_USEQ_FLUSH = 0xff25 +HWRM_DBG_USEQ_QCAPS = 0xff26 +HWRM_DBG_USEQ_CW_CFG = 0xff27 +HWRM_DBG_USEQ_SCHED_CFG = 0xff28 +HWRM_DBG_USEQ_RUN = 0xff29 +HWRM_DBG_USEQ_DELIVERY_REQ = 0xff2a +HWRM_DBG_USEQ_RESP_HDR = 0xff2b +HWRM_DBG_COREDUMP_CAPTURE = 0xff2c +HWRM_DBG_PTRACE = 0xff2d +HWRM_DBG_SIM_CABLE_STATE = 0xff2e +HWRM_DBG_TOKEN_QUERY_AUTH_IDS = 0xff2f +HWRM_DBG_TOKEN_CFG = 0xff30 +HWRM_NVM_GET_VPD_FIELD_INFO = 0xffea +HWRM_NVM_SET_VPD_FIELD_INFO = 0xffeb +HWRM_NVM_DEFRAG = 0xffec +HWRM_NVM_REQ_ARBITRATION = 0xffed +HWRM_NVM_FACTORY_DEFAULTS = 0xffee +HWRM_NVM_VALIDATE_OPTION = 0xffef +HWRM_NVM_FLUSH = 0xfff0 +HWRM_NVM_GET_VARIABLE = 0xfff1 +HWRM_NVM_SET_VARIABLE = 0xfff2 +HWRM_NVM_INSTALL_UPDATE = 0xfff3 +HWRM_NVM_MODIFY = 0xfff4 +HWRM_NVM_VERIFY_UPDATE = 0xfff5 +HWRM_NVM_GET_DEV_INFO = 0xfff6 +HWRM_NVM_ERASE_DIR_ENTRY = 0xfff7 +HWRM_NVM_MOD_DIR_ENTRY = 0xfff8 +HWRM_NVM_FIND_DIR_ENTRY = 0xfff9 +HWRM_NVM_GET_DIR_ENTRIES = 0xfffa +HWRM_NVM_GET_DIR_INFO = 0xfffb +HWRM_NVM_RAW_DUMP = 0xfffc +HWRM_NVM_READ = 0xfffd +HWRM_NVM_WRITE = 0xfffe +HWRM_NVM_RAW_WRITE_BLK = 0xffff +HWRM_LAST = HWRM_NVM_RAW_WRITE_BLK +HWRM_ERR_CODE_SUCCESS = 0x0 +HWRM_ERR_CODE_FAIL = 0x1 +HWRM_ERR_CODE_INVALID_PARAMS = 0x2 +HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED = 0x3 +HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR = 0x4 +HWRM_ERR_CODE_INVALID_FLAGS = 0x5 +HWRM_ERR_CODE_INVALID_ENABLES = 0x6 +HWRM_ERR_CODE_UNSUPPORTED_TLV = 0x7 +HWRM_ERR_CODE_NO_BUFFER = 0x8 +HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR = 0x9 +HWRM_ERR_CODE_HOT_RESET_PROGRESS = 0xa +HWRM_ERR_CODE_HOT_RESET_FAIL = 0xb +HWRM_ERR_CODE_NO_FLOW_COUNTER_DURING_ALLOC = 0xc +HWRM_ERR_CODE_KEY_HASH_COLLISION = 0xd +HWRM_ERR_CODE_KEY_ALREADY_EXISTS = 0xe +HWRM_ERR_CODE_HWRM_ERROR = 0xf +HWRM_ERR_CODE_BUSY = 0x10 +HWRM_ERR_CODE_RESOURCE_LOCKED = 0x11 +HWRM_ERR_CODE_PF_UNAVAILABLE = 0x12 +HWRM_ERR_CODE_ENTITY_NOT_PRESENT = 0x13 +HWRM_ERR_CODE_SECURE_SOC_ERROR = 0x14 +HWRM_ERR_CODE_TLV_ENCAPSULATED_RESPONSE = 0x8000 +HWRM_ERR_CODE_UNKNOWN_ERR = 0xfffe +HWRM_ERR_CODE_CMD_NOT_SUPPORTED = 0xffff +HWRM_ERR_CODE_LAST = HWRM_ERR_CODE_CMD_NOT_SUPPORTED +HWRM_MAX_REQ_LEN = 128 +HWRM_MAX_RESP_LEN = 704 +HWRM_RESP_VALID_KEY = 1 +HWRM_TARGET_ID_BONO = 0xFFF8 +HWRM_TARGET_ID_KONG = 0xFFF9 +HWRM_TARGET_ID_APE = 0xFFFA +HWRM_TARGET_ID_TOOLS = 0xFFFD +HWRM_VERSION_MAJOR = 1 +HWRM_VERSION_MINOR = 10 +HWRM_VERSION_UPDATE = 3 +HWRM_VERSION_RSVD = 133 +HWRM_VERSION_STR = "1.10.3.133" +FUNC_RESET_REQ_ENABLES_VF_ID_VALID = 0x1 +FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETALL = 0x0 +FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETME = 0x1 +FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETCHILDREN = 0x2 +FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF = 0x3 +FUNC_RESET_REQ_FUNC_RESET_LEVEL_LAST = FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF +FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED = 0x1 +FUNC_QCAPS_RESP_FLAGS_GLOBAL_MSIX_AUTOMASKING = 0x2 +FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED = 0x4 +FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED = 0x8 +FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED = 0x10 +FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED = 0x20 +FUNC_QCAPS_RESP_FLAGS_WOL_BMP_SUPPORTED = 0x40 +FUNC_QCAPS_RESP_FLAGS_TX_RING_RL_SUPPORTED = 0x80 +FUNC_QCAPS_RESP_FLAGS_TX_BW_CFG_SUPPORTED = 0x100 +FUNC_QCAPS_RESP_FLAGS_VF_TX_RING_RL_SUPPORTED = 0x200 +FUNC_QCAPS_RESP_FLAGS_VF_BW_CFG_SUPPORTED = 0x400 +FUNC_QCAPS_RESP_FLAGS_STD_TX_RING_MODE_SUPPORTED = 0x800 +FUNC_QCAPS_RESP_FLAGS_GENEVE_TUN_FLAGS_SUPPORTED = 0x1000 +FUNC_QCAPS_RESP_FLAGS_NVGRE_TUN_FLAGS_SUPPORTED = 0x2000 +FUNC_QCAPS_RESP_FLAGS_GRE_TUN_FLAGS_SUPPORTED = 0x4000 +FUNC_QCAPS_RESP_FLAGS_MPLS_TUN_FLAGS_SUPPORTED = 0x8000 +FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED = 0x10000 +FUNC_QCAPS_RESP_FLAGS_ADOPTED_PF_SUPPORTED = 0x20000 +FUNC_QCAPS_RESP_FLAGS_ADMIN_PF_SUPPORTED = 0x40000 +FUNC_QCAPS_RESP_FLAGS_LINK_ADMIN_STATUS_SUPPORTED = 0x80000 +FUNC_QCAPS_RESP_FLAGS_WCB_PUSH_MODE = 0x100000 +FUNC_QCAPS_RESP_FLAGS_DYNAMIC_TX_RING_ALLOC = 0x200000 +FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE = 0x400000 +FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE = 0x800000 +FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED = 0x1000000 +FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD = 0x2000000 +FUNC_QCAPS_RESP_FLAGS_NOTIFY_VF_DEF_VNIC_CHNG_SUPPORTED = 0x4000000 +FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED = 0x8000000 +FUNC_QCAPS_RESP_FLAGS_COREDUMP_CMD_SUPPORTED = 0x10000000 +FUNC_QCAPS_RESP_FLAGS_CRASHDUMP_CMD_SUPPORTED = 0x20000000 +FUNC_QCAPS_RESP_FLAGS_PFC_WD_STATS_SUPPORTED = 0x40000000 +FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED = 0x80000000 +FUNC_QCAPS_RESP_FLAGS_EXT_ECN_MARK_SUPPORTED = 0x1 +FUNC_QCAPS_RESP_FLAGS_EXT_ECN_STATS_SUPPORTED = 0x2 +FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED = 0x4 +FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT = 0x8 +FUNC_QCAPS_RESP_FLAGS_EXT_PROXY_MODE_SUPPORT = 0x10 +FUNC_QCAPS_RESP_FLAGS_EXT_TX_PROXY_SRC_INTF_OVERRIDE_SUPPORT = 0x20 +FUNC_QCAPS_RESP_FLAGS_EXT_SCHQ_SUPPORTED = 0x40 +FUNC_QCAPS_RESP_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED = 0x80 +FUNC_QCAPS_RESP_FLAGS_EXT_EVB_MODE_CFG_NOT_SUPPORTED = 0x100 +FUNC_QCAPS_RESP_FLAGS_EXT_SOC_SPD_SUPPORTED = 0x200 +FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED = 0x400 +FUNC_QCAPS_RESP_FLAGS_EXT_FAST_RESET_CAPABLE = 0x800 +FUNC_QCAPS_RESP_FLAGS_EXT_TX_METADATA_CFG_CAPABLE = 0x1000 +FUNC_QCAPS_RESP_FLAGS_EXT_NVM_OPTION_ACTION_SUPPORTED = 0x2000 +FUNC_QCAPS_RESP_FLAGS_EXT_BD_METADATA_SUPPORTED = 0x4000 +FUNC_QCAPS_RESP_FLAGS_EXT_ECHO_REQUEST_SUPPORTED = 0x8000 +FUNC_QCAPS_RESP_FLAGS_EXT_NPAR_1_2_SUPPORTED = 0x10000 +FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PTM_SUPPORTED = 0x20000 +FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED = 0x40000 +FUNC_QCAPS_RESP_FLAGS_EXT_VF_CFG_ASYNC_FOR_PF_SUPPORTED = 0x80000 +FUNC_QCAPS_RESP_FLAGS_EXT_PARTITION_BW_SUPPORTED = 0x100000 +FUNC_QCAPS_RESP_FLAGS_EXT_DFLT_VLAN_TPID_PCP_SUPPORTED = 0x200000 +FUNC_QCAPS_RESP_FLAGS_EXT_KTLS_SUPPORTED = 0x400000 +FUNC_QCAPS_RESP_FLAGS_EXT_EP_RATE_CONTROL = 0x800000 +FUNC_QCAPS_RESP_FLAGS_EXT_MIN_BW_SUPPORTED = 0x1000000 +FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP = 0x2000000 +FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED = 0x4000000 +FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_REQUIRED = 0x8000000 +FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED = 0x10000000 +FUNC_QCAPS_RESP_FLAGS_EXT_DBR_PACING_SUPPORTED = 0x20000000 +FUNC_QCAPS_RESP_FLAGS_EXT_HW_DBR_DROP_RECOV_SUPPORTED = 0x40000000 +FUNC_QCAPS_RESP_FLAGS_EXT_DISABLE_CQ_OVERFLOW_DETECTION_SUPPORTED = 0x80000000 +FUNC_QCAPS_RESP_MPC_CHNLS_CAP_TCE = 0x1 +FUNC_QCAPS_RESP_MPC_CHNLS_CAP_RCE = 0x2 +FUNC_QCAPS_RESP_MPC_CHNLS_CAP_TE_CFA = 0x4 +FUNC_QCAPS_RESP_MPC_CHNLS_CAP_RE_CFA = 0x8 +FUNC_QCAPS_RESP_MPC_CHNLS_CAP_PRIMATE = 0x10 +FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED = 0x1 +FUNC_QCAPS_RESP_FLAGS_EXT2_QUIC_SUPPORTED = 0x2 +FUNC_QCAPS_RESP_FLAGS_EXT2_KDNET_SUPPORTED = 0x4 +FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED = 0x8 +FUNC_QCAPS_RESP_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED = 0x10 +FUNC_QCAPS_RESP_FLAGS_EXT2_GENERIC_STATS_SUPPORTED = 0x20 +FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED = 0x40 +FUNC_QCAPS_RESP_FLAGS_EXT2_SYNCE_SUPPORTED = 0x80 +FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED = 0x100 +FUNC_QCAPS_RESP_FLAGS_EXT2_TX_PKT_TS_CMPL_SUPPORTED = 0x200 +FUNC_QCAPS_RESP_FLAGS_EXT2_HW_LAG_SUPPORTED = 0x400 +FUNC_QCAPS_RESP_FLAGS_EXT2_ON_CHIP_CTX_SUPPORTED = 0x800 +FUNC_QCAPS_RESP_FLAGS_EXT2_STEERING_TAG_SUPPORTED = 0x1000 +FUNC_QCAPS_RESP_FLAGS_EXT2_ENHANCED_VF_SCALE_SUPPORTED = 0x2000 +FUNC_QCAPS_RESP_FLAGS_EXT2_KEY_XID_PARTITION_SUPPORTED = 0x4000 +FUNC_QCAPS_RESP_FLAGS_EXT2_CONCURRENT_KTLS_QUIC_SUPPORTED = 0x8000 +FUNC_QCAPS_RESP_FLAGS_EXT2_SCHQ_CROSS_TC_CAP_SUPPORTED = 0x10000 +FUNC_QCAPS_RESP_FLAGS_EXT2_SCHQ_PER_TC_CAP_SUPPORTED = 0x20000 +FUNC_QCAPS_RESP_FLAGS_EXT2_SCHQ_PER_TC_RESERVATION_SUPPORTED = 0x40000 +FUNC_QCAPS_RESP_FLAGS_EXT2_DB_ERROR_STATS_SUPPORTED = 0x80000 +FUNC_QCAPS_RESP_FLAGS_EXT2_ROCE_VF_RESOURCE_MGMT_SUPPORTED = 0x100000 +FUNC_QCAPS_RESP_FLAGS_EXT2_UDCC_SUPPORTED = 0x200000 +FUNC_QCAPS_RESP_FLAGS_EXT2_TIMED_TX_SO_TXTIME_SUPPORTED = 0x400000 +FUNC_QCAPS_RESP_FLAGS_EXT2_SW_MAX_RESOURCE_LIMITS_SUPPORTED = 0x800000 +FUNC_QCAPS_RESP_FLAGS_EXT2_TF_INGRESS_NIC_FLOW_SUPPORTED = 0x1000000 +FUNC_QCAPS_RESP_FLAGS_EXT2_LPBK_STATS_SUPPORTED = 0x2000000 +FUNC_QCAPS_RESP_FLAGS_EXT2_TF_EGRESS_NIC_FLOW_SUPPORTED = 0x4000000 +FUNC_QCAPS_RESP_FLAGS_EXT2_MULTI_LOSSLESS_QUEUES_SUPPORTED = 0x8000000 +FUNC_QCAPS_RESP_FLAGS_EXT2_PEER_MMAP_SUPPORTED = 0x10000000 +FUNC_QCAPS_RESP_FLAGS_EXT2_TIMED_TX_PACING_SUPPORTED = 0x20000000 +FUNC_QCAPS_RESP_FLAGS_EXT2_VF_STAT_EJECTION_SUPPORTED = 0x40000000 +FUNC_QCAPS_RESP_FLAGS_EXT2_HOST_COREDUMP_SUPPORTED = 0x80000000 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_VXLAN = 0x1 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_NGE = 0x2 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_NVGRE = 0x4 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_L2GRE = 0x8 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_GRE = 0x10 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_IPINIP = 0x20 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_MPLS = 0x40 +FUNC_QCAPS_RESP_TUNNEL_DISABLE_FLAG_DISABLE_PPPOE = 0x80 +FUNC_QCAPS_RESP_XID_PARTITION_CAP_TX_CK = 0x1 +FUNC_QCAPS_RESP_XID_PARTITION_CAP_RX_CK = 0x2 +FUNC_QCAPS_RESP_FLAGS_EXT3_RM_RSV_WHILE_ALLOC_CAP = 0x1 +FUNC_QCAPS_RESP_FLAGS_EXT3_REQUIRE_L2_FILTER = 0x2 +FUNC_QCAPS_RESP_FLAGS_EXT3_MAX_ROCE_VFS_SUPPORTED = 0x4 +FUNC_QCAPS_RESP_FLAGS_EXT3_RX_RATE_PROFILE_SEL_SUPPORTED = 0x8 +FUNC_QCAPS_RESP_FLAGS_EXT3_BIDI_OPT_SUPPORTED = 0x10 +FUNC_QCAPS_RESP_FLAGS_EXT3_MIRROR_ON_ROCE_SUPPORTED = 0x20 +FUNC_QCAPS_RESP_FLAGS_EXT3_ROCE_VF_DYN_ALLOC_SUPPORT = 0x40 +FUNC_QCAPS_RESP_FLAGS_EXT3_CHANGE_UDP_SRCPORT_SUPPORT = 0x80 +FUNC_QCAPS_RESP_FLAGS_EXT3_PCIE_COMPLIANCE_SUPPORTED = 0x100 +FUNC_QCAPS_RESP_FLAGS_EXT3_MULTI_L2_DB_SUPPORTED = 0x200 +FUNC_QCAPS_RESP_FLAGS_EXT3_PCIE_SECURE_ATS_SUPPORTED = 0x400 +FUNC_QCAPS_RESP_FLAGS_EXT3_MBUF_STATS_SUPPORTED = 0x800 +FUNC_QCFG_RESP_FLAGS_OOB_WOL_MAGICPKT_ENABLED = 0x1 +FUNC_QCFG_RESP_FLAGS_OOB_WOL_BMP_ENABLED = 0x2 +FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED = 0x4 +FUNC_QCFG_RESP_FLAGS_STD_TX_RING_MODE_ENABLED = 0x8 +FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED = 0x10 +FUNC_QCFG_RESP_FLAGS_MULTI_HOST = 0x20 +FUNC_QCFG_RESP_FLAGS_TRUSTED_VF = 0x40 +FUNC_QCFG_RESP_FLAGS_SECURE_MODE_ENABLED = 0x80 +FUNC_QCFG_RESP_FLAGS_PREBOOT_LEGACY_L2_RINGS = 0x100 +FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED = 0x200 +FUNC_QCFG_RESP_FLAGS_PPP_PUSH_MODE_ENABLED = 0x400 +FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED = 0x800 +FUNC_QCFG_RESP_FLAGS_FAST_RESET_ALLOWED = 0x1000 +FUNC_QCFG_RESP_FLAGS_MULTI_ROOT = 0x2000 +FUNC_QCFG_RESP_FLAGS_ENABLE_RDMA_SRIOV = 0x4000 +FUNC_QCFG_RESP_FLAGS_ROCE_VNIC_ID_VALID = 0x8000 +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_SPF = 0x0 +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_MPFS = 0x1 +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0 = 0x2 +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5 = 0x3 +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0 = 0x4 +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_2 = 0x5 +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN = 0xff +FUNC_QCFG_RESP_PORT_PARTITION_TYPE_LAST = FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN +FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL = 0x0 +FUNC_QCFG_RESP_PORT_PF_CNT_LAST = FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_MASK = 0xfffffff +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_SFT = 0 +FUNC_QCFG_RESP_MIN_BW_SCALE = 0x10000000 +FUNC_QCFG_RESP_MIN_BW_SCALE_BITS = (0x0 << 28) +FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES = (0x1 << 28) +FUNC_QCFG_RESP_MIN_BW_SCALE_LAST = FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MASK = 0xe0000000 +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_SFT = 29 +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MEGA = (0x0 << 29) +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_KILO = (0x2 << 29) +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_BASE = (0x4 << 29) +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_GIGA = (0x6 << 29) +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 = (0x1 << 29) +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID = (0x7 << 29) +FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_LAST = FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_MASK = 0xfffffff +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_SFT = 0 +FUNC_QCFG_RESP_MAX_BW_SCALE = 0x10000000 +FUNC_QCFG_RESP_MAX_BW_SCALE_BITS = (0x0 << 28) +FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES = (0x1 << 28) +FUNC_QCFG_RESP_MAX_BW_SCALE_LAST = FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MASK = 0xe0000000 +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_SFT = 29 +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MEGA = (0x0 << 29) +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_KILO = (0x2 << 29) +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_BASE = (0x4 << 29) +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_GIGA = (0x6 << 29) +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 = (0x1 << 29) +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID = (0x7 << 29) +FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_LAST = FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID +FUNC_QCFG_RESP_EVB_MODE_NO_EVB = 0x0 +FUNC_QCFG_RESP_EVB_MODE_VEB = 0x1 +FUNC_QCFG_RESP_EVB_MODE_VEPA = 0x2 +FUNC_QCFG_RESP_EVB_MODE_LAST = FUNC_QCFG_RESP_EVB_MODE_VEPA +FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_MASK = 0x3 +FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SFT = 0 +FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SIZE_64 = 0x0 +FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SIZE_128 = 0x1 +FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_LAST = FUNC_QCFG_RESP_OPTIONS_CACHE_LINESIZE_SIZE_128 +FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_MASK = 0xc +FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_SFT = 2 +FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_FORCED_DOWN = (0x0 << 2) +FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_FORCED_UP = (0x1 << 2) +FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_AUTO = (0x2 << 2) +FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_LAST = FUNC_QCFG_RESP_OPTIONS_LINK_ADMIN_STATE_AUTO +FUNC_QCFG_RESP_OPTIONS_RSVD_MASK = 0xf0 +FUNC_QCFG_RESP_OPTIONS_RSVD_SFT = 4 +FUNC_QCFG_RESP_SVIF_INFO_SVIF_MASK = 0x7fff +FUNC_QCFG_RESP_SVIF_INFO_SVIF_SFT = 0 +FUNC_QCFG_RESP_SVIF_INFO_SVIF_VALID = 0x8000 +FUNC_QCFG_RESP_MPC_CHNLS_TCE_ENABLED = 0x1 +FUNC_QCFG_RESP_MPC_CHNLS_RCE_ENABLED = 0x2 +FUNC_QCFG_RESP_MPC_CHNLS_TE_CFA_ENABLED = 0x4 +FUNC_QCFG_RESP_MPC_CHNLS_RE_CFA_ENABLED = 0x8 +FUNC_QCFG_RESP_MPC_CHNLS_PRIMATE_ENABLED = 0x10 +FUNC_QCFG_RESP_DB_PAGE_SIZE_4KB = 0x0 +FUNC_QCFG_RESP_DB_PAGE_SIZE_8KB = 0x1 +FUNC_QCFG_RESP_DB_PAGE_SIZE_16KB = 0x2 +FUNC_QCFG_RESP_DB_PAGE_SIZE_32KB = 0x3 +FUNC_QCFG_RESP_DB_PAGE_SIZE_64KB = 0x4 +FUNC_QCFG_RESP_DB_PAGE_SIZE_128KB = 0x5 +FUNC_QCFG_RESP_DB_PAGE_SIZE_256KB = 0x6 +FUNC_QCFG_RESP_DB_PAGE_SIZE_512KB = 0x7 +FUNC_QCFG_RESP_DB_PAGE_SIZE_1MB = 0x8 +FUNC_QCFG_RESP_DB_PAGE_SIZE_2MB = 0x9 +FUNC_QCFG_RESP_DB_PAGE_SIZE_4MB = 0xa +FUNC_QCFG_RESP_DB_PAGE_SIZE_LAST = FUNC_QCFG_RESP_DB_PAGE_SIZE_4MB +FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_MASK = 0xfffffff +FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_SFT = 0 +FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE = 0x10000000 +FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BITS = (0x0 << 28) +FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BYTES = (0x1 << 28) +FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_LAST = FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BYTES +FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_MASK = 0xe0000000 +FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_SFT = 29 +FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 = (0x1 << 29) +FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_LAST = FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 +FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_MASK = 0xfffffff +FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_SFT = 0 +FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE = 0x10000000 +FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BITS = (0x0 << 28) +FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BYTES = (0x1 << 28) +FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_LAST = FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BYTES +FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_MASK = 0xe0000000 +FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_SFT = 29 +FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 = (0x1 << 29) +FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_LAST = FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 +FUNC_QCFG_RESP_FLAGS2_SRIOV_DSCP_INSERT_ENABLED = 0x1 +FUNC_QCFG_RESP_PORT_KDNET_MODE_DISABLED = 0x0 +FUNC_QCFG_RESP_PORT_KDNET_MODE_ENABLED = 0x1 +FUNC_QCFG_RESP_PORT_KDNET_MODE_LAST = FUNC_QCFG_RESP_PORT_KDNET_MODE_ENABLED +FUNC_QCFG_RESP_ROCE_BIDI_OPT_MODE_DISABLED = 0x1 +FUNC_QCFG_RESP_ROCE_BIDI_OPT_MODE_DEDICATED = 0x2 +FUNC_QCFG_RESP_ROCE_BIDI_OPT_MODE_SHARED = 0x4 +FUNC_QCFG_RESP_XID_PARTITION_CFG_TX_CK = 0x1 +FUNC_QCFG_RESP_XID_PARTITION_CFG_RX_CK = 0x2 +FUNC_QCFG_RESP_MAX_LINK_WIDTH_UNKNOWN = 0x0 +FUNC_QCFG_RESP_MAX_LINK_WIDTH_X1 = 0x1 +FUNC_QCFG_RESP_MAX_LINK_WIDTH_X2 = 0x2 +FUNC_QCFG_RESP_MAX_LINK_WIDTH_X4 = 0x4 +FUNC_QCFG_RESP_MAX_LINK_WIDTH_X8 = 0x8 +FUNC_QCFG_RESP_MAX_LINK_WIDTH_X16 = 0x10 +FUNC_QCFG_RESP_MAX_LINK_WIDTH_LAST = FUNC_QCFG_RESP_MAX_LINK_WIDTH_X16 +FUNC_QCFG_RESP_MAX_LINK_SPEED_UNKNOWN = 0x0 +FUNC_QCFG_RESP_MAX_LINK_SPEED_G1 = 0x1 +FUNC_QCFG_RESP_MAX_LINK_SPEED_G2 = 0x2 +FUNC_QCFG_RESP_MAX_LINK_SPEED_G3 = 0x3 +FUNC_QCFG_RESP_MAX_LINK_SPEED_G4 = 0x4 +FUNC_QCFG_RESP_MAX_LINK_SPEED_G5 = 0x5 +FUNC_QCFG_RESP_MAX_LINK_SPEED_LAST = FUNC_QCFG_RESP_MAX_LINK_SPEED_G5 +FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_UNKNOWN = 0x0 +FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X1 = 0x1 +FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X2 = 0x2 +FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X4 = 0x4 +FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X8 = 0x8 +FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X16 = 0x10 +FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_LAST = FUNC_QCFG_RESP_NEGOTIATED_LINK_WIDTH_X16 +FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_UNKNOWN = 0x0 +FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G1 = 0x1 +FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G2 = 0x2 +FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G3 = 0x3 +FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G4 = 0x4 +FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G5 = 0x5 +FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_LAST = FUNC_QCFG_RESP_NEGOTIATED_LINK_SPEED_G5 +FUNC_DRV_RGTR_REQ_FLAGS_FWD_ALL_MODE = 0x1 +FUNC_DRV_RGTR_REQ_FLAGS_FWD_NONE_MODE = 0x2 +FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE = 0x4 +FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE = 0x8 +FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT = 0x10 +FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT = 0x20 +FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT = 0x40 +FUNC_DRV_RGTR_REQ_FLAGS_FAST_RESET_SUPPORT = 0x80 +FUNC_DRV_RGTR_REQ_FLAGS_RSS_STRICT_HASH_TYPE_SUPPORT = 0x100 +FUNC_DRV_RGTR_REQ_FLAGS_NPAR_1_2_SUPPORT = 0x200 +FUNC_DRV_RGTR_REQ_FLAGS_ASYM_QUEUE_CFG_SUPPORT = 0x400 +FUNC_DRV_RGTR_REQ_FLAGS_TF_INGRESS_NIC_FLOW_MODE = 0x800 +FUNC_DRV_RGTR_REQ_FLAGS_TF_EGRESS_NIC_FLOW_MODE = 0x1000 +FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE = 0x1 +FUNC_DRV_RGTR_REQ_ENABLES_VER = 0x2 +FUNC_DRV_RGTR_REQ_ENABLES_TIMESTAMP = 0x4 +FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD = 0x8 +FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD = 0x10 +FUNC_DRV_RGTR_REQ_OS_TYPE_UNKNOWN = 0x0 +FUNC_DRV_RGTR_REQ_OS_TYPE_OTHER = 0x1 +FUNC_DRV_RGTR_REQ_OS_TYPE_MSDOS = 0xe +FUNC_DRV_RGTR_REQ_OS_TYPE_WINDOWS = 0x12 +FUNC_DRV_RGTR_REQ_OS_TYPE_SOLARIS = 0x1d +FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX = 0x24 +FUNC_DRV_RGTR_REQ_OS_TYPE_FREEBSD = 0x2a +FUNC_DRV_RGTR_REQ_OS_TYPE_ESXI = 0x68 +FUNC_DRV_RGTR_REQ_OS_TYPE_WIN864 = 0x73 +FUNC_DRV_RGTR_REQ_OS_TYPE_WIN2012R2 = 0x74 +FUNC_DRV_RGTR_REQ_OS_TYPE_UEFI = 0x8000 +FUNC_DRV_RGTR_REQ_OS_TYPE_LAST = FUNC_DRV_RGTR_REQ_OS_TYPE_UEFI +FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED = 0x1 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_QP = 0x1 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_SRQ = 0x2 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_CQ = 0x4 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_VNIC = 0x8 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_STAT = 0x10 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_MRAV = 0x20 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_TKC = 0x40 +FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_RKC = 0x80 +FUNC_BACKING_STORE_CFG_REQ_FLAGS_PREBOOT_MODE = 0x1 +FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT = 0x2 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP = 0x1 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ = 0x2 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ = 0x4 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC = 0x8 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT = 0x10 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP = 0x20 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING0 = 0x40 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING1 = 0x80 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING2 = 0x100 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING3 = 0x200 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING4 = 0x400 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING5 = 0x800 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING6 = 0x1000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING7 = 0x2000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV = 0x4000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM = 0x8000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING8 = 0x10000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING9 = 0x20000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING10 = 0x40000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_TKC = 0x80000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_RKC = 0x100000 +FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD = 0x200000 +FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_SRQ_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_CQ_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_CQ_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_VNIC_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_VNIC_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_STAT_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_STAT_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_SP_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_SP_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING0_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING1_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING2_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING3_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING4_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING5_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING6_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TQM_RING7_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_MRAV_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_MRAV_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TIM_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TIM_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_RING8_TQM_RING_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_RING9_TQM_RING_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_RING10_TQM_RING_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_MASK = 0xf +FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_SFT = 0 +FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LAST = FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_2 +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_LAST = FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_QP = 0x0 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRQ = 0x1 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CQ = 0x2 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_VNIC = 0x3 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_STAT = 0x4 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SP_TQM_RING = 0x5 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_FP_TQM_RING = 0x6 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_MRAV = 0xe +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TIM = 0xf +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TX_CK = 0x13 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RX_CK = 0x14 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_MP_TQM_RING = 0x15 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SQ_DB_SHADOW = 0x16 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RQ_DB_SHADOW = 0x17 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRQ_DB_SHADOW = 0x18 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CQ_DB_SHADOW = 0x19 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TBL_SCOPE = 0x1c +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_XID_PARTITION = 0x1d +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRT_TRACE = 0x1e +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_SRT2_TRACE = 0x1f +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CRT_TRACE = 0x20 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CRT2_TRACE = 0x21 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RIGP0_TRACE = 0x22 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_L2_HWRM_TRACE = 0x23 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_ROCE_HWRM_TRACE = 0x24 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_TTX_PACING_TQM_RING = 0x25 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CA0_TRACE = 0x26 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CA1_TRACE = 0x27 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_CA2_TRACE = 0x28 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_RIGP1_TRACE = 0x29 +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_AFM_KONG_HWRM_TRACE = 0x2a +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_ERR_QPC_TRACE = 0x2b +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_INVALID = 0xffff +FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_LAST = FUNC_BACKING_STORE_CFG_V2_REQ_TYPE_INVALID +FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_PREBOOT_MODE = 0x1 +FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE = 0x2 +FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_EXTEND = 0x4 +FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_MASK = 0xf +FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_SFT = 0 +FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_0 = 0x0 +FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_1 = 0x1 +FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_2 = 0x2 +FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LAST = FUNC_BACKING_STORE_CFG_V2_REQ_PBL_LEVEL_LVL_2 +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_SFT = 4 +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_LAST = FUNC_BACKING_STORE_CFG_V2_REQ_PAGE_SIZE_PG_1G +FUNC_BACKING_STORE_CFG_V2_REQ_ENABLES_NEXT_BS_OFFSET = 0x1 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_QP = 0x0 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRQ = 0x1 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CQ = 0x2 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_VNIC = 0x3 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_STAT = 0x4 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SP_TQM_RING = 0x5 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_FP_TQM_RING = 0x6 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_MRAV = 0xe +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TIM = 0xf +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TX_CK = 0x13 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RX_CK = 0x14 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_MP_TQM_RING = 0x15 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SQ_DB_SHADOW = 0x16 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RQ_DB_SHADOW = 0x17 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRQ_DB_SHADOW = 0x18 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CQ_DB_SHADOW = 0x19 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TBL_SCOPE = 0x1c +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_XID_PARTITION_TABLE = 0x1d +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRT_TRACE = 0x1e +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_SRT2_TRACE = 0x1f +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CRT_TRACE = 0x20 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CRT2_TRACE = 0x21 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RIGP0_TRACE = 0x22 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_L2_HWRM_TRACE = 0x23 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_ROCE_HWRM_TRACE = 0x24 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_TTX_PACING_TQM_RING = 0x25 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CA0_TRACE = 0x26 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CA1_TRACE = 0x27 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_CA2_TRACE = 0x28 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_RIGP1_TRACE = 0x29 +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_AFM_KONG_HWRM_TRACE = 0x2a +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_ERR_QPC_TRACE = 0x2b +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_INVALID = 0xffff +FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_LAST = FUNC_BACKING_STORE_QCFG_V2_REQ_TYPE_INVALID +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_QP = 0x0 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SRQ = 0x1 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CQ = 0x2 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_VNIC = 0x3 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_STAT = 0x4 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SP_TQM_RING = 0x5 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_FP_TQM_RING = 0x6 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_MRAV = 0xe +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TIM = 0xf +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TX_CK = 0x13 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_RX_CK = 0x14 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_MP_TQM_RING = 0x15 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TBL_SCOPE = 0x1c +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_XID_PARTITION = 0x1d +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SRT_TRACE = 0x1e +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_SRT2_TRACE = 0x1f +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CRT_TRACE = 0x20 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CRT2_TRACE = 0x21 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_RIGP0_TRACE = 0x22 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_L2_HWRM_TRACE = 0x23 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_ROCE_HWRM_TRACE = 0x24 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_TTX_PACING_TQM_RING = 0x25 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CA0_TRACE = 0x26 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CA1_TRACE = 0x27 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_CA2_TRACE = 0x28 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_RIGP1_TRACE = 0x29 +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_ERR_QPC_TRACE = 0x2a +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_INVALID = 0xffff +FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_LAST = FUNC_BACKING_STORE_QCFG_V2_RESP_TYPE_INVALID +FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_MASK = 0xf +FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_SFT = 0 +FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_0 = 0x0 +FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_1 = 0x1 +FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_2 = 0x2 +FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LAST = FUNC_BACKING_STORE_QCFG_V2_RESP_PBL_LEVEL_LVL_2 +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_MASK = 0xf0 +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_SFT = 4 +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_4K = (0x0 << 4) +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_8K = (0x1 << 4) +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_64K = (0x2 << 4) +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_2M = (0x3 << 4) +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_8M = (0x4 << 4) +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_1G = (0x5 << 4) +FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_LAST = FUNC_BACKING_STORE_QCFG_V2_RESP_PAGE_SIZE_PG_1G +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_QP = 0x0 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRQ = 0x1 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CQ = 0x2 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_VNIC = 0x3 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_STAT = 0x4 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SP_TQM_RING = 0x5 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_FP_TQM_RING = 0x6 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_MRAV = 0xe +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TIM = 0xf +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TX_CK = 0x13 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RX_CK = 0x14 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_MP_TQM_RING = 0x15 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SQ_DB_SHADOW = 0x16 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RQ_DB_SHADOW = 0x17 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRQ_DB_SHADOW = 0x18 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CQ_DB_SHADOW = 0x19 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TBL_SCOPE = 0x1c +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_XID_PARTITION = 0x1d +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRT_TRACE = 0x1e +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_SRT2_TRACE = 0x1f +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CRT_TRACE = 0x20 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CRT2_TRACE = 0x21 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RIGP0_TRACE = 0x22 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_L2_HWRM_TRACE = 0x23 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_ROCE_HWRM_TRACE = 0x24 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_TTX_PACING_TQM_RING = 0x25 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CA0_TRACE = 0x26 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CA1_TRACE = 0x27 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_CA2_TRACE = 0x28 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_RIGP1_TRACE = 0x29 +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_AFM_KONG_HWRM_TRACE = 0x2a +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_ERR_QPC_TRACE = 0x2b +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_INVALID = 0xffff +FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_LAST = FUNC_BACKING_STORE_QCAPS_V2_REQ_TYPE_INVALID +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_QP = 0x0 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRQ = 0x1 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CQ = 0x2 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_VNIC = 0x3 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_STAT = 0x4 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SP_TQM_RING = 0x5 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_FP_TQM_RING = 0x6 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_MRAV = 0xe +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TIM = 0xf +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TX_CK = 0x13 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RX_CK = 0x14 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_MP_TQM_RING = 0x15 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SQ_DB_SHADOW = 0x16 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RQ_DB_SHADOW = 0x17 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRQ_DB_SHADOW = 0x18 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CQ_DB_SHADOW = 0x19 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TBL_SCOPE = 0x1c +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_XID_PARTITION = 0x1d +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRT_TRACE = 0x1e +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_SRT2_TRACE = 0x1f +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CRT_TRACE = 0x20 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CRT2_TRACE = 0x21 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RIGP0_TRACE = 0x22 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_L2_HWRM_TRACE = 0x23 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_ROCE_HWRM_TRACE = 0x24 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_TTX_PACING_TQM_RING = 0x25 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CA0_TRACE = 0x26 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CA1_TRACE = 0x27 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_CA2_TRACE = 0x28 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_RIGP1_TRACE = 0x29 +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_AFM_KONG_HWRM_TRACE = 0x2a +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_ERR_QPC_TRACE = 0x2b +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_INVALID = 0xffff +FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_LAST = FUNC_BACKING_STORE_QCAPS_V2_RESP_TYPE_INVALID +FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT = 0x1 +FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID = 0x2 +FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_DRIVER_MANAGED_MEMORY = 0x4 +FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ROCE_QP_PSEUDO_STATIC_ALLOC = 0x8 +FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_FW_DBG_TRACE = 0x10 +FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_FW_BIN_DBG_TRACE = 0x20 +FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_NEXT_BS_OFFSET = 0x40 +FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_0_EXACT = 0x1 +FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_1_EXACT = 0x2 +FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_2_EXACT = 0x4 +FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_SPLIT_ENTRY_3_EXACT = 0x8 +FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_UNUSED_MASK = 0xf0 +FUNC_BACKING_STORE_QCAPS_V2_RESP_EXACT_CNT_BIT_MAP_UNUSED_SFT = 4 +PORT_PHY_CFG_REQ_FLAGS_RESET_PHY = 0x1 +PORT_PHY_CFG_REQ_FLAGS_DEPRECATED = 0x2 +PORT_PHY_CFG_REQ_FLAGS_FORCE = 0x4 +PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG = 0x8 +PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE = 0x10 +PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE = 0x20 +PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE = 0x40 +PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE = 0x80 +PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_ENABLE = 0x100 +PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_DISABLE = 0x200 +PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_ENABLE = 0x400 +PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_DISABLE = 0x800 +PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_ENABLE = 0x1000 +PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_DISABLE = 0x2000 +PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN = 0x4000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_ENABLE = 0x8000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_DISABLE = 0x10000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_ENABLE = 0x20000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_DISABLE = 0x40000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_ENABLE = 0x80000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_DISABLE = 0x100000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_ENABLE = 0x200000 +PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_DISABLE = 0x400000 +PORT_PHY_CFG_REQ_FLAGS_LINK_TRAINING_ENABLE = 0x800000 +PORT_PHY_CFG_REQ_FLAGS_LINK_TRAINING_DISABLE = 0x1000000 +PORT_PHY_CFG_REQ_FLAGS_PRECODING_ENABLE = 0x2000000 +PORT_PHY_CFG_REQ_FLAGS_PRECODING_DISABLE = 0x4000000 +PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE = 0x1 +PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX = 0x2 +PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE = 0x4 +PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED = 0x8 +PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK = 0x10 +PORT_PHY_CFG_REQ_ENABLES_WIRESPEED = 0x20 +PORT_PHY_CFG_REQ_ENABLES_LPBK = 0x40 +PORT_PHY_CFG_REQ_ENABLES_PREEMPHASIS = 0x80 +PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE = 0x100 +PORT_PHY_CFG_REQ_ENABLES_EEE_LINK_SPEED_MASK = 0x200 +PORT_PHY_CFG_REQ_ENABLES_TX_LPI_TIMER = 0x400 +PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED = 0x800 +PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK = 0x1000 +PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2 = 0x2000 +PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK = 0x4000 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB = 0x1 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB = 0xa +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2GB = 0x14 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2_5GB = 0x19 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB = 0x64 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_20GB = 0xc8 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB = 0xfa +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB = 0x190 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB = 0x1f4 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB = 0x3e8 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB = 0xffff +PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_LAST = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB +PORT_PHY_CFG_REQ_AUTO_MODE_NONE = 0x0 +PORT_PHY_CFG_REQ_AUTO_MODE_ALL_SPEEDS = 0x1 +PORT_PHY_CFG_REQ_AUTO_MODE_ONE_SPEED = 0x2 +PORT_PHY_CFG_REQ_AUTO_MODE_ONE_OR_BELOW = 0x3 +PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK = 0x4 +PORT_PHY_CFG_REQ_AUTO_MODE_LAST = PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK +PORT_PHY_CFG_REQ_AUTO_DUPLEX_HALF = 0x0 +PORT_PHY_CFG_REQ_AUTO_DUPLEX_FULL = 0x1 +PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH = 0x2 +PORT_PHY_CFG_REQ_AUTO_DUPLEX_LAST = PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH +PORT_PHY_CFG_REQ_AUTO_PAUSE_TX = 0x1 +PORT_PHY_CFG_REQ_AUTO_PAUSE_RX = 0x2 +PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE = 0x4 +PORT_PHY_CFG_REQ_MGMT_FLAG_LINK_RELEASE = 0x1 +PORT_PHY_CFG_REQ_MGMT_FLAG_MGMT_VALID = 0x80 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100MB = 0x1 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_1GB = 0xa +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2GB = 0x14 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2_5GB = 0x19 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10GB = 0x64 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_20GB = 0xc8 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_25GB = 0xfa +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_40GB = 0x190 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_50GB = 0x1f4 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100GB = 0x3e8 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB = 0xffff +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_LAST = PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MBHD = 0x1 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MB = 0x2 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GBHD = 0x4 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GB = 0x8 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2GB = 0x10 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2_5GB = 0x20 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10GB = 0x40 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_20GB = 0x80 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_25GB = 0x100 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_40GB = 0x200 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_50GB = 0x400 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100GB = 0x800 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MBHD = 0x1000 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MB = 0x2000 +PORT_PHY_CFG_REQ_WIRESPEED_OFF = 0x0 +PORT_PHY_CFG_REQ_WIRESPEED_ON = 0x1 +PORT_PHY_CFG_REQ_WIRESPEED_LAST = PORT_PHY_CFG_REQ_WIRESPEED_ON +PORT_PHY_CFG_REQ_LPBK_NONE = 0x0 +PORT_PHY_CFG_REQ_LPBK_LOCAL = 0x1 +PORT_PHY_CFG_REQ_LPBK_REMOTE = 0x2 +PORT_PHY_CFG_REQ_LPBK_EXTERNAL = 0x3 +PORT_PHY_CFG_REQ_LPBK_LAST = PORT_PHY_CFG_REQ_LPBK_EXTERNAL +PORT_PHY_CFG_REQ_FORCE_PAUSE_TX = 0x1 +PORT_PHY_CFG_REQ_FORCE_PAUSE_RX = 0x2 +PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD1 = 0x1 +PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_100MB = 0x2 +PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD2 = 0x4 +PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_1GB = 0x8 +PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD3 = 0x10 +PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD4 = 0x20 +PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_10GB = 0x40 +PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_50GB = 0x1f4 +PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_100GB = 0x3e8 +PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB = 0x7d0 +PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_LAST = PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB +PORT_PHY_CFG_REQ_TX_LPI_TIMER_MASK = 0xffffff +PORT_PHY_CFG_REQ_TX_LPI_TIMER_SFT = 0 +PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_50G = 0x1 +PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_100G = 0x2 +PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_200G = 0x4 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_1GB = 0xa +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_10GB = 0x64 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_25GB = 0xfa +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_40GB = 0x190 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB = 0x1f4 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB = 0x3e8 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB_PAM4_56 = 0x1f5 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_56 = 0x3e9 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_56 = 0x7d1 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_56 = 0xfa1 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_112 = 0x3ea +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_112 = 0x7d2 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112 = 0xfa2 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_800GB_PAM4_112 = 0x1f42 +PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_LAST = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_800GB_PAM4_112 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_1GB = 0x1 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_10GB = 0x2 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_25GB = 0x4 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_40GB = 0x8 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB = 0x10 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB = 0x20 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB_PAM4_56 = 0x40 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_56 = 0x80 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_56 = 0x100 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_56 = 0x200 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_112 = 0x400 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_112 = 0x800 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_112 = 0x1000 +PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_800GB_PAM4_112 = 0x2000 +PORT_PHY_CFG_CMD_ERR_CODE_UNKNOWN = 0x0 +PORT_PHY_CFG_CMD_ERR_CODE_ILLEGAL_SPEED = 0x1 +PORT_PHY_CFG_CMD_ERR_CODE_RETRY = 0x2 +PORT_PHY_CFG_CMD_ERR_CODE_LAST = PORT_PHY_CFG_CMD_ERR_CODE_RETRY +VNIC_ALLOC_REQ_FLAGS_DEFAULT = 0x1 +VNIC_ALLOC_REQ_FLAGS_VIRTIO_NET_FID_VALID = 0x2 +VNIC_ALLOC_REQ_FLAGS_VNIC_ID_VALID = 0x4 +VNIC_UPDATE_REQ_ENABLES_VNIC_STATE_VALID = 0x1 +VNIC_UPDATE_REQ_ENABLES_MRU_VALID = 0x2 +VNIC_UPDATE_REQ_ENABLES_METADATA_FORMAT_TYPE_VALID = 0x4 +VNIC_UPDATE_REQ_VNIC_STATE_NORMAL = 0x0 +VNIC_UPDATE_REQ_VNIC_STATE_DROP = 0x1 +VNIC_UPDATE_REQ_VNIC_STATE_LAST = VNIC_UPDATE_REQ_VNIC_STATE_DROP +VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_0 = 0x0 +VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_1 = 0x1 +VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_2 = 0x2 +VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_3 = 0x3 +VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_4 = 0x4 +VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_LAST = VNIC_UPDATE_REQ_METADATA_FORMAT_TYPE_4 +VNIC_CFG_REQ_FLAGS_DEFAULT = 0x1 +VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE = 0x2 +VNIC_CFG_REQ_FLAGS_BD_STALL_MODE = 0x4 +VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE = 0x8 +VNIC_CFG_REQ_FLAGS_ROCE_ONLY_VNIC_MODE = 0x10 +VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE = 0x20 +VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE = 0x40 +VNIC_CFG_REQ_FLAGS_PORTCOS_MAPPING_MODE = 0x80 +VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP = 0x1 +VNIC_CFG_REQ_ENABLES_RSS_RULE = 0x2 +VNIC_CFG_REQ_ENABLES_COS_RULE = 0x4 +VNIC_CFG_REQ_ENABLES_LB_RULE = 0x8 +VNIC_CFG_REQ_ENABLES_MRU = 0x10 +VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID = 0x20 +VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID = 0x40 +VNIC_CFG_REQ_ENABLES_QUEUE_ID = 0x80 +VNIC_CFG_REQ_ENABLES_RX_CSUM_V2_MODE = 0x100 +VNIC_CFG_REQ_ENABLES_L2_CQE_MODE = 0x200 +VNIC_CFG_REQ_ENABLES_RAW_QP_ID = 0x400 +VNIC_CFG_REQ_RX_CSUM_V2_MODE_DEFAULT = 0x0 +VNIC_CFG_REQ_RX_CSUM_V2_MODE_ALL_OK = 0x1 +VNIC_CFG_REQ_RX_CSUM_V2_MODE_MAX = 0x2 +VNIC_CFG_REQ_RX_CSUM_V2_MODE_LAST = VNIC_CFG_REQ_RX_CSUM_V2_MODE_MAX +VNIC_CFG_REQ_L2_CQE_MODE_DEFAULT = 0x0 +VNIC_CFG_REQ_L2_CQE_MODE_COMPRESSED = 0x1 +VNIC_CFG_REQ_L2_CQE_MODE_MIXED = 0x2 +VNIC_CFG_REQ_L2_CQE_MODE_LAST = VNIC_CFG_REQ_L2_CQE_MODE_MIXED +VNIC_QCAPS_RESP_FLAGS_UNUSED = 0x1 +VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP = 0x2 +VNIC_QCAPS_RESP_FLAGS_BD_STALL_CAP = 0x4 +VNIC_QCAPS_RESP_FLAGS_ROCE_DUAL_VNIC_CAP = 0x8 +VNIC_QCAPS_RESP_FLAGS_ROCE_ONLY_VNIC_CAP = 0x10 +VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP = 0x20 +VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP = 0x40 +VNIC_QCAPS_RESP_FLAGS_OUTERMOST_RSS_CAP = 0x80 +VNIC_QCAPS_RESP_FLAGS_COS_ASSIGNMENT_CAP = 0x100 +VNIC_QCAPS_RESP_FLAGS_RX_CMPL_V2_CAP = 0x200 +VNIC_QCAPS_RESP_FLAGS_VNIC_STATE_CAP = 0x400 +VNIC_QCAPS_RESP_FLAGS_VIRTIO_NET_VNIC_ALLOC_CAP = 0x800 +VNIC_QCAPS_RESP_FLAGS_METADATA_FORMAT_CAP = 0x1000 +VNIC_QCAPS_RESP_FLAGS_RSS_STRICT_HASH_TYPE_CAP = 0x2000 +VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP = 0x4000 +VNIC_QCAPS_RESP_FLAGS_RING_SELECT_MODE_TOEPLITZ_CAP = 0x8000 +VNIC_QCAPS_RESP_FLAGS_RING_SELECT_MODE_XOR_CAP = 0x10000 +VNIC_QCAPS_RESP_FLAGS_RING_SELECT_MODE_TOEPLITZ_CHKSM_CAP = 0x20000 +VNIC_QCAPS_RESP_FLAGS_RSS_IPV6_FLOW_LABEL_CAP = 0x40000 +VNIC_QCAPS_RESP_FLAGS_RX_CMPL_V3_CAP = 0x80000 +VNIC_QCAPS_RESP_FLAGS_L2_CQE_MODE_CAP = 0x100000 +VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV4_CAP = 0x200000 +VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV4_CAP = 0x400000 +VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV6_CAP = 0x800000 +VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV6_CAP = 0x1000000 +VNIC_QCAPS_RESP_FLAGS_OUTERMOST_RSS_TRUSTED_VF_CAP = 0x2000000 +VNIC_QCAPS_RESP_FLAGS_PORTCOS_MAPPING_MODE = 0x4000000 +VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED = 0x8000000 +VNIC_QCAPS_RESP_FLAGS_VNIC_RSS_HASH_MODE_CAP = 0x10000000 +VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP = 0x20000000 +VNIC_QCAPS_RESP_FLAGS_RE_FLUSH_CAP = 0x40000000 +VNIC_TPA_CFG_REQ_FLAGS_TPA = 0x1 +VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA = 0x2 +VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE = 0x4 +VNIC_TPA_CFG_REQ_FLAGS_GRO = 0x8 +VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN = 0x10 +VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ = 0x20 +VNIC_TPA_CFG_REQ_FLAGS_GRO_IPID_CHECK = 0x40 +VNIC_TPA_CFG_REQ_FLAGS_GRO_TTL_CHECK = 0x80 +VNIC_TPA_CFG_REQ_FLAGS_AGG_PACK_AS_GRO = 0x100 +VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS = 0x1 +VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS = 0x2 +VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_TIMER = 0x4 +VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN = 0x8 +VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN = 0x10 +VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_1 = 0x0 +VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_2 = 0x1 +VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_4 = 0x2 +VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_8 = 0x3 +VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX = 0x1f +VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_LAST = VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX +VNIC_TPA_CFG_REQ_MAX_AGGS_1 = 0x0 +VNIC_TPA_CFG_REQ_MAX_AGGS_2 = 0x1 +VNIC_TPA_CFG_REQ_MAX_AGGS_4 = 0x2 +VNIC_TPA_CFG_REQ_MAX_AGGS_8 = 0x3 +VNIC_TPA_CFG_REQ_MAX_AGGS_16 = 0x4 +VNIC_TPA_CFG_REQ_MAX_AGGS_MAX = 0x7 +VNIC_TPA_CFG_REQ_MAX_AGGS_LAST = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN = 0x1 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE = 0x2 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_NVGRE = 0x4 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE = 0x8 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 = 0x10 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6 = 0x20 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE = 0x40 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_CUST1 = 0x80 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE_CUST1 = 0x100 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR1 = 0x200 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR2 = 0x400 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR3 = 0x800 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR4 = 0x1000 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR5 = 0x2000 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR6 = 0x4000 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR7 = 0x8000 +VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_UPAR8 = 0x10000 +VNIC_TPA_QCFG_RESP_FLAGS_TPA = 0x1 +VNIC_TPA_QCFG_RESP_FLAGS_ENCAP_TPA = 0x2 +VNIC_TPA_QCFG_RESP_FLAGS_RSC_WND_UPDATE = 0x4 +VNIC_TPA_QCFG_RESP_FLAGS_GRO = 0x8 +VNIC_TPA_QCFG_RESP_FLAGS_AGG_WITH_ECN = 0x10 +VNIC_TPA_QCFG_RESP_FLAGS_AGG_WITH_SAME_GRE_SEQ = 0x20 +VNIC_TPA_QCFG_RESP_FLAGS_GRO_IPID_CHECK = 0x40 +VNIC_TPA_QCFG_RESP_FLAGS_GRO_TTL_CHECK = 0x80 +VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_1 = 0x0 +VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_2 = 0x1 +VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_4 = 0x2 +VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_8 = 0x3 +VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_MAX = 0x1f +VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_LAST = VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_MAX +VNIC_TPA_QCFG_RESP_MAX_AGGS_1 = 0x0 +VNIC_TPA_QCFG_RESP_MAX_AGGS_2 = 0x1 +VNIC_TPA_QCFG_RESP_MAX_AGGS_4 = 0x2 +VNIC_TPA_QCFG_RESP_MAX_AGGS_8 = 0x3 +VNIC_TPA_QCFG_RESP_MAX_AGGS_16 = 0x4 +VNIC_TPA_QCFG_RESP_MAX_AGGS_MAX = 0x7 +VNIC_TPA_QCFG_RESP_MAX_AGGS_LAST = VNIC_TPA_QCFG_RESP_MAX_AGGS_MAX +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_VXLAN = 0x1 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_GENEVE = 0x2 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_NVGRE = 0x4 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_GRE = 0x8 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_IPV4 = 0x10 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_IPV6 = 0x20 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_VXLAN_GPE = 0x40 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_VXLAN_CUST1 = 0x80 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_GRE_CUST1 = 0x100 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR1 = 0x200 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR2 = 0x400 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR3 = 0x800 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR4 = 0x1000 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR5 = 0x2000 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR6 = 0x4000 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR7 = 0x8000 +VNIC_TPA_QCFG_RESP_TNL_TPA_EN_BITMAP_UPAR8 = 0x10000 +VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 = 0x1 +VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 = 0x2 +VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 = 0x4 +VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 = 0x8 +VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6 = 0x10 +VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6 = 0x20 +VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6_FLOW_LABEL = 0x40 +VNIC_RSS_CFG_REQ_HASH_TYPE_AH_SPI_IPV4 = 0x80 +VNIC_RSS_CFG_REQ_HASH_TYPE_ESP_SPI_IPV4 = 0x100 +VNIC_RSS_CFG_REQ_HASH_TYPE_AH_SPI_IPV6 = 0x200 +VNIC_RSS_CFG_REQ_HASH_TYPE_ESP_SPI_IPV6 = 0x400 +VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT = 0x1 +VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_INNERMOST_4 = 0x2 +VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_INNERMOST_2 = 0x4 +VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_OUTERMOST_4 = 0x8 +VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_OUTERMOST_2 = 0x10 +VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE = 0x1 +VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE = 0x2 +VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT = 0x4 +VNIC_RSS_CFG_REQ_RING_SELECT_MODE_TOEPLITZ = 0x0 +VNIC_RSS_CFG_REQ_RING_SELECT_MODE_XOR = 0x1 +VNIC_RSS_CFG_REQ_RING_SELECT_MODE_TOEPLITZ_CHECKSUM = 0x2 +VNIC_RSS_CFG_REQ_RING_SELECT_MODE_LAST = VNIC_RSS_CFG_REQ_RING_SELECT_MODE_TOEPLITZ_CHECKSUM +VNIC_RSS_CFG_CMD_ERR_CODE_UNKNOWN = 0x0 +VNIC_RSS_CFG_CMD_ERR_CODE_INTERFACE_NOT_READY = 0x1 +VNIC_RSS_CFG_CMD_ERR_CODE_UNABLE_TO_GET_RSS_CFG = 0x2 +VNIC_RSS_CFG_CMD_ERR_CODE_HASH_TYPE_UNSUPPORTED = 0x3 +VNIC_RSS_CFG_CMD_ERR_CODE_HASH_TYPE_ERR = 0x4 +VNIC_RSS_CFG_CMD_ERR_CODE_HASH_MODE_FAIL = 0x5 +VNIC_RSS_CFG_CMD_ERR_CODE_RING_GRP_TABLE_ALLOC_ERR = 0x6 +VNIC_RSS_CFG_CMD_ERR_CODE_HASH_KEY_ALLOC_ERR = 0x7 +VNIC_RSS_CFG_CMD_ERR_CODE_DMA_FAILED = 0x8 +VNIC_RSS_CFG_CMD_ERR_CODE_RX_RING_ALLOC_ERR = 0x9 +VNIC_RSS_CFG_CMD_ERR_CODE_CMPL_RING_ALLOC_ERR = 0xa +VNIC_RSS_CFG_CMD_ERR_CODE_HW_SET_RSS_FAILED = 0xb +VNIC_RSS_CFG_CMD_ERR_CODE_CTX_INVALID = 0xc +VNIC_RSS_CFG_CMD_ERR_CODE_VNIC_INVALID = 0xd +VNIC_RSS_CFG_CMD_ERR_CODE_VNIC_RING_TABLE_PAIR_INVALID = 0xe +VNIC_RSS_CFG_CMD_ERR_CODE_LAST = VNIC_RSS_CFG_CMD_ERR_CODE_VNIC_RING_TABLE_PAIR_INVALID +VNIC_RSS_QCFG_RESP_HASH_TYPE_IPV4 = 0x1 +VNIC_RSS_QCFG_RESP_HASH_TYPE_TCP_IPV4 = 0x2 +VNIC_RSS_QCFG_RESP_HASH_TYPE_UDP_IPV4 = 0x4 +VNIC_RSS_QCFG_RESP_HASH_TYPE_IPV6 = 0x8 +VNIC_RSS_QCFG_RESP_HASH_TYPE_TCP_IPV6 = 0x10 +VNIC_RSS_QCFG_RESP_HASH_TYPE_UDP_IPV6 = 0x20 +VNIC_RSS_QCFG_RESP_HASH_TYPE_IPV6_FLOW_LABEL = 0x40 +VNIC_RSS_QCFG_RESP_HASH_TYPE_AH_SPI_IPV4 = 0x80 +VNIC_RSS_QCFG_RESP_HASH_TYPE_ESP_SPI_IPV4 = 0x100 +VNIC_RSS_QCFG_RESP_HASH_TYPE_AH_SPI_IPV6 = 0x200 +VNIC_RSS_QCFG_RESP_HASH_TYPE_ESP_SPI_IPV6 = 0x400 +VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_DEFAULT = 0x1 +VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_INNERMOST_4 = 0x2 +VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_INNERMOST_2 = 0x4 +VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_OUTERMOST_4 = 0x8 +VNIC_RSS_QCFG_RESP_HASH_MODE_FLAGS_OUTERMOST_2 = 0x10 +VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_TOEPLITZ = 0x0 +VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_XOR = 0x1 +VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_TOEPLITZ_CHECKSUM = 0x2 +VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_LAST = VNIC_RSS_QCFG_RESP_RING_SELECT_MODE_TOEPLITZ_CHECKSUM +VNIC_PLCMODES_CFG_REQ_FLAGS_REGULAR_PLACEMENT = 0x1 +VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT = 0x2 +VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 = 0x4 +VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6 = 0x8 +VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_FCOE = 0x10 +VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_ROCE = 0x20 +VNIC_PLCMODES_CFG_REQ_FLAGS_VIRTIO_PLACEMENT = 0x40 +VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID = 0x1 +VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_OFFSET_VALID = 0x2 +VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID = 0x4 +VNIC_PLCMODES_CFG_REQ_ENABLES_MAX_BDS_VALID = 0x8 +VNIC_PLCMODES_CFG_CMD_ERR_CODE_UNKNOWN = 0x0 +VNIC_PLCMODES_CFG_CMD_ERR_CODE_INVALID_HDS_THRESHOLD = 0x1 +VNIC_PLCMODES_CFG_CMD_ERR_CODE_LAST = VNIC_PLCMODES_CFG_CMD_ERR_CODE_INVALID_HDS_THRESHOLD +RING_ALLOC_REQ_ENABLES_RING_ARB_CFG = 0x2 +RING_ALLOC_REQ_ENABLES_STAT_CTX_ID_VALID = 0x8 +RING_ALLOC_REQ_ENABLES_MAX_BW_VALID = 0x20 +RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID = 0x40 +RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID = 0x80 +RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID = 0x100 +RING_ALLOC_REQ_ENABLES_SCHQ_ID = 0x200 +RING_ALLOC_REQ_ENABLES_MPC_CHNLS_TYPE = 0x400 +RING_ALLOC_REQ_ENABLES_STEERING_TAG_VALID = 0x800 +RING_ALLOC_REQ_ENABLES_RX_RATE_PROFILE_VALID = 0x1000 +RING_ALLOC_REQ_ENABLES_DPI_VALID = 0x2000 +RING_ALLOC_REQ_RING_TYPE_L2_CMPL = 0x0 +RING_ALLOC_REQ_RING_TYPE_TX = 0x1 +RING_ALLOC_REQ_RING_TYPE_RX = 0x2 +RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL = 0x3 +RING_ALLOC_REQ_RING_TYPE_RX_AGG = 0x4 +RING_ALLOC_REQ_RING_TYPE_NQ = 0x5 +RING_ALLOC_REQ_RING_TYPE_LAST = RING_ALLOC_REQ_RING_TYPE_NQ +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_OFF = 0x0 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_4 = 0x1 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_8 = 0x2 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_12 = 0x3 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_16 = 0x4 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_24 = 0x5 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_32 = 0x6 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_48 = 0x7 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64 = 0x8 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_96 = 0x9 +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_128 = 0xa +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_192 = 0xb +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_256 = 0xc +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_320 = 0xd +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_384 = 0xe +RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_MAX = 0xf +RING_ALLOC_REQ_CMPL_COAL_CNT_LAST = RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_MAX +RING_ALLOC_REQ_FLAGS_RX_SOP_PAD = 0x1 +RING_ALLOC_REQ_FLAGS_DISABLE_CQ_OVERFLOW_DETECTION = 0x2 +RING_ALLOC_REQ_FLAGS_NQ_DBR_PACING = 0x4 +RING_ALLOC_REQ_FLAGS_TX_PKT_TS_CMPL_ENABLE = 0x8 +RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_MASK = 0xf +RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SFT = 0 +RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SP = 0x1 +RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ = 0x2 +RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_LAST = RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ +RING_ALLOC_REQ_RING_ARB_CFG_RSVD_MASK = 0xf0 +RING_ALLOC_REQ_RING_ARB_CFG_RSVD_SFT = 4 +RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_MASK = 0xff00 +RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_SFT = 8 +RING_ALLOC_REQ_MAX_BW_BW_VALUE_MASK = 0xfffffff +RING_ALLOC_REQ_MAX_BW_BW_VALUE_SFT = 0 +RING_ALLOC_REQ_MAX_BW_SCALE = 0x10000000 +RING_ALLOC_REQ_MAX_BW_SCALE_BITS = (0x0 << 28) +RING_ALLOC_REQ_MAX_BW_SCALE_BYTES = (0x1 << 28) +RING_ALLOC_REQ_MAX_BW_SCALE_LAST = RING_ALLOC_REQ_MAX_BW_SCALE_BYTES +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MASK = 0xe0000000 +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_SFT = 29 +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MEGA = (0x0 << 29) +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_KILO = (0x2 << 29) +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_BASE = (0x4 << 29) +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_GIGA = (0x6 << 29) +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 = (0x1 << 29) +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID = (0x7 << 29) +RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_LAST = RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID +RING_ALLOC_REQ_INT_MODE_LEGACY = 0x0 +RING_ALLOC_REQ_INT_MODE_RSVD = 0x1 +RING_ALLOC_REQ_INT_MODE_MSIX = 0x2 +RING_ALLOC_REQ_INT_MODE_POLL = 0x3 +RING_ALLOC_REQ_INT_MODE_LAST = RING_ALLOC_REQ_INT_MODE_POLL +RING_ALLOC_REQ_MPC_CHNLS_TYPE_TCE = 0x0 +RING_ALLOC_REQ_MPC_CHNLS_TYPE_RCE = 0x1 +RING_ALLOC_REQ_MPC_CHNLS_TYPE_TE_CFA = 0x2 +RING_ALLOC_REQ_MPC_CHNLS_TYPE_RE_CFA = 0x3 +RING_ALLOC_REQ_MPC_CHNLS_TYPE_PRIMATE = 0x4 +RING_ALLOC_REQ_MPC_CHNLS_TYPE_LAST = RING_ALLOC_REQ_MPC_CHNLS_TYPE_PRIMATE +RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_DEFAULT = 0x0 +RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_POLL_MODE = 0x1 +RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_LAST = RING_ALLOC_REQ_RX_RATE_PROFILE_SEL_POLL_MODE +RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PING_BUFFER = 0x0 +RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PONG_BUFFER = 0x1 +RING_ALLOC_RESP_PUSH_BUFFER_INDEX_LAST = RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PONG_BUFFER +CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH = 0x1 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_TX = 0x0 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX = 0x1 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_LAST = CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX +CFA_L2_FILTER_ALLOC_REQ_FLAGS_LOOPBACK = 0x2 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_DROP = 0x4 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST = 0x8 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_MASK = 0x30 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_SFT = 4 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_NO_ROCE_L2 = (0x0 << 4) +CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_L2 = (0x1 << 4) +CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_ROCE = (0x2 << 4) +CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_LAST = CFA_L2_FILTER_ALLOC_REQ_FLAGS_TRAFFIC_ROCE +CFA_L2_FILTER_ALLOC_REQ_FLAGS_XDP_DISABLE = 0x40 +CFA_L2_FILTER_ALLOC_REQ_FLAGS_SOURCE_VALID = 0x80 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR = 0x1 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK = 0x2 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN = 0x4 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN_MASK = 0x8 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN = 0x10 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK = 0x20 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR = 0x40 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR_MASK = 0x80 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN = 0x100 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN_MASK = 0x200 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN = 0x400 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN_MASK = 0x800 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_TYPE = 0x1000 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_ID = 0x2000 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE = 0x4000 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID = 0x8000 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID = 0x10000 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS = 0x20000 +CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_NUM_VLANS = 0x40000 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_NPORT = 0x0 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_PF = 0x1 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VF = 0x2 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VNIC = 0x3 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_KONG = 0x4 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_APE = 0x5 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_BONO = 0x6 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG = 0x7 +CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_LAST = CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL = 0x0 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN = 0x1 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE = 0x2 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE = 0x3 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP = 0x4 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE = 0x5 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS = 0x6 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT = 0x7 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE = 0x8 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4 = 0x9 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1 = 0xa +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE = 0xb +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 = 0xc +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE = 0x10 +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL = 0xff +CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST = CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL +CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER = 0x0 +CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_ABOVE_FILTER = 0x1 +CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_BELOW_FILTER = 0x2 +CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MAX = 0x3 +CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN = 0x4 +CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_LAST = CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_VALUE_MASK = 0x3fffffff +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_VALUE_SFT = 0 +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE = 0x40000000 +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_INT = (0x0 << 30) +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT = (0x1 << 30) +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_LAST = CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_TYPE_EXT +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR = 0x80000000 +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_RX = (0x0 << 31) +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX = (0x1 << 31) +CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_LAST = CFA_L2_FILTER_ALLOC_RESP_FLOW_ID_DIR_TX +CFA_L2_FILTER_CFG_REQ_FLAGS_PATH = 0x1 +CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_TX = 0x0 +CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX = 0x1 +CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_LAST = CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX +CFA_L2_FILTER_CFG_REQ_FLAGS_DROP = 0x2 +CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_MASK = 0xc +CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_SFT = 2 +CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_NO_ROCE_L2 = (0x0 << 2) +CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_L2 = (0x1 << 2) +CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_ROCE = (0x2 << 2) +CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_LAST = CFA_L2_FILTER_CFG_REQ_FLAGS_TRAFFIC_ROCE +CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_MASK = 0x30 +CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_SFT = 4 +CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_NO_UPDATE = (0x0 << 4) +CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_BYPASS_LKUP = (0x1 << 4) +CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_ENABLE_LKUP = (0x2 << 4) +CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_RESTORE_FW_OP = (0x3 << 4) +CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_LAST = CFA_L2_FILTER_CFG_REQ_FLAGS_REMAP_OP_RESTORE_FW_OP +CFA_L2_FILTER_CFG_REQ_ENABLES_DST_ID = 0x1 +CFA_L2_FILTER_CFG_REQ_ENABLES_NEW_MIRROR_VNIC_ID = 0x2 +CFA_L2_FILTER_CFG_REQ_ENABLES_PROF_FUNC = 0x4 +CFA_L2_FILTER_CFG_REQ_ENABLES_L2_CONTEXT_ID = 0x8 +STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE = 0x1 +STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_DUP_HOST_BUF = 0x2 +STAT_CTX_ALLOC_REQ_FLAGS_STEERING_TAG_VALID = 0x1 +STAT_CTX_QUERY_REQ_FLAGS_COUNTER_MASK = 0x1 +DBC_DBC_INDEX_MASK = 0xffffff +DBC_DBC_INDEX_SFT = 0 +DBC_DBC_EPOCH = 0x1000000 +DBC_DBC_TOGGLE_MASK = 0x6000000 +DBC_DBC_TOGGLE_SFT = 25 +DBC_DBC_XID_MASK = 0xfffff +DBC_DBC_XID_SFT = 0 +DBC_DBC_PATH_MASK = 0x3000000 +DBC_DBC_PATH_SFT = 24 +DBC_DBC_PATH_ROCE = (0x0 << 24) +DBC_DBC_PATH_L2 = (0x1 << 24) +DBC_DBC_PATH_ENGINE = (0x2 << 24) +DBC_DBC_PATH_LAST = DBC_DBC_PATH_ENGINE +DBC_DBC_VALID = 0x4000000 +DBC_DBC_DEBUG_TRACE = 0x8000000 +DBC_DBC_TYPE_MASK = 0xf0000000 +DBC_DBC_TYPE_SFT = 28 +DBC_DBC_TYPE_SQ = (0x0 << 28) +DBC_DBC_TYPE_RQ = (0x1 << 28) +DBC_DBC_TYPE_SRQ = (0x2 << 28) +DBC_DBC_TYPE_SRQ_ARM = (0x3 << 28) +DBC_DBC_TYPE_CQ = (0x4 << 28) +DBC_DBC_TYPE_CQ_ARMSE = (0x5 << 28) +DBC_DBC_TYPE_CQ_ARMALL = (0x6 << 28) +DBC_DBC_TYPE_CQ_ARMENA = (0x7 << 28) +DBC_DBC_TYPE_SRQ_ARMENA = (0x8 << 28) +DBC_DBC_TYPE_CQ_CUTOFF_ACK = (0x9 << 28) +DBC_DBC_TYPE_NQ = (0xa << 28) +DBC_DBC_TYPE_NQ_ARM = (0xb << 28) +DBC_DBC_TYPE_NQ_MASK = (0xe << 28) +DBC_DBC_TYPE_NULL = (0xf << 28) +DBC_DBC_TYPE_LAST = DBC_DBC_TYPE_NULL +CMDQ_INIT_CMDQ_LVL_MASK = 0x3 +CMDQ_INIT_CMDQ_LVL_SFT = 0 +CMDQ_INIT_CMDQ_SIZE_MASK = 0xfffc +CMDQ_INIT_CMDQ_SIZE_SFT = 2 +CMDQ_BASE_OPCODE_CREATE_QP = 0x1 +CMDQ_BASE_OPCODE_DESTROY_QP = 0x2 +CMDQ_BASE_OPCODE_MODIFY_QP = 0x3 +CMDQ_BASE_OPCODE_QUERY_QP = 0x4 +CMDQ_BASE_OPCODE_CREATE_SRQ = 0x5 +CMDQ_BASE_OPCODE_DESTROY_SRQ = 0x6 +CMDQ_BASE_OPCODE_QUERY_SRQ = 0x8 +CMDQ_BASE_OPCODE_CREATE_CQ = 0x9 +CMDQ_BASE_OPCODE_DESTROY_CQ = 0xa +CMDQ_BASE_OPCODE_RESIZE_CQ = 0xc +CMDQ_BASE_OPCODE_ALLOCATE_MRW = 0xd +CMDQ_BASE_OPCODE_DEALLOCATE_KEY = 0xe +CMDQ_BASE_OPCODE_REGISTER_MR = 0xf +CMDQ_BASE_OPCODE_DEREGISTER_MR = 0x10 +CMDQ_BASE_OPCODE_ADD_GID = 0x11 +CMDQ_BASE_OPCODE_DELETE_GID = 0x12 +CMDQ_BASE_OPCODE_MODIFY_GID = 0x17 +CMDQ_BASE_OPCODE_QUERY_GID = 0x18 +CMDQ_BASE_OPCODE_CREATE_QP1 = 0x13 +CMDQ_BASE_OPCODE_DESTROY_QP1 = 0x14 +CMDQ_BASE_OPCODE_CREATE_AH = 0x15 +CMDQ_BASE_OPCODE_DESTROY_AH = 0x16 +CMDQ_BASE_OPCODE_INITIALIZE_FW = 0x80 +CMDQ_BASE_OPCODE_DEINITIALIZE_FW = 0x81 +CMDQ_BASE_OPCODE_STOP_FUNC = 0x82 +CMDQ_BASE_OPCODE_QUERY_FUNC = 0x83 +CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES = 0x84 +CMDQ_BASE_OPCODE_READ_CONTEXT = 0x85 +CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST = 0x86 +CMDQ_BASE_OPCODE_READ_VF_MEMORY = 0x87 +CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST = 0x88 +CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY = 0x89 +CMDQ_BASE_OPCODE_MAP_TC_TO_COS = 0x8a +CMDQ_BASE_OPCODE_QUERY_VERSION = 0x8b +CMDQ_BASE_OPCODE_MODIFY_ROCE_CC = 0x8c +CMDQ_BASE_OPCODE_QUERY_ROCE_CC = 0x8d +CMDQ_BASE_OPCODE_QUERY_ROCE_STATS = 0x8e +CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE = 0x8f +CMDQ_BASE_OPCODE_MODIFY_CQ = 0x90 +CMDQ_BASE_OPCODE_QUERY_QP_EXTEND = 0x91 +CMDQ_BASE_OPCODE_QUERY_ROCE_STATS_EXT = 0x92 +CMDQ_BASE_OPCODE_ROCE_MIRROR_CFG = 0x99 +CMDQ_BASE_OPCODE_LAST = CMDQ_BASE_OPCODE_ROCE_MIRROR_CFG +CREQ_BASE_TYPE_MASK = 0x3f +CREQ_BASE_TYPE_SFT = 0 +CREQ_BASE_TYPE_QP_EVENT = 0x38 +CREQ_BASE_TYPE_FUNC_EVENT = 0x3a +CREQ_BASE_TYPE_LAST = CREQ_BASE_TYPE_FUNC_EVENT +CREQ_BASE_V = 0x1 +CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION = 0x8b +CMDQ_QUERY_VERSION_OPCODE_LAST = CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION +CREQ_QUERY_VERSION_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_VERSION_RESP_TYPE_SFT = 0 +CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_VERSION_RESP_TYPE_LAST = CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT +CREQ_QUERY_VERSION_RESP_V = 0x1 +CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION = 0x8b +CREQ_QUERY_VERSION_RESP_EVENT_LAST = CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION +CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW = 0x80 +CMDQ_INITIALIZE_FW_OPCODE_LAST = CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW +CMDQ_INITIALIZE_FW_FLAGS_MRAV_RESERVATION_SPLIT = 0x1 +CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED = 0x2 +CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED = 0x8 +CMDQ_INITIALIZE_FW_FLAGS_L2_VF_RESOURCE_MGMT = 0x10 +CMDQ_INITIALIZE_FW_FLAGS_MIRROR_ON_ROCE_SUPPORTED = 0x80 +CMDQ_INITIALIZE_FW_QPC_LVL_MASK = 0xf +CMDQ_INITIALIZE_FW_QPC_LVL_SFT = 0 +CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 = 0x0 +CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 = 0x1 +CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 = 0x2 +CMDQ_INITIALIZE_FW_QPC_LVL_LAST = CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK = 0xf0 +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT = 4 +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_INITIALIZE_FW_QPC_PG_SIZE_LAST = CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G +CMDQ_INITIALIZE_FW_MRW_LVL_MASK = 0xf +CMDQ_INITIALIZE_FW_MRW_LVL_SFT = 0 +CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 = 0x0 +CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 = 0x1 +CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 = 0x2 +CMDQ_INITIALIZE_FW_MRW_LVL_LAST = CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK = 0xf0 +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT = 4 +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_INITIALIZE_FW_MRW_PG_SIZE_LAST = CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G +CMDQ_INITIALIZE_FW_SRQ_LVL_MASK = 0xf +CMDQ_INITIALIZE_FW_SRQ_LVL_SFT = 0 +CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 = 0x0 +CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 = 0x1 +CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 = 0x2 +CMDQ_INITIALIZE_FW_SRQ_LVL_LAST = CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK = 0xf0 +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT = 4 +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_LAST = CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G +CMDQ_INITIALIZE_FW_CQ_LVL_MASK = 0xf +CMDQ_INITIALIZE_FW_CQ_LVL_SFT = 0 +CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 = 0x0 +CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 = 0x1 +CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 = 0x2 +CMDQ_INITIALIZE_FW_CQ_LVL_LAST = CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK = 0xf0 +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT = 4 +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_INITIALIZE_FW_CQ_PG_SIZE_LAST = CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G +CMDQ_INITIALIZE_FW_TQM_LVL_MASK = 0xf +CMDQ_INITIALIZE_FW_TQM_LVL_SFT = 0 +CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 = 0x0 +CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 = 0x1 +CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 = 0x2 +CMDQ_INITIALIZE_FW_TQM_LVL_LAST = CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK = 0xf0 +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT = 4 +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_INITIALIZE_FW_TQM_PG_SIZE_LAST = CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G +CMDQ_INITIALIZE_FW_TIM_LVL_MASK = 0xf +CMDQ_INITIALIZE_FW_TIM_LVL_SFT = 0 +CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 = 0x0 +CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 = 0x1 +CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 = 0x2 +CMDQ_INITIALIZE_FW_TIM_LVL_LAST = CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK = 0xf0 +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT = 4 +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_INITIALIZE_FW_TIM_PG_SIZE_LAST = CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_MASK = 0xf +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_SFT = 0 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4K = 0x0 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8K = 0x1 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16K = 0x2 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32K = 0x3 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64K = 0x4 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128K = 0x5 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_256K = 0x6 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_512K = 0x7 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_1M = 0x8 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_2M = 0x9 +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4M = 0xa +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8M = 0xb +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16M = 0xc +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32M = 0xd +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64M = 0xe +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M = 0xf +CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_LAST = CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M +CMDQ_INITIALIZE_FW_RSVD_MASK = 0xfff0 +CMDQ_INITIALIZE_FW_RSVD_SFT = 4 +CREQ_INITIALIZE_FW_RESP_TYPE_MASK = 0x3f +CREQ_INITIALIZE_FW_RESP_TYPE_SFT = 0 +CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT = 0x38 +CREQ_INITIALIZE_FW_RESP_TYPE_LAST = CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT +CREQ_INITIALIZE_FW_RESP_V = 0x1 +CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW = 0x80 +CREQ_INITIALIZE_FW_RESP_EVENT_LAST = CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW +CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW = 0x81 +CMDQ_DEINITIALIZE_FW_OPCODE_LAST = CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW +CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK = 0x3f +CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT = 0 +CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DEINITIALIZE_FW_RESP_TYPE_LAST = CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT +CREQ_DEINITIALIZE_FW_RESP_V = 0x1 +CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW = 0x81 +CREQ_DEINITIALIZE_FW_RESP_EVENT_LAST = CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW +CMDQ_CREATE_QP_OPCODE_CREATE_QP = 0x1 +CMDQ_CREATE_QP_OPCODE_LAST = CMDQ_CREATE_QP_OPCODE_CREATE_QP +CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED = 0x1 +CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION = 0x2 +CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE = 0x4 +CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED = 0x8 +CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED = 0x10 +CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED = 0x20 +CMDQ_CREATE_QP_QP_FLAGS_RESPONDER_UD_CQE_WITH_CFA = 0x40 +CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED = 0x80 +CMDQ_CREATE_QP_QP_FLAGS_EXPRESS_MODE_ENABLED = 0x100 +CMDQ_CREATE_QP_QP_FLAGS_STEERING_TAG_VALID = 0x200 +CMDQ_CREATE_QP_QP_FLAGS_RDMA_READ_OR_ATOMICS_USED = 0x400 +CMDQ_CREATE_QP_QP_FLAGS_LAST = CMDQ_CREATE_QP_QP_FLAGS_RDMA_READ_OR_ATOMICS_USED +CMDQ_CREATE_QP_TYPE_RC = 0x2 +CMDQ_CREATE_QP_TYPE_UD = 0x4 +CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE = 0x6 +CMDQ_CREATE_QP_TYPE_GSI = 0x7 +CMDQ_CREATE_QP_TYPE_LAST = CMDQ_CREATE_QP_TYPE_GSI +CMDQ_CREATE_QP_SQ_LVL_MASK = 0xf +CMDQ_CREATE_QP_SQ_LVL_SFT = 0 +CMDQ_CREATE_QP_SQ_LVL_LVL_0 = 0x0 +CMDQ_CREATE_QP_SQ_LVL_LVL_1 = 0x1 +CMDQ_CREATE_QP_SQ_LVL_LVL_2 = 0x2 +CMDQ_CREATE_QP_SQ_LVL_LAST = CMDQ_CREATE_QP_SQ_LVL_LVL_2 +CMDQ_CREATE_QP_SQ_PG_SIZE_MASK = 0xf0 +CMDQ_CREATE_QP_SQ_PG_SIZE_SFT = 4 +CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_CREATE_QP_SQ_PG_SIZE_LAST = CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G +CMDQ_CREATE_QP_RQ_LVL_MASK = 0xf +CMDQ_CREATE_QP_RQ_LVL_SFT = 0 +CMDQ_CREATE_QP_RQ_LVL_LVL_0 = 0x0 +CMDQ_CREATE_QP_RQ_LVL_LVL_1 = 0x1 +CMDQ_CREATE_QP_RQ_LVL_LVL_2 = 0x2 +CMDQ_CREATE_QP_RQ_LVL_LAST = CMDQ_CREATE_QP_RQ_LVL_LVL_2 +CMDQ_CREATE_QP_RQ_PG_SIZE_MASK = 0xf0 +CMDQ_CREATE_QP_RQ_PG_SIZE_SFT = 4 +CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_CREATE_QP_RQ_PG_SIZE_LAST = CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G +CMDQ_CREATE_QP_SQ_SGE_MASK = 0xf +CMDQ_CREATE_QP_SQ_SGE_SFT = 0 +CMDQ_CREATE_QP_SQ_FWO_MASK = 0xfff0 +CMDQ_CREATE_QP_SQ_FWO_SFT = 4 +CMDQ_CREATE_QP_RQ_SGE_MASK = 0xf +CMDQ_CREATE_QP_RQ_SGE_SFT = 0 +CMDQ_CREATE_QP_RQ_FWO_MASK = 0xfff0 +CMDQ_CREATE_QP_RQ_FWO_SFT = 4 +CREQ_CREATE_QP_RESP_TYPE_MASK = 0x3f +CREQ_CREATE_QP_RESP_TYPE_SFT = 0 +CREQ_CREATE_QP_RESP_TYPE_QP_EVENT = 0x38 +CREQ_CREATE_QP_RESP_TYPE_LAST = CREQ_CREATE_QP_RESP_TYPE_QP_EVENT +CREQ_CREATE_QP_RESP_V = 0x1 +CREQ_CREATE_QP_RESP_EVENT_CREATE_QP = 0x1 +CREQ_CREATE_QP_RESP_EVENT_LAST = CREQ_CREATE_QP_RESP_EVENT_CREATE_QP +CMDQ_DESTROY_QP_OPCODE_DESTROY_QP = 0x2 +CMDQ_DESTROY_QP_OPCODE_LAST = CMDQ_DESTROY_QP_OPCODE_DESTROY_QP +CREQ_DESTROY_QP_RESP_TYPE_MASK = 0x3f +CREQ_DESTROY_QP_RESP_TYPE_SFT = 0 +CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DESTROY_QP_RESP_TYPE_LAST = CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT +CREQ_DESTROY_QP_RESP_V = 0x1 +CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP = 0x2 +CREQ_DESTROY_QP_RESP_EVENT_LAST = CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP +CMDQ_MODIFY_QP_OPCODE_MODIFY_QP = 0x3 +CMDQ_MODIFY_QP_OPCODE_LAST = CMDQ_MODIFY_QP_OPCODE_MODIFY_QP +CMDQ_MODIFY_QP_FLAGS_SRQ_USED = 0x1 +CMDQ_MODIFY_QP_QP_TYPE_RC = 0x2 +CMDQ_MODIFY_QP_QP_TYPE_UD = 0x4 +CMDQ_MODIFY_QP_QP_TYPE_RAW_ETHERTYPE = 0x6 +CMDQ_MODIFY_QP_QP_TYPE_GSI = 0x7 +CMDQ_MODIFY_QP_QP_TYPE_LAST = CMDQ_MODIFY_QP_QP_TYPE_GSI +CMDQ_MODIFY_QP_MODIFY_MASK_STATE = 0x1 +CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY = 0x2 +CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS = 0x4 +CMDQ_MODIFY_QP_MODIFY_MASK_PKEY = 0x8 +CMDQ_MODIFY_QP_MODIFY_MASK_QKEY = 0x10 +CMDQ_MODIFY_QP_MODIFY_MASK_DGID = 0x20 +CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL = 0x40 +CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX = 0x80 +CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT = 0x100 +CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS = 0x200 +CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC = 0x400 +CMDQ_MODIFY_QP_MODIFY_MASK_PINGPONG_PUSH_MODE = 0x800 +CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU = 0x1000 +CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT = 0x2000 +CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT = 0x4000 +CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY = 0x8000 +CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN = 0x10000 +CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC = 0x20000 +CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER = 0x40000 +CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN = 0x80000 +CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC = 0x100000 +CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE = 0x200000 +CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE = 0x400000 +CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE = 0x800000 +CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE = 0x1000000 +CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA = 0x2000000 +CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID = 0x4000000 +CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC = 0x8000000 +CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID = 0x10000000 +CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC = 0x20000000 +CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN = 0x40000000 +CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP = 0x80000000 +CMDQ_MODIFY_QP_NEW_STATE_MASK = 0xf +CMDQ_MODIFY_QP_NEW_STATE_SFT = 0 +CMDQ_MODIFY_QP_NEW_STATE_RESET = 0x0 +CMDQ_MODIFY_QP_NEW_STATE_INIT = 0x1 +CMDQ_MODIFY_QP_NEW_STATE_RTR = 0x2 +CMDQ_MODIFY_QP_NEW_STATE_RTS = 0x3 +CMDQ_MODIFY_QP_NEW_STATE_SQD = 0x4 +CMDQ_MODIFY_QP_NEW_STATE_SQE = 0x5 +CMDQ_MODIFY_QP_NEW_STATE_ERR = 0x6 +CMDQ_MODIFY_QP_NEW_STATE_LAST = CMDQ_MODIFY_QP_NEW_STATE_ERR +CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY = 0x10 +CMDQ_MODIFY_QP_UNUSED1 = 0x20 +CMDQ_MODIFY_QP_NETWORK_TYPE_MASK = 0xc0 +CMDQ_MODIFY_QP_NETWORK_TYPE_SFT = 6 +CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 = (0x0 << 6) +CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 = (0x2 << 6) +CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 = (0x3 << 6) +CMDQ_MODIFY_QP_NETWORK_TYPE_LAST = CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 +CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_MASK = 0xff +CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_SFT = 0 +CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE = 0x1 +CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE = 0x2 +CMDQ_MODIFY_QP_ACCESS_REMOTE_READ = 0x4 +CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC = 0x8 +CMDQ_MODIFY_QP_TOS_ECN_MASK = 0x3 +CMDQ_MODIFY_QP_TOS_ECN_SFT = 0 +CMDQ_MODIFY_QP_TOS_DSCP_MASK = 0xfc +CMDQ_MODIFY_QP_TOS_DSCP_SFT = 2 +CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE = 0x1 +CMDQ_MODIFY_QP_UNUSED3_MASK = 0xe +CMDQ_MODIFY_QP_UNUSED3_SFT = 1 +CMDQ_MODIFY_QP_PATH_MTU_MASK = 0xf0 +CMDQ_MODIFY_QP_PATH_MTU_SFT = 4 +CMDQ_MODIFY_QP_PATH_MTU_MTU_256 = (0x0 << 4) +CMDQ_MODIFY_QP_PATH_MTU_MTU_512 = (0x1 << 4) +CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 = (0x2 << 4) +CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 = (0x3 << 4) +CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 = (0x4 << 4) +CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 = (0x5 << 4) +CMDQ_MODIFY_QP_PATH_MTU_LAST = CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 +CMDQ_MODIFY_QP_ENABLE_CC = 0x1 +CMDQ_MODIFY_QP_UNUSED15_MASK = 0xfffe +CMDQ_MODIFY_QP_UNUSED15_SFT = 1 +CMDQ_MODIFY_QP_VLAN_ID_MASK = 0xfff +CMDQ_MODIFY_QP_VLAN_ID_SFT = 0 +CMDQ_MODIFY_QP_VLAN_DEI = 0x1000 +CMDQ_MODIFY_QP_VLAN_PCP_MASK = 0xe000 +CMDQ_MODIFY_QP_VLAN_PCP_SFT = 13 +CMDQ_MODIFY_QP_EXT_MODIFY_MASK_EXT_STATS_CTX = 0x1 +CMDQ_MODIFY_QP_EXT_MODIFY_MASK_SCHQ_ID_VALID = 0x2 +CREQ_MODIFY_QP_RESP_TYPE_MASK = 0x3f +CREQ_MODIFY_QP_RESP_TYPE_SFT = 0 +CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT = 0x38 +CREQ_MODIFY_QP_RESP_TYPE_LAST = CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT +CREQ_MODIFY_QP_RESP_V = 0x1 +CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP = 0x3 +CREQ_MODIFY_QP_RESP_EVENT_LAST = CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP +CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED = 0x1 +CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_INDEX_MASK = 0xe +CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_INDEX_SFT = 1 +CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_STATE = 0x10 +CMDQ_QUERY_QP_OPCODE_QUERY_QP = 0x4 +CMDQ_QUERY_QP_OPCODE_LAST = CMDQ_QUERY_QP_OPCODE_QUERY_QP +CREQ_QUERY_QP_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_QP_RESP_TYPE_SFT = 0 +CREQ_QUERY_QP_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_QP_RESP_TYPE_LAST = CREQ_QUERY_QP_RESP_TYPE_QP_EVENT +CREQ_QUERY_QP_RESP_V = 0x1 +CREQ_QUERY_QP_RESP_EVENT_QUERY_QP = 0x4 +CREQ_QUERY_QP_RESP_EVENT_LAST = CREQ_QUERY_QP_RESP_EVENT_QUERY_QP +CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP = 0x4 +CREQ_QUERY_QP_RESP_SB_OPCODE_LAST = CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP +CREQ_QUERY_QP_RESP_SB_STATE_MASK = 0xf +CREQ_QUERY_QP_RESP_SB_STATE_SFT = 0 +CREQ_QUERY_QP_RESP_SB_STATE_RESET = 0x0 +CREQ_QUERY_QP_RESP_SB_STATE_INIT = 0x1 +CREQ_QUERY_QP_RESP_SB_STATE_RTR = 0x2 +CREQ_QUERY_QP_RESP_SB_STATE_RTS = 0x3 +CREQ_QUERY_QP_RESP_SB_STATE_SQD = 0x4 +CREQ_QUERY_QP_RESP_SB_STATE_SQE = 0x5 +CREQ_QUERY_QP_RESP_SB_STATE_ERR = 0x6 +CREQ_QUERY_QP_RESP_SB_STATE_LAST = CREQ_QUERY_QP_RESP_SB_STATE_ERR +CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY = 0x10 +CREQ_QUERY_QP_RESP_SB_UNUSED3_MASK = 0xe0 +CREQ_QUERY_QP_RESP_SB_UNUSED3_SFT = 5 +CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_MASK = 0xff +CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC_REMOTE_READ_REMOTE_WRITE_LOCAL_WRITE_SFT = 0 +CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE = 0x1 +CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE = 0x2 +CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ = 0x4 +CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC = 0x8 +CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK = 0xfff +CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT = 0 +CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK = 0xf000 +CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT = 12 +CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 = (0x0 << 12) +CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 = (0x1 << 12) +CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 = (0x2 << 12) +CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 = (0x3 << 12) +CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 = (0x4 << 12) +CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 = (0x5 << 12) +CREQ_QUERY_QP_RESP_SB_PATH_MTU_LAST = CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 +CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK = 0x3 +CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT = 0 +CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK = 0xfc +CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT = 2 +CREQ_QUERY_QP_RESP_SB_ENABLE_CC = 0x1 +CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK = 0xfff +CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT = 0 +CREQ_QUERY_QP_RESP_SB_VLAN_DEI = 0x1000 +CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK = 0xe000 +CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT = 13 +CMDQ_QUERY_QP_EXTEND_OPCODE_QUERY_QP_EXTEND = 0x91 +CMDQ_QUERY_QP_EXTEND_OPCODE_LAST = CMDQ_QUERY_QP_EXTEND_OPCODE_QUERY_QP_EXTEND +CMDQ_QUERY_QP_EXTEND_PF_NUM_MASK = 0xff +CMDQ_QUERY_QP_EXTEND_PF_NUM_SFT = 0 +CMDQ_QUERY_QP_EXTEND_VF_NUM_MASK = 0xffff00 +CMDQ_QUERY_QP_EXTEND_VF_NUM_SFT = 8 +CMDQ_QUERY_QP_EXTEND_VF_VALID = 0x1000000 +CREQ_QUERY_QP_EXTEND_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_QP_EXTEND_RESP_TYPE_SFT = 0 +CREQ_QUERY_QP_EXTEND_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_QP_EXTEND_RESP_TYPE_LAST = CREQ_QUERY_QP_EXTEND_RESP_TYPE_QP_EVENT +CREQ_QUERY_QP_EXTEND_RESP_V = 0x1 +CREQ_QUERY_QP_EXTEND_RESP_EVENT_QUERY_QP_EXTEND = 0x91 +CREQ_QUERY_QP_EXTEND_RESP_EVENT_LAST = CREQ_QUERY_QP_EXTEND_RESP_EVENT_QUERY_QP_EXTEND +CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_QUERY_QP_EXTEND = 0x91 +CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_LAST = CREQ_QUERY_QP_EXTEND_RESP_SB_OPCODE_QUERY_QP_EXTEND +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_MASK = 0xf +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SFT = 0 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RESET = 0x0 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_INIT = 0x1 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RTR = 0x2 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_RTS = 0x3 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SQD = 0x4 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_SQE = 0x5 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_ERR = 0x6 +CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_LAST = CREQ_QUERY_QP_EXTEND_RESP_SB_STATE_ERR +CREQ_QUERY_QP_EXTEND_RESP_SB_UNUSED4_MASK = 0xf0 +CREQ_QUERY_QP_EXTEND_RESP_SB_UNUSED4_SFT = 4 +CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV1 = 0x0 +CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV4 = 0x2 +CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV6 = 0x3 +CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_LAST = CREQ_QUERY_QP_EXTEND_RESP_SB_NETWORK_TYPE_ROCEV2_IPV6 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE = 0x1 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE_LAST = 0x0 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST = 0x1 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED = 0x2 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO = (0x0 << 1) +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES = (0x1 << 1) +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST = CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_QUERY_QP_EXTEND = 0x91 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_LAST = CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_OPCODE_QUERY_QP_EXTEND +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_MASK = 0xf +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SFT = 0 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RESET = 0x0 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_INIT = 0x1 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RTR = 0x2 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_RTS = 0x3 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SQD = 0x4 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_SQE = 0x5 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_ERR = 0x6 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_LAST = CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_STATE_ERR +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_UNUSED4_MASK = 0xf0 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_UNUSED4_SFT = 4 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV1 = 0x0 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV4 = 0x2 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV6 = 0x3 +CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_LAST = CREQ_QUERY_QP_EXTEND_RESP_SB_TLV_NETWORK_TYPE_ROCEV2_IPV6 +CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ = 0x5 +CMDQ_CREATE_SRQ_OPCODE_LAST = CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ +CMDQ_CREATE_SRQ_FLAGS_STEERING_TAG_VALID = 0x1 +CMDQ_CREATE_SRQ_LVL_MASK = 0x3 +CMDQ_CREATE_SRQ_LVL_SFT = 0 +CMDQ_CREATE_SRQ_LVL_LVL_0 = 0x0 +CMDQ_CREATE_SRQ_LVL_LVL_1 = 0x1 +CMDQ_CREATE_SRQ_LVL_LVL_2 = 0x2 +CMDQ_CREATE_SRQ_LVL_LAST = CMDQ_CREATE_SRQ_LVL_LVL_2 +CMDQ_CREATE_SRQ_PG_SIZE_MASK = 0x1c +CMDQ_CREATE_SRQ_PG_SIZE_SFT = 2 +CMDQ_CREATE_SRQ_PG_SIZE_PG_4K = (0x0 << 2) +CMDQ_CREATE_SRQ_PG_SIZE_PG_8K = (0x1 << 2) +CMDQ_CREATE_SRQ_PG_SIZE_PG_64K = (0x2 << 2) +CMDQ_CREATE_SRQ_PG_SIZE_PG_2M = (0x3 << 2) +CMDQ_CREATE_SRQ_PG_SIZE_PG_8M = (0x4 << 2) +CMDQ_CREATE_SRQ_PG_SIZE_PG_1G = (0x5 << 2) +CMDQ_CREATE_SRQ_PG_SIZE_LAST = CMDQ_CREATE_SRQ_PG_SIZE_PG_1G +CMDQ_CREATE_SRQ_UNUSED11_MASK = 0xffe0 +CMDQ_CREATE_SRQ_UNUSED11_SFT = 5 +CMDQ_CREATE_SRQ_EVENTQ_ID_MASK = 0xfff +CMDQ_CREATE_SRQ_EVENTQ_ID_SFT = 0 +CMDQ_CREATE_SRQ_UNUSED4_MASK = 0xf000 +CMDQ_CREATE_SRQ_UNUSED4_SFT = 12 +CREQ_CREATE_SRQ_RESP_TYPE_MASK = 0x3f +CREQ_CREATE_SRQ_RESP_TYPE_SFT = 0 +CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT = 0x38 +CREQ_CREATE_SRQ_RESP_TYPE_LAST = CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT +CREQ_CREATE_SRQ_RESP_V = 0x1 +CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ = 0x5 +CREQ_CREATE_SRQ_RESP_EVENT_LAST = CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ +CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ = 0x6 +CMDQ_DESTROY_SRQ_OPCODE_LAST = CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ +CREQ_DESTROY_SRQ_RESP_TYPE_MASK = 0x3f +CREQ_DESTROY_SRQ_RESP_TYPE_SFT = 0 +CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DESTROY_SRQ_RESP_TYPE_LAST = CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT +CREQ_DESTROY_SRQ_RESP_V = 0x1 +CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ = 0x6 +CREQ_DESTROY_SRQ_RESP_EVENT_LAST = CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ +CREQ_DESTROY_SRQ_RESP_UNUSED0_MASK = 0xffff +CREQ_DESTROY_SRQ_RESP_UNUSED0_SFT = 0 +CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK = 0x30000 +CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT = 16 +CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ = 0x8 +CMDQ_QUERY_SRQ_OPCODE_LAST = CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ +CREQ_QUERY_SRQ_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_SRQ_RESP_TYPE_SFT = 0 +CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_SRQ_RESP_TYPE_LAST = CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT +CREQ_QUERY_SRQ_RESP_V = 0x1 +CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ = 0x8 +CREQ_QUERY_SRQ_RESP_EVENT_LAST = CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ +CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ = 0x8 +CREQ_QUERY_SRQ_RESP_SB_OPCODE_LAST = CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ +CMDQ_CREATE_CQ_OPCODE_CREATE_CQ = 0x9 +CMDQ_CREATE_CQ_OPCODE_LAST = CMDQ_CREATE_CQ_OPCODE_CREATE_CQ +CMDQ_CREATE_CQ_FLAGS_DISABLE_CQ_OVERFLOW_DETECTION = 0x1 +CMDQ_CREATE_CQ_FLAGS_STEERING_TAG_VALID = 0x2 +CMDQ_CREATE_CQ_FLAGS_INFINITE_CQ_MODE = 0x4 +CMDQ_CREATE_CQ_FLAGS_COALESCING_VALID = 0x8 +CMDQ_CREATE_CQ_LVL_MASK = 0x3 +CMDQ_CREATE_CQ_LVL_SFT = 0 +CMDQ_CREATE_CQ_LVL_LVL_0 = 0x0 +CMDQ_CREATE_CQ_LVL_LVL_1 = 0x1 +CMDQ_CREATE_CQ_LVL_LVL_2 = 0x2 +CMDQ_CREATE_CQ_LVL_LAST = CMDQ_CREATE_CQ_LVL_LVL_2 +CMDQ_CREATE_CQ_PG_SIZE_MASK = 0x1c +CMDQ_CREATE_CQ_PG_SIZE_SFT = 2 +CMDQ_CREATE_CQ_PG_SIZE_PG_4K = (0x0 << 2) +CMDQ_CREATE_CQ_PG_SIZE_PG_8K = (0x1 << 2) +CMDQ_CREATE_CQ_PG_SIZE_PG_64K = (0x2 << 2) +CMDQ_CREATE_CQ_PG_SIZE_PG_2M = (0x3 << 2) +CMDQ_CREATE_CQ_PG_SIZE_PG_8M = (0x4 << 2) +CMDQ_CREATE_CQ_PG_SIZE_PG_1G = (0x5 << 2) +CMDQ_CREATE_CQ_PG_SIZE_LAST = CMDQ_CREATE_CQ_PG_SIZE_PG_1G +CMDQ_CREATE_CQ_UNUSED27_MASK = 0xffffffe0 +CMDQ_CREATE_CQ_UNUSED27_SFT = 5 +CMDQ_CREATE_CQ_CNQ_ID_MASK = 0xfff +CMDQ_CREATE_CQ_CNQ_ID_SFT = 0 +CMDQ_CREATE_CQ_CQ_FCO_MASK = 0xfffff000 +CMDQ_CREATE_CQ_CQ_FCO_SFT = 12 +CMDQ_CREATE_CQ_BUF_MAXTIME_MASK = 0x1ff +CMDQ_CREATE_CQ_BUF_MAXTIME_SFT = 0 +CMDQ_CREATE_CQ_NORMAL_MAXBUF_MASK = 0x3e00 +CMDQ_CREATE_CQ_NORMAL_MAXBUF_SFT = 9 +CMDQ_CREATE_CQ_DURING_MAXBUF_MASK = 0x7c000 +CMDQ_CREATE_CQ_DURING_MAXBUF_SFT = 14 +CMDQ_CREATE_CQ_ENABLE_RING_IDLE_MODE = 0x80000 +CMDQ_CREATE_CQ_UNUSED12_MASK = 0xfff00000 +CMDQ_CREATE_CQ_UNUSED12_SFT = 20 +CREQ_CREATE_CQ_RESP_TYPE_MASK = 0x3f +CREQ_CREATE_CQ_RESP_TYPE_SFT = 0 +CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT = 0x38 +CREQ_CREATE_CQ_RESP_TYPE_LAST = CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT +CREQ_CREATE_CQ_RESP_V = 0x1 +CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ = 0x9 +CREQ_CREATE_CQ_RESP_EVENT_LAST = CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ +CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ = 0xa +CMDQ_DESTROY_CQ_OPCODE_LAST = CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ +CREQ_DESTROY_CQ_RESP_TYPE_MASK = 0x3f +CREQ_DESTROY_CQ_RESP_TYPE_SFT = 0 +CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DESTROY_CQ_RESP_TYPE_LAST = CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT +CREQ_DESTROY_CQ_RESP_V = 0x1 +CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ = 0xa +CREQ_DESTROY_CQ_RESP_EVENT_LAST = CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ +CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK = 0x3 +CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT = 0 +CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ = 0xc +CMDQ_RESIZE_CQ_OPCODE_LAST = CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ +CMDQ_RESIZE_CQ_LVL_MASK = 0x3 +CMDQ_RESIZE_CQ_LVL_SFT = 0 +CMDQ_RESIZE_CQ_LVL_LVL_0 = 0x0 +CMDQ_RESIZE_CQ_LVL_LVL_1 = 0x1 +CMDQ_RESIZE_CQ_LVL_LVL_2 = 0x2 +CMDQ_RESIZE_CQ_LVL_LAST = CMDQ_RESIZE_CQ_LVL_LVL_2 +CMDQ_RESIZE_CQ_PG_SIZE_MASK = 0x1c +CMDQ_RESIZE_CQ_PG_SIZE_SFT = 2 +CMDQ_RESIZE_CQ_PG_SIZE_PG_4K = (0x0 << 2) +CMDQ_RESIZE_CQ_PG_SIZE_PG_8K = (0x1 << 2) +CMDQ_RESIZE_CQ_PG_SIZE_PG_64K = (0x2 << 2) +CMDQ_RESIZE_CQ_PG_SIZE_PG_2M = (0x3 << 2) +CMDQ_RESIZE_CQ_PG_SIZE_PG_8M = (0x4 << 2) +CMDQ_RESIZE_CQ_PG_SIZE_PG_1G = (0x5 << 2) +CMDQ_RESIZE_CQ_PG_SIZE_LAST = CMDQ_RESIZE_CQ_PG_SIZE_PG_1G +CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK = 0x1fffffe0 +CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT = 5 +CREQ_RESIZE_CQ_RESP_TYPE_MASK = 0x3f +CREQ_RESIZE_CQ_RESP_TYPE_SFT = 0 +CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT = 0x38 +CREQ_RESIZE_CQ_RESP_TYPE_LAST = CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT +CREQ_RESIZE_CQ_RESP_V = 0x1 +CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ = 0xc +CREQ_RESIZE_CQ_RESP_EVENT_LAST = CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ +CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW = 0xd +CMDQ_ALLOCATE_MRW_OPCODE_LAST = CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW +CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK = 0xf +CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT = 0 +CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR = 0x0 +CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR = 0x1 +CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 = 0x2 +CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A = 0x3 +CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B = 0x4 +CMDQ_ALLOCATE_MRW_MRW_FLAGS_LAST = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B +CMDQ_ALLOCATE_MRW_STEERING_TAG_VALID = 0x10 +CMDQ_ALLOCATE_MRW_UNUSED4_MASK = 0xe0 +CMDQ_ALLOCATE_MRW_UNUSED4_SFT = 5 +CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY = 0x20 +CREQ_ALLOCATE_MRW_RESP_TYPE_MASK = 0x3f +CREQ_ALLOCATE_MRW_RESP_TYPE_SFT = 0 +CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT = 0x38 +CREQ_ALLOCATE_MRW_RESP_TYPE_LAST = CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT +CREQ_ALLOCATE_MRW_RESP_V = 0x1 +CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW = 0xd +CREQ_ALLOCATE_MRW_RESP_EVENT_LAST = CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW +CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY = 0xe +CMDQ_DEALLOCATE_KEY_OPCODE_LAST = CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK = 0xf +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT = 0 +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR = 0x0 +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR = 0x1 +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 = 0x2 +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A = 0x3 +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B = 0x4 +CMDQ_DEALLOCATE_KEY_MRW_FLAGS_LAST = CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B +CMDQ_DEALLOCATE_KEY_UNUSED4_MASK = 0xf0 +CMDQ_DEALLOCATE_KEY_UNUSED4_SFT = 4 +CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK = 0x3f +CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT = 0 +CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DEALLOCATE_KEY_RESP_TYPE_LAST = CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT +CREQ_DEALLOCATE_KEY_RESP_V = 0x1 +CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY = 0xe +CREQ_DEALLOCATE_KEY_RESP_EVENT_LAST = CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY +CMDQ_REGISTER_MR_OPCODE_REGISTER_MR = 0xf +CMDQ_REGISTER_MR_OPCODE_LAST = CMDQ_REGISTER_MR_OPCODE_REGISTER_MR +CMDQ_REGISTER_MR_FLAGS_ALLOC_MR = 0x1 +CMDQ_REGISTER_MR_FLAGS_STEERING_TAG_VALID = 0x2 +CMDQ_REGISTER_MR_FLAGS_ENABLE_RO = 0x4 +CMDQ_REGISTER_MR_LVL_MASK = 0x3 +CMDQ_REGISTER_MR_LVL_SFT = 0 +CMDQ_REGISTER_MR_LVL_LVL_0 = 0x0 +CMDQ_REGISTER_MR_LVL_LVL_1 = 0x1 +CMDQ_REGISTER_MR_LVL_LVL_2 = 0x2 +CMDQ_REGISTER_MR_LVL_LAST = CMDQ_REGISTER_MR_LVL_LVL_2 +CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK = 0x7c +CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT = 2 +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4K = (0xc << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_8K = (0xd << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_64K = (0x10 << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_256K = (0x12 << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1M = (0x14 << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_2M = (0x15 << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4M = (0x16 << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G = (0x1e << 2) +CMDQ_REGISTER_MR_LOG2_PG_SIZE_LAST = CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G +CMDQ_REGISTER_MR_UNUSED1 = 0x80 +CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE = 0x1 +CMDQ_REGISTER_MR_ACCESS_REMOTE_READ = 0x2 +CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE = 0x4 +CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC = 0x8 +CMDQ_REGISTER_MR_ACCESS_MW_BIND = 0x10 +CMDQ_REGISTER_MR_ACCESS_ZERO_BASED = 0x20 +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK = 0x1f +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT = 0 +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K = 0xc +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K = 0xd +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K = 0x10 +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K = 0x12 +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M = 0x14 +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M = 0x15 +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M = 0x16 +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G = 0x1e +CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_LAST = CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G +CMDQ_REGISTER_MR_UNUSED11_MASK = 0xffe0 +CMDQ_REGISTER_MR_UNUSED11_SFT = 5 +CREQ_REGISTER_MR_RESP_TYPE_MASK = 0x3f +CREQ_REGISTER_MR_RESP_TYPE_SFT = 0 +CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT = 0x38 +CREQ_REGISTER_MR_RESP_TYPE_LAST = CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT +CREQ_REGISTER_MR_RESP_V = 0x1 +CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR = 0xf +CREQ_REGISTER_MR_RESP_EVENT_LAST = CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR +CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR = 0x10 +CMDQ_DEREGISTER_MR_OPCODE_LAST = CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR +CREQ_DEREGISTER_MR_RESP_TYPE_MASK = 0x3f +CREQ_DEREGISTER_MR_RESP_TYPE_SFT = 0 +CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DEREGISTER_MR_RESP_TYPE_LAST = CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT +CREQ_DEREGISTER_MR_RESP_V = 0x1 +CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR = 0x10 +CREQ_DEREGISTER_MR_RESP_EVENT_LAST = CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR +CMDQ_ADD_GID_OPCODE_ADD_GID = 0x11 +CMDQ_ADD_GID_OPCODE_LAST = CMDQ_ADD_GID_OPCODE_ADD_GID +CMDQ_ADD_GID_VLAN_VLAN_EN_TPID_VLAN_ID_MASK = 0xffff +CMDQ_ADD_GID_VLAN_VLAN_EN_TPID_VLAN_ID_SFT = 0 +CMDQ_ADD_GID_VLAN_VLAN_ID_MASK = 0xfff +CMDQ_ADD_GID_VLAN_VLAN_ID_SFT = 0 +CMDQ_ADD_GID_VLAN_TPID_MASK = 0x7000 +CMDQ_ADD_GID_VLAN_TPID_SFT = 12 +CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 = (0x0 << 12) +CMDQ_ADD_GID_VLAN_TPID_TPID_8100 = (0x1 << 12) +CMDQ_ADD_GID_VLAN_TPID_TPID_9100 = (0x2 << 12) +CMDQ_ADD_GID_VLAN_TPID_TPID_9200 = (0x3 << 12) +CMDQ_ADD_GID_VLAN_TPID_TPID_9300 = (0x4 << 12) +CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 = (0x5 << 12) +CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 = (0x6 << 12) +CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 = (0x7 << 12) +CMDQ_ADD_GID_VLAN_TPID_LAST = CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 +CMDQ_ADD_GID_VLAN_VLAN_EN = 0x8000 +CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID_STATS_CTX_ID_MASK = 0xffff +CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID_STATS_CTX_ID_SFT = 0 +CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK = 0x7fff +CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT = 0 +CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID = 0x8000 +CREQ_ADD_GID_RESP_TYPE_MASK = 0x3f +CREQ_ADD_GID_RESP_TYPE_SFT = 0 +CREQ_ADD_GID_RESP_TYPE_QP_EVENT = 0x38 +CREQ_ADD_GID_RESP_TYPE_LAST = CREQ_ADD_GID_RESP_TYPE_QP_EVENT +CREQ_ADD_GID_RESP_V = 0x1 +CREQ_ADD_GID_RESP_EVENT_ADD_GID = 0x11 +CREQ_ADD_GID_RESP_EVENT_LAST = CREQ_ADD_GID_RESP_EVENT_ADD_GID +CMDQ_DELETE_GID_OPCODE_DELETE_GID = 0x12 +CMDQ_DELETE_GID_OPCODE_LAST = CMDQ_DELETE_GID_OPCODE_DELETE_GID +CREQ_DELETE_GID_RESP_TYPE_MASK = 0x3f +CREQ_DELETE_GID_RESP_TYPE_SFT = 0 +CREQ_DELETE_GID_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DELETE_GID_RESP_TYPE_LAST = CREQ_DELETE_GID_RESP_TYPE_QP_EVENT +CREQ_DELETE_GID_RESP_V = 0x1 +CREQ_DELETE_GID_RESP_EVENT_DELETE_GID = 0x12 +CREQ_DELETE_GID_RESP_EVENT_LAST = CREQ_DELETE_GID_RESP_EVENT_DELETE_GID +CMDQ_MODIFY_GID_OPCODE_MODIFY_GID = 0x17 +CMDQ_MODIFY_GID_OPCODE_LAST = CMDQ_MODIFY_GID_OPCODE_MODIFY_GID +CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK = 0xfff +CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT = 0 +CMDQ_MODIFY_GID_VLAN_TPID_MASK = 0x7000 +CMDQ_MODIFY_GID_VLAN_TPID_SFT = 12 +CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 = (0x0 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 = (0x1 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 = (0x2 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 = (0x3 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 = (0x4 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 = (0x5 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 = (0x6 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 = (0x7 << 12) +CMDQ_MODIFY_GID_VLAN_TPID_LAST = CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 +CMDQ_MODIFY_GID_VLAN_VLAN_EN = 0x8000 +CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK = 0x7fff +CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT = 0 +CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID = 0x8000 +CREQ_MODIFY_GID_RESP_TYPE_MASK = 0x3f +CREQ_MODIFY_GID_RESP_TYPE_SFT = 0 +CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT = 0x38 +CREQ_MODIFY_GID_RESP_TYPE_LAST = CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT +CREQ_MODIFY_GID_RESP_V = 0x1 +CREQ_MODIFY_GID_RESP_EVENT_ADD_GID = 0x11 +CREQ_MODIFY_GID_RESP_EVENT_LAST = CREQ_MODIFY_GID_RESP_EVENT_ADD_GID +CMDQ_QUERY_GID_OPCODE_QUERY_GID = 0x18 +CMDQ_QUERY_GID_OPCODE_LAST = CMDQ_QUERY_GID_OPCODE_QUERY_GID +CREQ_QUERY_GID_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_GID_RESP_TYPE_SFT = 0 +CREQ_QUERY_GID_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_GID_RESP_TYPE_LAST = CREQ_QUERY_GID_RESP_TYPE_QP_EVENT +CREQ_QUERY_GID_RESP_V = 0x1 +CREQ_QUERY_GID_RESP_EVENT_QUERY_GID = 0x18 +CREQ_QUERY_GID_RESP_EVENT_LAST = CREQ_QUERY_GID_RESP_EVENT_QUERY_GID +CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID = 0x18 +CREQ_QUERY_GID_RESP_SB_OPCODE_LAST = CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID +CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN_TPID_VLAN_ID_MASK = 0xffff +CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN_TPID_VLAN_ID_SFT = 0 +CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK = 0xfff +CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT = 0 +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK = 0x7000 +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT = 12 +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 = (0x0 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 = (0x1 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 = (0x2 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 = (0x3 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 = (0x4 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 = (0x5 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 = (0x6 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 = (0x7 << 12) +CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST = CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 +CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN = 0x8000 +CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 = 0x13 +CMDQ_CREATE_QP1_OPCODE_LAST = CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 +CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED = 0x1 +CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION = 0x2 +CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE = 0x4 +CMDQ_CREATE_QP1_QP_FLAGS_LAST = CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE +CMDQ_CREATE_QP1_TYPE_GSI = 0x1 +CMDQ_CREATE_QP1_TYPE_LAST = CMDQ_CREATE_QP1_TYPE_GSI +CMDQ_CREATE_QP1_SQ_LVL_MASK = 0xf +CMDQ_CREATE_QP1_SQ_LVL_SFT = 0 +CMDQ_CREATE_QP1_SQ_LVL_LVL_0 = 0x0 +CMDQ_CREATE_QP1_SQ_LVL_LVL_1 = 0x1 +CMDQ_CREATE_QP1_SQ_LVL_LVL_2 = 0x2 +CMDQ_CREATE_QP1_SQ_LVL_LAST = CMDQ_CREATE_QP1_SQ_LVL_LVL_2 +CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK = 0xf0 +CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT = 4 +CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_CREATE_QP1_SQ_PG_SIZE_LAST = CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G +CMDQ_CREATE_QP1_RQ_LVL_MASK = 0xf +CMDQ_CREATE_QP1_RQ_LVL_SFT = 0 +CMDQ_CREATE_QP1_RQ_LVL_LVL_0 = 0x0 +CMDQ_CREATE_QP1_RQ_LVL_LVL_1 = 0x1 +CMDQ_CREATE_QP1_RQ_LVL_LVL_2 = 0x2 +CMDQ_CREATE_QP1_RQ_LVL_LAST = CMDQ_CREATE_QP1_RQ_LVL_LVL_2 +CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK = 0xf0 +CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT = 4 +CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K = (0x0 << 4) +CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K = (0x1 << 4) +CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K = (0x2 << 4) +CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M = (0x3 << 4) +CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M = (0x4 << 4) +CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G = (0x5 << 4) +CMDQ_CREATE_QP1_RQ_PG_SIZE_LAST = CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G +CMDQ_CREATE_QP1_SQ_SGE_MASK = 0xf +CMDQ_CREATE_QP1_SQ_SGE_SFT = 0 +CMDQ_CREATE_QP1_SQ_FWO_MASK = 0xfff0 +CMDQ_CREATE_QP1_SQ_FWO_SFT = 4 +CMDQ_CREATE_QP1_RQ_SGE_MASK = 0xf +CMDQ_CREATE_QP1_RQ_SGE_SFT = 0 +CMDQ_CREATE_QP1_RQ_FWO_MASK = 0xfff0 +CMDQ_CREATE_QP1_RQ_FWO_SFT = 4 +CREQ_CREATE_QP1_RESP_TYPE_MASK = 0x3f +CREQ_CREATE_QP1_RESP_TYPE_SFT = 0 +CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT = 0x38 +CREQ_CREATE_QP1_RESP_TYPE_LAST = CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT +CREQ_CREATE_QP1_RESP_V = 0x1 +CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 = 0x13 +CREQ_CREATE_QP1_RESP_EVENT_LAST = CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 +CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 = 0x14 +CMDQ_DESTROY_QP1_OPCODE_LAST = CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 +CREQ_DESTROY_QP1_RESP_TYPE_MASK = 0x3f +CREQ_DESTROY_QP1_RESP_TYPE_SFT = 0 +CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DESTROY_QP1_RESP_TYPE_LAST = CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT +CREQ_DESTROY_QP1_RESP_V = 0x1 +CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 = 0x14 +CREQ_DESTROY_QP1_RESP_EVENT_LAST = CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 +CMDQ_CREATE_AH_OPCODE_CREATE_AH = 0x15 +CMDQ_CREATE_AH_OPCODE_LAST = CMDQ_CREATE_AH_OPCODE_CREATE_AH +CMDQ_CREATE_AH_TYPE_V1 = 0x0 +CMDQ_CREATE_AH_TYPE_V2IPV4 = 0x2 +CMDQ_CREATE_AH_TYPE_V2IPV6 = 0x3 +CMDQ_CREATE_AH_TYPE_LAST = CMDQ_CREATE_AH_TYPE_V2IPV6 +CMDQ_CREATE_AH_FLOW_LABEL_MASK = 0xfffff +CMDQ_CREATE_AH_FLOW_LABEL_SFT = 0 +CMDQ_CREATE_AH_DEST_VLAN_ID_MASK = 0xfff00000 +CMDQ_CREATE_AH_DEST_VLAN_ID_SFT = 20 +CMDQ_CREATE_AH_ENABLE_CC = 0x1 +CREQ_CREATE_AH_RESP_TYPE_MASK = 0x3f +CREQ_CREATE_AH_RESP_TYPE_SFT = 0 +CREQ_CREATE_AH_RESP_TYPE_QP_EVENT = 0x38 +CREQ_CREATE_AH_RESP_TYPE_LAST = CREQ_CREATE_AH_RESP_TYPE_QP_EVENT +CREQ_CREATE_AH_RESP_V = 0x1 +CREQ_CREATE_AH_RESP_EVENT_CREATE_AH = 0x15 +CREQ_CREATE_AH_RESP_EVENT_LAST = CREQ_CREATE_AH_RESP_EVENT_CREATE_AH +CMDQ_DESTROY_AH_OPCODE_DESTROY_AH = 0x16 +CMDQ_DESTROY_AH_OPCODE_LAST = CMDQ_DESTROY_AH_OPCODE_DESTROY_AH +CREQ_DESTROY_AH_RESP_TYPE_MASK = 0x3f +CREQ_DESTROY_AH_RESP_TYPE_SFT = 0 +CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT = 0x38 +CREQ_DESTROY_AH_RESP_TYPE_LAST = CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT +CREQ_DESTROY_AH_RESP_V = 0x1 +CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH = 0x16 +CREQ_DESTROY_AH_RESP_EVENT_LAST = CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH +CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS = 0x8e +CMDQ_QUERY_ROCE_STATS_OPCODE_LAST = CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS +CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID = 0x1 +CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID = 0x2 +CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK = 0xff +CMDQ_QUERY_ROCE_STATS_PF_NUM_SFT = 0 +CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK = 0xffff00 +CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT = 8 +CMDQ_QUERY_ROCE_STATS_VF_VALID = 0x1000000 +CREQ_QUERY_ROCE_STATS_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_ROCE_STATS_RESP_TYPE_SFT = 0 +CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_ROCE_STATS_RESP_TYPE_LAST = CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT +CREQ_QUERY_ROCE_STATS_RESP_V = 0x1 +CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS = 0x8e +CREQ_QUERY_ROCE_STATS_RESP_EVENT_LAST = CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS +CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS = 0x8e +CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_LAST = CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS +CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS = 0x92 +CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_LAST = CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS +CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_COLLECTION_ID = 0x1 +CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID = 0x2 +CMDQ_QUERY_ROCE_STATS_EXT_PF_NUM_MASK = 0xff +CMDQ_QUERY_ROCE_STATS_EXT_PF_NUM_SFT = 0 +CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_MASK = 0xffff00 +CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT = 8 +CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID = 0x1000000 +CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_SFT = 0 +CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_LAST = CREQ_QUERY_ROCE_STATS_EXT_RESP_TYPE_QP_EVENT +CREQ_QUERY_ROCE_STATS_EXT_RESP_V = 0x1 +CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_QUERY_ROCE_STATS_EXT = 0x92 +CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_LAST = CREQ_QUERY_ROCE_STATS_EXT_RESP_EVENT_QUERY_ROCE_STATS_EXT +CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_QUERY_ROCE_STATS_EXT = 0x92 +CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_LAST = CREQ_QUERY_ROCE_STATS_EXT_RESP_SB_OPCODE_QUERY_ROCE_STATS_EXT +CMDQ_ROCE_MIRROR_CFG_OPCODE_ROCE_MIRROR_CFG = 0x99 +CMDQ_ROCE_MIRROR_CFG_OPCODE_LAST = CMDQ_ROCE_MIRROR_CFG_OPCODE_ROCE_MIRROR_CFG +CMDQ_ROCE_MIRROR_CFG_MIRROR_ENABLE = 0x1 +CREQ_ROCE_MIRROR_CFG_RESP_TYPE_MASK = 0x3f +CREQ_ROCE_MIRROR_CFG_RESP_TYPE_SFT = 0 +CREQ_ROCE_MIRROR_CFG_RESP_TYPE_QP_EVENT = 0x38 +CREQ_ROCE_MIRROR_CFG_RESP_TYPE_LAST = CREQ_ROCE_MIRROR_CFG_RESP_TYPE_QP_EVENT +CREQ_ROCE_MIRROR_CFG_RESP_V = 0x1 +CREQ_ROCE_MIRROR_CFG_RESP_EVENT_ROCE_MIRROR_CFG = 0x99 +CREQ_ROCE_MIRROR_CFG_RESP_EVENT_LAST = CREQ_ROCE_MIRROR_CFG_RESP_EVENT_ROCE_MIRROR_CFG +CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC = 0x83 +CMDQ_QUERY_FUNC_OPCODE_LAST = CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC +CREQ_QUERY_FUNC_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_FUNC_RESP_TYPE_SFT = 0 +CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_FUNC_RESP_TYPE_LAST = CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT +CREQ_QUERY_FUNC_RESP_V = 0x1 +CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC = 0x83 +CREQ_QUERY_FUNC_RESP_EVENT_LAST = CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC +CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC = 0x83 +CREQ_QUERY_FUNC_RESP_SB_OPCODE_LAST = CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC +CREQ_QUERY_FUNC_RESP_SB_RESIZE_QP = 0x1 +CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_MASK = 0xe +CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_SFT = 1 +CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN0 = (0x0 << 1) +CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1 = (0x1 << 1) +CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1_EXT = (0x2 << 1) +CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_LAST = CREQ_QUERY_FUNC_RESP_SB_CC_GENERATION_CC_GEN1_EXT +CREQ_QUERY_FUNC_RESP_SB_EXT_STATS = 0x10 +CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC = 0x20 +CREQ_QUERY_FUNC_RESP_SB_OPTIMIZED_TRANSMIT_ENABLED = 0x40 +CREQ_QUERY_FUNC_RESP_SB_CQE_V2 = 0x80 +CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE = 0x100 +CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED = 0x200 +CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED = 0x400 +CREQ_QUERY_FUNC_RESP_SB_ATOMIC_OPS_NOT_SUPPORTED = 0x1 +CREQ_QUERY_FUNC_RESP_SB_DRV_VERSION_RGTR_SUPPORTED = 0x2 +CREQ_QUERY_FUNC_RESP_SB_CREATE_QP_BATCH_SUPPORTED = 0x4 +CREQ_QUERY_FUNC_RESP_SB_DESTROY_QP_BATCH_SUPPORTED = 0x8 +CREQ_QUERY_FUNC_RESP_SB_ROCE_STATS_EXT_CTX_SUPPORTED = 0x10 +CREQ_QUERY_FUNC_RESP_SB_CREATE_SRQ_SGE_SUPPORTED = 0x20 +CREQ_QUERY_FUNC_RESP_SB_FIXED_SIZE_WQE_DISABLED = 0x40 +CREQ_QUERY_FUNC_RESP_SB_DCN_SUPPORTED = 0x80 +CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED = 0x1 +CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED = 0x2 +CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED = 0x4 +CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED = 0x8 +CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK = 0x30 +CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_SFT = 4 +CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_PSN_TABLE = (0x0 << 4) +CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE = (0x1 << 4) +CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE = (0x2 << 4) +CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_LAST = CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE +CREQ_QUERY_FUNC_RESP_SB_MAX_SRQ_EXTENDED = 0x40 +CREQ_QUERY_FUNC_RESP_SB_MIN_RNR_RTR_RTS_OPT_SUPPORTED = 0x1000 +CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES = 0x84 +CMDQ_SET_FUNC_RESOURCES_OPCODE_LAST = CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES +CMDQ_SET_FUNC_RESOURCES_FLAGS_MRAV_RESERVATION_SPLIT = 0x1 +CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK = 0x3f +CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT = 0 +CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT = 0x38 +CREQ_SET_FUNC_RESOURCES_RESP_TYPE_LAST = CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT +CREQ_SET_FUNC_RESOURCES_RESP_V = 0x1 +CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES = 0x84 +CREQ_SET_FUNC_RESOURCES_RESP_EVENT_LAST = CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES +CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT = 0x85 +CMDQ_READ_CONTEXT_OPCODE_LAST = CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT +CMDQ_READ_CONTEXT_TYPE_QPC = 0x0 +CMDQ_READ_CONTEXT_TYPE_CQ = 0x1 +CMDQ_READ_CONTEXT_TYPE_MRW = 0x2 +CMDQ_READ_CONTEXT_TYPE_SRQ = 0x3 +CMDQ_READ_CONTEXT_TYPE_LAST = CMDQ_READ_CONTEXT_TYPE_SRQ +CREQ_READ_CONTEXT_TYPE_MASK = 0x3f +CREQ_READ_CONTEXT_TYPE_SFT = 0 +CREQ_READ_CONTEXT_TYPE_QP_EVENT = 0x38 +CREQ_READ_CONTEXT_TYPE_LAST = CREQ_READ_CONTEXT_TYPE_QP_EVENT +CREQ_READ_CONTEXT_V = 0x1 +CREQ_READ_CONTEXT_EVENT_READ_CONTEXT = 0x85 +CREQ_READ_CONTEXT_EVENT_LAST = CREQ_READ_CONTEXT_EVENT_READ_CONTEXT +CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS = 0x8a +CMDQ_MAP_TC_TO_COS_OPCODE_LAST = CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS +CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE = 0xffff +CMDQ_MAP_TC_TO_COS_COS0_LAST = CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE +CMDQ_MAP_TC_TO_COS_COS1_DISABLE = 0x8000 +CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE = 0xffff +CMDQ_MAP_TC_TO_COS_COS1_LAST = CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE +CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK = 0x3f +CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT = 0 +CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT = 0x38 +CREQ_MAP_TC_TO_COS_RESP_TYPE_LAST = CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT +CREQ_MAP_TC_TO_COS_RESP_V = 0x1 +CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS = 0x8a +CREQ_MAP_TC_TO_COS_RESP_EVENT_LAST = CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS +CMDQ_QUERY_ROCE_CC_OPCODE_QUERY_ROCE_CC = 0x8d +CMDQ_QUERY_ROCE_CC_OPCODE_LAST = CMDQ_QUERY_ROCE_CC_OPCODE_QUERY_ROCE_CC +CREQ_QUERY_ROCE_CC_RESP_TYPE_MASK = 0x3f +CREQ_QUERY_ROCE_CC_RESP_TYPE_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_TYPE_QP_EVENT = 0x38 +CREQ_QUERY_ROCE_CC_RESP_TYPE_LAST = CREQ_QUERY_ROCE_CC_RESP_TYPE_QP_EVENT +CREQ_QUERY_ROCE_CC_RESP_V = 0x1 +CREQ_QUERY_ROCE_CC_RESP_EVENT_QUERY_ROCE_CC = 0x8d +CREQ_QUERY_ROCE_CC_RESP_EVENT_LAST = CREQ_QUERY_ROCE_CC_RESP_EVENT_QUERY_ROCE_CC +CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_QUERY_ROCE_CC = 0x8d +CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_LAST = CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_QUERY_ROCE_CC +CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC = 0x1 +CREQ_QUERY_ROCE_CC_RESP_SB_UNUSED7_MASK = 0xfe +CREQ_QUERY_ROCE_CC_RESP_SB_UNUSED7_SFT = 1 +CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK = 0x3 +CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK = 0xfc +CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT = 2 +CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_MASK = 0x7 +CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_MASK = 0xf8 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_SFT = 3 +CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_MASK = 0x3f +CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_MASK = 0xc0 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_SFT = 6 +CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_DCTCP = 0x0 +CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_PROBABILISTIC = 0x1 +CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_LAST = CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE_PROBABILISTIC +CREQ_QUERY_ROCE_CC_RESP_SB_RTT_MASK = 0x3fff +CREQ_QUERY_ROCE_CC_RESP_SB_RTT_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_MASK = 0xc000 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_SFT = 14 +CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_MASK = 0x3ff +CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_MASK = 0xfc00 +CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_SFT = 10 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE = 0x1 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE_LAST = 0x0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST = 0x1 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED = 0x2 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO = (0x0 << 1) +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES = (0x1 << 1) +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST = CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_QUERY_ROCE_CC = 0x8d +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_LAST = CREQ_QUERY_ROCE_CC_RESP_SB_TLV_OPCODE_QUERY_ROCE_CC +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ENABLE_CC = 0x1 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_UNUSED7_MASK = 0xfe +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_UNUSED7_SFT = 1 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_ECN_MASK = 0x3 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_ECN_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_DSCP_MASK = 0xfc +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TOS_DSCP_SFT = 2 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_VLAN_PCP_MASK = 0x7 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_VLAN_PCP_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD1_MASK = 0xf8 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD1_SFT = 3 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_TOS_DSCP_MASK = 0x3f +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_ALT_TOS_DSCP_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD4_MASK = 0xc0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD4_SFT = 6 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_DCTCP = 0x0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_PROBABILISTIC = 0x1 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_LAST = CREQ_QUERY_ROCE_CC_RESP_SB_TLV_CC_MODE_PROBABILISTIC +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RTT_MASK = 0x3fff +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RTT_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD5_MASK = 0xc000 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD5_SFT = 14 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TCP_CP_MASK = 0x3ff +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_TCP_CP_SFT = 0 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD6_MASK = 0xfc00 +CREQ_QUERY_ROCE_CC_RESP_SB_TLV_RSVD6_SFT = 10 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE = 0x1 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE_LAST = 0x0 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_MORE_NOT_LAST = 0x1 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED = 0x2 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_NO = (0x0 << 1) +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES = (0x1 << 1) +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_LAST = CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_TLV_FLAGS_REQUIRED_YES +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_NOT_ECT = 0x0 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_1 = 0x1 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_0 = 0x2 +CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_LAST = CREQ_QUERY_ROCE_CC_GEN1_RESP_SB_TLV_CNP_ECN_ECT_0 +CMDQ_MODIFY_ROCE_CC_OPCODE_MODIFY_ROCE_CC = 0x8c +CMDQ_MODIFY_ROCE_CC_OPCODE_LAST = CMDQ_MODIFY_ROCE_CC_OPCODE_MODIFY_ROCE_CC +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC = 0x1 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_G = 0x2 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_NUMPHASEPERSTATE = 0x4 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_CR = 0x8 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_TR = 0x10 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN = 0x20 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP = 0x40 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP = 0x80 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP = 0x100 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_RTT = 0x200 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE = 0x400 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TCP_CP = 0x800 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TX_QUEUE = 0x1000 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP = 0x2000 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TIME_PER_PHASE = 0x4000 +CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_PKTS_PER_PHASE = 0x8000 +CMDQ_MODIFY_ROCE_CC_ENABLE_CC = 0x1 +CMDQ_MODIFY_ROCE_CC_RSVD1_MASK = 0xfe +CMDQ_MODIFY_ROCE_CC_RSVD1_SFT = 1 +CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK = 0x3 +CMDQ_MODIFY_ROCE_CC_TOS_ECN_SFT = 0 +CMDQ_MODIFY_ROCE_CC_TOS_DSCP_MASK = 0xfc +CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT = 2 +CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_MASK = 0x7 +CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_SFT = 0 +CMDQ_MODIFY_ROCE_CC_RSVD3_MASK = 0xf8 +CMDQ_MODIFY_ROCE_CC_RSVD3_SFT = 3 +CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_MASK = 0x3f +CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_SFT = 0 +CMDQ_MODIFY_ROCE_CC_RSVD4_MASK = 0xffc0 +CMDQ_MODIFY_ROCE_CC_RSVD4_SFT = 6 +CMDQ_MODIFY_ROCE_CC_RTT_MASK = 0x3fff +CMDQ_MODIFY_ROCE_CC_RTT_SFT = 0 +CMDQ_MODIFY_ROCE_CC_RSVD5_MASK = 0xc000 +CMDQ_MODIFY_ROCE_CC_RSVD5_SFT = 14 +CMDQ_MODIFY_ROCE_CC_TCP_CP_MASK = 0x3ff +CMDQ_MODIFY_ROCE_CC_TCP_CP_SFT = 0 +CMDQ_MODIFY_ROCE_CC_RSVD6_MASK = 0xfc00 +CMDQ_MODIFY_ROCE_CC_RSVD6_SFT = 10 +CMDQ_MODIFY_ROCE_CC_CC_MODE_DCTCP_CC_MODE = 0x0 +CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE = 0x1 +CMDQ_MODIFY_ROCE_CC_CC_MODE_LAST = CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE +CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE = 0x1 +CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE_LAST = 0x0 +CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_MORE_NOT_LAST = 0x1 +CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED = 0x2 +CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_NO = (0x0 << 1) +CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_YES = (0x1 << 1) +CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_LAST = CMDQ_MODIFY_ROCE_CC_TLV_TLV_FLAGS_REQUIRED_YES +CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_MODIFY_ROCE_CC = 0x8c +CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_LAST = CMDQ_MODIFY_ROCE_CC_TLV_OPCODE_MODIFY_ROCE_CC +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ENABLE_CC = 0x1 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_G = 0x2 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_NUMPHASEPERSTATE = 0x4 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INIT_CR = 0x8 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INIT_TR = 0x10 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TOS_ECN = 0x20 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TOS_DSCP = 0x40 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ALT_VLAN_PCP = 0x80 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_ALT_TOS_DSCP = 0x100 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_RTT = 0x200 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_CC_MODE = 0x400 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TCP_CP = 0x800 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TX_QUEUE = 0x1000 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_INACTIVITY_CP = 0x2000 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_TIME_PER_PHASE = 0x4000 +CMDQ_MODIFY_ROCE_CC_TLV_MODIFY_MASK_PKTS_PER_PHASE = 0x8000 +CMDQ_MODIFY_ROCE_CC_TLV_ENABLE_CC = 0x1 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD1_MASK = 0xfe +CMDQ_MODIFY_ROCE_CC_TLV_RSVD1_SFT = 1 +CMDQ_MODIFY_ROCE_CC_TLV_TOS_ECN_MASK = 0x3 +CMDQ_MODIFY_ROCE_CC_TLV_TOS_ECN_SFT = 0 +CMDQ_MODIFY_ROCE_CC_TLV_TOS_DSCP_MASK = 0xfc +CMDQ_MODIFY_ROCE_CC_TLV_TOS_DSCP_SFT = 2 +CMDQ_MODIFY_ROCE_CC_TLV_ALT_VLAN_PCP_MASK = 0x7 +CMDQ_MODIFY_ROCE_CC_TLV_ALT_VLAN_PCP_SFT = 0 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD3_MASK = 0xf8 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD3_SFT = 3 +CMDQ_MODIFY_ROCE_CC_TLV_ALT_TOS_DSCP_MASK = 0x3f +CMDQ_MODIFY_ROCE_CC_TLV_ALT_TOS_DSCP_SFT = 0 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD4_MASK = 0xffc0 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD4_SFT = 6 +CMDQ_MODIFY_ROCE_CC_TLV_RTT_MASK = 0x3fff +CMDQ_MODIFY_ROCE_CC_TLV_RTT_SFT = 0 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD5_MASK = 0xc000 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD5_SFT = 14 +CMDQ_MODIFY_ROCE_CC_TLV_TCP_CP_MASK = 0x3ff +CMDQ_MODIFY_ROCE_CC_TLV_TCP_CP_SFT = 0 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD6_MASK = 0xfc00 +CMDQ_MODIFY_ROCE_CC_TLV_RSVD6_SFT = 10 +CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_DCTCP_CC_MODE = 0x0 +CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_PROBABILISTIC_CC_MODE = 0x1 +CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_LAST = CMDQ_MODIFY_ROCE_CC_TLV_CC_MODE_PROBABILISTIC_CC_MODE +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE = 0x1 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE_LAST = 0x0 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_MORE_NOT_LAST = 0x1 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED = 0x2 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_NO = (0x0 << 1) +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_YES = (0x1 << 1) +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_LAST = CMDQ_MODIFY_ROCE_CC_GEN1_TLV_TLV_FLAGS_REQUIRED_YES +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_MIN_TIME_BETWEEN_CNPS = 0x1 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_INIT_CP = 0x2 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_UPDATE_MODE = 0x4 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_UPDATE_CYCLES = 0x8 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_FR_NUM_RTTS = 0x10 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_AI_RATE_INCREASE = 0x20 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCTION_RELAX_RTTS_TH = 0x40 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ADDITIONAL_RELAX_CR_TH = 0x80 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_MIN_TH = 0x100 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_BW_AVG_WEIGHT = 0x200 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_FACTOR = 0x400 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_MAX_CP_CR_TH = 0x800 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_BIAS_EN = 0x1000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_BIAS = 0x2000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CNP_ECN = 0x4000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RTT_JITTER_EN = 0x8000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_LINK_BYTES_PER_USEC = 0x10000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RESET_CC_CR_TH = 0x20000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_WIDTH = 0x40000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_MIN = 0x80000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_MAX = 0x100000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_ABS_MAX = 0x200000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_LOWER_BOUND = 0x400000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CR_PROB_FACTOR = 0x800000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_TR_PROB_FACTOR = 0x1000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_FAIRNESS_CR_TH = 0x2000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RED_DIV = 0x4000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CNP_RATIO_TH = 0x8000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_EXP_AI_RTTS = 0x10000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_EXP_AI_CR_CP_RATIO = 0x20000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CP_EXP_UPDATE_TH = 0x40000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_HIGH_EXP_AI_RTTS_TH1 = 0x80000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_HIGH_EXP_AI_RTTS_TH2 = 0x100000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_USE_RATE_TABLE = 0x200000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_LINK64B_PER_RTT = 0x400000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_CONG_FREE_RTTS_TH = 0x800000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_SEVERE_CONG_CR_TH1 = 0x1000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_SEVERE_CONG_CR_TH2 = 0x2000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_CC_ACK_BYTES = 0x4000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCE_INIT_EN = 0x8000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_REDUCE_INIT_CONG_FREE_RTTS_TH = 0x10000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_RANDOM_NO_RED_EN = 0x20000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_ACTUAL_CR_SHIFT_CORRECTION_EN = 0x40000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_MODIFY_MASK_QUOTA_PERIOD_ADJUST_EN = 0x80000000000 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_NOT_ECT = 0x0 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_1 = 0x1 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_0 = 0x2 +CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_LAST = CMDQ_MODIFY_ROCE_CC_GEN1_TLV_CNP_ECN_ECT_0 +CREQ_MODIFY_ROCE_CC_RESP_TYPE_MASK = 0x3f +CREQ_MODIFY_ROCE_CC_RESP_TYPE_SFT = 0 +CREQ_MODIFY_ROCE_CC_RESP_TYPE_QP_EVENT = 0x38 +CREQ_MODIFY_ROCE_CC_RESP_TYPE_LAST = CREQ_MODIFY_ROCE_CC_RESP_TYPE_QP_EVENT +CREQ_MODIFY_ROCE_CC_RESP_V = 0x1 +CREQ_MODIFY_ROCE_CC_RESP_EVENT_MODIFY_ROCE_CC = 0x8c +CREQ_MODIFY_ROCE_CC_RESP_EVENT_LAST = CREQ_MODIFY_ROCE_CC_RESP_EVENT_MODIFY_ROCE_CC +CMDQ_SET_LINK_AGGR_MODE_OPCODE_SET_LINK_AGGR_MODE = 0x8f +CMDQ_SET_LINK_AGGR_MODE_OPCODE_LAST = CMDQ_SET_LINK_AGGR_MODE_OPCODE_SET_LINK_AGGR_MODE +CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN = 0x1 +CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP = 0x2 +CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP = 0x4 +CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE = 0x8 +CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID = 0x10 +CMDQ_SET_LINK_AGGR_MODE_AGGR_ENABLE = 0x1 +CMDQ_SET_LINK_AGGR_MODE_RSVD1_MASK = 0xfe +CMDQ_SET_LINK_AGGR_MODE_RSVD1_SFT = 1 +CMDQ_SET_LINK_AGGR_MODE_ACTIVE_PORT_MAP_MASK = 0xf +CMDQ_SET_LINK_AGGR_MODE_ACTIVE_PORT_MAP_SFT = 0 +CMDQ_SET_LINK_AGGR_MODE_RSVD2_MASK = 0xf0 +CMDQ_SET_LINK_AGGR_MODE_RSVD2_SFT = 4 +CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_ACTIVE_ACTIVE = 0x1 +CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_ACTIVE_BACKUP = 0x2 +CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_BALANCE_XOR = 0x3 +CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_802_3_AD = 0x4 +CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_LAST = CMDQ_SET_LINK_AGGR_MODE_AGGR_MODE_802_3_AD +CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_MASK = 0x3f +CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_SFT = 0 +CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_QP_EVENT = 0x38 +CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_LAST = CREQ_SET_LINK_AGGR_MODE_RESP_TYPE_QP_EVENT +CREQ_SET_LINK_AGGR_MODE_RESP_V = 0x1 +CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_SET_LINK_AGGR_MODE = 0x8f +CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_LAST = CREQ_SET_LINK_AGGR_MODE_RESP_EVENT_SET_LINK_AGGR_MODE +CREQ_FUNC_EVENT_TYPE_MASK = 0x3f +CREQ_FUNC_EVENT_TYPE_SFT = 0 +CREQ_FUNC_EVENT_TYPE_FUNC_EVENT = 0x3a +CREQ_FUNC_EVENT_TYPE_LAST = CREQ_FUNC_EVENT_TYPE_FUNC_EVENT +CREQ_FUNC_EVENT_V = 0x1 +CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR = 0x1 +CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR = 0x2 +CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR = 0x3 +CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR = 0x4 +CREQ_FUNC_EVENT_EVENT_CQ_ERROR = 0x5 +CREQ_FUNC_EVENT_EVENT_TQM_ERROR = 0x6 +CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR = 0x7 +CREQ_FUNC_EVENT_EVENT_CFCS_ERROR = 0x8 +CREQ_FUNC_EVENT_EVENT_CFCC_ERROR = 0x9 +CREQ_FUNC_EVENT_EVENT_CFCM_ERROR = 0xa +CREQ_FUNC_EVENT_EVENT_TIM_ERROR = 0xb +CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST = 0x80 +CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED = 0x81 +CREQ_FUNC_EVENT_EVENT_LAST = CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED +CREQ_QP_EVENT_TYPE_MASK = 0x3f +CREQ_QP_EVENT_TYPE_SFT = 0 +CREQ_QP_EVENT_TYPE_QP_EVENT = 0x38 +CREQ_QP_EVENT_TYPE_LAST = CREQ_QP_EVENT_TYPE_QP_EVENT +CREQ_QP_EVENT_STATUS_SUCCESS = 0x0 +CREQ_QP_EVENT_STATUS_FAIL = 0x1 +CREQ_QP_EVENT_STATUS_RESOURCES = 0x2 +CREQ_QP_EVENT_STATUS_INVALID_CMD = 0x3 +CREQ_QP_EVENT_STATUS_NOT_IMPLEMENTED = 0x4 +CREQ_QP_EVENT_STATUS_INVALID_PARAMETER = 0x5 +CREQ_QP_EVENT_STATUS_HARDWARE_ERROR = 0x6 +CREQ_QP_EVENT_STATUS_INTERNAL_ERROR = 0x7 +CREQ_QP_EVENT_STATUS_LAST = CREQ_QP_EVENT_STATUS_INTERNAL_ERROR +CREQ_QP_EVENT_V = 0x1 +CREQ_QP_EVENT_EVENT_CREATE_QP = 0x1 +CREQ_QP_EVENT_EVENT_DESTROY_QP = 0x2 +CREQ_QP_EVENT_EVENT_MODIFY_QP = 0x3 +CREQ_QP_EVENT_EVENT_QUERY_QP = 0x4 +CREQ_QP_EVENT_EVENT_CREATE_SRQ = 0x5 +CREQ_QP_EVENT_EVENT_DESTROY_SRQ = 0x6 +CREQ_QP_EVENT_EVENT_QUERY_SRQ = 0x8 +CREQ_QP_EVENT_EVENT_CREATE_CQ = 0x9 +CREQ_QP_EVENT_EVENT_DESTROY_CQ = 0xa +CREQ_QP_EVENT_EVENT_RESIZE_CQ = 0xc +CREQ_QP_EVENT_EVENT_ALLOCATE_MRW = 0xd +CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY = 0xe +CREQ_QP_EVENT_EVENT_REGISTER_MR = 0xf +CREQ_QP_EVENT_EVENT_DEREGISTER_MR = 0x10 +CREQ_QP_EVENT_EVENT_ADD_GID = 0x11 +CREQ_QP_EVENT_EVENT_DELETE_GID = 0x12 +CREQ_QP_EVENT_EVENT_MODIFY_GID = 0x17 +CREQ_QP_EVENT_EVENT_QUERY_GID = 0x18 +CREQ_QP_EVENT_EVENT_CREATE_QP1 = 0x13 +CREQ_QP_EVENT_EVENT_DESTROY_QP1 = 0x14 +CREQ_QP_EVENT_EVENT_CREATE_AH = 0x15 +CREQ_QP_EVENT_EVENT_DESTROY_AH = 0x16 +CREQ_QP_EVENT_EVENT_INITIALIZE_FW = 0x80 +CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW = 0x81 +CREQ_QP_EVENT_EVENT_STOP_FUNC = 0x82 +CREQ_QP_EVENT_EVENT_QUERY_FUNC = 0x83 +CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES = 0x84 +CREQ_QP_EVENT_EVENT_READ_CONTEXT = 0x85 +CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS = 0x8a +CREQ_QP_EVENT_EVENT_QUERY_VERSION = 0x8b +CREQ_QP_EVENT_EVENT_MODIFY_CC = 0x8c +CREQ_QP_EVENT_EVENT_QUERY_CC = 0x8d +CREQ_QP_EVENT_EVENT_QUERY_ROCE_STATS = 0x8e +CREQ_QP_EVENT_EVENT_SET_LINK_AGGR_MODE = 0x8f +CREQ_QP_EVENT_EVENT_QUERY_QP_EXTEND = 0x91 +CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION = 0xc0 +CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION = 0xc1 +CREQ_QP_EVENT_EVENT_LAST = CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION +CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK = 0x3f +CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT = 0 +CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT = 0x38 +CREQ_QP_ERROR_NOTIFICATION_TYPE_LAST = CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_NO_ERROR = 0X0 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_OPCODE_ERROR = 0X1 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TIMEOUT_RETRY_LIMIT = 0X2 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RNR_TIMEOUT_RETRY_LIMIT = 0X3 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_1 = 0X4 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_2 = 0X5 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_3 = 0X6 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_4 = 0X7 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_MEMORY_ERROR = 0X8 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_MEMORY_ERROR = 0X9 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_READ_RESP_LENGTH = 0XA +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_READ_RESP = 0XB +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_BIND = 0XC +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_FAST_REG = 0XD +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_INVALIDATE = 0XE +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CMP_ERROR = 0XF +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETRAN_LOCAL_ERROR = 0X10 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_WQE_FORMAT_ERROR = 0X11 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ORRQ_FORMAT_ERROR = 0X12 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_AVID_ERROR = 0X13 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_AV_DOMAIN_ERROR = 0X14 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CQ_LOAD_ERROR = 0X15 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_SERV_TYPE_ERROR = 0X16 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_OP_ERROR = 0X17 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_PCI_ERROR = 0X18 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_PCI_ERROR = 0X19 +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PROD_WQE_MSMTCH_ERROR = 0X1A +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PSN_RANGE_CHECK_ERROR = 0X1B +CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETX_SETUP_ERROR = 0X1C +CREQ_QP_ERROR_NOTIFICATION_V = 0x1 +CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION = 0xc0 +CREQ_QP_ERROR_NOTIFICATION_EVENT_LAST = CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_NO_ERROR = 0x0 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEED_MAX = 0x1 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH = 0x2 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE = 0x3 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_OPCODE_ERROR = 0x4 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT = 0x5 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY = 0x6 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR = 0x7 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION = 0x8 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR = 0x9 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY = 0xa +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR = 0xb +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION = 0xc +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR = 0xd +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW = 0xe +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE = 0xf +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC = 0x10 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_REM_INVALIDATE = 0x11 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_MEMORY_ERROR = 0x12 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_ERROR = 0x13 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CMP_ERROR = 0x14 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_INVALID_DUP_RKEY = 0x15 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR = 0x16 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_FORMAT_ERROR = 0x17 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR = 0x18 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR = 0x19 +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR = 0x1b +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR = 0x1c +CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_NOT_FOUND = 0x1d +CREQ_CQ_ERROR_NOTIFICATION_TYPE_MASK = 0x3f +CREQ_CQ_ERROR_NOTIFICATION_TYPE_SFT = 0 +CREQ_CQ_ERROR_NOTIFICATION_TYPE_CQ_EVENT = 0x38 +CREQ_CQ_ERROR_NOTIFICATION_TYPE_LAST = CREQ_CQ_ERROR_NOTIFICATION_TYPE_CQ_EVENT +CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR = 0x1 +CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR = 0x2 +CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR = 0x3 +CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR = 0x4 +CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR = 0x5 +CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR = 0x6 +CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_LAST = CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR +CREQ_CQ_ERROR_NOTIFICATION_V = 0x1 +CREQ_CQ_ERROR_NOTIFICATION_EVENT_CQ_ERROR_NOTIFICATION = 0xc1 +CREQ_CQ_ERROR_NOTIFICATION_EVENT_LAST = CREQ_CQ_ERROR_NOTIFICATION_EVENT_CQ_ERROR_NOTIFICATION +SQ_BASE_WQE_TYPE_SEND = 0x0 +SQ_BASE_WQE_TYPE_SEND_W_IMMEAD = 0x1 +SQ_BASE_WQE_TYPE_SEND_W_INVALID = 0x2 +SQ_BASE_WQE_TYPE_WRITE_WQE = 0x4 +SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD = 0x5 +SQ_BASE_WQE_TYPE_READ_WQE = 0x6 +SQ_BASE_WQE_TYPE_ATOMIC_CS = 0x8 +SQ_BASE_WQE_TYPE_ATOMIC_FA = 0xb +SQ_BASE_WQE_TYPE_LOCAL_INVALID = 0xc +SQ_BASE_WQE_TYPE_FR_PMR = 0xd +SQ_BASE_WQE_TYPE_BIND = 0xe +SQ_BASE_WQE_TYPE_FR_PPMR = 0xf +SQ_BASE_WQE_TYPE_LAST = SQ_BASE_WQE_TYPE_FR_PPMR +SQ_PSN_SEARCH_START_PSN_MASK = 0xffffff +SQ_PSN_SEARCH_START_PSN_SFT = 0 +SQ_PSN_SEARCH_OPCODE_MASK = 0xff000000 +SQ_PSN_SEARCH_OPCODE_SFT = 24 +SQ_PSN_SEARCH_NEXT_PSN_MASK = 0xffffff +SQ_PSN_SEARCH_NEXT_PSN_SFT = 0 +SQ_PSN_SEARCH_FLAGS_MASK = 0xff000000 +SQ_PSN_SEARCH_FLAGS_SFT = 24 +SQ_PSN_SEARCH_EXT_START_PSN_MASK = 0xffffff +SQ_PSN_SEARCH_EXT_START_PSN_SFT = 0 +SQ_PSN_SEARCH_EXT_OPCODE_MASK = 0xff000000 +SQ_PSN_SEARCH_EXT_OPCODE_SFT = 24 +SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK = 0xffffff +SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT = 0 +SQ_PSN_SEARCH_EXT_FLAGS_MASK = 0xff000000 +SQ_PSN_SEARCH_EXT_FLAGS_SFT = 24 +SQ_MSN_SEARCH_START_PSN_MASK = 0xffffff +SQ_MSN_SEARCH_START_PSN_SFT = 0 +SQ_MSN_SEARCH_NEXT_PSN_MASK = 0xffffff000000 +SQ_MSN_SEARCH_NEXT_PSN_SFT = 24 +SQ_MSN_SEARCH_START_IDX_MASK = 0xffff000000000000 +SQ_MSN_SEARCH_START_IDX_SFT = 48 +SQ_SEND_WQE_TYPE_SEND = 0x0 +SQ_SEND_WQE_TYPE_SEND_W_IMMEAD = 0x1 +SQ_SEND_WQE_TYPE_SEND_W_INVALID = 0x2 +SQ_SEND_WQE_TYPE_LAST = SQ_SEND_WQE_TYPE_SEND_W_INVALID +SQ_SEND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_SEND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_SEND_FLAGS_SIGNAL_COMP = 0x1 +SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_SEND_FLAGS_UC_FENCE = 0x4 +SQ_SEND_FLAGS_SE = 0x8 +SQ_SEND_FLAGS_INLINE = 0x10 +SQ_SEND_FLAGS_WQE_TS_EN = 0x20 +SQ_SEND_FLAGS_DEBUG_TRACE = 0x40 +SQ_SEND_DST_QP_MASK = 0xffffff +SQ_SEND_DST_QP_SFT = 0 +SQ_SEND_AVID_MASK = 0xfffff +SQ_SEND_AVID_SFT = 0 +SQ_SEND_TIMESTAMP_MASK = 0xffffff +SQ_SEND_TIMESTAMP_SFT = 0 +SQ_SEND_HDR_WQE_TYPE_SEND = 0x0 +SQ_SEND_HDR_WQE_TYPE_SEND_W_IMMEAD = 0x1 +SQ_SEND_HDR_WQE_TYPE_SEND_W_INVALID = 0x2 +SQ_SEND_HDR_WQE_TYPE_LAST = SQ_SEND_HDR_WQE_TYPE_SEND_W_INVALID +SQ_SEND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_SEND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_SEND_HDR_FLAGS_SIGNAL_COMP = 0x1 +SQ_SEND_HDR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_SEND_HDR_FLAGS_UC_FENCE = 0x4 +SQ_SEND_HDR_FLAGS_SE = 0x8 +SQ_SEND_HDR_FLAGS_INLINE = 0x10 +SQ_SEND_HDR_FLAGS_WQE_TS_EN = 0x20 +SQ_SEND_HDR_FLAGS_DEBUG_TRACE = 0x40 +SQ_SEND_HDR_DST_QP_MASK = 0xffffff +SQ_SEND_HDR_DST_QP_SFT = 0 +SQ_SEND_HDR_AVID_MASK = 0xfffff +SQ_SEND_HDR_AVID_SFT = 0 +SQ_SEND_HDR_TIMESTAMP_MASK = 0xffffff +SQ_SEND_HDR_TIMESTAMP_SFT = 0 +SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND = 0x0 +SQ_SEND_RAWETH_QP1_WQE_TYPE_LAST = SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND +SQ_SEND_RAWETH_QP1_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_SEND_RAWETH_QP1_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP = 0x1 +SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE = 0x4 +SQ_SEND_RAWETH_QP1_FLAGS_SE = 0x8 +SQ_SEND_RAWETH_QP1_FLAGS_INLINE = 0x10 +SQ_SEND_RAWETH_QP1_FLAGS_WQE_TS_EN = 0x20 +SQ_SEND_RAWETH_QP1_FLAGS_DEBUG_TRACE = 0x40 +SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM = 0x1 +SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM = 0x2 +SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC = 0x4 +SQ_SEND_RAWETH_QP1_LFLAGS_STAMP = 0x8 +SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM = 0x10 +SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC = 0x100 +SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC = 0x200 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK = 0xfff +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT = 0 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE = 0x1000 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK = 0xe000 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT = 13 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK = 0x70000 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT = 16 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 = (0x0 << 16) +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 = (0x1 << 16) +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 = (0x2 << 16) +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 = (0x3 << 16) +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 = (0x4 << 16) +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG = (0x5 << 16) +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST = SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK = 0xff80000 +SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT = 19 +SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK = 0xf0000000 +SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT = 28 +SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE = (0x0 << 28) +SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG = (0x1 << 28) +SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST = SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG +SQ_SEND_RAWETH_QP1_TIMESTAMP_MASK = 0xffffff +SQ_SEND_RAWETH_QP1_TIMESTAMP_SFT = 0 +SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_SEND = 0x0 +SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_LAST = SQ_SEND_RAWETH_QP1_HDR_WQE_TYPE_SEND +SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_SEND_RAWETH_QP1_HDR_FLAGS_SIGNAL_COMP = 0x1 +SQ_SEND_RAWETH_QP1_HDR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_SEND_RAWETH_QP1_HDR_FLAGS_UC_FENCE = 0x4 +SQ_SEND_RAWETH_QP1_HDR_FLAGS_SE = 0x8 +SQ_SEND_RAWETH_QP1_HDR_FLAGS_INLINE = 0x10 +SQ_SEND_RAWETH_QP1_HDR_FLAGS_WQE_TS_EN = 0x20 +SQ_SEND_RAWETH_QP1_HDR_FLAGS_DEBUG_TRACE = 0x40 +SQ_SEND_RAWETH_QP1_HDR_LFLAGS_TCP_UDP_CHKSUM = 0x1 +SQ_SEND_RAWETH_QP1_HDR_LFLAGS_IP_CHKSUM = 0x2 +SQ_SEND_RAWETH_QP1_HDR_LFLAGS_NOCRC = 0x4 +SQ_SEND_RAWETH_QP1_HDR_LFLAGS_STAMP = 0x8 +SQ_SEND_RAWETH_QP1_HDR_LFLAGS_T_IP_CHKSUM = 0x10 +SQ_SEND_RAWETH_QP1_HDR_LFLAGS_ROCE_CRC = 0x100 +SQ_SEND_RAWETH_QP1_HDR_LFLAGS_FCOE_CRC = 0x200 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_VID_MASK = 0xfff +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_VID_SFT = 0 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_DE = 0x1000 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_PRI_MASK = 0xe000 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_PRI_SFT = 13 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_MASK = 0x70000 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_SFT = 16 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID88A8 = (0x0 << 16) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID8100 = (0x1 << 16) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9100 = (0x2 << 16) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9200 = (0x3 << 16) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPID9300 = (0x4 << 16) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPIDCFG = (0x5 << 16) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_LAST = SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_TPID_TPIDCFG +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_RESERVED_MASK = 0xff80000 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_VLAN_RESERVED_SFT = 19 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_MASK = 0xf0000000 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_SFT = 28 +SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_NONE = (0x0 << 28) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_VLAN_TAG = (0x1 << 28) +SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_LAST = SQ_SEND_RAWETH_QP1_HDR_CFA_META_KEY_VLAN_TAG +SQ_SEND_RAWETH_QP1_HDR_TIMESTAMP_MASK = 0xffffff +SQ_SEND_RAWETH_QP1_HDR_TIMESTAMP_SFT = 0 +SQ_RDMA_WQE_TYPE_WRITE_WQE = 0x4 +SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD = 0x5 +SQ_RDMA_WQE_TYPE_READ_WQE = 0x6 +SQ_RDMA_WQE_TYPE_LAST = SQ_RDMA_WQE_TYPE_READ_WQE +SQ_RDMA_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_RDMA_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_RDMA_FLAGS_SIGNAL_COMP = 0x1 +SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_RDMA_FLAGS_UC_FENCE = 0x4 +SQ_RDMA_FLAGS_SE = 0x8 +SQ_RDMA_FLAGS_INLINE = 0x10 +SQ_RDMA_FLAGS_WQE_TS_EN = 0x20 +SQ_RDMA_FLAGS_DEBUG_TRACE = 0x40 +SQ_RDMA_TIMESTAMP_MASK = 0xffffff +SQ_RDMA_TIMESTAMP_SFT = 0 +SQ_RDMA_HDR_WQE_TYPE_WRITE_WQE = 0x4 +SQ_RDMA_HDR_WQE_TYPE_WRITE_W_IMMEAD = 0x5 +SQ_RDMA_HDR_WQE_TYPE_READ_WQE = 0x6 +SQ_RDMA_HDR_WQE_TYPE_LAST = SQ_RDMA_HDR_WQE_TYPE_READ_WQE +SQ_RDMA_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_RDMA_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_RDMA_HDR_FLAGS_SIGNAL_COMP = 0x1 +SQ_RDMA_HDR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_RDMA_HDR_FLAGS_UC_FENCE = 0x4 +SQ_RDMA_HDR_FLAGS_SE = 0x8 +SQ_RDMA_HDR_FLAGS_INLINE = 0x10 +SQ_RDMA_HDR_FLAGS_WQE_TS_EN = 0x20 +SQ_RDMA_HDR_FLAGS_DEBUG_TRACE = 0x40 +SQ_RDMA_HDR_TIMESTAMP_MASK = 0xffffff +SQ_RDMA_HDR_TIMESTAMP_SFT = 0 +SQ_ATOMIC_WQE_TYPE_ATOMIC_CS = 0x8 +SQ_ATOMIC_WQE_TYPE_ATOMIC_FA = 0xb +SQ_ATOMIC_WQE_TYPE_LAST = SQ_ATOMIC_WQE_TYPE_ATOMIC_FA +SQ_ATOMIC_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_ATOMIC_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_ATOMIC_FLAGS_SIGNAL_COMP = 0x1 +SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_ATOMIC_FLAGS_UC_FENCE = 0x4 +SQ_ATOMIC_FLAGS_SE = 0x8 +SQ_ATOMIC_FLAGS_INLINE = 0x10 +SQ_ATOMIC_FLAGS_WQE_TS_EN = 0x20 +SQ_ATOMIC_FLAGS_DEBUG_TRACE = 0x40 +SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_CS = 0x8 +SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_FA = 0xb +SQ_ATOMIC_HDR_WQE_TYPE_LAST = SQ_ATOMIC_HDR_WQE_TYPE_ATOMIC_FA +SQ_ATOMIC_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_ATOMIC_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_ATOMIC_HDR_FLAGS_SIGNAL_COMP = 0x1 +SQ_ATOMIC_HDR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_ATOMIC_HDR_FLAGS_UC_FENCE = 0x4 +SQ_ATOMIC_HDR_FLAGS_SE = 0x8 +SQ_ATOMIC_HDR_FLAGS_INLINE = 0x10 +SQ_ATOMIC_HDR_FLAGS_WQE_TS_EN = 0x20 +SQ_ATOMIC_HDR_FLAGS_DEBUG_TRACE = 0x40 +SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID = 0xc +SQ_LOCALINVALIDATE_WQE_TYPE_LAST = SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID +SQ_LOCALINVALIDATE_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_LOCALINVALIDATE_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP = 0x1 +SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_LOCALINVALIDATE_FLAGS_UC_FENCE = 0x4 +SQ_LOCALINVALIDATE_FLAGS_SE = 0x8 +SQ_LOCALINVALIDATE_FLAGS_INLINE = 0x10 +SQ_LOCALINVALIDATE_FLAGS_WQE_TS_EN = 0x20 +SQ_LOCALINVALIDATE_FLAGS_DEBUG_TRACE = 0x40 +SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LOCAL_INVALID = 0xc +SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LAST = SQ_LOCALINVALIDATE_HDR_WQE_TYPE_LOCAL_INVALID +SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_LOCALINVALIDATE_HDR_FLAGS_SIGNAL_COMP = 0x1 +SQ_LOCALINVALIDATE_HDR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_LOCALINVALIDATE_HDR_FLAGS_UC_FENCE = 0x4 +SQ_LOCALINVALIDATE_HDR_FLAGS_SE = 0x8 +SQ_LOCALINVALIDATE_HDR_FLAGS_INLINE = 0x10 +SQ_LOCALINVALIDATE_HDR_FLAGS_WQE_TS_EN = 0x20 +SQ_LOCALINVALIDATE_HDR_FLAGS_DEBUG_TRACE = 0x40 +SQ_FR_PMR_WQE_TYPE_FR_PMR = 0xd +SQ_FR_PMR_WQE_TYPE_LAST = SQ_FR_PMR_WQE_TYPE_FR_PMR +SQ_FR_PMR_FLAGS_SIGNAL_COMP = 0x1 +SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_FR_PMR_FLAGS_UC_FENCE = 0x4 +SQ_FR_PMR_FLAGS_SE = 0x8 +SQ_FR_PMR_FLAGS_INLINE = 0x10 +SQ_FR_PMR_FLAGS_WQE_TS_EN = 0x20 +SQ_FR_PMR_FLAGS_DEBUG_TRACE = 0x40 +SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE = 0x1 +SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ = 0x2 +SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE = 0x4 +SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC = 0x8 +SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND = 0x10 +SQ_FR_PMR_PAGE_SIZE_LOG_MASK = 0x1f +SQ_FR_PMR_PAGE_SIZE_LOG_SFT = 0 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K = 0x0 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K = 0x1 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16K = 0x2 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32K = 0x3 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K = 0x4 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128K = 0x5 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K = 0x6 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512K = 0x7 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M = 0x8 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M = 0x9 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M = 0xa +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8M = 0xb +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16M = 0xc +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32M = 0xd +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64M = 0xe +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128M = 0xf +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256M = 0x10 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512M = 0x11 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G = 0x12 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2G = 0x13 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4G = 0x14 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8G = 0x15 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_16G = 0x16 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_32G = 0x17 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64G = 0x18 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_128G = 0x19 +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256G = 0x1a +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_512G = 0x1b +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1T = 0x1c +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2T = 0x1d +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4T = 0x1e +SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8T = 0x1f +SQ_FR_PMR_PAGE_SIZE_LOG_LAST = SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8T +SQ_FR_PMR_ZERO_BASED = 0x20 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK = 0x1f +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT = 0 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K = 0x0 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K = 0x1 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16K = 0x2 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32K = 0x3 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K = 0x4 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128K = 0x5 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K = 0x6 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512K = 0x7 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M = 0x8 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M = 0x9 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M = 0xa +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8M = 0xb +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16M = 0xc +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32M = 0xd +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64M = 0xe +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128M = 0xf +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256M = 0x10 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512M = 0x11 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G = 0x12 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2G = 0x13 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4G = 0x14 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8G = 0x15 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_16G = 0x16 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_32G = 0x17 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64G = 0x18 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_128G = 0x19 +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256G = 0x1a +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_512G = 0x1b +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1T = 0x1c +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2T = 0x1d +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4T = 0x1e +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8T = 0x1f +SQ_FR_PMR_PBL_PAGE_SIZE_LOG_LAST = SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8T +SQ_FR_PMR_NUMLEVELS_MASK = 0xc0 +SQ_FR_PMR_NUMLEVELS_SFT = 6 +SQ_FR_PMR_NUMLEVELS_PHYSICAL = (0x0 << 6) +SQ_FR_PMR_NUMLEVELS_LAYER1 = (0x1 << 6) +SQ_FR_PMR_NUMLEVELS_LAYER2 = (0x2 << 6) +SQ_FR_PMR_NUMLEVELS_LAST = SQ_FR_PMR_NUMLEVELS_LAYER2 +SQ_FR_PMR_HDR_WQE_TYPE_FR_PMR = 0xd +SQ_FR_PMR_HDR_WQE_TYPE_LAST = SQ_FR_PMR_HDR_WQE_TYPE_FR_PMR +SQ_FR_PMR_HDR_FLAGS_SIGNAL_COMP = 0x1 +SQ_FR_PMR_HDR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_FR_PMR_HDR_FLAGS_UC_FENCE = 0x4 +SQ_FR_PMR_HDR_FLAGS_SE = 0x8 +SQ_FR_PMR_HDR_FLAGS_INLINE = 0x10 +SQ_FR_PMR_HDR_FLAGS_WQE_TS_EN = 0x20 +SQ_FR_PMR_HDR_FLAGS_DEBUG_TRACE = 0x40 +SQ_FR_PMR_HDR_ACCESS_CNTL_LOCAL_WRITE = 0x1 +SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_READ = 0x2 +SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_WRITE = 0x4 +SQ_FR_PMR_HDR_ACCESS_CNTL_REMOTE_ATOMIC = 0x8 +SQ_FR_PMR_HDR_ACCESS_CNTL_WINDOW_BIND = 0x10 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_MASK = 0x1f +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_SFT = 0 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4K = 0x0 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8K = 0x1 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16K = 0x2 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32K = 0x3 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64K = 0x4 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128K = 0x5 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256K = 0x6 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512K = 0x7 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1M = 0x8 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2M = 0x9 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4M = 0xa +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8M = 0xb +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16M = 0xc +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32M = 0xd +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64M = 0xe +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128M = 0xf +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256M = 0x10 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512M = 0x11 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1G = 0x12 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2G = 0x13 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4G = 0x14 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8G = 0x15 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_16G = 0x16 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_32G = 0x17 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_64G = 0x18 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_128G = 0x19 +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_256G = 0x1a +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_512G = 0x1b +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_1T = 0x1c +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_2T = 0x1d +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_4T = 0x1e +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8T = 0x1f +SQ_FR_PMR_HDR_PAGE_SIZE_LOG_LAST = SQ_FR_PMR_HDR_PAGE_SIZE_LOG_PGSZ_8T +SQ_FR_PMR_HDR_ZERO_BASED = 0x20 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_MASK = 0x1f +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_SFT = 0 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4K = 0x0 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8K = 0x1 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16K = 0x2 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32K = 0x3 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64K = 0x4 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128K = 0x5 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256K = 0x6 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512K = 0x7 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1M = 0x8 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2M = 0x9 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4M = 0xa +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8M = 0xb +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16M = 0xc +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32M = 0xd +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64M = 0xe +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128M = 0xf +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256M = 0x10 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512M = 0x11 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1G = 0x12 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2G = 0x13 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4G = 0x14 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8G = 0x15 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_16G = 0x16 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_32G = 0x17 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_64G = 0x18 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_128G = 0x19 +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_256G = 0x1a +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_512G = 0x1b +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_1T = 0x1c +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_2T = 0x1d +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_4T = 0x1e +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8T = 0x1f +SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_LAST = SQ_FR_PMR_HDR_PBL_PAGE_SIZE_LOG_PGSZ_8T +SQ_FR_PMR_HDR_NUMLEVELS_MASK = 0xc0 +SQ_FR_PMR_HDR_NUMLEVELS_SFT = 6 +SQ_FR_PMR_HDR_NUMLEVELS_PHYSICAL = (0x0 << 6) +SQ_FR_PMR_HDR_NUMLEVELS_LAYER1 = (0x1 << 6) +SQ_FR_PMR_HDR_NUMLEVELS_LAYER2 = (0x2 << 6) +SQ_FR_PMR_HDR_NUMLEVELS_LAST = SQ_FR_PMR_HDR_NUMLEVELS_LAYER2 +SQ_BIND_WQE_TYPE_BIND = 0xe +SQ_BIND_WQE_TYPE_LAST = SQ_BIND_WQE_TYPE_BIND +SQ_BIND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_BIND_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_BIND_FLAGS_SIGNAL_COMP = 0x1 +SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_BIND_FLAGS_UC_FENCE = 0x4 +SQ_BIND_FLAGS_SE = 0x8 +SQ_BIND_FLAGS_INLINE = 0x10 +SQ_BIND_FLAGS_WQE_TS_EN = 0x20 +SQ_BIND_FLAGS_DEBUG_TRACE = 0x40 +SQ_BIND_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_MASK = 0xff +SQ_BIND_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_SFT = 0 +SQ_BIND_ACCESS_CNTL_LOCAL_WRITE = 0x1 +SQ_BIND_ACCESS_CNTL_REMOTE_READ = 0x2 +SQ_BIND_ACCESS_CNTL_REMOTE_WRITE = 0x4 +SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC = 0x8 +SQ_BIND_ACCESS_CNTL_WINDOW_BIND = 0x10 +SQ_BIND_ZERO_BASED = 0x1 +SQ_BIND_MW_TYPE = 0x2 +SQ_BIND_MW_TYPE_TYPE1 = (0x0 << 1) +SQ_BIND_MW_TYPE_TYPE2 = (0x1 << 1) +SQ_BIND_MW_TYPE_LAST = SQ_BIND_MW_TYPE_TYPE2 +SQ_BIND_HDR_WQE_TYPE_BIND = 0xe +SQ_BIND_HDR_WQE_TYPE_LAST = SQ_BIND_HDR_WQE_TYPE_BIND +SQ_BIND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_MASK = 0xff +SQ_BIND_HDR_FLAGS_INLINE_SE_UC_FENCE_RD_OR_ATOMIC_FENCE_SIGNAL_COMP_SFT = 0 +SQ_BIND_HDR_FLAGS_SIGNAL_COMP = 0x1 +SQ_BIND_HDR_FLAGS_RD_OR_ATOMIC_FENCE = 0x2 +SQ_BIND_HDR_FLAGS_UC_FENCE = 0x4 +SQ_BIND_HDR_FLAGS_SE = 0x8 +SQ_BIND_HDR_FLAGS_INLINE = 0x10 +SQ_BIND_HDR_FLAGS_WQE_TS_EN = 0x20 +SQ_BIND_HDR_FLAGS_DEBUG_TRACE = 0x40 +SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_MASK = 0xff +SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND_REMOTE_ATOMIC_REMOTE_WRITE_REMOTE_READ_LOCAL_WRITE_SFT = 0 +SQ_BIND_HDR_ACCESS_CNTL_LOCAL_WRITE = 0x1 +SQ_BIND_HDR_ACCESS_CNTL_REMOTE_READ = 0x2 +SQ_BIND_HDR_ACCESS_CNTL_REMOTE_WRITE = 0x4 +SQ_BIND_HDR_ACCESS_CNTL_REMOTE_ATOMIC = 0x8 +SQ_BIND_HDR_ACCESS_CNTL_WINDOW_BIND = 0x10 +SQ_BIND_HDR_ZERO_BASED = 0x1 +SQ_BIND_HDR_MW_TYPE = 0x2 +SQ_BIND_HDR_MW_TYPE_TYPE1 = (0x0 << 1) +SQ_BIND_HDR_MW_TYPE_TYPE2 = (0x1 << 1) +SQ_BIND_HDR_MW_TYPE_LAST = SQ_BIND_HDR_MW_TYPE_TYPE2 +CQ_BASE_TOGGLE = 0x1 +CQ_BASE_CQE_TYPE_MASK = 0x1e +CQ_BASE_CQE_TYPE_SFT = 1 +CQ_BASE_CQE_TYPE_REQ = (0x0 << 1) +CQ_BASE_CQE_TYPE_RES_RC = (0x1 << 1) +CQ_BASE_CQE_TYPE_RES_UD = (0x2 << 1) +CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 = (0x3 << 1) +CQ_BASE_CQE_TYPE_RES_UD_CFA = (0x4 << 1) +CQ_BASE_CQE_TYPE_REQ_V3 = (0x8 << 1) +CQ_BASE_CQE_TYPE_RES_RC_V3 = (0x9 << 1) +CQ_BASE_CQE_TYPE_RES_UD_V3 = (0xa << 1) +CQ_BASE_CQE_TYPE_RES_RAWETH_QP1_V3 = (0xb << 1) +CQ_BASE_CQE_TYPE_RES_UD_CFA_V3 = (0xc << 1) +CQ_BASE_CQE_TYPE_NO_OP = (0xd << 1) +CQ_BASE_CQE_TYPE_TERMINAL = (0xe << 1) +CQ_BASE_CQE_TYPE_CUT_OFF = (0xf << 1) +CQ_BASE_CQE_TYPE_LAST = CQ_BASE_CQE_TYPE_CUT_OFF +CQ_BASE_STATUS_OK = 0x0 +CQ_BASE_STATUS_BAD_RESPONSE_ERR = 0x1 +CQ_BASE_STATUS_LOCAL_LENGTH_ERR = 0x2 +CQ_BASE_STATUS_HW_LOCAL_LENGTH_ERR = 0x3 +CQ_BASE_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_BASE_STATUS_LOCAL_PROTECTION_ERR = 0x5 +CQ_BASE_STATUS_LOCAL_ACCESS_ERROR = 0x6 +CQ_BASE_STATUS_MEMORY_MGT_OPERATION_ERR = 0x7 +CQ_BASE_STATUS_REMOTE_INVALID_REQUEST_ERR = 0x8 +CQ_BASE_STATUS_REMOTE_ACCESS_ERR = 0x9 +CQ_BASE_STATUS_REMOTE_OPERATION_ERR = 0xa +CQ_BASE_STATUS_RNR_NAK_RETRY_CNT_ERR = 0xb +CQ_BASE_STATUS_TRANSPORT_RETRY_CNT_ERR = 0xc +CQ_BASE_STATUS_WORK_REQUEST_FLUSHED_ERR = 0xd +CQ_BASE_STATUS_HW_FLUSH_ERR = 0xe +CQ_BASE_STATUS_OVERFLOW_ERR = 0xf +CQ_BASE_STATUS_LAST = CQ_BASE_STATUS_OVERFLOW_ERR +CQ_REQ_TOGGLE = 0x1 +CQ_REQ_CQE_TYPE_MASK = 0x1e +CQ_REQ_CQE_TYPE_SFT = 1 +CQ_REQ_CQE_TYPE_REQ = (0x0 << 1) +CQ_REQ_CQE_TYPE_LAST = CQ_REQ_CQE_TYPE_REQ +CQ_REQ_PUSH = 0x20 +CQ_REQ_STATUS_OK = 0x0 +CQ_REQ_STATUS_BAD_RESPONSE_ERR = 0x1 +CQ_REQ_STATUS_LOCAL_LENGTH_ERR = 0x2 +CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR = 0x3 +CQ_REQ_STATUS_LOCAL_PROTECTION_ERR = 0x4 +CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR = 0x6 +CQ_REQ_STATUS_REMOTE_ACCESS_ERR = 0x7 +CQ_REQ_STATUS_REMOTE_OPERATION_ERR = 0x8 +CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR = 0x9 +CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR = 0xa +CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR = 0xb +CQ_REQ_STATUS_LAST = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR +CQ_RES_RC_TOGGLE = 0x1 +CQ_RES_RC_CQE_TYPE_MASK = 0x1e +CQ_RES_RC_CQE_TYPE_SFT = 1 +CQ_RES_RC_CQE_TYPE_RES_RC = (0x1 << 1) +CQ_RES_RC_CQE_TYPE_LAST = CQ_RES_RC_CQE_TYPE_RES_RC +CQ_RES_RC_STATUS_OK = 0x0 +CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR = 0x1 +CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR = 0x2 +CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR = 0x3 +CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR = 0x6 +CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR = 0x7 +CQ_RES_RC_STATUS_HW_FLUSH_ERR = 0x8 +CQ_RES_RC_STATUS_LAST = CQ_RES_RC_STATUS_HW_FLUSH_ERR +CQ_RES_RC_FLAGS_SRQ = 0x1 +CQ_RES_RC_FLAGS_SRQ_RQ = 0x0 +CQ_RES_RC_FLAGS_SRQ_SRQ = 0x1 +CQ_RES_RC_FLAGS_SRQ_LAST = CQ_RES_RC_FLAGS_SRQ_SRQ +CQ_RES_RC_FLAGS_IMM = 0x2 +CQ_RES_RC_FLAGS_INV = 0x4 +CQ_RES_RC_FLAGS_RDMA = 0x8 +CQ_RES_RC_FLAGS_RDMA_SEND = (0x0 << 3) +CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE = (0x1 << 3) +CQ_RES_RC_FLAGS_RDMA_LAST = CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE +CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK = 0xfffff +CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT = 0 +CQ_RES_UD_LENGTH_MASK = 0x3fff +CQ_RES_UD_LENGTH_SFT = 0 +CQ_RES_UD_CFA_METADATA_VID_MASK = 0xfff +CQ_RES_UD_CFA_METADATA_VID_SFT = 0 +CQ_RES_UD_CFA_METADATA_DE = 0x1000 +CQ_RES_UD_CFA_METADATA_PRI_MASK = 0xe000 +CQ_RES_UD_CFA_METADATA_PRI_SFT = 13 +CQ_RES_UD_TOGGLE = 0x1 +CQ_RES_UD_CQE_TYPE_MASK = 0x1e +CQ_RES_UD_CQE_TYPE_SFT = 1 +CQ_RES_UD_CQE_TYPE_RES_UD = (0x2 << 1) +CQ_RES_UD_CQE_TYPE_LAST = CQ_RES_UD_CQE_TYPE_RES_UD +CQ_RES_UD_STATUS_OK = 0x0 +CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR = 0x1 +CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR = 0x2 +CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR = 0x3 +CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR = 0x7 +CQ_RES_UD_STATUS_HW_FLUSH_ERR = 0x8 +CQ_RES_UD_STATUS_LAST = CQ_RES_UD_STATUS_HW_FLUSH_ERR +CQ_RES_UD_FLAGS_SRQ = 0x1 +CQ_RES_UD_FLAGS_SRQ_RQ = 0x0 +CQ_RES_UD_FLAGS_SRQ_SRQ = 0x1 +CQ_RES_UD_FLAGS_SRQ_LAST = CQ_RES_UD_FLAGS_SRQ_SRQ +CQ_RES_UD_FLAGS_IMM = 0x2 +CQ_RES_UD_FLAGS_UNUSED_MASK = 0xc +CQ_RES_UD_FLAGS_UNUSED_SFT = 2 +CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK = 0x30 +CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT = 4 +CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 = (0x0 << 4) +CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 = (0x2 << 4) +CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 = (0x3 << 4) +CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST = CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 +CQ_RES_UD_FLAGS_META_FORMAT_MASK = 0x3c0 +CQ_RES_UD_FLAGS_META_FORMAT_SFT = 6 +CQ_RES_UD_FLAGS_META_FORMAT_NONE = (0x0 << 6) +CQ_RES_UD_FLAGS_META_FORMAT_VLAN = (0x1 << 6) +CQ_RES_UD_FLAGS_META_FORMAT_TUNNEL_ID = (0x2 << 6) +CQ_RES_UD_FLAGS_META_FORMAT_CHDR_DATA = (0x3 << 6) +CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET = (0x4 << 6) +CQ_RES_UD_FLAGS_META_FORMAT_LAST = CQ_RES_UD_FLAGS_META_FORMAT_HDR_OFFSET +CQ_RES_UD_FLAGS_EXT_META_FORMAT_MASK = 0xc00 +CQ_RES_UD_FLAGS_EXT_META_FORMAT_SFT = 10 +CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK = 0xfffff +CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT = 0 +CQ_RES_UD_SRC_QP_HIGH_MASK = 0xff000000 +CQ_RES_UD_SRC_QP_HIGH_SFT = 24 +CQ_RES_UD_V2_LENGTH_MASK = 0x3fff +CQ_RES_UD_V2_LENGTH_SFT = 0 +CQ_RES_UD_V2_CFA_METADATA0_VID_MASK = 0xfff +CQ_RES_UD_V2_CFA_METADATA0_VID_SFT = 0 +CQ_RES_UD_V2_CFA_METADATA0_DE = 0x1000 +CQ_RES_UD_V2_CFA_METADATA0_PRI_MASK = 0xe000 +CQ_RES_UD_V2_CFA_METADATA0_PRI_SFT = 13 +CQ_RES_UD_V2_TOGGLE = 0x1 +CQ_RES_UD_V2_CQE_TYPE_MASK = 0x1e +CQ_RES_UD_V2_CQE_TYPE_SFT = 1 +CQ_RES_UD_V2_CQE_TYPE_RES_UD = (0x2 << 1) +CQ_RES_UD_V2_CQE_TYPE_LAST = CQ_RES_UD_V2_CQE_TYPE_RES_UD +CQ_RES_UD_V2_STATUS_OK = 0x0 +CQ_RES_UD_V2_STATUS_LOCAL_ACCESS_ERROR = 0x1 +CQ_RES_UD_V2_STATUS_HW_LOCAL_LENGTH_ERR = 0x2 +CQ_RES_UD_V2_STATUS_LOCAL_PROTECTION_ERR = 0x3 +CQ_RES_UD_V2_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_RES_UD_V2_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_RES_UD_V2_STATUS_WORK_REQUEST_FLUSHED_ERR = 0x7 +CQ_RES_UD_V2_STATUS_HW_FLUSH_ERR = 0x8 +CQ_RES_UD_V2_STATUS_LAST = CQ_RES_UD_V2_STATUS_HW_FLUSH_ERR +CQ_RES_UD_V2_FLAGS_SRQ = 0x1 +CQ_RES_UD_V2_FLAGS_SRQ_RQ = 0x0 +CQ_RES_UD_V2_FLAGS_SRQ_SRQ = 0x1 +CQ_RES_UD_V2_FLAGS_SRQ_LAST = CQ_RES_UD_V2_FLAGS_SRQ_SRQ +CQ_RES_UD_V2_FLAGS_IMM = 0x2 +CQ_RES_UD_V2_FLAGS_UNUSED_MASK = 0xc +CQ_RES_UD_V2_FLAGS_UNUSED_SFT = 2 +CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_MASK = 0x30 +CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_SFT = 4 +CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V1 = (0x0 << 4) +CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV4 = (0x2 << 4) +CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV6 = (0x3 << 4) +CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_LAST = CQ_RES_UD_V2_FLAGS_ROCE_IP_VER_V2IPV6 +CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK = 0x3c0 +CQ_RES_UD_V2_FLAGS_META_FORMAT_SFT = 6 +CQ_RES_UD_V2_FLAGS_META_FORMAT_NONE = (0x0 << 6) +CQ_RES_UD_V2_FLAGS_META_FORMAT_ACT_REC_PTR = (0x1 << 6) +CQ_RES_UD_V2_FLAGS_META_FORMAT_TUNNEL_ID = (0x2 << 6) +CQ_RES_UD_V2_FLAGS_META_FORMAT_CHDR_DATA = (0x3 << 6) +CQ_RES_UD_V2_FLAGS_META_FORMAT_HDR_OFFSET = (0x4 << 6) +CQ_RES_UD_V2_FLAGS_META_FORMAT_LAST = CQ_RES_UD_V2_FLAGS_META_FORMAT_HDR_OFFSET +CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK = 0xfffff +CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_SFT = 0 +CQ_RES_UD_V2_CFA_METADATA1_MASK = 0xf00000 +CQ_RES_UD_V2_CFA_METADATA1_SFT = 20 +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_MASK = 0x700000 +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_SFT = 20 +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID88A8 = (0x0 << 20) +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID8100 = (0x1 << 20) +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9100 = (0x2 << 20) +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9200 = (0x3 << 20) +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPID9300 = (0x4 << 20) +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPIDCFG = (0x5 << 20) +CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_LAST = CQ_RES_UD_V2_CFA_METADATA1_TPID_SEL_TPIDCFG +CQ_RES_UD_V2_CFA_METADATA1_VALID = 0x800000 +CQ_RES_UD_V2_SRC_QP_HIGH_MASK = 0xff000000 +CQ_RES_UD_V2_SRC_QP_HIGH_SFT = 24 +CQ_RES_UD_CFA_LENGTH_MASK = 0x3fff +CQ_RES_UD_CFA_LENGTH_SFT = 0 +CQ_RES_UD_CFA_QID_MASK = 0xfffff +CQ_RES_UD_CFA_QID_SFT = 0 +CQ_RES_UD_CFA_CFA_METADATA_VID_MASK = 0xfff +CQ_RES_UD_CFA_CFA_METADATA_VID_SFT = 0 +CQ_RES_UD_CFA_CFA_METADATA_DE = 0x1000 +CQ_RES_UD_CFA_CFA_METADATA_PRI_MASK = 0xe000 +CQ_RES_UD_CFA_CFA_METADATA_PRI_SFT = 13 +CQ_RES_UD_CFA_CFA_METADATA_TPID_MASK = 0xffff0000 +CQ_RES_UD_CFA_CFA_METADATA_TPID_SFT = 16 +CQ_RES_UD_CFA_TOGGLE = 0x1 +CQ_RES_UD_CFA_CQE_TYPE_MASK = 0x1e +CQ_RES_UD_CFA_CQE_TYPE_SFT = 1 +CQ_RES_UD_CFA_CQE_TYPE_RES_UD_CFA = (0x4 << 1) +CQ_RES_UD_CFA_CQE_TYPE_LAST = CQ_RES_UD_CFA_CQE_TYPE_RES_UD_CFA +CQ_RES_UD_CFA_STATUS_OK = 0x0 +CQ_RES_UD_CFA_STATUS_LOCAL_ACCESS_ERROR = 0x1 +CQ_RES_UD_CFA_STATUS_HW_LOCAL_LENGTH_ERR = 0x2 +CQ_RES_UD_CFA_STATUS_LOCAL_PROTECTION_ERR = 0x3 +CQ_RES_UD_CFA_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_RES_UD_CFA_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_RES_UD_CFA_STATUS_WORK_REQUEST_FLUSHED_ERR = 0x7 +CQ_RES_UD_CFA_STATUS_HW_FLUSH_ERR = 0x8 +CQ_RES_UD_CFA_STATUS_LAST = CQ_RES_UD_CFA_STATUS_HW_FLUSH_ERR +CQ_RES_UD_CFA_FLAGS_SRQ = 0x1 +CQ_RES_UD_CFA_FLAGS_SRQ_RQ = 0x0 +CQ_RES_UD_CFA_FLAGS_SRQ_SRQ = 0x1 +CQ_RES_UD_CFA_FLAGS_SRQ_LAST = CQ_RES_UD_CFA_FLAGS_SRQ_SRQ +CQ_RES_UD_CFA_FLAGS_IMM = 0x2 +CQ_RES_UD_CFA_FLAGS_UNUSED_MASK = 0xc +CQ_RES_UD_CFA_FLAGS_UNUSED_SFT = 2 +CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_MASK = 0x30 +CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_SFT = 4 +CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V1 = (0x0 << 4) +CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV4 = (0x2 << 4) +CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV6 = (0x3 << 4) +CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_LAST = CQ_RES_UD_CFA_FLAGS_ROCE_IP_VER_V2IPV6 +CQ_RES_UD_CFA_FLAGS_META_FORMAT_MASK = 0x3c0 +CQ_RES_UD_CFA_FLAGS_META_FORMAT_SFT = 6 +CQ_RES_UD_CFA_FLAGS_META_FORMAT_NONE = (0x0 << 6) +CQ_RES_UD_CFA_FLAGS_META_FORMAT_VLAN = (0x1 << 6) +CQ_RES_UD_CFA_FLAGS_META_FORMAT_TUNNEL_ID = (0x2 << 6) +CQ_RES_UD_CFA_FLAGS_META_FORMAT_CHDR_DATA = (0x3 << 6) +CQ_RES_UD_CFA_FLAGS_META_FORMAT_HDR_OFFSET = (0x4 << 6) +CQ_RES_UD_CFA_FLAGS_META_FORMAT_LAST = CQ_RES_UD_CFA_FLAGS_META_FORMAT_HDR_OFFSET +CQ_RES_UD_CFA_FLAGS_EXT_META_FORMAT_MASK = 0xc00 +CQ_RES_UD_CFA_FLAGS_EXT_META_FORMAT_SFT = 10 +CQ_RES_UD_CFA_SRQ_OR_RQ_WR_ID_MASK = 0xfffff +CQ_RES_UD_CFA_SRQ_OR_RQ_WR_ID_SFT = 0 +CQ_RES_UD_CFA_SRC_QP_HIGH_MASK = 0xff000000 +CQ_RES_UD_CFA_SRC_QP_HIGH_SFT = 24 +CQ_RES_UD_CFA_V2_LENGTH_MASK = 0x3fff +CQ_RES_UD_CFA_V2_LENGTH_SFT = 0 +CQ_RES_UD_CFA_V2_CFA_METADATA0_VID_MASK = 0xfff +CQ_RES_UD_CFA_V2_CFA_METADATA0_VID_SFT = 0 +CQ_RES_UD_CFA_V2_CFA_METADATA0_DE = 0x1000 +CQ_RES_UD_CFA_V2_CFA_METADATA0_PRI_MASK = 0xe000 +CQ_RES_UD_CFA_V2_CFA_METADATA0_PRI_SFT = 13 +CQ_RES_UD_CFA_V2_QID_MASK = 0xfffff +CQ_RES_UD_CFA_V2_QID_SFT = 0 +CQ_RES_UD_CFA_V2_TOGGLE = 0x1 +CQ_RES_UD_CFA_V2_CQE_TYPE_MASK = 0x1e +CQ_RES_UD_CFA_V2_CQE_TYPE_SFT = 1 +CQ_RES_UD_CFA_V2_CQE_TYPE_RES_UD_CFA = (0x4 << 1) +CQ_RES_UD_CFA_V2_CQE_TYPE_LAST = CQ_RES_UD_CFA_V2_CQE_TYPE_RES_UD_CFA +CQ_RES_UD_CFA_V2_STATUS_OK = 0x0 +CQ_RES_UD_CFA_V2_STATUS_LOCAL_ACCESS_ERROR = 0x1 +CQ_RES_UD_CFA_V2_STATUS_HW_LOCAL_LENGTH_ERR = 0x2 +CQ_RES_UD_CFA_V2_STATUS_LOCAL_PROTECTION_ERR = 0x3 +CQ_RES_UD_CFA_V2_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_RES_UD_CFA_V2_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_RES_UD_CFA_V2_STATUS_WORK_REQUEST_FLUSHED_ERR = 0x7 +CQ_RES_UD_CFA_V2_STATUS_HW_FLUSH_ERR = 0x8 +CQ_RES_UD_CFA_V2_STATUS_LAST = CQ_RES_UD_CFA_V2_STATUS_HW_FLUSH_ERR +CQ_RES_UD_CFA_V2_FLAGS_SRQ = 0x1 +CQ_RES_UD_CFA_V2_FLAGS_SRQ_RQ = 0x0 +CQ_RES_UD_CFA_V2_FLAGS_SRQ_SRQ = 0x1 +CQ_RES_UD_CFA_V2_FLAGS_SRQ_LAST = CQ_RES_UD_CFA_V2_FLAGS_SRQ_SRQ +CQ_RES_UD_CFA_V2_FLAGS_IMM = 0x2 +CQ_RES_UD_CFA_V2_FLAGS_UNUSED_MASK = 0xc +CQ_RES_UD_CFA_V2_FLAGS_UNUSED_SFT = 2 +CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_MASK = 0x30 +CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_SFT = 4 +CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V1 = (0x0 << 4) +CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV4 = (0x2 << 4) +CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV6 = (0x3 << 4) +CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_LAST = CQ_RES_UD_CFA_V2_FLAGS_ROCE_IP_VER_V2IPV6 +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_MASK = 0x3c0 +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_SFT = 6 +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_NONE = (0x0 << 6) +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_ACT_REC_PTR = (0x1 << 6) +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_TUNNEL_ID = (0x2 << 6) +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_CHDR_DATA = (0x3 << 6) +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_HDR_OFFSET = (0x4 << 6) +CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_LAST = CQ_RES_UD_CFA_V2_FLAGS_META_FORMAT_HDR_OFFSET +CQ_RES_UD_CFA_V2_SRQ_OR_RQ_WR_ID_MASK = 0xfffff +CQ_RES_UD_CFA_V2_SRQ_OR_RQ_WR_ID_SFT = 0 +CQ_RES_UD_CFA_V2_CFA_METADATA1_MASK = 0xf00000 +CQ_RES_UD_CFA_V2_CFA_METADATA1_SFT = 20 +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_MASK = 0x700000 +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_SFT = 20 +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID88A8 = (0x0 << 20) +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID8100 = (0x1 << 20) +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9100 = (0x2 << 20) +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9200 = (0x3 << 20) +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPID9300 = (0x4 << 20) +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPIDCFG = (0x5 << 20) +CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_LAST = CQ_RES_UD_CFA_V2_CFA_METADATA1_TPID_SEL_TPIDCFG +CQ_RES_UD_CFA_V2_CFA_METADATA1_VALID = 0x800000 +CQ_RES_UD_CFA_V2_SRC_QP_HIGH_MASK = 0xff000000 +CQ_RES_UD_CFA_V2_SRC_QP_HIGH_SFT = 24 +CQ_RES_RAWETH_QP1_LENGTH_MASK = 0x3fff +CQ_RES_RAWETH_QP1_LENGTH_SFT = 0 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK = 0x3ff +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT = 0 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR = 0x1 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK = 0x3c0 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT = 6 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN = (0x0 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP = (0x1 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP = (0x2 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP = (0x3 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE = (0x4 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE = (0x5 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP = (0x7 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP = (0x8 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP = (0x9 << 6) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST = CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR = 0x10 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR = 0x20 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR = 0x40 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR = 0x80 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR = 0x100 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK = 0xe00 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT = 9 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR = (0x0 << 9) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION = (0x1 << 9) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN = (0x2 << 9) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR = (0x3 << 9) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR = (0x4 << 9) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR = (0x5 << 9) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL = (0x6 << 9) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST = CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK = 0xf000 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT = 12 +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR = (0x0 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION = (0x1 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN = (0x2 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL = (0x3 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR = (0x4 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR = (0x5 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN = (0x6 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL = (0x7 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN = (0x8 << 12) +CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST = CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC = 0x1 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC = 0x2 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC = 0x4 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC = 0x8 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK = 0xf0 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT = 4 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE = (0x0 << 4) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN = (0x1 << 4) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_TUNNEL_ID = (0x2 << 4) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_CHDR_DATA = (0x3 << 4) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET = (0x4 << 4) +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST = CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE = 0x100 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_CALC = 0x200 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_EXT_META_FORMAT_MASK = 0xc00 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_EXT_META_FORMAT_SFT = 10 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_MASK = 0xffff0000 +CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_SFT = 16 +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_DE_VID_MASK = 0xffff +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_DE_VID_SFT = 0 +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK = 0xfff +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT = 0 +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE = 0x1000 +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK = 0xe000 +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT = 13 +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK = 0xffff0000 +CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT = 16 +CQ_RES_RAWETH_QP1_TOGGLE = 0x1 +CQ_RES_RAWETH_QP1_CQE_TYPE_MASK = 0x1e +CQ_RES_RAWETH_QP1_CQE_TYPE_SFT = 1 +CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 = (0x3 << 1) +CQ_RES_RAWETH_QP1_CQE_TYPE_LAST = CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 +CQ_RES_RAWETH_QP1_STATUS_OK = 0x0 +CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR = 0x1 +CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR = 0x2 +CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR = 0x3 +CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR = 0x7 +CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR = 0x8 +CQ_RES_RAWETH_QP1_STATUS_LAST = CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR +CQ_RES_RAWETH_QP1_FLAGS_SRQ = 0x1 +CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ = 0x0 +CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ = 0x1 +CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST = CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ +CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK = 0xfffff +CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT = 0 +CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK = 0xff000000 +CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT = 24 +CQ_RES_RAWETH_QP1_V2_LENGTH_MASK = 0x3fff +CQ_RES_RAWETH_QP1_V2_LENGTH_SFT = 0 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_MASK = 0x3ff +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_SFT = 0 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ERROR = 0x1 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_MASK = 0x3c0 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_SFT = 6 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN = (0x0 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_IP = (0x1 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_TCP = (0x2 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_UDP = (0x3 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_FCOE = (0x4 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_ROCE = (0x5 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_ICMP = (0x7 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP = (0x8 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP = (0x9 << 6) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_LAST = CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_IP_CS_ERROR = 0x10 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_L4_CS_ERROR = 0x20 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_IP_CS_ERROR = 0x40 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_L4_CS_ERROR = 0x80 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_CRC_ERROR = 0x100 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK = 0xe00 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT = 9 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR = (0x0 << 9) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION = (0x1 << 9) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN = (0x2 << 9) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR = (0x3 << 9) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR = (0x4 << 9) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR = (0x5 << 9) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL = (0x6 << 9) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST = CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_MASK = 0xf000 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_SFT = 12 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR = (0x0 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION = (0x1 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN = (0x2 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL = (0x3 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR = (0x4 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR = (0x5 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN = (0x6 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL = (0x7 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN = (0x8 << 12) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_LAST = CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN +CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_VID_MASK = 0xfff +CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_VID_SFT = 0 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_DE = 0x1000 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_PRI_MASK = 0xe000 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA0_PRI_SFT = 13 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_ALL_OK_MODE = 0x8 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_MASK = 0xf0 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_SFT = 4 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_NONE = (0x0 << 4) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_ACT_REC_PTR = (0x1 << 4) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_TUNNEL_ID = (0x2 << 4) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_CHDR_DATA = (0x3 << 4) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET = (0x4 << 4) +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_LAST = CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_META_FORMAT_HDR_OFFSET +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_IP_TYPE = 0x100 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_CALC = 0x200 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_OK_MASK = 0xfc00 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_CS_OK_SFT = 10 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_MASK = 0xffff0000 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_FLAGS2_COMPLETE_CHECKSUM_SFT = 16 +CQ_RES_RAWETH_QP1_V2_TOGGLE = 0x1 +CQ_RES_RAWETH_QP1_V2_CQE_TYPE_MASK = 0x1e +CQ_RES_RAWETH_QP1_V2_CQE_TYPE_SFT = 1 +CQ_RES_RAWETH_QP1_V2_CQE_TYPE_RES_RAWETH_QP1 = (0x3 << 1) +CQ_RES_RAWETH_QP1_V2_CQE_TYPE_LAST = CQ_RES_RAWETH_QP1_V2_CQE_TYPE_RES_RAWETH_QP1 +CQ_RES_RAWETH_QP1_V2_STATUS_OK = 0x0 +CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_ACCESS_ERROR = 0x1 +CQ_RES_RAWETH_QP1_V2_STATUS_HW_LOCAL_LENGTH_ERR = 0x2 +CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_PROTECTION_ERR = 0x3 +CQ_RES_RAWETH_QP1_V2_STATUS_LOCAL_QP_OPERATION_ERR = 0x4 +CQ_RES_RAWETH_QP1_V2_STATUS_MEMORY_MGT_OPERATION_ERR = 0x5 +CQ_RES_RAWETH_QP1_V2_STATUS_WORK_REQUEST_FLUSHED_ERR = 0x7 +CQ_RES_RAWETH_QP1_V2_STATUS_HW_FLUSH_ERR = 0x8 +CQ_RES_RAWETH_QP1_V2_STATUS_LAST = CQ_RES_RAWETH_QP1_V2_STATUS_HW_FLUSH_ERR +CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ = 0x1 +CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_RQ = 0x0 +CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_SRQ = 0x1 +CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_LAST = CQ_RES_RAWETH_QP1_V2_FLAGS_SRQ_SRQ +CQ_RES_RAWETH_QP1_V2_SRQ_OR_RQ_WR_ID_MASK = 0xfffff +CQ_RES_RAWETH_QP1_V2_SRQ_OR_RQ_WR_ID_SFT = 0 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_MASK = 0xf00000 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_SFT = 20 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_MASK = 0x700000 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_SFT = 20 +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID88A8 = (0x0 << 20) +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID8100 = (0x1 << 20) +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9100 = (0x2 << 20) +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9200 = (0x3 << 20) +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPID9300 = (0x4 << 20) +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPIDCFG = (0x5 << 20) +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_LAST = CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_TPID_SEL_TPIDCFG +CQ_RES_RAWETH_QP1_V2_CFA_METADATA1_VALID = 0x800000 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_PAYLOAD_OFFSET_MASK = 0xff000000 +CQ_RES_RAWETH_QP1_V2_RAWETH_QP1_PAYLOAD_OFFSET_SFT = 24 +CQ_TERMINAL_TOGGLE = 0x1 +CQ_TERMINAL_CQE_TYPE_MASK = 0x1e +CQ_TERMINAL_CQE_TYPE_SFT = 1 +CQ_TERMINAL_CQE_TYPE_TERMINAL = (0xe << 1) +CQ_TERMINAL_CQE_TYPE_LAST = CQ_TERMINAL_CQE_TYPE_TERMINAL +CQ_TERMINAL_STATUS_OK = 0x0 +CQ_TERMINAL_STATUS_LAST = CQ_TERMINAL_STATUS_OK +CQ_CUTOFF_TOGGLE = 0x1 +CQ_CUTOFF_CQE_TYPE_MASK = 0x1e +CQ_CUTOFF_CQE_TYPE_SFT = 1 +CQ_CUTOFF_CQE_TYPE_CUT_OFF = (0xf << 1) +CQ_CUTOFF_CQE_TYPE_LAST = CQ_CUTOFF_CQE_TYPE_CUT_OFF +CQ_CUTOFF_RESIZE_TOGGLE_MASK = 0x60 +CQ_CUTOFF_RESIZE_TOGGLE_SFT = 5 +CQ_CUTOFF_STATUS_OK = 0x0 +CQ_CUTOFF_STATUS_LAST = CQ_CUTOFF_STATUS_OK +PTU_PTE_VALID = 0x1 +PTU_PTE_LAST = 0x2 +PTU_PTE_NEXT_TO_LAST = 0x4 +PTU_PTE_UNUSED_MASK = 0xff8 +PTU_PTE_UNUSED_SFT = 3 +PTU_PTE_PAGE_MASK = 0xfffff000 +PTU_PTE_PAGE_SFT = 12 +PTU_PDE_VALID = 0x1 +PTU_PDE_UNUSED_MASK = 0xffe +PTU_PDE_UNUSED_SFT = 1 +PTU_PDE_PAGE_MASK = 0xfffff000 +PTU_PDE_PAGE_SFT = 12 +RCFW_CMDQ_TRIG_VAL = 1 +RCFW_COMM_PCI_BAR_REGION = 0 +RCFW_COMM_CONS_PCI_BAR_REGION = 2 +RCFW_COMM_BASE_OFFSET = 0x600 +RCFW_PF_VF_COMM_PROD_OFFSET = 0xc +RCFW_COMM_TRIG_OFFSET = 0x100 +RCFW_COMM_SIZE = 0x104 +RCFW_DBR_PCI_BAR_REGION = 2 +RCFW_DBR_BASE_PAGE_SHIFT = 12 +RCFW_FW_STALL_MAX_TIMEOUT = 40 +RCFW_CMD_NON_BLOCKING_SHADOW_QD = 64 +RCFW_CMD_WAIT_TIME_MS = 20000 +BNXT_QPLIB_CMDQE_MAX_CNT = 8192 +BNXT_QPLIB_CMDQE_BYTES = lambda depth: ((depth) * BNXT_QPLIB_CMDQE_UNITS) # type: ignore +RCFW_MAX_COOKIE_VALUE = (BNXT_QPLIB_CMDQE_MAX_CNT - 1) +RCFW_CMD_IS_BLOCKING = 0x8000 +HWRM_VERSION_DEV_ATTR_MAX_DPI = 0x1000A0000000D +HWRM_VERSION_READ_CTX = 0x1000A00030012 +BNXT_QPLIB_CREQE_MAX_CNT = (64 * 1024) +BNXT_QPLIB_CREQE_UNITS = 16 +CREQ_ENTRY_POLL_BUDGET = 0x100 +BNXT_QPLIB_OOS_COUNT_MASK = 0xFFFFFFFF +FIRMWARE_FIRST_FLAG = (31) +BNXT_RE_MAX_QPC_COUNT = (64 * 1024) +BNXT_RE_MAX_MRW_COUNT = (64 * 1024) +BNXT_RE_MAX_SRQC_COUNT = (64 * 1024) +BNXT_RE_MAX_CQ_COUNT = (64 * 1024) +BNXT_RE_MAX_MRW_COUNT_64K = (64 * 1024) +BNXT_RE_MAX_MRW_COUNT_256K = (256 * 1024) +BNXT_QPLIB_DBR_VALID = (0x1 << 26) +BNXT_QPLIB_DBR_EPOCH_SHIFT = 24 +BNXT_QPLIB_DBR_TOGGLE_SHIFT = 25 +BNXT_QPLIB_DBR_PF_DB_OFFSET = 0x10000 +BNXT_QPLIB_DBR_VF_DB_OFFSET = 0x4000 +BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE = 448 +BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE = 64 +BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE = 64 +BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE = 128 +BNXT_QPLIB_INIT_DBHDR = lambda xid,type,indx,toggle: (((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) | (((u32)(toggle)) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) # type: ignore +BNXT_RE_HW_RETX = lambda a: _is_hw_retx_supported((a)) # type: ignore +HWRM_CMD_MAX_TIMEOUT = 60000 +BNXT_HWRM_TARGET = 0xffff +BNXT_HWRM_NO_CMPL_RING = -1 +BNXT_HWRM_REQ_MAX_SIZE = 128 +BNXT_HWRM_DMA_ALIGN = 16 +BNXT_HWRM_SENTINEL = 0xb6e1f68a12e9a7eb +HWRM_SHORT_MIN_TIMEOUT = 3 +HWRM_SHORT_MAX_TIMEOUT = 10 +HWRM_SHORT_TIMEOUT_COUNTER = 5 +HWRM_MIN_TIMEOUT = 25 +HWRM_MAX_TIMEOUT = 40 +HWRM_VALID_BIT_DELAY_USEC = 50000 \ No newline at end of file diff --git a/tinygrad_repo/tinygrad/runtime/autogen/mesa.py b/tinygrad_repo/tinygrad/runtime/autogen/mesa.py index 6b74bf01d..98d7ee2c6 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/mesa.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/mesa.py @@ -5,7 +5,7 @@ from typing import Literal, TypeAlias from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR from tinygrad.runtime.support import c from tinygrad.helpers import DEV -import gzip, base64, platform, sysconfig, os +import gzip, base64, sysconfig, os dll = c.DLL('mesa', 'tinymesa_cpu' if DEV.renderer == 'LVP' else 'tinymesa', os.path.join(sysconfig.get_paths()['platlib'], 'tinymesa'), emsg='pip install tinymesa==25.2.7.2') class struct_u_printf_info(c.Struct): pass u_printf_info: TypeAlias = struct_u_printf_info diff --git a/tinygrad_repo/tinygrad/runtime/graph/hcq.py b/tinygrad_repo/tinygrad/runtime/graph/hcq.py index dbe321159..d8aecce5a 100644 --- a/tinygrad_repo/tinygrad/runtime/graph/hcq.py +++ b/tinygrad_repo/tinygrad/runtime/graph/hcq.py @@ -6,7 +6,6 @@ from tinygrad.device import Buffer, BufferSpec, Compiled, Device, MultiBuffer, P from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, Ops, Variable from tinygrad.engine.jit import GraphRunner, MultiGraphRunner -from tinygrad.runtime.ops_rdma import RDMACopyQueue class HCQGraph(MultiGraphRunner): def __init__(self, *args, **kwargs): @@ -50,7 +49,7 @@ class HCQGraph(MultiGraphRunner): self.comp_queues: dict[HCQCompiled, HWQueue] = {dev: unwrap(dev.hw_compute_queue_t)() for dev in self.devices} self.copy_queues: dict[tuple[HCQCompiled, int], HWQueue] = {} # lazy allocation, keyed by (device, queue_idx) - self.rdma_queues: dict[tuple[HCQCompiled, HCQCompiled], RDMACopyQueue] = {} # lazy allocation, keyed by device pair + self.rdma_queues: dict[tuple[HCQCompiled, HCQCompiled], "RDMACopyQueue"] = {} # lazy allocation, keyed by device pair self.num_copy_queues: int = getenv("HCQ_NUM_SDMA", min(len(self.devices), 8) if ALL2ALL >= 1 else 1) self.num_rdma_ops: dict[tuple[HCQCompiled, HCQCompiled], int] = collections.defaultdict(int) @@ -97,13 +96,13 @@ class HCQGraph(MultiGraphRunner): # set any fixedvars on the device self.device_vars[enqueue_dev] = merge_dicts([self.device_vars.get(enqueue_dev, {}), device_vars]) - if runtime is not None: self.device_vars[enqueue_dev] = merge_dicts([self.device_vars[enqueue_dev], {k: 0 for k in ast.arg.runtimevars}]) if runtime is not None: enqueue_queue = self.comp_queues[enqueue_dev] elif is_rdma: enqueue_queue = self.comp_queues[enqueue_dev] rdma_key = (cast(HCQCompiled, Device[bufs[0].device]).rdma_dev(), enqueue_dev.rdma_dev()) + from tinygrad.runtime.ops_rdma import RDMACopyQueue self.rdma_queues.setdefault(rdma_key, RDMACopyQueue(enqueue_dev.rdma_dev())) else: assert (enqueue_dev.hw_copy_queue_t is not None), "device must implement a copy queue" @@ -139,7 +138,8 @@ class HCQGraph(MultiGraphRunner): prof_ji_desc = runtime.name if runtime is not None else TracingKey(f"{bufs[1].device} -> {bufs[0].device}", ret=bufs[0].nbytes) prof_name = enqueue_dev.device if runtime is not None else f"{enqueue_dev.device}:SDMA:{queue_idx}" - self.prof_graph_entries.append(ProfileGraphEntry(prof_name, prof_ji_desc, sig_st, j * 2 + 1)) + self.prof_graph_entries.append(ProfileGraphEntry(prof_name, prof_ji_desc, sig_st, j * 2 + 1, + runtime.profile_key if runtime is not None else None)) self.prof_graph_deps.append([d - 1 for _, d in rdeps]) self.last_j[enqueue_queue] = j @@ -172,7 +172,7 @@ class HCQGraph(MultiGraphRunner): # Encode main commands based on ji type. if runtime is not None: - enqueue_queue.exec(runtime, self.ji_args[j], ast.arg.global_size or (1,1,1), ast.arg.local_size or (1,1,1)) + enqueue_queue.exec(runtime, self.ji_args[j], ast.arg.global_size, ast.arg.local_size) elif j in self.rdma_deps: dest_queue, dest_deps, dest_out_signal, dest_out_val = self.rdma_deps[j] for sig, val in dest_deps: dest_queue.wait(sig, val) diff --git a/tinygrad_repo/tinygrad/runtime/graph/metal.py b/tinygrad_repo/tinygrad/runtime/graph/metal.py index 409cfb973..39e33e809 100644 --- a/tinygrad_repo/tinygrad/runtime/graph/metal.py +++ b/tinygrad_repo/tinygrad/runtime/graph/metal.py @@ -102,7 +102,7 @@ class MetalGraph(GraphRunner): def collect_timestamps(self): # create a graph event and evenly space each program st, en = decimal.Decimal(self.command_buffer.GPUStartTime()) * 1000000, decimal.Decimal(self.command_buffer.GPUEndTime()) * 1000000 - ents = [ProfileGraphEntry(self.device, rt.name, i, i+1) for i, rt in enumerate(self.runtimes) if rt is not None] + ents = [ProfileGraphEntry(self.device, rt.name, i, i+1, rt.profile_key) for i, rt in enumerate(self.runtimes) if rt is not None] self.dev.profile_events += [ProfileGraphEvent(ents, [], [st + (en-st)/len(ents)*i for i in range(len(ents)+1)])] def __del__(self): @@ -113,5 +113,6 @@ class MetalGraph(GraphRunner): @staticmethod def supports_uop(batch_devs, new_call:UOp) -> bool: # Metal ICB replay encodes offsets as uint32; reject if any Metal buffer offset exceeds 32-bit range. - if any(b.op is Ops.SLICE and b.src[1].val * b.src[0].dtype.itemsize > 0xFFFFFFFF for b in new_call.src[1:]): return False + for shrink in [s for src in new_call.src[1:] if (s:=src.src[0] if src.op is Ops.BITCAST else src).op is Ops.SHRINK]: + if shrink.src[1].val * shrink.src[0].dtype.itemsize > 0xFFFFFFFF: return False return GraphRunner.supports_uop(batch_devs, new_call) diff --git a/tinygrad_repo/tinygrad/runtime/ops_amd.py b/tinygrad_repo/tinygrad/runtime/ops_amd.py index a9248428b..f73b3cf0a 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_amd.py +++ b/tinygrad_repo/tinygrad/runtime/ops_amd.py @@ -8,7 +8,7 @@ from tinygrad.runtime.support.hcq import MMIOInterface, BumpAllocator, hcq_filte from tinygrad.uop.ops import sint from tinygrad.device import Compiled, BufferSpec, TinyELF from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, ProfileEvent, lo32, hi32, colored, prod, ContextVar, TracingKey -from tinygrad.helpers import VIZ, ceildiv, unwrap, pluralize +from tinygrad.helpers import VIZ, HCQ2, ceildiv, unwrap, pluralize from tinygrad.renderer.cstyle import HIPRenderer, HIPCCRenderer from tinygrad.renderer.llvmir import AMDLLVMRenderer from tinygrad.runtime.autogen import kfd, hsa, sqtt, amdgpu_kd, amdgpu_drm @@ -651,6 +651,59 @@ class AMDAllocator(HCQAllocator['AMDDevice']): def _do_map(self, buf:HCQBuffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) + def _copyin(self, dest:HCQBuffer, src:memoryview): + if not self.dev.is_usb(): return super()._copyin(dest, src) + from tinygrad.runtime.support.usb import alloc_cbuffer + # Pipelined copyin over the 0xF2 engine. ~256KB chunks stream into two alternating 256KB SRAM bounce windows; the + # engine can't signal data landing, so each chunk's wire image ends in a 4B sentinel tagged with its sequence number. + # A prebuilt SDMA ring polls each chunk's sentinel before copying it to VRAM, then bumps a drain fence; the host + # waits on that fence before re-arming a window. No timing is assumed in either direction. + dev, usb, ts, sdma = self.dev, self.dev.iface.pci_dev.usb, self.dev.timeline_signal, self.dev.sdma + CHUNK, src_mv = 0x40000 - 4, src.cast('B') # payload per chunk: the 256KB window minus the 4B trailing sentinel + nchunks = ceildiv(src.nbytes, CHUNK) + FENCE = 0xA800 # drain fence: the GPU writes it via sys_buf (PCIe 0x820800), the host reads it here (xdata) + if not hasattr(self, '_usb_seq'): # one-time: clear the fence and zero both windows so garbage can't match a sentinel + self._usb_seq, self._usb_stage = 0, [alloc_cbuffer(0x40000) for _ in range(2)] # (backing array, memoryview) pairs + self._usb_wins = (self.b[0].offset(0, 0x40000), self.b[0].offset(0x40000, 0x40000)) # two windows, engine slots 0/16 + usb.write(FENCE, bytes(8)) + for bi in range(2): usb.scsi_write(bytes(0x40000), slot_start=bi * 16) + + def wait_drain(count): # spin until the drain fence reaches count, i.e. chunks 0..count-1 are fully in VRAM + t0 = time.perf_counter() + while int.from_bytes(usb.read(FENCE, 8), 'little') < count: + if time.perf_counter() - t0 > 10: raise RuntimeError(f"GPU failed to drain USB copyin chunk {count - 1} (10s, hung GPU?)") + + # build the whole ring upfront: per chunk, poll the sentinel, copy SRAM->VRAM, bump the fence; then one doorbell + POLL_EQ = sdma.SDMA_OP_POLL_REGMEM | sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1) + POLL_DW5 = sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) + q = dev.hw_copy_queue_t().wait(ts, dev.timeline_value - 1) + for c in range(nchunks): + seq, size = self._usb_seq + c, min(CHUNK, src.nbytes - c * CHUNK) + q.q(POLL_EQ, *data64_le(self._usb_wins[seq & 1].va_addr + round_up(size + 4, 512) - 4), 0x51000000 | (seq & 0xFFFFFF), 0xFFFFFFFF, POLL_DW5) + q.copy(dest.offset(c * CHUNK), self._usb_wins[seq & 1], size) + q.write(dev.iface.sys_buf.offset(0x800, 8), seq + 1, b64=True) + q.signal(ts, dev.next_timeline()).submit(dev) + + # stream the chunks: stage the wire image [payload][sentinel], arm the window, send. A window is reusable once + # its previous occupant (seq-2) is both fully sent (tag reaped) and fully drained to VRAM (the fence). + inflight = [None, None] + for c in range(nchunks): + seq, size = self._usb_seq + c, min(CHUNK, src.nbytes - c * CHUNK) + if inflight[seq & 1] is not None: usb.usb.bulk_wait(inflight[seq & 1]) + buf = self._usb_stage[seq & 1][1] + buf[:size] = src_mv[c * CHUNK : c * CHUNK + size] + wire = round_up(size + 4, 512) # payload plus the sentinel, padded to 512B sectors (full window for max chunks) + struct.pack_into(' HCQBuffer: + def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, zero=False, **kwargs) -> HCQBuffer: # usb allocates uncached and cpu_access in vram. vram writes are faster than sram writes - if host and self.sys_next_off + size < self.sys_buf.size: - self.sys_next_off += size - return self.sys_buf.offset(self.sys_next_off - size, size) + # NOTE: host allocs deliberately do NOT use sys_buf (the 0x820000 NVMe SQ region): the GPU's signal writes there + # collide with the 0xF2 engine mid-stream. Signals in VRAM are read back via 0xF0 streaming reads instead. # force devmem - return super().alloc(size, host=False, uncached=uncached, cpu_access=cpu_access, contiguous=contiguous, force_devmem=True, **kwargs) + return super().alloc(size, host=False, uncached=uncached, cpu_access=cpu_access, contiguous=contiguous, force_devmem=True, zero=zero, **kwargs) def sleep(self, timeout): pass @@ -946,9 +998,7 @@ class AMDDevice(HCQCompiled): def is_usb(self) -> bool: return isinstance(self.iface, USBIface) def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - - self.iface = self._select_iface() + self.iface = self._select_iface(device) self.target:tuple[int, ...] = ((trgt:=self.iface.props['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100) self.arch = "gfx%d%x%x" % self.target @@ -1015,7 +1065,8 @@ class AMDDevice(HCQCompiled): for k in (PMC_COUNTERS:=getenv("PMC_COUNTERS", pmc_default).split(",")): if k not in self.pmc_counters: raise RuntimeError(f"PMC counter {k} is not supported. Available: {','.join(self.pmc_counters.keys())}") - cast(AMDComputeQueue, unwrap(self.hw_compute_queue_t)()).pmc_start([(k, *self.pmc_counters[k]) for k in PMC_COUNTERS]).submit(self) + with (q:=cast(AMDComputeQueue, unwrap(self.hw_compute_queue_t)())).pred_exec((1 << self.xccs) - 1): + q.pmc_start([(k, *self.pmc_counters[k]) for k in PMC_COUNTERS]).submit(self) self.pmc_buffer = self.allocator.alloc(self.pmc_sched[-1].off + self.pmc_sched[-1].size, BufferSpec(nolru=True, uncached=True)) self.allocator._copyin(self.pmc_buffer, memoryview(bytearray(self.pmc_buffer.size))) # zero pmc buffers, some counters have only lo part. @@ -1029,6 +1080,10 @@ class AMDDevice(HCQCompiled): self.sqtt_wptrs = self.allocator.alloc(round_up(self.se_cnt * self.xccs * 4, 0x1000), BufferSpec(cpu_access=True, nolru=True)) self.sqtt_next_cmd_id = itertools.count(0) + if self.is_am(): + self.iface.dev_impl.gmc.vf_owner = self + if self.iface.dev_impl.vf_access: self.iface.dev_impl.release_vf_access() + def create_queue(self, queue_type, ring_size, ctx_save_restore_size=0, eop_buffer_size=0, ctl_stack_size=0, debug_memory_size=0, idx=0): ring = self.iface.alloc(ring_size, uncached=True, cpu_access=True) gart = self.iface.alloc(0x100, uncached=True, cpu_access=True) @@ -1052,7 +1107,8 @@ class AMDDevice(HCQCompiled): if getenv("AMD_DISABLE_SDMA"): return None if idx in self.sdma_queues: return self.sdma_queues[idx] with contextlib.suppress(OSError): - self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x200 if self.is_usb() else (16 << 20), idx=idx) + # USB: a copyin submits its whole ring at once (3 packets per 240KB chunk), so it needs more than the 0x200 default + self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, (1 << 20) if self.is_usb() else (16 << 20), idx=idx) return self.sdma_queues.get(idx, None) def _ensure_has_local_memory(self, private_segment_size): @@ -1102,4 +1158,4 @@ class AMDDevice(HCQCompiled): def hw_copy_queues(self): return [(f"SDMA:{i}", functools.partial(unwrap(self.hw_copy_queue_t), queue_idx=i)) for i in self.sdma_queues] -if getenv("HCQ2"): from extra.hcq2.ops_amd2 import * # noqa: F401, F403 # pylint: disable=unused-import +if HCQ2: from extra.hcq2.ops_amd2 import * # noqa: F401, F403 # pylint: disable=unused-import diff --git a/tinygrad_repo/tinygrad/runtime/ops_cl.py b/tinygrad_repo/tinygrad/runtime/ops_cl.py index 5087996e3..061f1eee2 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_cl.py +++ b/tinygrad_repo/tinygrad/runtime/ops_cl.py @@ -24,10 +24,10 @@ class CLCompiler(Compiler): super().__init__(f"compile_cl_{compile_key}") def compile(self, src:str) -> bytes: program = checked(cl.clCreateProgramWithSource(self.dev.context, 1, to_char_p_p([src.encode()]), None, status := ctypes.c_int32()), status) - build_status: int = cl.clBuildProgram(program, 1, self.dev.device_id, None, BP_CB(), None) + build_status: int = cl.clBuildProgram(program, 1, self.dev.cl_dev, None, BP_CB(), None) if build_status != 0: - cl.clGetProgramBuildInfo(program, self.dev.device_id, cl.CL_PROGRAM_BUILD_LOG, 0, None, log_size := ctypes.c_size_t()) - cl.clGetProgramBuildInfo(program, self.dev.device_id, cl.CL_PROGRAM_BUILD_LOG, + cl.clGetProgramBuildInfo(program, self.dev.cl_dev, cl.CL_PROGRAM_BUILD_LOG, 0, None, log_size := ctypes.c_size_t()) + cl.clGetProgramBuildInfo(program, self.dev.cl_dev, cl.CL_PROGRAM_BUILD_LOG, log_size.value, mstr := ctypes.create_string_buffer(log_size.value), None) raise CompileError(f"OpenCL Compile Error\n\n{mstr.value.decode()}") check(cl.clGetProgramInfo(program, cl.CL_PROGRAM_BINARY_SIZES, ctypes.sizeof(ctypes.c_size_t), binary_sizes := (ctypes.c_size_t * 1)(), None)) @@ -39,11 +39,11 @@ class CLCompiler(Compiler): class CLProgram(Program['CLDevice']): def __init__(self, device:CLDevice, obj:TinyELF): self.dev, self.lib, self.signature = device, device.cl_compiler.compile_cached(obj.lib.decode()), obj.signature - self.program = checked(cl.clCreateProgramWithBinary(device.context, 1, device.device_id, (ctypes.c_size_t * 1)(len(self.lib)), + self.program = checked(cl.clCreateProgramWithBinary(device.context, 1, device.cl_dev, (ctypes.c_size_t * 1)(len(self.lib)), to_char_p_p([self.lib], ctypes.c_ubyte), binary_status := ctypes.c_int32(), errcode_ret := ctypes.c_int32()), errcode_ret) check(binary_status.value) - check(cl.clBuildProgram(self.program, 1, device.device_id, None, BP_CB(), None)) # NOTE: OSX requires this + check(cl.clBuildProgram(self.program, 1, device.cl_dev, None, BP_CB(), None)) # NOTE: OSX requires this self.kernel = checked(cl.clCreateKernel(self.program, obj.name.encode(), status := ctypes.c_int32()), status) def __del__(self): @@ -52,7 +52,7 @@ class CLProgram(Program['CLDevice']): try: check(cl.clReleaseProgram(self.program)) except (TypeError, AttributeError): pass - def __call__(self, *bufs:cl.cl_mem, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]|None=None, vals:tuple[int, ...]=(), + def __call__(self, *bufs:cl.cl_mem, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw) -> float|None: for i, (_, slot, dt, shape) in enumerate(self.signature): b = bufs[slot] if slot < len(bufs) else getattr(ctypes, f"c_int{dt.bitsize}")(vals[slot-len(bufs)]) @@ -63,10 +63,10 @@ class CLProgram(Program['CLDevice']): img = checked(cl.clCreateImage(self.dev.context, cl.CL_MEM_READ_WRITE, fmt, desc, None, status:=ctypes.c_int32()), status) check(cl.clSetKernelArg(self.kernel, i, ctypes.sizeof(img), ctypes.byref(img))) else: check(cl.clSetKernelArg(self.kernel, i, ctypes.sizeof(b), ctypes.byref(b))) - if local_size is not None: global_size = cast(tuple[int,int,int], tuple(int(g*l) for g,l in zip(global_size, local_size))) + global_size = cast(tuple[int,int,int], tuple(int(g*l) for g,l in zip(global_size, local_size))) event = cl.cl_event() if wait else None check(cl.clEnqueueNDRangeKernel(self.dev.queue, self.kernel, len(global_size), None, (ctypes.c_size_t * len(global_size))(*global_size), - (ctypes.c_size_t * len(local_size))(*local_size) if local_size else None, 0, None, event)) + (ctypes.c_size_t * len(local_size))(*local_size), 0, None, event)) if wait: assert event is not None check(cl.clWaitForEvents(1, event)) @@ -101,17 +101,17 @@ class CLDevice(Compiled): CLDevice.device_ids = c.init_c_var((cl.cl_device_id * num_devices.value), lambda x: check(cl.clGetDeviceIDs(platform_ids[0], device_type, num_devices, x, None))) - self.device_id = CLDevice.device_ids[0 if ":" not in device else int(device.split(":")[1])] - self.device_name = (cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_NAME, 256, + self.cl_dev = CLDevice.device_ids[0 if ":" not in device else int(device.split(":")[1])] + self.device_name = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_NAME, 256, buf:=ctypes.create_string_buffer(256), None), buf.value.decode())[1] - self.driver_version = (cl.clGetDeviceInfo(self.device_id, cl.CL_DRIVER_VERSION, 256, + self.driver_version = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DRIVER_VERSION, 256, buf:=ctypes.create_string_buffer(256), None), buf.value.decode())[1] if DEBUG >= 1: print(f"CLDevice: opening {self.device_name} with version {self.driver_version}") - self.context = checked(cl.clCreateContext(None, 1, self.device_id, CC_CB(), None, status := ctypes.c_int32()), status) - self.queue = checked(cl.clCreateCommandQueue(self.context, self.device_id, cl.CL_QUEUE_PROFILING_ENABLE, status), status) + self.context = checked(cl.clCreateContext(None, 1, self.cl_dev, CC_CB(), None, status := ctypes.c_int32()), status) + self.queue = checked(cl.clCreateCommandQueue(self.context, self.cl_dev, cl.CL_QUEUE_PROFILING_ENABLE, status), status) self.pending_copyin: list[memoryview] = [] - check(cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_EXTENSIONS, 0, None, ctypes.byref(exts_len:=ctypes.c_size_t()))) - self.device_exts = (cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_EXTENSIONS, exts_len.value, + check(cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_EXTENSIONS, 0, None, ctypes.byref(exts_len:=ctypes.c_size_t()))) + self.device_exts = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_EXTENSIONS, exts_len.value, ctypes.byref(buf := ctypes.create_string_buffer(exts_len.value)), None), ctypes.string_at(buf).decode().split())[1] @@ -119,7 +119,7 @@ class CLDevice(Compiled): arch = ",".join(self.device_exts) if "cl_khr_image2d_from_buffer" in self.device_exts: - check(cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_IMAGE_PITCH_ALIGNMENT, 4, ctypes.byref(ipa := ctypes.c_uint32()), None)) + check(cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_IMAGE_PITCH_ALIGNMENT, 4, ctypes.byref(ipa := ctypes.c_uint32()), None)) arch += f",IMAGE_PITCH_ALIGNMENT={ipa.value}" super().__init__(device, CLAllocator(self), [OpenCLRenderer], CLProgram, arch=arch) diff --git a/tinygrad_repo/tinygrad/runtime/ops_cpu.py b/tinygrad_repo/tinygrad/runtime/ops_cpu.py index 3ecb49a54..ae9967a3d 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_cpu.py +++ b/tinygrad_repo/tinygrad/runtime/ops_cpu.py @@ -1,11 +1,12 @@ from __future__ import annotations -import platform, sys, os, ctypes, functools, mmap, threading, array, itertools +import platform, sys, ctypes, functools, mmap, array, struct, time from dataclasses import replace -from typing import cast -from tinygrad.helpers import to_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le, partition -from tinygrad.device import Buffer, BufferSpec, TinyELF -from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQBuffer, HWQueue, HCQArgsState, HCQSignal, HCQProgram, MMIOInterface -from tinygrad.runtime.support.hcq import CLikeArgsState +from typing import cast, Callable +from tinygrad.helpers import to_mv, from_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le +from tinygrad.device import Buffer, BufferSpec, TinyELF, Program, Device +from tinygrad.runtime.support.hcq import HCQBuffer, MMIOInterface +from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_buf, hcq_size_var +from tinygrad.runtime.support.c import DLL from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.renderer.llvmir import CPULLVMRenderer from tinygrad.renderer.nir import LVPRenderer @@ -13,111 +14,91 @@ from tinygrad.renderer.isa.x86 import X86Renderer from tinygrad.runtime.support.elf import jit_loader from tinygrad.runtime.autogen import libc from tinygrad.codegen import do_to_program +from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops, get_runtime from tinygrad import UOp, dtypes from tinygrad.dtype import AddrSpace -from tinygrad.uop.ops import sint, KernelInfo, Ops, UPat, PatternMatcher, graph_rewrite +from tinygrad.uop.ops import KernelInfo, Ops, UPat, PatternMatcher -MAX_ARGS, CMD_SIZE, RING_SLOTS = 63, 64, (16 << 10) +MAX_ARGS, CMD_SIZE, FUNCS = 63, 64, (() if WIN else ('clock_gettime',)) + +# ***************** +# 1. signal programs def signal_prog(): val = UOp.param(1, dtypes.int, (), vmin_vmax=(0, dtypes.int.max), name="value", addrspace=AddrSpace.ALU) - return UOp.param(0, dtypes.uint32, (1,))[0].store(val.cast(dtypes.uint32)) + return UOp.param(0, dtypes.uint32, 1)[0].store(val.cast(dtypes.uint32)) def wait_prog(): val = UOp.param(1, dtypes.int, (), vmin_vmax=(0, dtypes.int.max), name="value", addrspace=AddrSpace.ALU) - return (v:=UOp.param(0, dtypes.uint32, (1,), volatile=True).after(l:=UOp.loop(0))[0].load()).end(l, v < val.cast(dtypes.uint32)) + return (v:=UOp.param(0, dtypes.uint32, 1, volatile=True).after(l:=UOp.loop(0))[0].load()).end(l, v < val.cast(dtypes.uint32)) def timestamp_prog(): if WIN: val = UOp.const(0, dtypes.uint64) else: - fn, ts = UOp.param(1, dtypes.uint64, (1,)), UOp.placeholder((2,), dtypes.uint64, slot=0, addrspace=AddrSpace.REG) + fn, ts = UOp.param(1, dtypes.uint64, 1), UOp.placeholder((2,), dtypes.uint64, slot=0, addrspace=AddrSpace.REG) call = fn[0].load().call(UOp.const(6 if OSX else 1, dtypes.int), ts[0], ret_dtype=dtypes.void) # clock_gettime(CLOCK_MONOTONIC, &ts) val = ts.after(call)[0].load() * 1_000_000_000 + ts.after(call)[1].load() - return UOp.param(0, dtypes.uint64, (1,))[0].store(val) + return UOp.param(0, dtypes.uint64, 1)[0].store(val) -def quit_prog(): - fn = UOp.param(0, dtypes.uint64, (1 if WIN else 3,)) - if WIN: return fn[0].load().call(UOp.const(0, dtypes.uint64), ret_dtype=dtypes.void) # ExitThread(0) - sem = UOp.param(1, dtypes.uint64, (1,)) +# ***************** +# 2. queue encoders - close = fn[2].load().call(sem[0], ret_dtype=dtypes.void) # sem_close(sem) - return fn.after(close)[0].load().call(UOp.const(0, dtypes.uint64), ret_dtype=dtypes.void) # pthread_exit(0) +def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: + progs = [get_runtime(d, prog) if isinstance(prog, UOp) else cast(CPUDevice, Device[d]).prgs[prog] for d in devs] + addrs = tuple(UOp.const(p.addr, dtypes.uint64) for p in progs) + words = ((addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, src=addrs)),) + args + return UOp(Ops.LINEAR, src=words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words))) -def worker_prog(): - ring = UOp.param(0, dtypes.uint64, (RING_SLOTS * CMD_SIZE,), volatile=True) - wait, sem = UOp.param(1, dtypes.uint64, (1,), volatile=True), UOp.param(2, dtypes.uint64, (1,)) - cur = UOp.range(2**64-1, 0, dtype=dtypes.uint64) +def cpu_exec(ctx, call:UOp, prg:UOp) -> UOp: + devs = ctx.devs + args = [get_call_arg_uops(call)[i].getaddr(devs) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] + return cpu_cmd(devs, prg, *args) - # spin on windows, sem_wait to sleep on posix - if WIN: ready = (v:=wait.after(lw:=UOp.loop(1), cur)[0].load()).end(lw, v <= cur) - else: ready = (rv:=wait.after(lw:=UOp.loop(1), cur)[0].load().call(sem.after(cur)[0], ret_dtype=dtypes.int)).end(lw, rv != 0) +pm_cpu_opsel = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cpu_exec), - entry = [ring.after(ready).index((cur % RING_SLOTS) * CMD_SIZE + i).load() for i in range(CMD_SIZE)] - return entry[0].call(*entry[1:], ret_dtype=dtypes.void).end(cur) + (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.LINEAR)), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), + lambda ctx, dst, val: cpu_cmd(ctx.devs, wait_prog, dst.getaddr(ctx.devs), val.cast(dtypes.uint64))), + (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), + lambda ctx, dst, val: cpu_cmd(ctx.devs, signal_prog, dst.getaddr(ctx.devs), val.cast(dtypes.uint64))), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), lambda ctx, dst: cpu_cmd(ctx.devs, timestamp_prog, dst.getaddr(ctx.devs), + *(() if WIN else (make_buf(ctx.devs, tag="func:clock_gettime").getaddr(ctx.devs),)))), +]) -def host_wait(ctx, dst:UOp, val:UOp) -> UOp: - return (cur:=dst.after(loop:=UOp.loop(next(ctx))).index(UOp.const(0, dtypes.int)).load()).end(loop, cur < val) +def cpu_submit(ctx, cmdbuf:UOp) -> UOp: + # run the cmd entries inline on the submitting thread, the cpu has no worker threads + cb, cnt = cmdbuf.bitcast(dtypes.uint64), hcq_size_var(cmdbuf) // (CMD_SIZE * 8) + e = UOp.range(cnt, 10, dtype=dtypes.int, src=(cmdbuf,)) + entry = [cb.index(e*CMD_SIZE + i).load() for i in range(CMD_SIZE)] + return entry[0].call(*entry[1:], ret_dtype=dtypes.void).end(e) -pm_host_opsel = PatternMatcher([(UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val"))), host_wait)]) +pm_cpu_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), cpu_submit)]) -def encode_host_queue(q:UOp) -> UOp: - # TODO: subset of hcq2 for now - spins, (store,) = partition(graph_rewrite(q, pm_host_opsel, ctx=itertools.count(), walk=True, name="host opsel").src, lambda u: u.op is Ops.END) - assert store.op is Ops.INS and store.arg == "store", f"host queue cannot encode {store.op} {store.arg}" - return store.src[0].after(*spins).index(UOp.const(0, dtypes.int)).store(store.src[1]) - -class CPUComputeQueue(HWQueue): - def __init__(self, dev): - super().__init__() - self.dev = dev - def _cmd(self, prog, args=(), vals=()): return self.exec(prg:=self.dev.prgs[prog], prg.fill_kernargs(args, vals), None, None) - def memory_barrier(self): return self - def exec(self, prg:CPUProgram, args_state:HCQArgsState, global_size, local_size): - if (lvp:=isinstance(args_state, LVPArgsState)): self.bind_args_state(args_state) - args:list[sint|None] = [args_state.buf.va_addr] if lvp else [*[x.va_addr for x in args_state.bufs], *args_state.vals] - assert len(args) <= MAX_ARGS, f"CPU programs support at most {MAX_ARGS} arguments, got {len(args)}" - for tid in range(1 if lvp else (global_size or (1,))[0]): - if not lvp and 'core_id' in prg.runtimevars: args[prg.runtimevars['core_id']] = tid - self.q(prg, *[unwrap(x) for x in args], *([0] * (MAX_ARGS - len(args)))) - return self - def wait(self, signal, value=0): return self._cmd(wait_prog, (signal.base_buf,), (value,)) - def timestamp(self, signal): return self._cmd(timestamp_prog, (signal.base_buf.offset(8, 8), self.dev.func_table._buf.offset(0, 8))) - def signal(self, signal, value:sint=0): return self._cmd(signal_prog, (signal.base_buf,), (value,)) - def _submit(self, dev): - dev.ensure_worker() - ring_view = dev.ring.as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') - for off in range(0, len(self._q), CMD_SIZE): - entry = [self._q[off].addr, *self._q[off+1:off+CMD_SIZE]] - ring_view[(base:=(dev.ring_pos % RING_SLOTS) * CMD_SIZE):base+CMD_SIZE] = array.array('Q', (int(x) & ((1<<64)-1) for x in entry)) - dev.ring_pos += 1 - if WIN: dev.sys.as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] = dev.ring_pos - else: assert libc.sem_post(dev.sem) == 0 - -class LVPArgsState(CLikeArgsState): - def __init__(self, buf, prg, bufs, vals=()): super().__init__(buf, prg, bufs, vals, [*data64_le(buf.va_addr + 12), (len(bufs) + len(vals)) * 2]) +# ***************** # NOTE: MAP_JIT is added to mmap module in python 3.13 MAP_JIT = 0x0800 -class CPUProgram(HCQProgram['CPUDevice']): - rt_lib = None - try: rt_lib = ctypes.CDLL(ctypes.util.find_library('System' if OSX else 'kernel32') if OSX or WIN else 'libgcc_s.so.1') - except OSError: pass +class CPUProgram(Program['CPUDevice']): + rt_lib, libm = DLL('rt', 'System' if OSX else 'kernel' if WIN else 'gcc_s'), DLL('m', 'm') + + def _load(self, lib, base=0): return lib if lib[:4] != libc.ELFMAG.encode() else jit_loader(lib, base=base, link_libs=[self.libm, self.rt_lib]) def __init__(self, dev:CPUDevice, obj:TinyELF): - self.signature, self.runtimevars = obj.signature, {name:slot for name,slot,*_ in obj.signature if name == 'core_id'} + self.dev, self.name, self.signature = dev, obj.name, obj.signature + self.lvp = obj.target.renderer == "LVP" - LVP = obj.target.renderer == "LVP" if sys.platform == "win32": # mypy doesn't understand when WIN is used here PAGE_EXECUTE_READWRITE, MEM_COMMIT, MEM_RESERVE = 0x40, 0x1000, 0x2000 ctypes.windll.kernel32.VirtualAlloc.restype = ctypes.c_void_p self.addr = ctypes.windll.kernel32.VirtualAlloc(ctypes.c_void_p(0), ctypes.c_size_t(len(obj.lib)), MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE) - ctypes.memmove(self.addr, obj.lib, len(obj.lib)) + ctypes.memmove(self.addr, (loaded:=self._load(obj.lib, self.addr)), len(loaded)) ctypes.windll.kernel32.GetCurrentProcess.restype = ctypes.c_void_p proc = ctypes.windll.kernel32.GetCurrentProcess() - ctypes.windll.kernel32.FlushInstructionCache(ctypes.c_void_p(proc), ctypes.c_void_p(self.addr), ctypes.c_size_t(len(obj.lib))) - self.fxn = ctypes.CFUNCTYPE(None)(self.addr) + ctypes.windll.kernel32.FlushInstructionCache(ctypes.c_void_p(proc), ctypes.c_void_p(self.addr), ctypes.c_size_t(len(loaded))) + self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) else: # On apple silicon with SPRR enabled (it always is in macos) RWX pages are unrepresentable: https://blog.svenpeter.dev/posts/m1_sprr_gxf/ # MAP_JIT allows us to easily flip pages from RW- to R-X and vice versa. It is a noop on intel cpus. (man pthread_jit_write_protect_np) @@ -125,28 +106,40 @@ class CPUProgram(HCQProgram['CPUDevice']): self.addr = mv_address(self.mem) if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(False) - lib = jit_loader(obj.lib, base=ctypes.addressof(ctypes.c_void_p.from_buffer(self.mem)), link_libs=['m']) if LVP else obj.lib - self.mem.write(lib) + self.mem.write(loaded:=self._load(obj.lib, mv_address(self.mem))) if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(True) # __clear_cache isn't a normal libc function, but a compiler support routine found in libgcc_s for gcc and compiler-rt for clang. # libgcc_s comes as shared library but compiler-rt is only a bunch of static library archives which we can't directly load, but fortunately # it somehow found its way into libSystem on macos (likely because it used __builtin_clear_cache) and libgcc_s is ~always present on linux # Using ["name"] instead of .name because otherwise name is getting mangled: https://docs.python.org/3.12/reference/expressions.html#index-5 - if CPUProgram.rt_lib is not None: CPUProgram.rt_lib["__clear_cache"](ctypes.c_void_p(self.addr), ctypes.c_void_p(self.addr + len(lib))) + if 'rt' in DLL._loaded_: CPUProgram.rt_lib["__clear_cache"](ctypes.c_void_p(self.addr), ctypes.c_void_p(self.addr + len(loaded))) else: # msync should be a universal POSIX way to do this - libc.msync(ctypes.c_void_p(self.addr), len(lib), libc.MS_SYNC | libc.MS_INVALIDATE) + libc.msync(ctypes.c_void_p(self.addr), len(loaded), libc.MS_SYNC | libc.MS_INVALIDATE) - self.fxn = ctypes.CFUNCTYPE(None)(self.addr) + self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) - super().__init__(LVPArgsState if LVP else HCQArgsState, dev, obj, kernargs_alloc_size=12+256 if LVP else 0) + def __call__(self, *bufs:HCQBuffer, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), + vals:tuple[int|None, ...]=(), wait:bool=False, timeout:int|None=None) -> float|None: + st = time.perf_counter() + if self.lvp: + lvp_args = bytearray(12 + (len(bufs) + len(vals)) * 8) + addr = mv_address(lvp_args) + struct.pack_into(f'<3I{len(bufs)}Q', lvp_args, 0, *data64_le(addr+12), (len(bufs)+len(vals))*2, *[b.va_addr for b in bufs]) + for v,(off,dt) in zip(vals, TinyELF.iter_sig(self.signature[-len(vals):], len(bufs)*8)): struct.pack_into(f'<{dt.fmt}', lvp_args, 12+off, v) + self.fxn(addr) + else: + args = [*[cast(int, b.va_addr) for b in bufs], *cast(tuple[int, ...], vals)] + assert len(args) <= MAX_ARGS, f"CPU programs support at most {MAX_ARGS} arguments, got {len(args)}" + self.fxn(*[ctypes.c_uint64(x) for x in args]) + return time.perf_counter() - st if wait else None @suppress_finalizing def __del__(self): if sys.platform == 'win32': ctypes.windll.kernel32.VirtualFree(ctypes.c_void_p(self.addr), ctypes.c_size_t(0), 0x8000) #0x8000 - MEM_RELEASE -class CPUAllocator(HCQAllocator): +class CPUAllocator(HCQAllocator['CPUDevice']): def __init__(self, dev:CPUDevice): super().__init__(dev, supports_copy_from_disk=False, supports_transfer=False) def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: if options.external_ptr is not None: addr, buf = options.external_ptr, None @@ -154,68 +147,38 @@ class CPUAllocator(HCQAllocator): else: addr = mv_address(buf:=mmap.mmap(-1, size, mmap.MAP_ANON | mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE)) return HCQBuffer(va:=addr, sz:=size, meta=buf, view=MMIOInterface(va, sz, fmt='B'), owner=self.dev) def _as_buffer(self, src) -> memoryview: return to_mv(src.va_addr, src.size) + def _copyin(self, dest:HCQBuffer, src:memoryview): + self.dev.synchronize() + ctypes.memmove(int(dest.va_addr), from_mv(src), len(src)) + def _copyout(self, dest:memoryview, src:HCQBuffer): + self.dev.synchronize() + ctypes.memmove(from_mv(dest), int(src.va_addr), len(dest)) def _do_map(self, buf:HCQBuffer): if buf.view is None or not isinstance(buf.view, MMIOInterface): raise RuntimeError("Cannot map buffer without view to cpu") return HCQBuffer(buf.view.addr, buf.size, view=buf.view, owner=buf.owner) def _unmap(self, mb): pass # CPU _do_map returns a view wrapper, nothing to release -class CPUDevice(HCQCompiled): - pm_lower = PatternMatcher([ - (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_host_queue)]) - - pm_bufferize = PatternMatcher([ - (UPat(Ops.PARAM, tag="sentinel_signal"), lambda ctx: ctx[0].signal("sentinel", (1 << 64) - 1)), - (UPat(Ops.PARAM, tag="timeline_signal"), lambda ctx: ctx[0].signal("timeline")), - (UPat(Ops.PARAM, tag="timeline_value"), lambda ctx: ctx[0].signal("value", 1)), - (UPat(Ops.PARAM, tag="signal", name="b"), lambda ctx, b: ctx[0].signal(b.arg.slot)), - ]) - - @functools.cache - def signal(self, name:str|int, init_value:int=0) -> Buffer: - (buf:=Buffer(self.device, 1, dtypes.uint64, preallocate=True)).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] = init_value - return buf +class CPUDevice(HCQ2Compiled): + wait_timeout_ms, has_copy_queue = 30000, False + pm_encode, pm_lower = {"COMPUTE": pm_cpu_opsel, "SUBMIT": pm_cpu_opsel}, {"COMPUTE": pm_cpu_submit, "SUBMIT": pm_cpu_submit} def __init__(self, device:str=""): - super().__init__(device, CPUAllocator(self), [ClangRenderer, CPULLVMRenderer, LVPRenderer, X86Renderer], CPUProgram, HCQSignal, - functools.partial(CPUComputeQueue, self), arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") + super().__init__(device, CPUAllocator(self), [ClangRenderer, CPULLVMRenderer, LVPRenderer, X86Renderer], CPUProgram, + arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") - self.ring_pos = 0 + self.pm_bufferize = PatternMatcher( + [(UPat(Ops.PARAM, tag=f"func:{f}"), lambda ctx, f=f: ctx[0].func_ptr(f)) for f in FUNCS]) + self.pm_bufferize - # posix uses sem to put cpus into sleep - self.sem_addr = 0 - if not WIN: - self.sem = libc.sem_open(sem_name:=f"/tinygrad-{os.getpid()}-{id(self):x}".encode(), os.O_CREAT|os.O_EXCL, 0o600, 0) # type: ignore[call-arg] - self.sem_addr = unwrap(ctypes.cast(self.sem, ctypes.c_void_p).value) - if self.sem_addr == ctypes.c_void_p(-1).value or libc.sem_unlink(sem_name): raise OSError(ctypes.get_errno(), "semaphore") - - # TODO: move to hcq2 with Context(EMULATED_DTYPES="", TRACK_MATCH_STATS=0): - prgs = {f: f().sink(arg=KernelInfo(f.__name__), tag=1) for f in (signal_prog, wait_prog, timestamp_prog, quit_prog, worker_prog)} - self.prgs = {f: self.runtime(do_to_program(v, ClangRenderer(replace(self.renderer.target, renderer="CLANG"))).to_elf()) for f,v in prgs.items()} + clang = ClangRenderer(replace(self.renderer.target, renderer="CLANG")) + self.prgs:dict[Callable, CPUProgram] = {f: CPUProgram(self, do_to_program(f().sink(arg=KernelInfo(f.__name__), tag=1), clang).to_elf()) + for f in (signal_prog, wait_prog, timestamp_prog)} - @functools.cached_property - def ring(self) -> Buffer: return Buffer(self.device, RING_SLOTS * CMD_SIZE, dtypes.uint64, preallocate=True) - @functools.cached_property - def sys(self) -> Buffer: return Buffer(self.device, 1, dtypes.uint64, preallocate=True) - @functools.cached_property - def sem_buf(self) -> Buffer: return Buffer(self.device, 1, dtypes.uint8, options=BufferSpec(external_ptr=self.sem_addr), preallocate=True) + def func_ptr(self, name:str) -> Buffer: return self.func_table.view(1, dtypes.uint64, FUNCS.index(name)*8).ensure_allocated() - # TODO: move to hcq2 infra @functools.cached_property def func_table(self) -> Buffer: - fns = ([0, ctypes.windll.kernel32.ExitThread, 0, 0] if WIN else # type: ignore[attr-defined] - [libc.dll.clock_gettime, libc.dll.pthread_exit, libc.dll.sem_wait, libc.dll.sem_close]) - addrs = array.array('Q', [unwrap(ctypes.cast(f, ctypes.c_void_p).value) if f else 0 for f in fns]) - (ft:=Buffer(self.device, len(fns), dtypes.uint64, preallocate=True)).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[:] = addrs + lib = ctypes.windll.kernel32 if sys.platform == "win32" else libc.dll # type: ignore[attr-defined] + (ft:=Buffer(self.device, len(FUNCS), dtypes.uint64, preallocate=True))._buf.cpu_view().view(fmt='Q')[:] = \ + array.array('Q', [unwrap(ctypes.cast(getattr(lib, f), ctypes.c_void_p).value) for f in FUNCS]) return ft - - @functools.cache - def ensure_worker(self): - threading.Thread(target=cast(CPUProgram, self.prgs[worker_prog]).fxn, daemon=True, args=[ctypes.c_uint64(x) for x in - [self.ring._buf.va_addr, self.sys._buf.va_addr if WIN else self.func_table._buf.va_addr+16, self.sem_addr]]).start() - - def finalize(self): - if self.ring_pos == 0: return # the worker starts with the first submit - ft = self.func_table._buf - CPUComputeQueue(self)._cmd(quit_prog, (ft.offset(8, 8),) if WIN else (ft.offset(8, 24), self.sem_buf._buf)).submit(self) - self.ring_pos = 0 diff --git a/tinygrad_repo/tinygrad/runtime/ops_dsp.py b/tinygrad_repo/tinygrad/runtime/ops_dsp.py index e8ffa2ef8..2de37e791 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_dsp.py +++ b/tinygrad_repo/tinygrad/runtime/ops_dsp.py @@ -9,18 +9,9 @@ from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.runtime.autogen import libc, qcom_dsp if getenv("IOCTL"): import extra.dsp.run # noqa: F401 # pylint: disable=unused-import -from tinygrad.uop.ops import PatternMatcher, UPat - -# NOTE: this just increases readability of the generated code -dsp_string = PatternMatcher([ - (UPat(Ops.CONST, (dtypes.int8, dtypes.uint8), name="x"), lambda ctx,x: str(x.val)), -]) - class DSPRenderer(ClangRenderer): - has_threads = False buffer_suffix = " restrict __attribute__((align_value(128)))" kernel_typedef = "__attribute__((noinline)) void" - string_rewrite = dsp_string+ClangRenderer.string_rewrite type_map = { **ClangRenderer.type_map, dtypes.uint64: "unsigned long long", dtypes.int64: "long long" } code_for_op = {k:v for k,v in ClangRenderer.code_for_op.items() if k != Ops.SQRT} @@ -105,7 +96,7 @@ class DSPAllocator(Allocator['DSPDevice']): class DSPCompiler(Compiler): def __init__(self, mock:bool=False): - compiler_args = "--target=hexagon -mcpu=hexagonv65 -fuse-ld=lld -nostdlib -mhvx=v65 -mhvx-length=128b" + self.mock, compiler_args = mock, "--target=hexagon -mcpu=hexagonv65 -fuse-ld=lld -nostdlib -mhvx=v65 -mhvx-length=128b" if mock: self.args = f"-static {compiler_args}" else: # Generate link script to pass into clang. Aligning all used sections to 4k fixes invoke problem. @@ -120,6 +111,9 @@ class DSPCompiler(Compiler): super().__init__(None if mock else "compile_dsp") + def __del__(self): + if not self.mock: os.unlink(self.link_ld.name) + def compile(self, src:str) -> bytes: # TODO: remove file write. sadly clang doesn't like the use of /dev/stdout here with tempfile.NamedTemporaryFile(delete=True) as f: diff --git a/tinygrad_repo/tinygrad/runtime/ops_metal.py b/tinygrad_repo/tinygrad/runtime/ops_metal.py index 3a0fe42d2..2c00be72c 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_metal.py +++ b/tinygrad_repo/tinygrad/runtime/ops_metal.py @@ -34,6 +34,7 @@ class MetalDevice(Compiled): self.mtl_queue = self.sysdevice.newCommandQueueWithMaxCommandBufferCount(1024) if self.mtl_queue is None: raise RuntimeError("Cannot allocate a new command queue") self.mtl_buffers_in_flight: list[metal.MTLCommandBuffer] = [] + self.mtl_profile_keys: dict[int, bytes] = {} self.timeline_signal = self.sysdevice.newSharedEvent() self.timeline_value = 0 @@ -55,7 +56,7 @@ class MetalDevice(Compiled): st, en = decimal.Decimal(cbuf.GPUStartTime()) * 1000000, decimal.Decimal(cbuf.GPUEndTime()) * 1000000 # NOTE: command buffers from MetalGraph are not profiled here if PROFILE and (lb:=cmdbuf_label(cbuf)) is not None and not lb.startswith("batched"): - Compiled.profile_events += [ProfileRangeEvent(self.device, lb, st, en)] + Compiled.profile_events += [ProfileRangeEvent(self.device, lb, st, en, self.mtl_profile_keys.pop(id(cbuf), None))] self.mtl_buffers_in_flight.clear() class MetalCompiler(Compiler): @@ -113,7 +114,7 @@ class MetalCompiler(Compiler): class MetalProgram(Program[MetalDevice]): def __init__(self, dev:MetalDevice, obj:TinyELF): - self.dev, self.name, self.lib, self.signature = dev, obj.name, obj.lib, obj.signature + self.dev, self.name, self.lib, self.signature, self.profile_key = dev, obj.name, obj.lib, obj.signature, obj.profile_key data = objc.dispatch_data_create(obj.lib, len(obj.lib), None, None) self.library = self.dev.sysdevice.newLibraryWithData_error(data, ctypes.byref(error_lib:=metal.NSError().retained())).retained() error_check(error_lib) @@ -145,6 +146,7 @@ class MetalProgram(Program[MetalDevice]): command_buffer.setLabel(to_ns_str(self.name)) # TODO: is this always needed? command_buffer.commit() self.dev.mtl_buffers_in_flight.append(command_buffer) + if PROFILE and self.profile_key is not None: self.dev.mtl_profile_keys[id(command_buffer)] = self.profile_key if wait: wait_check(command_buffer) return command_buffer.GPUEndTime() - command_buffer.GPUStartTime() diff --git a/tinygrad_repo/tinygrad/runtime/ops_npy.py b/tinygrad_repo/tinygrad/runtime/ops_npy.py index b7ef4e47f..2e5196ed6 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_npy.py +++ b/tinygrad_repo/tinygrad/runtime/ops_npy.py @@ -6,6 +6,8 @@ class NpyAllocator(Allocator['NpyDevice']): def _alloc(self, size:int, options=None) -> np.ndarray: return np.empty(size, dtype=np.uint8) def _as_buffer(self, src:np.ndarray) -> memoryview: return flat_mv(np.require(src, requirements='C').data) def _copyout(self, dest:memoryview, src:np.ndarray): dest[:] = self._as_buffer(src) + def _offset(self, buf:np.ndarray, size:int, offset:int) -> np.ndarray: + return np.require(buf, requirements='C').reshape(-1).view(np.uint8)[offset:offset+size] class NpyDevice(Compiled): def __init__(self, device:str): super().__init__(device, NpyAllocator(self), [], None) diff --git a/tinygrad_repo/tinygrad/runtime/ops_null.py b/tinygrad_repo/tinygrad/runtime/ops_null.py index 228ef917d..e7c8d97c4 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_null.py +++ b/tinygrad_repo/tinygrad/runtime/ops_null.py @@ -17,9 +17,9 @@ class NullRenderer(CStyleLanguage): return assemble_linear(prg, lin, self.target.arch) class NullProgram(Program['NullDevice']): - def __init__(self, dev:'NullDevice', obj:TinyELF): self.device, self.name = dev.device, obj.name + def __init__(self, dev:'NullDevice', obj:TinyELF): self.device, self.name, self.profile_key = dev.device, obj.name, obj.profile_key def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw): - with cpu_profile(self.name, self.device): return 1e-3 + with cpu_profile(self.name, self.device, profile_key=self.profile_key): return 1e-3 class NullAllocator(Allocator['NullDevice']): def _alloc(self, size, options): pass @@ -38,13 +38,14 @@ class NullGraph(MultiGraphRunner): for (_,_,bufs,_),runtime in zip(self.calls, self.runtimes): # description based on command, copied from HCQ graph device = runtime.device if runtime is not None else f"{bufs[1].device}:SDMA:0" - descs.append((device, runtime.name if runtime is not None else f"{bufs[1].device} -> {bufs[0].device}", count:=event_count.get(device, 0))) + descs.append((device, runtime.name if runtime is not None else f"{bufs[1].device} -> {bufs[0].device}", + runtime.profile_key if runtime is not None else None, count:=event_count.get(device, 0))) event_count[device] = count+1 # pack events evenly per device dur, sigs, ents = max(1, math.ceil((perf_counter_us()-st)/max(event_count.values()))), [], [] - for i,(device,name,count) in enumerate(descs): + for i,(device,name,profile_key,count) in enumerate(descs): sigs += [st+count*dur, st+(count+1)*dur] - ents.append(ProfileGraphEntry(device, name, 2*i, 2*i+1)) + ents.append(ProfileGraphEntry(device, name, 2*i, 2*i+1, profile_key)) cpu_events.append(ProfileGraphEvent(ents, [], sigs)) return 1e-1 diff --git a/tinygrad_repo/tinygrad/runtime/ops_nv.py b/tinygrad_repo/tinygrad/runtime/ops_nv.py index 2c274a6dd..f814a752b 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_nv.py +++ b/tinygrad_repo/tinygrad/runtime/ops_nv.py @@ -22,7 +22,7 @@ nv_gpu = nv_570 # default to 570 PMA = ContextVar("PMA", abs(VIZ.value)>=2) @dataclass(frozen=True) -class ProfilePMAEvent(ProfileEvent): device:str; kern:str; blob:bytes; exec_tag:int # noqa: E702 +class ProfilePMAEvent(ProfileEvent): device:str; kern:str; blob:bytes; exec_tag:int; profile_key:bytes|None=None # noqa: E702 class NVSignal(HCQSignal): def _sleep(self, time_spent_since_last_sleep_ms:int): @@ -335,12 +335,12 @@ class NVProgram(HCQProgram['NVDevice']): if self.dev.pma_enabled: self.dev.synchronize() if pma_blob:=self.dev._prof_readback(): - Compiled.profile_events += [ProfilePMAEvent(self.dev.device, self.name, pma_blob, self.dev.prof_exec_counter)] + Compiled.profile_events += [ProfilePMAEvent(self.dev.device, self.name, pma_blob, self.dev.prof_exec_counter, self.profile_key)] return res class NVAllocator(HCQAllocator['NVDevice']): def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: - return self.dev.iface.alloc(size, cpu_access=options.cpu_access, host=options.host) + return self.dev.iface.alloc(size, cpu_access=options.cpu_access, host=options.host, zero=options.zero) def _do_free(self, opaque:HCQBuffer, options:BufferSpec): self.dev.iface.free(opaque) @@ -565,7 +565,7 @@ class PCIIface(PCIIfaceBase): # Setup classes for the GPU self.gpfifo_class, self.compute_class, self.dma_class = (gsp:=self.dev_impl.gsp).gpfifo_class, gsp.compute_class, gsp.dma_class - self.viddec_class = None + self.viddec_class = gsp.viddec_class def setup_usermode(self): return 0xce000000, self.pci_dev.map_bar(bar=0, fmt='I', off=0xbb0000, size=0x10000) def setup_vm(self, vaspace): pass @@ -588,8 +588,7 @@ class NVDevice(HCQCompiled[NVSignal]): def is_nvd(self) -> bool: return isinstance(self.iface, PCIIface) def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - self.iface = self._select_iface() + self.iface = self._select_iface(device) device_params = nv_gpu.NV0080_ALLOC_PARAMETERS(deviceId=self.iface.gpu_instance, hClientShare=self.iface.root, vaMode=nv_gpu.NV_DEVICE_ALLOCATION_VAMODE_OPTIONAL_MULTIPLE_VASPACES) @@ -604,7 +603,7 @@ class NVDevice(HCQCompiled[NVSignal]): vaspace_params = nv_gpu.NV_VASPACE_ALLOCATION_PARAMETERS(vaBase=0x1000, vaSize=0x1fffffb000000, flags=nv_gpu.NV_VASPACE_ALLOCATION_FLAGS_ENABLE_PAGE_FAULTING | nv_gpu.NV_VASPACE_ALLOCATION_FLAGS_IS_EXTERNALLY_OWNED) - vaspace = self.iface.rm_alloc(self.nvdevice, nv_gpu.FERMI_VASPACE_A, vaspace_params) + self.vaspace = vaspace = self.iface.rm_alloc(self.nvdevice, nv_gpu.FERMI_VASPACE_A, vaspace_params) self.iface.setup_vm(vaspace) @@ -644,7 +643,8 @@ class NVDevice(HCQCompiled[NVSignal]): notifier = self.iface.alloc(48 << 20, uncached=True) params = nv_gpu.NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS(gpFifoOffset=gpfifo_area.va_addr+offset, gpFifoEntries=entries, hContextShare=ctxshare, hObjectError=notifier.meta.hMemory, hObjectBuffer=self.virtmem if video else gpfifo_area.meta.hMemory, - hUserdMemory=(ctypes.c_uint32*8)(gpfifo_area.meta.hMemory), userdOffset=(ctypes.c_uint64*8)(entries*8+offset), engineType=19 if video else 0) + hUserdMemory=(ctypes.c_uint32*8)(gpfifo_area.meta.hMemory), userdOffset=(ctypes.c_uint64*8)(entries*8+offset), engineType=19 if video else 0, + hVASpace=self.vaspace if video and self.is_nvd() else 0) # gsp has no default vaspace, rm maps the decoder ctx into its own gpfifo = self.iface.rm_alloc(channel_group, self.iface.gpfifo_class, params) if compute: @@ -710,22 +710,22 @@ class NVDevice(HCQCompiled[NVSignal]): def _ensure_has_vid_hw(self, w, h): if self.iface.viddec_class is None: raise RuntimeError(f"{self.device} Video decoder class not available.") - coloc_size = round_up((round_up(h, 64) * round_up(h, 64)) + (round_up(w, 64) * round_up(h, 64) // 16), 2 << 20) + coloc_sz = round_up((round_up(h, 64) * round_up(h, 64)) + (round_up(w, 64) * round_up(h, 64) // 16), 2 << 20) self.intra_top_off = round_up(h, 64) * (608 + 4864 + 152 + 2000) intra_unk_size = ((2 << 20) if self.iface.viddec_class >= nv_gpu.NVCFB0_VIDEO_DECODER else 0) self.intra_unk_off = (round_up(self.intra_top_off, 0x10000) + (64 << 10)) if intra_unk_size > 0 else None - filter_size = round_up(round_up(self.intra_top_off, 0x10000) + (64 << 10) + intra_unk_size, 2 << 20) + filter_sz = round_up(round_up(self.intra_top_off, 0x10000) + (64 << 10) + intra_unk_size, 2 << 20) if not hasattr(self, 'vid_gpfifo'): self.vid_gpfifo = self._new_gpu_fifo(self.gpfifo_area, 0, self.nvdevice, offset=0x200000, entries=2048, compute=False, video=True) - self.vid_coloc_buf, self.vid_filter_buf = self.allocator.alloc(coloc_size), self.allocator.alloc(filter_size) - self.vid_stat_buf = self.allocator.alloc(0x1000) + self.vid_coloc_buf, self.vid_filter_buf = (self.allocator.alloc(sz, BufferSpec(zero=True)) for sz in [coloc_sz, filter_sz]) + self.vid_stat_buf = self.allocator.alloc(0x1000, BufferSpec(zero=True)) NVVideoQueue().wait(self.timeline_signal, self.timeline_value - 1) \ .setup(copy_class=self.iface.viddec_class) \ .signal(self.timeline_signal, self.next_timeline()).submit(self) else: - if coloc_size > self.vid_coloc_buf.size: self.vid_coloc_buf, _ = self._realloc(self.vid_coloc_buf, coloc_size, force=True) - if filter_size > self.vid_filter_buf.size: self.vid_filter_buf, _ = self._realloc(self.vid_filter_buf, filter_size, force=True) + if coloc_sz > self.vid_coloc_buf.size: self.vid_coloc_buf,_= self._realloc(self.vid_coloc_buf, coloc_sz, BufferSpec(zero=True), force=True) + if filter_sz > self.vid_filter_buf.size: self.vid_filter_buf,_= self._realloc(self.vid_filter_buf, filter_sz, BufferSpec(zero=True), force=True) def hw_copy_queues(self): return super().hw_copy_queues() + ([("NVDEC:0", NVVideoQueue)] if hasattr(self, 'vid_gpfifo') else []) diff --git a/tinygrad_repo/tinygrad/runtime/ops_python.py b/tinygrad_repo/tinygrad/runtime/ops_python.py index aa1390f04..f7ad33ddc 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_python.py +++ b/tinygrad_repo/tinygrad/runtime/ops_python.py @@ -3,19 +3,20 @@ # works to test the tensor cores, and all the uops in general # this is the (living) definition of uops from typing import Any, TYPE_CHECKING -import pickle, base64, itertools, time, sys, functools +import pickle, base64, itertools, time, sys, functools, ctypes from dataclasses import replace from tinygrad.dtype import bitcast, DType, dtypes, AddrSpace, truncate, storage_fmt_for_dtype, to_storage_scalar, from_storage_scalar -from tinygrad.helpers import all_same, getenv, flatten, Target, IMAGE, is_image_shape, cpu_profile +from tinygrad.helpers import all_same, getenv, flatten, Target, IMAGE, is_image_shape, cpu_profile, mv_address from tinygrad.device import Buffer, Compiled, Compiler, Allocator, Program, TinyELF -from tinygrad.codegen.opt import tc +from tinygrad.renderer import tc from tinygrad.uop.ops import exec_alu, python_alu, Ops, UOp, GroupOp from tinygrad.renderer import Renderer def _load(m, i, dtype: DType): if i is None: return 0.0 if i < 0 or i >= len(m): raise IndexError(f"load out of bounds, size is {len(m)} and access is {i}") - return from_storage_scalar(m[i], dtype) + if (w:=m.nbytes // len(m)) >= dtype.itemsize: return from_storage_scalar(m[i], dtype) + return sum(m[i+k] << (8*w*k) for k in range(dtype.itemsize // w)) # a bitcast can read wider than the buffer, _store splits it the same way def load(inp, j, dtype: DType): if len(inp) >= 3: return [_load(m, x+j if x is not None else None, dtype) if gate else default for (m,x),default,gate in zip(*inp[:3])] @@ -23,7 +24,9 @@ def load(inp, j, dtype: DType): def _store(m, i, v, dtype: DType): if i < 0 or i >= len(m): raise IndexError(f"store out of bounds, size is {len(m)}, access is {i}, value is {v}") - m[i] = to_storage_scalar(v, dtype) + if (w:=m.nbytes // len(m)) >= dtype.itemsize: m[i] = to_storage_scalar(v, dtype) + else: + for k in range(dtype.itemsize // w): m[i+k] = (v >> 8*w*k) & ((1 << 8*w) - 1) # here are the models for the WMMA instruction on the different hardware def generic_wmma_helper(inp, warp_size, WARP_THREADS, K, NUM_A, NUM_B, NUM_C, a_elem, b_elem, c_map): @@ -78,7 +81,6 @@ class PythonProgram(Program['PythonDevice']): # in the python emulator, the warp is always in sync i += 1 continue - assert u.dtype is not None, f"{u.op} is missing a dtype" if u.op is Ops.STORE: assert len(src_values) == 2, f"STORE must be lowered to 2 srcs, got {len(src_values)}" store_gate = exec_masks[-1] @@ -87,7 +89,7 @@ class PythonProgram(Program['PythonDevice']): if g: _store(m, o+j, v, src_dtypes[1]) i += 1 continue - if u.op is Ops.AFTER: values[u] = src_values[0] + if u.op is Ops.AFTER or (u.op is Ops.BITCAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)): values[u] = src_values[0] elif u.op is Ops.PARAM and u.addrspace is AddrSpace.ALU: values[u] = [pvals.pop(0)] * warp_size elif u.op in {Ops.PARAM, Ops.BUFFER}: storage_fmt = storage_fmt_for_dtype(u.dtype) @@ -112,7 +114,8 @@ class PythonProgram(Program['PythonDevice']): if ox < 0 or ox >= u.src[0]._shape[1] or oy < 0 or oy >= u.src[0]._shape[0]: ret.append((m, None)) else: ret.append((m, ox*4 + oy*u.src[0]._shape[1]*4)) else: - for m,o in zip(src_values[0], src_values[1]): ret.append((m,o)) + scale = u.src[0].dtype.itemsize // u.src[0].src[0].dtype.itemsize if u.src[0].op is Ops.BITCAST else 1 + for m,o in zip(src_values[0], src_values[1]): ret.append((m[0], m[1]+o*scale) if isinstance(m, tuple) else (m, o*scale)) values[u] = ret elif u.op is Ops.RANGE: if u not in values: values[u] = [0] * warp_size @@ -134,6 +137,13 @@ class PythonProgram(Program['PythonDevice']): for k in range(len(src_values))], j, u.dtype) for j in range(load_sz)] else: values[u] = load(src_values, 0, u.dtype) + elif u.op is Ops.CALL: + assert u.dtype is dtypes.void + cfunc = ctypes.CFUNCTYPE(None, *[ctypes.c_uint64] * (len(src_values)-1)) + values[u] = [] + for args,gate in zip(zip(*src_values), exec_masks[-1]): + call_args = [(mv_address(x[0]) + x[1]*dt.itemsize) if isinstance(x, tuple) else x for x,dt in zip(args, src_dtypes)] + values[u].append(cfunc(call_args[0])(*call_args[1:]) if gate else None) elif u.op is Ops.WMMA: first_src_dtype = u.src[0].dtype assert isinstance(first_src_dtype, DType) # mypy diff --git a/tinygrad_repo/tinygrad/runtime/ops_qcom.py b/tinygrad_repo/tinygrad/runtime/ops_qcom.py index 3535207b2..63fc9bade 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_qcom.py +++ b/tinygrad_repo/tinygrad/runtime/ops_qcom.py @@ -20,11 +20,12 @@ BUFTYPE_BUF, BUFTYPE_TEX, BUFTYPE_IBO = 0, 1, 2 def dcache_flush(): from tinygrad.uop.ops import UOp, Ops, KernelInfo from tinygrad.codegen import to_program - buf, n = UOp.param(0, dtypes.uint8, shape=(1,)), UOp.param(1, dtypes.int, shape=(), name="n", addrspace=AddrSpace.ALU) + buf, n = UOp.param(0, dtypes.uint8, 1), UOp.param(1, dtypes.int, shape=(), name="n", addrspace=AddrSpace.ALU) i = UOp.range(n, 0, dtype=dtypes.int) - flush = UOp(Ops.CUSTOM, src=(buf.index(i * 64),), arg='__asm__ volatile("dc cvac, %0" :: "r"({0}) : "memory");') - sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg='__asm__ volatile("dsb sy" ::: "memory");'), arg=KernelInfo(name="dcache_flush")) - prg = to_program(UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(sink.toposort())))), Device["CPU"].renderer) + flush = UOp(Ops.CUSTOM, src=(buf.index(i * 64),), arg=('__asm__ volatile("dc cvac, %0" :: "r"({0}) : "memory");', dtypes.void)) + sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg=('__asm__ volatile("dsb sy" ::: "memory");', dtypes.void)), + arg=KernelInfo(name="dcache_flush"), tag=1) + prg = to_program(sink, Device["CPU"].renderer) return Device["CPU"].runtime(prg.to_elf()) #Parse C-style defines: ___SHIFT and ___MASK from the adreno module into the following format: diff --git a/tinygrad_repo/tinygrad/runtime/ops_rdma.py b/tinygrad_repo/tinygrad/runtime/ops_rdma.py index 6f7a46600..c4be2d0f9 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_rdma.py +++ b/tinygrad_repo/tinygrad/runtime/ops_rdma.py @@ -101,6 +101,5 @@ class RDMAAllocator(HCQAllocatorBase): class RDMADevice(HCQCompiled): def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - self.iface = MLXIface(self, self.device_id) + self.iface = MLXIface(self, int(device.split(":")[1]) if ":" in device else 0) super().__init__(device, RDMAAllocator(self), [], None, signal_t=None) diff --git a/tinygrad_repo/tinygrad/runtime/ops_tinyfs.py b/tinygrad_repo/tinygrad/runtime/ops_tinyfs.py deleted file mode 100644 index c041a65de..000000000 --- a/tinygrad_repo/tinygrad/runtime/ops_tinyfs.py +++ /dev/null @@ -1,128 +0,0 @@ -import socket, json, asyncio, threading, math -from contextlib import asynccontextmanager -from tinygrad.device import Compiled, Allocator -from tinygrad.helpers import DEBUG, getenv, CHUNK_SIZE - -TINYFS_ENDPOINT = getenv("TINYFS_ENDPOINT", "localhost:6767") -TINYFS_TIMEOUT = getenv("TINYFS_TIMEOUT", 60) - -class TinyFSDevice(Compiled): - def __init__(self, device:str): - self.op = device[len("tinyfs:"):].upper() - super().__init__(device, TinyFSAllocator(self), [], None, None) - - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.connect((TINYFS_ENDPOINT.rsplit(":", 1)[0], int(TINYFS_ENDPOINT.rsplit(":", 1)[1]))) - self.sock.settimeout(TINYFS_TIMEOUT) - self.sfile = self.sock.makefile("rwb") - - # fetch node info - self.sfile.write(b"INFO\r\n") - self.sfile.flush() - info = self.sfile.readline() - self.node_info = json.loads(info) - if DEBUG >= 2: print(f"nodes: {self.node_info}") - - # spawn thread for async copyout - self.start_event = threading.Event() - self.t = threading.Thread(target=self._start_thread, daemon=True) - self.t.start() - self.start_event.wait() - - # connection pools - self.conn_pools: dict[str, asyncio.Queue] = {} - self.conn_pools_lock = asyncio.Lock() - - def finalize(self): - self.sfile.close() - - for pool in self.conn_pools.values(): - while not pool.empty(): - _, w = pool.get_nowait() - w.close() - asyncio.run_coroutine_threadsafe(w.wait_closed(), self.loop).result() - - if hasattr(self, "loop"): - self.loop.call_soon_threadsafe(self.loop.stop) - self.t.join() - - def _start_thread(self): - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - - self.start_event.set() - self.loop.run_forever() - self.loop.close() - - @asynccontextmanager - async def connection(self, loc): - if loc not in self.conn_pools: - await self.conn_pools_lock.acquire() - if loc not in self.conn_pools: - self.conn_pools[loc] = asyncio.Queue(nw:=getenv("ASYNC_COPY_WORKERS", 4)) - conn_tasks = [asyncio.open_connection(*self.node_info[loc][-1].rsplit(":", 1)) for _ in range(nw)] - connections = await asyncio.gather(*conn_tasks) - for reader, writer in connections: self.conn_pools[loc].put_nowait((reader, writer)) - self.conn_pools_lock.release() - - reader, writer = await self.conn_pools[loc].get() - try: - yield reader, writer - finally: - await self.conn_pools[loc].put((reader, writer)) - -class TinyFSBuffer: - def __init__(self, device:TinyFSDevice, size:int, offset=0, copyout_queue=None, hash_buf=None): - self.device, self.size, self.offset = device, size, offset - self.copyout_queue = [] if copyout_queue is None else copyout_queue - self.hash_buf = bytearray() if hash_buf is None else hash_buf - def __repr__(self): return f"" - -class TinyFSAllocator(Allocator[TinyFSDevice]): - def _alloc(self, size, options): - return TinyFSBuffer(self.dev, size) - - def _copyin(self, dest:TinyFSBuffer, src:memoryview): - if DEBUG >= 2: print(f"Copying in {dest.size} bytes to TINYFS:{dest.device.op}") - self.dev.sfile.write(f"{dest.device.op}_IN {dest.size}\r\n".encode()) - - self.dev.sfile.write(src) - self.dev.sfile.flush() - - if dest.device.op == "LOAD": - locs = self.dev.sfile.readline() - dest.copyout_queue[:] = json.loads(locs) - dest.hash_buf[:] = src.tobytes() - elif dest.device.op == "STORE": - expected_hashes = math.ceil(dest.size / CHUNK_SIZE) - dest.hash_buf[:] = bytearray(expected_hashes * 16) - self.dev.sfile.readinto(dest.hash_buf) - - def _copyout(self, dest:memoryview, src:TinyFSBuffer): - if DEBUG >= 2: print(f"Copying out {src.size} bytes from TINYFS:{src.device.op}") - if src.device.op == "LOAD": - asyncio.run_coroutine_threadsafe(self._copyout_async(dest, src), src.device.loop).result() - elif src.device.op == "STORE": - dest[:] = src.hash_buf - - async def _copyout_async(self, dest:memoryview, src:TinyFSBuffer): - async def _worker(i, loc): - async with self.dev.connection(loc) as (reader, writer): - ptr = i * CHUNK_SIZE - size = min(len(dest[ptr:ptr+CHUNK_SIZE]), CHUNK_SIZE) - - writer.write(f"CHUNK_OUT {size}\r\n".encode()) - writer.write(src.hash_buf[i*16:(i+1)*16]) - await asyncio.wait_for(writer.drain(), timeout=TINYFS_TIMEOUT) - - chunk = await asyncio.wait_for(reader.readexactly(size), timeout=TINYFS_TIMEOUT) - - view = dest[ptr:ptr+len(chunk)] - view[:] = chunk - del view - - workers = [asyncio.create_task(_worker(i, loc)) for i, loc in enumerate(src.copyout_queue)] - await asyncio.gather(*workers) - - def _offset(self, buf:TinyFSBuffer, size:int, offset:int): - return TinyFSBuffer(buf.device, size, offset, buf.copyout_queue, buf.hash_buf) diff --git a/tinygrad_repo/tinygrad/runtime/ops_webgpu.py b/tinygrad_repo/tinygrad/runtime/ops_webgpu.py index dc3466e4a..de2103f28 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_webgpu.py +++ b/tinygrad_repo/tinygrad/runtime/ops_webgpu.py @@ -201,7 +201,7 @@ class WebGpuDevice(Compiled): def create_uniform(self, val:int|float) -> webgpu.WGPUBuffer: buf = webgpu.wgpuDeviceCreateBuffer(self.device_res, webgpu.WGPUBufferDescriptor(size=4, usage=webgpu.WGPUBufferUsage_Uniform | webgpu.WGPUBufferUsage_CopyDst)) - self.write_buffer(buf, val.to_bytes(4, "little") if isinstance(val, int) else struct.pack(' webgpu.WGPUBuffer: size = webgpu.wgpuBufferGetSize(buf) diff --git a/tinygrad_repo/tinygrad/runtime/support/am/amdev.py b/tinygrad_repo/tinygrad/runtime/support/am/amdev.py index d4b577123..9e7d02452 100644 --- a/tinygrad_repo/tinygrad/runtime/support/am/amdev.py +++ b/tinygrad_repo/tinygrad/runtime/support/am/amdev.py @@ -1,6 +1,6 @@ from __future__ import annotations -import ctypes, collections, dataclasses, functools, hashlib, array -from tinygrad.helpers import mv_address, getenv, DEBUG, lo32, hi32, fetch_fw +import ctypes, collections, dataclasses, functools, hashlib, array, contextlib +from tinygrad.helpers import mv_address, getenv, DEBUG, lo32, hi32, fetch_fw, to_mv, wait_cond from tinygrad.runtime.autogen import pci from tinygrad.runtime.autogen.am import am, fw from tinygrad.runtime.support.amd import AMDReg, import_module, import_asic_regs @@ -14,10 +14,11 @@ AM_DEBUG = getenv("AM_DEBUG", 0) class AMRegister(AMDReg): adev:AMDev - def read(self, inst=0): return self.adev.rreg(self.addr[inst]) + def read(self, inst=0, direct=False): return self.adev.rreg(self.addr[inst], inst=inst, direct=direct) def read_bitfields(self, inst=0) -> dict[str, int]: return self.decode(self.read(inst=inst)) - def write(self, _am_val:int=0, inst=0, **kwargs): self.adev.wreg(self.addr[inst], _am_val | self.encode(**kwargs)) + def write(self, _am_val:int=0, inst=0, direct=False, **kwargs): + self.adev.wreg(self.addr[inst], _am_val|self.encode(**kwargs), inst=inst, direct=direct) def update(self, inst=0, **kwargs): self.write(self.read(inst=inst) & ~self.fields_mask(*kwargs.keys()), inst=inst, **kwargs) @@ -145,10 +146,26 @@ class AMMemoryManager(MemoryManager): class AMDev: Version = 0xA0000008 + def _disable_aspm(self): + # L1 across retimers makes reads oscillate to 0xffffffff; power on defaults it enabled. Clearing the GPU endpoint + # alone suffices: L1 only engages when both ends of the link enable it. + cap, seen = self.pci_dev.read_config(0x34, 1) & 0xfc, set() # bound the walk: a dead link can return 0xff pointers forever + while cap and cap not in seen and self.pci_dev.read_config(cap, 1) != 0x10: + seen.add(cap) + cap = self.pci_dev.read_config(cap + 1, 1) & 0xfc + if cap and cap not in seen: self.pci_dev.write_config_flush(cap + 0x10, self.pci_dev.read_config(cap + 0x10, 2) & ~3, 2) # PCIe cap lnkctl + def __init__(self, pci_dev:PCIDevice, reset_mode=False): self.pci_dev, self.devfmt = pci_dev, pci_dev.pcibus + self._disable_aspm() self.vram, self.doorbell64, self.mmio = self.pci_dev.map_bar(0), self.pci_dev.map_bar(2, fmt='Q'), self.pci_dev.map_bar(5, fmt='I') + # VF related + self.is_vf = bool(self.mmio[am.mmRCC_IOV_FUNC_IDENTIFIER] & 1) + self.vf_mailbox = self.mmio.view(am.NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE, 2, fmt='B') + self.vf_access = self._vf_mailbox_request(am.IDH_REQ_GPU_INIT_ACCESS) if self.is_vf else 0 + self.vf_rlc_gated:list[tuple[int, int]] = [] + self._run_discovery() self._build_regs() @@ -170,16 +187,19 @@ class AMDev: if DEBUG >= 2: print(f"am {self.devfmt}: Malformed state. Issuing a full reset.") self.partial_boot = False + # aqua (gc 9.5.0): full boot over live state can kill the fabric (power cycle recovers); partial boot+reset_mec is the deepest safe reset + if self.ip_ver[am.GC_HWIP] == (9,5,0) and self.reg("regSCRATCH_REG7").read() == AMDev.Version: self.partial_boot = True + # Init hw for IP blocks where it is needed if not self.partial_boot: - if self.psp.is_sos_alive() and self.smu.is_smu_alive(): + if not self.is_vf and self.psp.is_sos_alive() and self.smu.is_smu_alive(): # skip in vf mode, these are pf funcs. self.pci_dev.write_config_flush(pci.PCI_COMMAND, self.pci_dev.read_config(pci.PCI_COMMAND, 2) & ~pci.PCI_COMMAND_MASTER, 2) if self.is_hive(): if reset_mode: return # in reset mode, do not raise raise RuntimeError("Malformed state. Use extra/amdpci/hive_reset.py to reset the hive") self.smu.mode1_reset() self.pci_dev.write_config_flush(pci.PCI_COMMAND, self.pci_dev.read_config(pci.PCI_COMMAND, 2) | pci.PCI_COMMAND_MASTER, 2) - self.init_hw(self.soc, self.gmc, self.ih, self.psp, self.smu) + self.init_hw(self.soc, self.gmc, self.ih, *(() if self.is_vf else (self.psp, self.smu))) # Booting done self.is_booting = False @@ -187,13 +207,15 @@ class AMDev: # Re-initialize main blocks self.init_hw(self.gfx, self.sdma) - if (max_power:=getenv("AM_POWER_LIMIT", 0.0)) > 0: - self.smu.set_power_limit(max_power) - self.smu.set_clocks(level=None) - else: self.smu.set_clocks(level=-1) # last level, max perf. - for ip in [self.soc, self.gfx]: ip.set_clockgating_state() - self.reg("regSCRATCH_REG7").write(AMDev.Version) - self.reg("regSCRATCH_REG6").write(1) # set initialized state. + if not self.is_vf: # skip in vf mode, these are pf funcs. + if (max_power:=getenv("AM_POWER_LIMIT", 0.0)) > 0: + self.smu.set_power_limit(max_power) + self.smu.set_clocks(level=None) + else: self.smu.set_clocks(level=-1) # last level, max perf. + for ip in [self.soc, self.gfx]: ip.set_clockgating_state() + self.reg("regSCRATCH_REG7").write(AMDev.Version) + self.reg("regSCRATCH_REG6").write(1) # set initialized state. + if DEBUG >= 2: print(f"am {self.devfmt}: boot done") def init_sw(self, smi_dev=False): @@ -224,10 +246,33 @@ class AMDev: def fini(self): if DEBUG >= 2: print(f"am {self.devfmt}: Finalizing") + # a VF may only touch the engines inside an access window, take one so the host does not have to FLR the VF later + if self.is_vf and not self.vf_access: + with contextlib.suppress(TimeoutError): self.vf_access = self._vf_mailbox_request(am.IDH_REQ_GPU_FINI_ACCESS) for ip in [self.sdma, self.gfx]: ip.fini_hw() - self.smu.set_clocks(level=0) + if not self.is_vf: self.smu.set_clocks(level=0) self.ih.interrupt_handler() - self.reg("regSCRATCH_REG6").write(self.is_err_state) # set finalized state. + if not self.is_vf: self.reg("regSCRATCH_REG6").write(self.is_err_state) # set finalized state. + if self.vf_access: self.release_vf_access() + + def release_vf_access(self): + rel, self.vf_access = self.vf_access, 0 # give back the same lease that was taken + with contextlib.suppress(TimeoutError): self._vf_mailbox_request(rel, wait_ready=False) + + def _vf_mailbox_request(self, req:int, wait_ready=True) -> int: + self.vf_mailbox[0] = 0 # drop TRN_MSG_VALID + + wait_cond(lambda: self.vf_mailbox[0] & 2, value=0, timeout_ms=1000, msg="VF mailbox acknowledgement did not clear") + for i, val in enumerate((req, 0, 0, 0)): self.mmio[am.mmMAILBOX_MSGBUF_TRN_DW0 + i] = val + + self.vf_mailbox[0] = 1 # set TRN_MSG_VALID + wait_cond(lambda: self.vf_mailbox[0] & 2, value=2, timeout_ms=am.NV_MAILBOX_POLL_ACK_TIMEDOUT, msg=f"VF mailbox request {req:#x} was not acked") + self.vf_mailbox[0] = 0 + if wait_ready: + wait_cond(lambda: self.mmio[am.mmMAILBOX_MSGBUF_RCV_DW0], value=am.IDH_READY_TO_ACCESS_GPU, timeout_ms=am.NV_MAILBOX_POLL_MSG_TIMEDOUT, + msg="VF mailbox: the pf never granted access") + self.vf_mailbox[1] = 2 # ack + return req + 1 def recover(self, force=False) -> bool: if not force and not self.is_err_state: return False @@ -238,7 +283,8 @@ class AMDev: if DEBUG >= 3: print(f"am {self.devfmt}: Recovery complete") return True - def is_hive(self) -> bool: return self.gmc.xgmi_seg_sz > 0 + # a hive has multiple XGMI regions; single-node parts (like MI350P) may still program LFB_SIZE with region 0 only + def is_hive(self) -> bool: return self.gmc.xgmi_seg_sz > 0 and self.gmc.xgmi_max_region > 0 def paddr2mc(self, paddr:int) -> int: return self.gmc.mc_base + paddr def paddr2xgmi(self, paddr:int) -> int: return self.gmc.paddr_base + paddr @@ -246,20 +292,36 @@ class AMDev: def reg(self, reg:str) -> AMRegister: return self.__dict__[reg] - def rreg(self, reg:int) -> int: + def rreg(self, reg:int, inst=0, direct=False) -> int: + if not direct and any(lo <= reg <= hi for lo, hi in self.vf_rlc_gated): return self.rlcg_rw(reg, 0, inst, read=True) val = self.indirect_rreg(reg) if reg >= len(self.mmio) else self.mmio[reg] if AM_DEBUG >= 4 and getattr(self, '_prev_rreg', None) != (reg, val): print(f"am {self.devfmt}: Reading register {reg:#x} with value {val:#x}") self._prev_rreg = (reg, val) return val - def wreg(self, reg:int, val:int): + def wreg(self, reg:int, val:int, inst=0, direct=False): if AM_DEBUG >= 4: print(f"am {self.devfmt}: Writing register {reg:#x} with value {val:#x}") - if reg >= len(self.mmio): self.indirect_wreg(reg, val) + if not direct and any(lo <= reg <= hi for lo, hi in self.vf_rlc_gated): self.rlcg_rw(reg, val, inst) + elif reg >= len(self.mmio): self.indirect_wreg(reg, val) else: self.mmio[reg] = val - def wreg_pair(self, reg_base:str, lo_suffix:str, hi_suffix:str, val:int, inst:int=0): - self.reg(f"{reg_base}{lo_suffix}").write(lo32(val), inst=inst) - self.reg(f"{reg_base}{hi_suffix}").write(hi32(val), inst=inst) + def rlcg_rw(self, addr:int, val:int, inst:int, read=False) -> int: + # the rlc gateway takes the grbm selection through its own scratch registers + if addr in {self.reg("regGRBM_GFX_CNTL").addr[inst], self.reg("regGRBM_GFX_INDEX").addr[inst]}: + self.reg("regSCRATCH_REG2" if addr == self.reg("regGRBM_GFX_CNTL").addr[inst] else "regSCRATCH_REG3").write(val, inst=inst, direct=True) + return val + + self.wreg_pair("regSCRATCH_REG", "0", "1", (addr | (0x1 << 28 if read else 0)) << 32 | val, inst=inst, direct=True) + self.reg("regRLC_SPARE_INT").write(1, inst=inst, direct=True) + wait_cond(lambda: self.reg("regSCRATCH_REG1").read(inst=inst, direct=True) & 0xFFFFF, value=0, msg=f"RLC gateway timeout on {addr:#x}") + + if AM_DEBUG >= 1 and (err:=self.reg("regSCRATCH_REG1").read(inst=inst, direct=True) & 0xF000000): + print(f"am {self.devfmt}: RLC gateway refused {addr:#x}: {err:#x}") + return self.reg("regSCRATCH_REG0").read(inst=inst, direct=True) + + def wreg_pair(self, reg_base:str, lo_suffix:str, hi_suffix:str, val:int, inst:int=0, direct=False): + self.reg(f"{reg_base}{lo_suffix}").write(lo32(val), inst=inst, direct=direct) + self.reg(f"{reg_base}{hi_suffix}").write(hi32(val), inst=inst, direct=direct) def indirect_rreg(self, reg:int) -> int: self.reg("regBIF_BX_PF0_RSMU_INDEX").write(reg * 4) @@ -315,6 +377,15 @@ class AMDev: ip_offset += 8 + (8 if ihdr.base_addr_64_bit else 4) * ip.num_base_address + # HARV(EST) table: harvested instances must be excluded (like amdgpu_discovery_harvest_ip) + # layout: u32 signature, u16 version, u16 size, then 32 entries of {hw_id:u16, inst:u8, rsv:u8} + self.harvested:dict[int, set[int]] = collections.defaultdict(set) + if (harv_off:=self.bhdr.table_list[am.HARVEST_INFO].offset) != 0 and \ + (blob:=to_mv(ctypes.addressof(self.bhdr) + harv_off, 8 + 32*4).cast('I'))[0] == am.HARVEST_TABLE_SIGNATURE: + inv_hw_id = {hw_id: hw_ip for hw_ip, hw_id in am.hw_id_map.items()} + for ent in blob[2:]: + if (ip_:=inv_hw_id.get(ent & 0xffff)) is not None: self.harvested[ip_].add((ent >> 16) & 0xff) + gc_info = am.struct_gc_info_v1_0.from_address(gc_addr:=ctypes.addressof(self.bhdr) + self.bhdr.table_list[am.GC].offset) self.gc_info = getattr(am, f"struct_gc_info_v{gc_info.header.version_major}_{gc_info.header.version_minor}").from_address(gc_addr) self.reserved_vram_size = (384 << 20) if self.ip_ver[am.GC_HWIP][:2] in {(9,4), (9,5)} else (64 << 20) @@ -330,5 +401,15 @@ class AMDev: if self.ip_ver[am.SDMA0_HWIP] in {(4,4,2), (4,4,4)}: mods += [("sdma", am.SDMA0_HWIP)] for prefix, hwip in mods: - self.__dict__.update(import_asic_regs(prefix, self.ip_ver[hwip], cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[hwip]))) + regs = import_asic_regs(prefix, self.ip_ver[hwip], cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[hwip])) + self.__dict__.update(regs) + if prefix == "gc" and self.is_vf: + ext = {seg: max(r.offset for r in regs.values() if r.segment == seg) for seg in {r.segment for r in regs.values()}} + self.vf_rlc_gated = sorted((bases[seg], bases[seg] + off) for bases in self.regs_offset[hwip].values() for seg, off in ext.items()) self.__dict__.update(import_asic_regs('mp', (11, 0, 0), cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[am.MP1_HWIP]))) + + # Live AIDs like the kernel: 4 SDMAs per AID; the AID lives iff its group's alive-mask is 0xf/0x3/0xc. + # Dead AIDs must never be touched via the indirect window: writes poison the whole fabric. + live_sdma = {k for k in self.regs_offset[am.SDMA0_HWIP] if k not in self.harvested[am.SDMA0_HWIP]} + max_aid = max((k >> 2 for k in self.regs_offset[am.SDMA0_HWIP]), default=0) + self.aids = [0] + [aid for aid in range(1, max_aid + 1) if sum(1 << (i & 3) for i in live_sdma if i >> 2 == aid) in {0xf, 0x3, 0xc}] diff --git a/tinygrad_repo/tinygrad/runtime/support/am/ip.py b/tinygrad_repo/tinygrad/runtime/support/am/ip.py index fcebddd30..6155542c9 100644 --- a/tinygrad_repo/tinygrad/runtime/support/am/ip.py +++ b/tinygrad_repo/tinygrad/runtime/support/am/ip.py @@ -1,5 +1,5 @@ import ctypes, time, contextlib, functools -from typing import Literal +from typing import Any, Iterable, Literal from tinygrad.helpers import to_mv, data64, lo32, hi32, DEBUG, wait_cond, pad_bytes, getbits from tinygrad.runtime.autogen.am import am from tinygrad.runtime.support.amd import import_soc @@ -29,8 +29,10 @@ class AM_SOC(AM_IP): def init_hw(self): if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)}: - self.adev.regXCC_DOORBELL_FENCE.write(0x0) - for aid in range(1, self.adev.gmc.vmhubs): + # fence doorbells for harvested xccs (0xff & ~xcc_mask in the kernel); a fully-unharvested chip keeps the previous 0x0 + live_xccs = sum(1 << i for i in self.adev.regs_offset[am.GC_HWIP] if i not in self.adev.harvested[am.GC_HWIP] and i < 8) + self.adev.regXCC_DOORBELL_FENCE.write(0xff & ~live_xccs) + for aid in self.adev.aids[1:]: self.adev.indirect_wreg_pcie(self.adev.regXCC_DOORBELL_FENCE.addr[0], self.adev.regXCC_DOORBELL_FENCE.encode(shub_slv_mode=1), aid=aid) self.adev.regBIFC_GFX_INT_MONITOR_MASK.write(0x7ff) self.adev.regBIFC_DOORBELL_ACCESS_EN_PF.write(0xfffff) @@ -48,12 +50,14 @@ class AM_SOC(AM_IP): else: reg.write(val) class AM_GMC(AM_IP): + vf_owner:Any = None + def init_sw(self): self.vmhubs = len(self.adev.regs_offset[am.MMHUB_HWIP]) - # XGMI (for supported systems) - self.xgmi_phys_id = self.adev.regMMMC_VM_XGMI_LFB_CNTL.read_bitfields()['pf_lfb_region'] if hasattr(self.adev, 'regMMMC_VM_XGMI_LFB_CNTL') else 0 - self.xgmi_seg_sz = self.adev.regMMMC_VM_XGMI_LFB_SIZE.read_bitfields()['pf_lfb_size']<<24 if hasattr(self.adev, 'regMMMC_VM_XGMI_LFB_SIZE') else 0 + xgmi_lfb_cntl = self.adev.regGCMC_VM_XGMI_LFB_CNTL.read_bitfields() if hasattr(self.adev, 'regGCMC_VM_XGMI_LFB_CNTL') else {} + self.xgmi_phys_id, self.xgmi_max_region = xgmi_lfb_cntl.get('pf_lfb_region', 0), xgmi_lfb_cntl.get('pf_max_region', 0) + self.xgmi_seg_sz = self.adev.regGCMC_VM_XGMI_LFB_SIZE.read_bitfields()['pf_lfb_size']<<24 if hasattr(self.adev, 'regGCMC_VM_XGMI_LFB_CNTL') else 0 self.paddr_base = self.xgmi_phys_id * self.xgmi_seg_sz @@ -78,26 +82,42 @@ class AM_GMC(AM_IP): # MM hub is inited before any tlb flushes and is still valid during partial_boot, so set it to true self.hub_initted = {"MM": True, "GC": False} + self.mm_insts = self.adev.aids if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)} else list(range(self.vmhubs)) # dead mmhubs hang us + self.pf_status_reg = lambda ip: f"reg{ip}VM_L2_PROTECTION_FAULT_STATUS{'_LO32' if self.adev.ip_ver[am.GC_HWIP] >= (12,0,0) else ''}" - def init_hw(self): self.init_hub("MM", inst_cnt=self.vmhubs) + def init_hw(self): self.init_hub("MM", insts=self.mm_insts) + + def flush_hdp(self): + if self.adev.is_vf: self.adev.reg("regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL").write(0x0) + else: self.adev.wreg(self.adev.reg("regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL").read() // 4, 0x0) - def flush_hdp(self): self.adev.wreg(self.adev.reg("regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL").read() // 4, 0x0) def flush_tlb(self, ip:Literal["MM", "GC"], vmid, flush_type=0): self.flush_hdp() # Can't issue TLB invalidation if the hub isn't initialized. if not self.hub_initted[ip]: return - for inst in range(self.adev.gmc.vmhubs if ip == "MM" else self.adev.gfx.xccs): - if ip == "MM": wait_cond(lambda: self.adev.regMMVM_INVALIDATE_ENG17_SEM.read(inst=inst) & 0x1, value=1, msg="mm flush_tlb timeout") + req = self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_REQ").encode(flush_type=flush_type, per_vmid_invalidate_req=(1 << vmid), + invalidate_l2_ptes=1, invalidate_l2_pde0=1, invalidate_l2_pde1=1, invalidate_l2_pde2=1, invalidate_l1_ptes=1, + clear_protection_fault_status_addr=0) - self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_REQ").write(flush_type=flush_type, per_vmid_invalidate_req=(1 << vmid), invalidate_l2_ptes=1, - invalidate_l2_pde0=1, invalidate_l2_pde1=1, invalidate_l2_pde2=1, invalidate_l1_ptes=1, clear_protection_fault_status_addr=0, inst=inst) + if ip == "GC" and self.adev.is_vf and (dev:=self.vf_owner) is not None: # the cp runs invalidations once its queues are up + from tinygrad.runtime.ops_amd import WAIT_REG_MEM_FUNCTION_EQ + dev.hw_compute_queue_t().wait_reg_mem(req, mask=1 << vmid, reg_done=self.adev.regGCVM_INVALIDATE_ENG17_ACK.addr[0], + reg=self.adev.regGCVM_INVALIDATE_ENG17_REQ.addr[0], op=WAIT_REG_MEM_FUNCTION_EQ).signal(dev.timeline_signal, dev.next_timeline()).submit(dev) + dev.timeline_signal.wait(dev.timeline_value - 1) + return + + use_sema = ip == "MM" and not self.adev.is_vf # vf can't use sema + for inst in (self.adev.gmc.mm_insts if ip == "MM" else range(self.adev.gfx.xccs)): + if use_sema: wait_cond(lambda: self.adev.regMMVM_INVALIDATE_ENG17_SEM.read(inst=inst) & 0x1, value=1, msg="mm flush_tlb timeout") + + self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_REQ").write(req, inst=inst) wait_cond(lambda: self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_ACK").read(inst=inst) & (1 << vmid), value=(1 << vmid), msg="flush_tlb timeout") - if ip == "MM": self.adev.regMMVM_INVALIDATE_ENG17_SEM.write(0x0, inst=inst) + if use_sema: self.adev.regMMVM_INVALIDATE_ENG17_SEM.write(0x0, inst=inst) if self.adev.ip_ver[am.GC_HWIP] >= (11,0,0) and ip == "MM": self.adev.regMMVM_L2_BANK_SELECT_RESERVED_CID2.update(reserved_cache_private_invalidation=1, inst=inst) @@ -114,9 +134,9 @@ class AM_GMC(AM_IP): self.adev.reg(f"reg{ip}VM_CONTEXT{vmid}_CNTL").write(0x1800000, **fault_flags, **en_def_flags, enable_context=1, page_table_depth=((2 if self.trans_futher else 3) - page_table.lv), page_table_block_size=9 if self.trans_futher else 0, inst=inst) - def init_hub(self, ip:Literal["MM", "GC"], inst_cnt:int): + def init_hub(self, ip:Literal["MM", "GC"], insts:Iterable[int]): # Init system apertures - for inst in range(inst_cnt): + for inst in insts: self.adev.reg(f"reg{ip}MC_VM_AGP_BASE").write(0, inst=inst) self.adev.reg(f"reg{ip}MC_VM_AGP_BOT").write(0xffffffffffff >> 24, inst=inst) # disable AGP self.adev.reg(f"reg{ip}MC_VM_AGP_TOP").write(0, inst=inst) @@ -189,13 +209,13 @@ class AM_SMU(AM_IP): if DEBUG >= 2: print(f"am {self.adev.devfmt}: mode1 reset") if self.adev.ip_ver[am.MP0_HWIP] >= (14,0,0) or self.adev.ip_ver[am.MP0_HWIP] in {(13,0,0), (13,0,7), (13,0,10)}: self._send_msg(__DEBUGSMC_MSG_Mode1Reset:=2, 0, debug=True) - elif self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6), (13,0,12)}: self._send_msg(self.smu_mod.PPSMC_MSG_GfxDriverReset, 1) + elif self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6), (13,0,12), (13,0,15)}: self._send_msg(self.smu_mod.PPSMC_MSG_GfxDriverReset, 1) else: self._send_msg(self.smu_mod.PPSMC_MSG_Mode1Reset, 0) if not self.adev.is_hive(): time.sleep(0.5) # 500ms def read_table(self, table_t, arg): - if self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6),(13,0,12)}: self._send_msg(self.smu_mod.PPSMC_MSG_GetMetricsTable, arg) + if self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6),(13,0,12),(13,0,15)}: self._send_msg(self.smu_mod.PPSMC_MSG_GetMetricsTable, arg) else: self._send_msg(self.smu_mod.PPSMC_MSG_TransferTableSmu2Dram, arg) return table_t.from_buffer(bytearray(self.adev.vram.view(self.driver_table_paddr, ctypes.sizeof(table_t))[:])) @@ -206,7 +226,7 @@ class AM_SMU(AM_IP): def set_clocks(self, level:int|None): clks = tuple([self.smu_mod.PPCLK_UCLK, self.smu_mod.PPCLK_FCLK, self.smu_mod.PPCLK_SOCCLK]) - if self.adev.ip_ver[am.MP0_HWIP] not in {(13,0,6), (13,0,12)}: clks += (self.smu_mod.PPCLK_GFXCLK,) + if self.adev.ip_ver[am.MP0_HWIP] not in {(13,0,6), (13,0,12), (13,0,15)}: clks += (self.smu_mod.PPCLK_GFXCLK,) if level is None: for clck in clks: @@ -246,16 +266,17 @@ class AM_SMU(AM_IP): class AM_GFX(AM_IP): def init_sw(self): - self.xccs = len(self.adev.regs_offset[am.GC_HWIP]) + self.xccs = sum(1 for i in self.adev.regs_offset[am.GC_HWIP] if i not in self.adev.harvested[am.GC_HWIP]) self.mqd_paddr = [self.adev.mm.palloc(0x1000 * self.xccs, zero=False, boot=True) for i in range(2)] self.mqd_mc = [self.adev.paddr2mc(mqd_paddr) for mqd_paddr in self.mqd_paddr] def init_hw(self): # Wait for RLC autoload to complete - wait_cond(lambda: self.adev.regCP_STAT.read() == 0 or self.adev.regRLC_RLCS_BOOTLOAD_STATUS.read_bitfields()['bootload_complete'] == 0, - value=True, msg="RLC autoload timeout") + if not self.adev.is_vf: # VF boots with the RLC already up + wait_cond(lambda: self.adev.regCP_STAT.read() == 0 or self.adev.regRLC_RLCS_BOOTLOAD_STATUS.read_bitfields()['bootload_complete'] == 0, + value=True, msg="RLC autoload timeout") - self.adev.gmc.init_hub("GC", inst_cnt=self.xccs) + self.adev.gmc.init_hub("GC", insts=range(self.xccs)) if self.adev.partial_boot: return self.reset_mec() self._config_mec() @@ -297,8 +318,11 @@ class AM_GFX(AM_IP): self._enable_mec() - # Set 1 partition - if self.xccs > 1: self.adev.psp._spatial_partition_cmd(1) + if self.adev.is_vf: # the host PF shuts a VF down when it leaves its access window with no cp scheduler, point the RLC at the kiq slot + for xcc in range(self.xccs): self.adev.reg("regRLC_CP_SCHEDULERS").update(scheduler0=(2 << 5) | (1 << 3) | 0x80, inst=xcc) + + # set 1 partition on bare metal. a VF uses the spatial partition its host PF assigned. + if self.xccs > 1 and not self.adev.is_vf: self.adev.psp._spatial_partition_cmd(1) def fini_hw(self): self._dequeue_hqds() @@ -340,7 +364,7 @@ class AM_GFX(AM_IP): mqd_st_mv = to_mv(ctypes.addressof(mqd_struct), ctypes.sizeof(mqd_struct)).cast('I') for i, reg in enumerate(range(self.adev.regCP_MQD_BASE_ADDR.addr[xcc], self.adev.regCP_HQD_PQ_WPTR_HI.addr[xcc] + 1)): - self.adev.wreg(reg, mqd_st_mv[0x80 + i]) + self.adev.wreg(reg, mqd_st_mv[0x80 + i], inst=xcc) self.adev.regCP_HQD_ACTIVE.write(0x1, inst=xcc) self.adev.gmc.flush_hdp() @@ -403,7 +427,11 @@ class AM_GFX(AM_IP): if self.adev.regCP_HQD_ACTIVE.read(inst=xcc) & 1: self.adev.regCP_HQD_DEQUEUE_REQUEST.write(0x2, inst=xcc) # 1 - DRAIN_PIPE; 2 - RESET_WAVES self.adev.regSPI_COMPUTE_QUEUE_RESET.write(0x1, inst=xcc) - if not self.adev.is_err_state: wait_cond(lambda: self.adev.regCP_HQD_ACTIVE.read(inst=xcc) & 1, value=0, msg="HQD dequeue timeout") + if not self.adev.is_err_state: + try: wait_cond(lambda: self.adev.regCP_HQD_ACTIVE.read(inst=xcc) & 1, value=0, msg="HQD dequeue timeout") + # kernel tolerates this too; a wedged wave can survive RESET_WAVES + except TimeoutError: + if DEBUG >= 2: print(f"am {self.adev.devfmt}: HQD dequeue timeout xcc{xcc} q{q}, continuing") self._grbm_select() class AM_IH(AM_IP): @@ -481,6 +509,7 @@ class AM_IH(AM_IP): self.drain() + if self.adev.is_vf: return # fatal RAS events are handled by the host PF bif_intr = self.adev.regBIF_BX0_BIF_DOORBELL_INT_CNTL.read_bitfields() athub_err, cntlr_err = bif_intr['ras_athub_err_event_interrupt_status'], bif_intr['ras_cntlr_interrupt_status'] if athub_err or cntlr_err: @@ -514,7 +543,7 @@ class AM_SDMA(AM_IP): **({'utc_l1_enable':1} if self.adev.ip_ver[am.SDMA0_HWIP] <= (5,2,0) else {}), inst=inst) if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)}: - for aid_id in range(4): + for aid_id in self.adev.aids: for dev_inst, (port, awid, offset, awaddr) in enumerate([(1, 0xe, 0xe, 0x1), (2, 0x8, 0x8, 0x2), (5, 0x9, 0x9, 0x8), (6, 0xa, 0xa, 0x9)]): entry = dev_inst + 1 + 4 * aid_id self.adev.reg(f"regDOORBELL0_CTRL_ENTRY_{entry}").write(**{f"bif_doorbell{entry}_range_size_entry": 20, diff --git a/tinygrad_repo/tinygrad/runtime/support/c.py b/tinygrad_repo/tinygrad/runtime/support/c.py index 009bf60e7..aade78160 100644 --- a/tinygrad_repo/tinygrad/runtime/support/c.py +++ b/tinygrad_repo/tinygrad/runtime/support/c.py @@ -91,7 +91,7 @@ class DLL(ctypes.CDLL): @staticmethod def findlib(nm:str, paths:list[str], extra_paths=[]): - if nm == 'libc' and OSX: return '/usr/lib/libc.dylib' + if nm in ('libc', 'm') and OSX: return f'/usr/lib/lib{nm.removeprefix("lib")}.dylib' if pathlib.Path(path:=getenv(nm.replace('-', '_').upper()+"_PATH", '')).is_file(): return path for p in paths: libpaths = {"posix": [d for d in os.environ.get('LD_LIBRARY_PATH', '').split(os.pathsep) if d] + ["/usr/lib64", "/usr/lib", "/usr/local/lib"], diff --git a/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py b/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py index 7bb24f5f3..9fb5212c8 100644 --- a/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py +++ b/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py @@ -1,10 +1,9 @@ import subprocess from tinygrad.device import Compiler -from tinygrad.helpers import getenv, capstone_flatdump -from tinygrad.runtime.support.elf import jit_loader +from tinygrad.helpers import getenv, capstone_flatdump, cpu_objdump class ClangCompiler(Compiler): - def __init__(self, arch:list[str], cachekey="compile_clang_jit"): + def __init__(self, arch:list[str], cachekey="compile_clang_obj"): assert len(arch) >= 2, f"invalid arch string: {','.join(arch)!r}, expected ',,[]' (eg. 'x86_64,znver2')" self.arch, cpu, *feats = arch match self.arch: @@ -16,15 +15,13 @@ class ClangCompiler(Compiler): case _: raise RuntimeError(f"unsupported arch: {self.arch!r}") super().__init__(f"{cachekey}_{'_'.join(arch)}") - def compile_to_obj(self, src:str) -> bytes: + def compile(self, src:str) -> bytes: """Compile C source to ELF object file (before linking).""" # -fno-math-errno is required for __builtin_sqrt to become an instruction instead of a function call return subprocess.check_output([getenv("CC", 'clang'), '-c', '-x', 'c', '-O2', '-fPIC', '-ffreestanding', '-fno-math-errno', '-nostdlib', '-fno-ident', f'--target={self.arch}-none-unknown-elf', *self.args, '-', '-o', '-'], input=src.encode('utf-8')) - def compile(self, src:str) -> bytes: return jit_loader(self.compile_to_obj(src)) - - def disassemble(self, lib:bytes): return capstone_flatdump(lib, self.arch) + def disassemble(self, lib: bytes): cpu_objdump(lib) class X86Compiler(Compiler): diff --git a/tinygrad_repo/tinygrad/runtime/support/compiler_cuda.py b/tinygrad_repo/tinygrad/runtime/support/compiler_cuda.py index 1ba245a9c..81124b116 100644 --- a/tinygrad_repo/tinygrad/runtime/support/compiler_cuda.py +++ b/tinygrad_repo/tinygrad/runtime/support/compiler_cuda.py @@ -1,10 +1,12 @@ import hashlib, tempfile, ctypes, re, pathlib -from tinygrad.helpers import to_char_p_p, colored, getenv, system +from tinygrad.helpers import to_char_p_p, colored, getenv, system, OSX from tinygrad.runtime.support.c import init_c_var from tinygrad.runtime.autogen import nvrtc, nvjitlink as jitlink from tinygrad.device import Compiler, CompileError CUDA_PATH = getenv("CUDA_PATH", "") +root = pathlib.Path(__file__).parents[3] +osx_docker_cmd = f"docker run --rm -i -v {root}:{root} -e PYTHONPATH={root} ghcr.io/tinygrad/cuda-arm64:v2.3" def _get_bytes(arg, get_str, get_sz, check) -> bytes: x = ctypes.create_string_buffer(init_c_var(ctypes.c_size_t, lambda x: check(get_sz(arg, ctypes.byref(x)))).value) @@ -44,11 +46,14 @@ def cuda_disassemble(lib:bytes, arch:str, ptx=False): class NVRTCCompiler(Compiler): def __init__(self, arch:str, ptx=True, cache_key:str="cuda"): self.ptx, self.arch, self.compile_options = ptx, arch, [f'--gpu-architecture={arch}'] - self.compile_options += [f"-I{CUDA_PATH}/include"] if CUDA_PATH else ["-I/usr/local/cuda/include", "-I/usr/include", "-I/opt/cuda/include"] - nvrtc_check(nvrtc.nvrtcVersion((nvrtcMajor := ctypes.c_int()), (nvrtcMinor := ctypes.c_int()))) - if (nvrtcMajor.value, nvrtcMinor.value) >= (12, 4): self.compile_options.append("--minimal") + if OSX: self.compiler_process = self.server(osx_docker_cmd, arch, ptx) + else: + self.compile_options += [f"-I{CUDA_PATH}/include"] if CUDA_PATH else ["-I/usr/local/cuda/include", "-I/usr/include", "-I/opt/cuda/include"] + nvrtc_check(nvrtc.nvrtcVersion((nvrtcMajor := ctypes.c_int()), (nvrtcMinor := ctypes.c_int()))) + if (nvrtcMajor.value, nvrtcMinor.value) >= (12, 4): self.compile_options.append("--minimal") super().__init__(f"compile_{cache_key}_{self.arch}") def compile(self, src:str) -> bytes: + if OSX: return self.compile_server(src, self.compiler_process) nvrtc_check(nvrtc.nvrtcCreateProgram(ctypes.byref(prog := nvrtc.nvrtcProgram()), src.encode(), "".encode(), 0, None, None)) nvrtc_check(nvrtc.nvrtcCompileProgram(prog, len(self.compile_options), to_char_p_p([o.encode() for o in self.compile_options])), prog) data = _get_bytes(prog, nvrtc.nvrtcGetPTX if self.ptx else nvrtc.nvrtcGetCUBIN, @@ -80,9 +85,11 @@ class PTXCompiler(Compiler): class NVPTXCompiler(PTXCompiler): def __init__(self, arch:str): - jitlink_check(jitlink.nvJitLinkVersion(ctypes.byref(ctypes.c_uint()), ctypes.byref(ctypes.c_uint()))) + if OSX: self.compiler_process = self.server(osx_docker_cmd, arch) + else: jitlink_check(jitlink.nvJitLinkVersion(ctypes.byref(ctypes.c_uint()), ctypes.byref(ctypes.c_uint()))) super().__init__(arch, cache_key="nv_ptx") def compile(self, src:str) -> bytes: + if OSX: return self.compile_server(src, self.compiler_process) jitlink_check(jitlink.nvJitLinkCreate(handle := jitlink.nvJitLinkHandle(), 1, to_char_p_p([f'-arch={self.arch}'.encode()])), handle) jitlink_check(jitlink.nvJitLinkAddData(handle, jitlink.NVJITLINK_INPUT_PTX, ptxsrc:=super().compile(src), len(ptxsrc), "".encode()), handle) jitlink_check(jitlink.nvJitLinkComplete(handle), handle) diff --git a/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py b/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py index be012441a..56a874602 100644 --- a/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py +++ b/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py @@ -1,7 +1,6 @@ import ctypes from tinygrad.device import Compiler, CompileError -from tinygrad.helpers import getenv, capstone_flatdump, amdgpu_disassemble, unwrap, DEBUG -from tinygrad.runtime.support.elf import jit_loader +from tinygrad.helpers import getenv, cpu_objdump, amdgpu_disassemble, unwrap, DEBUG from tinygrad.runtime.autogen import llvm def cerr(): return ctypes.pointer(ctypes.pointer(ctypes.c_char())) @@ -11,7 +10,6 @@ def expect(x, err, ret=None): return ret class LLVMCompiler(Compiler): - jit = True def __init__(self, arch:str, processor:str, feats:str, cache_key=None): for component in ['Target', 'TargetInfo', 'TargetMC', 'AsmParser', 'AsmPrinter']: getattr(llvm, "LLVMInitialize" + {'arm64': 'AArch64', 'x86_64': 'X86', 'riscv64': 'riscv64'}.get(arch, "AMDGPU") + component)() @@ -43,13 +41,13 @@ class LLVMCompiler(Compiler): self.diag_msgs.append(msg) self.handle_diag = handle_diag llvm.LLVMContextSetDiagnosticHandler(self.context, handle_diag, None) - super().__init__(cache_key or f"compile_llvm_{processor}_{feats}{'_jit' if self.jit else ''}{'_opt' if opt else ''}") + super().__init__(cache_key or f"compile_llvm_{processor}_{feats}{'_opt' if opt else ''}") def __del__(self): if hasattr(self, 'pbo'): llvm.LLVMDisposePassBuilderOptions(self.pbo) if hasattr(self, 'context'): llvm.LLVMContextDispose(self.context) - def compile_to_obj(self, src:str) -> bytes: + def compile(self, src:str) -> bytes: self.diag_msgs.clear() src_buf = llvm.LLVMCreateMemoryBufferWithMemoryRangeCopy(ctypes.create_string_buffer(src_bytes:=src.encode()), len(src_bytes), b'src') mod = expect(llvm.LLVMParseIRInContext(self.context, src_buf, ctypes.pointer(m:=llvm.LLVMModuleRef()), err:=cerr()), err, m) @@ -64,9 +62,6 @@ class LLVMCompiler(Compiler): if self.diag_msgs: raise RuntimeError("llvm diagnostic: " + "\n".join(self.diag_msgs)) return obj - def compile(self, src:str) -> bytes: return jit_loader(self.compile_to_obj(src)) if self.jit else self.compile_to_obj(src) - - class CPULLVMCompiler(LLVMCompiler): def __init__(self, arch:list[str], cache_key=None): assert len(arch) >= 2, f"invalid arch string: {','.join(arch)!r}, expected ',,[]' (eg. 'x86_64,znver2')" @@ -78,10 +73,9 @@ class CPULLVMCompiler(LLVMCompiler): # +reserve-x18 here does the same thing as -ffixed-x18 in ClangCompiler, see comments there for why it's needed on arm osx super().__init__(self.arch, cpu, ('+reserve-x18,' if self.arch == "arm64" else '') + featstr, cache_key) - def disassemble(self, lib:bytes): capstone_flatdump(lib, self.arch) + def disassemble(self, lib: bytes): cpu_objdump(lib) class AMDLLVMCompiler(LLVMCompiler): - jit = False def __init__(self, arch: str): self.arch = arch super().__init__("AMDGPU", self.arch, "+cumode") diff --git a/tinygrad_repo/tinygrad/runtime/support/compiler_qcom.py b/tinygrad_repo/tinygrad/runtime/support/compiler_qcom.py index ceb8f270b..62693ea5e 100644 --- a/tinygrad_repo/tinygrad/runtime/support/compiler_qcom.py +++ b/tinygrad_repo/tinygrad/runtime/support/compiler_qcom.py @@ -1,6 +1,6 @@ -import ctypes, struct +import ctypes, struct, platform, pathlib, shutil from tinygrad.device import Compiler -from tinygrad.helpers import DEBUG, system +from tinygrad.helpers import DEBUG, system, fetch from tinygrad.runtime.support.compiler_mesa import disas_adreno # see https://github.com/sirhcm/tinydreno from tinygrad.runtime.autogen import llvm_qcom @@ -10,10 +10,17 @@ def _read_lib(lib, off) -> int: return struct.unpack("I", lib[off:off+4])[0] class QCOMCompiler(Compiler): def __init__(self, arch:str): assert arch.split(',')[0] == "a630", "only a630 supported" - self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() + if platform.machine() == "aarch64": self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() + else: + # extract once into the download cache, all processes share the rootfs (extract=True) + self.arch, self.chip_id = arch, 0x6030001 + fs, root = fetch('https://git.tinygrad.win/sirhcm/images/releases/download/v2/qcomcl.tar.gz', extract=True), pathlib.Path(__file__).parents[3] + self.compiler_process = self.server(f"{qemu} -cpu max,pauth=off -L {fs} {fs}/usr/bin/python3" if (qemu:=shutil.which("qemu-aarch64-static")) + else (f"docker run --rm -i --platform linux/aarch64 -v {fs}/usr:/usr -v {root}:{root} " + f"-e PYTHONPATH={root} -e QEMU_CPU=max,pauth=off gcr.io/distroless/static python3"), arch) super().__init__(f"compile_qcomcl_{arch}") - def __del__(self): llvm_qcom.cl_compiler_destroy_llvm_instance(self.llvm_inst) + def __del__(self): llvm_qcom.cl_compiler_destroy_llvm_instance(self.llvm_inst) if platform.machine() == "aarch64" else self.compiler_process.kill() def __reduce__(self): return QCOMCompiler, (self.arch,) @@ -25,6 +32,7 @@ class QCOMCompiler(Compiler): return handle def compile(self, src) -> bytes: + if platform.machine() != "aarch64": return self.compile_server(src, self.compiler_process) ch = self.checked(llvm_qcom.cl_compiler_compile_source(self.llvm_inst, self.chip_id, llvm_qcom.CL_MODE_64BIT, b"", 0, 0, 0, src.encode(), 0, llvm_qcom.CL_SRC_STR, None)) if DEBUG >= 8: print(system("llvm-dis", input=ctypes.string_at((comp:=ch.contents.compiled.contents).llvm_bitcode, comp.llvm_bitcode_size))) @@ -36,3 +44,4 @@ class QCOMCompiler(Compiler): return ret def disassemble(self, lib: bytes): disas_adreno(lib[(ofs:=_read_lib(lib, 0xc0)):ofs+_read_lib(lib, 0x100)], self.chip_id) + diff --git a/tinygrad_repo/tinygrad/runtime/support/compileserver.py b/tinygrad_repo/tinygrad/runtime/support/compileserver.py new file mode 100644 index 000000000..b41bd7ab6 --- /dev/null +++ b/tinygrad_repo/tinygrad/runtime/support/compileserver.py @@ -0,0 +1,13 @@ +import ast, struct, sys +from tinygrad.helpers import fromimport + +if __name__ == "__main__": + assert len(sys.argv) >= 3, f"usage: {sys.argv[0]} []" + compiler = fromimport(*sys.argv[1].split(':'))(sys.argv[2], *(ast.literal_eval(arg) for arg in sys.argv[3:])) + while (amt:=sys.stdin.buffer.read(4)): + try: lib = compiler.compile(sys.stdin.buffer.read(struct.unpack("I", amt)[0]).decode()) + except Exception as e: + lib = b"" + print(e, file=sys.stderr, flush=True) + sys.stdout.buffer.write(struct.pack("I", len(lib)) + lib) + sys.stdout.buffer.flush() diff --git a/tinygrad_repo/tinygrad/runtime/support/elf.py b/tinygrad_repo/tinygrad/runtime/support/elf.py index 41917b2cb..5dbaff7b0 100644 --- a/tinygrad_repo/tinygrad/runtime/support/elf.py +++ b/tinygrad_repo/tinygrad/runtime/support/elf.py @@ -1,4 +1,4 @@ -import struct, ctypes, ctypes.util +import struct, ctypes from dataclasses import dataclass from tinygrad.helpers import getbits, i2u, unwrap from tinygrad.runtime.autogen import libc @@ -6,13 +6,13 @@ from tinygrad.runtime.autogen import libc @dataclass(frozen=True) class ElfSection: name:str; header:libc.Elf64_Shdr|libc.Elf32_Shdr; content:bytes # noqa: E702 -def link_sym(sym:str, libs:list[str]) -> int: +def link_sym(sym:str, libs:list[ctypes.CDLL]) -> int: for lib in libs: - try: return unwrap(ctypes.cast(getattr(ctypes.CDLL(ctypes.util.find_library(lib)), sym), ctypes.c_void_p).value) + try: return unwrap(ctypes.cast(getattr(lib, sym), ctypes.c_void_p).value) except (OSError, AttributeError): pass raise RuntimeError(f'Attempting to relocate against an undefined symbol {sym}') -def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[str]|None=None) -> tuple[memoryview, list[ElfSection], list[tuple]]: +def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[ctypes.CDLL]|None=None) -> tuple[memoryview, list[ElfSection], list[tuple]]: assert blob[:4] == libc.ELFMAG.encode(), "blob is not an ELF, missing magic bytes" ecls = {libc.ELFCLASS32: "Elf32", libc.ELFCLASS64: "Elf64"}[blob[libc.EI_CLASS]] @@ -49,7 +49,7 @@ def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[str]|None=N return memoryview(image), sections, relocs -def jit_loader(obj: bytes, base:int=0, link_libs:list[str]|None=None) -> bytes: +def jit_loader(obj: bytes, base:int=0, link_libs:list[ctypes.CDLL]|None=None) -> bytes: image_, _, relocs = elf_loader(obj, link_libs=link_libs) image = bytearray(image_) diff --git a/tinygrad_repo/tinygrad/runtime/support/hcq.py b/tinygrad_repo/tinygrad/runtime/support/hcq.py index e22a436b0..14db571e3 100644 --- a/tinygrad_repo/tinygrad/runtime/support/hcq.py +++ b/tinygrad_repo/tinygrad/runtime/support/hcq.py @@ -1,25 +1,16 @@ from __future__ import annotations from typing import cast, Callable, Type, TypeVar, Generic, Any -import contextlib, decimal, statistics, time, ctypes, array, os, struct, collections, functools, itertools -from dataclasses import replace +import contextlib, decimal, statistics, time, ctypes, array, os, collections, itertools try: import fcntl # windows misses that except ImportError: fcntl = None #type:ignore[assignment] -from tinygrad.helpers import DEV, PROFILE, getenv, to_mv, from_mv, cpu_profile, ProfileRangeEvent, select_first_inited, select_by_name, unwrap +from tinygrad.helpers import DEV, PROFILE, getenv, from_mv, cpu_profile, ProfileRangeEvent, unwrap from tinygrad.helpers import suppress_finalizing, pluralize, TracingKey from tinygrad.device import Device, BufferSpec, Compiled, LRUAllocator, ProfileDeviceEvent, ProfileProgramEvent, Program, TinyELF from tinygrad.uop.ops import sym_infer, sint, UOp from tinygrad.runtime.autogen import libc -from tinygrad.runtime.support.memory import BumpAllocator +from tinygrad.runtime.support.memory import BumpAllocator, MMIOInterface from tinygrad.renderer import Renderer -class MMIOInterface: - def __init__(self, addr:int, nbytes:int, fmt='B'): self.mv, self.addr, self.nbytes, self.fmt = to_mv(addr, nbytes).cast(fmt), addr, nbytes, fmt - def __len__(self): return self.nbytes // struct.calcsize(self.fmt) - def __getitem__(self, k): return (self.mv[k] if self.fmt == 'B' else self.mv[k].tolist()) if isinstance(k, slice) else self.mv[k] - def __setitem__(self, k, v): self.mv[k] = v - def view(self, offset:int=0, size:int|None=None, fmt=None) -> MMIOInterface: - return MMIOInterface(self.addr+offset, (self.nbytes - offset) if size is None else size, fmt=fmt or self.fmt) - class FileIOInterface: """ Hardware Abstraction Layer for HCQ devices. The class provides a unified interface for interacting with hardware devices. @@ -296,7 +287,8 @@ class HCQSignal(Generic[HCQDeviceType]): if not_passed and self.value < value: raise RuntimeError(f"Wait timeout: {timeout} ms! (the signal is not set to {value}, but {self.value})") @contextlib.contextmanager -def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Callable[[], HWQueue]|None=None, queue:HWQueue|None=None, dev_suff:str|None=None): +def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Callable[[], HWQueue]|None=None, queue:HWQueue|None=None, dev_suff:str|None=None, + profile_key:bytes|None=None): st, en = (dev.new_signal(), dev.new_signal()) if enabled else (None, None) assert queue is not None or queue_type is not None, "Either queue or queue_type must be provided" @@ -310,7 +302,8 @@ def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Callable[[], HWQueue] elif enabled and queue_type is not None: queue_type().wait(dev.timeline_signal, dev.timeline_value - 1).timestamp(en).signal(dev.timeline_signal, dev.next_timeline()).submit(dev) - if enabled and PROFILE: dev.sig_prof_records.append((unwrap(st), unwrap(en), desc, f"{dev.device}:{dev_suff}" if dev_suff else dev.device)) + if enabled and PROFILE: dev.sig_prof_records.append((unwrap(st), unwrap(en), desc, f"{dev.device}:{dev_suff}" if dev_suff else dev.device, + profile_key)) class HCQArgsState(Generic[ProgramType]): def __init__(self, buf:HCQBuffer, prg:ProgramType, bufs:tuple[HCQBuffer, ...], vals:tuple[sint|None, ...]=()): @@ -333,8 +326,9 @@ class CLikeArgsState(HCQArgsState[ProgramType]): class HCQProgram(Program[HCQDeviceType]): def __init__(self, args_state_t:Type[HCQArgsState], dev:HCQDeviceType, obj:TinyELF, kernargs_alloc_size:int, base:int|None=None): self.args_state_t, self.dev, self.name, self.signature, self.kernargs_alloc_size = args_state_t, dev, obj.name, obj.signature, kernargs_alloc_size + self.profile_key = obj.profile_key self.prof_prg_counter = next(self.dev.prof_prg_counter) - if PROFILE: Compiled.profile_events += [ProfileProgramEvent(dev.device, obj.name, obj.lib, base, self.prof_prg_counter)] + if PROFILE: Compiled.profile_events += [ProfileProgramEvent(dev.device, obj.name, obj.lib, base, self.prof_prg_counter, self.profile_key)] @staticmethod def _fini(dev, buf, spec): dev.allocator.free(buf, buf.size, spec) @@ -373,7 +367,7 @@ class HCQProgram(Program[HCQDeviceType]): q = unwrap(self.dev.hw_compute_queue_t)().wait(self.dev.timeline_signal, self.dev.timeline_value - 1).memory_barrier() self.dev.prof_exec_counter += 1 - with hcq_profile(self.dev, queue=q, desc=self.name, enabled=wait or PROFILE) as (sig_st, sig_en): + with hcq_profile(self.dev, queue=q, desc=self.name, enabled=wait or PROFILE, profile_key=self.profile_key) as (sig_st, sig_en): q.exec(self, kernargs, global_size, local_size) q.signal(self.dev.timeline_signal, self.dev.next_timeline()).submit(self.dev) @@ -393,8 +387,6 @@ class HCQCompiled(Compiled, Generic[SignalType]): def __init__(self, device:str, allocator:HCQAllocatorBase, compilers:list[type[Renderer]], runtime:type[Program]|None, signal_t:Type[SignalType]|None=None, comp_queue_t:Callable[..., HWQueue]|None=None, copy_queue_t:Callable[..., HWQueue]|None=None, kernargs_size=(16 << 20), sigalloc_size=0x1000, can_recover:bool=False, arch=None): - self.device_id:int = int(device.split(":")[1]) if ":" in device else 0 - from tinygrad.runtime.graph.hcq import HCQGraph super().__init__(device, allocator, compilers, runtime, HCQGraph, arch=arch) @@ -404,7 +396,7 @@ class HCQCompiled(Compiled, Generic[SignalType]): self.signal_t, self.hw_compute_queue_t, self.hw_copy_queue_t = signal_t, comp_queue_t, copy_queue_t self.timeline_value:int = 1 - self.sig_prof_records:list[tuple[HCQSignal, HCQSignal, str|TracingKey, str]] = [] + self.sig_prof_records:list[tuple[HCQSignal, HCQSignal, str|TracingKey, str, bytes|None]] = [] self.prof_exec_counter:int = 0 self.prof_prg_counter = itertools.count(0) @@ -424,8 +416,6 @@ class HCQCompiled(Compiled, Generic[SignalType]): if self._is_cpu(): HCQCompiled.cpu_devices.append(self) - def count(self) -> int: return self.iface.count if hasattr(self, 'iface') else 1 - def synchronize(self, timeout:int|None=None): if self.error_state is not None: raise self.error_state if not hasattr(self, 'timeline_signal'): return @@ -442,7 +432,7 @@ class HCQCompiled(Compiled, Generic[SignalType]): if self.timeline_value > (1 << 31): self._wrap_timeline_signal() if PROFILE: - Compiled.profile_events += [ProfileRangeEvent(dev, name, st.timestamp, en.timestamp) for st,en,name,dev in self.sig_prof_records] + Compiled.profile_events += [ProfileRangeEvent(dev, name, st.timestamp, en.timestamp, pk) for st,en,name,dev,pk in self.sig_prof_records] self.sig_prof_records = [] def next_timeline(self): @@ -491,16 +481,6 @@ class HCQCompiled(Compiled, Generic[SignalType]): buf, realloced = self.allocator.alloc(oldbuf.size if oldbuf is not None else new_size, options=options), False return buf, realloced - def _select_iface(self): - assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \ - f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead" - assert hasattr(self, "ifaces"), "must have ifaces to select an iface" - t = DEV.target(dev:=type(self).__name__[:-6]) - filtered = select_by_name(self.ifaces, lambda i: i.__name__[:-5], t.interface, f"{dev} has no interface {t.interface!r}") - filtered = [i for i in filtered if t.interface.startswith("MOCK") or not i.__name__[:-5].startswith("MOCK")] # never fallback to mock ifaces - return select_first_inited([functools.partial(cast(Callable, iface), self, self.device_id) for iface in filtered], - f"No interface for {dev}:{self.device_id} is available") - def _is_cpu(self) -> bool: return hasattr(self, 'device') and self.device.split(":")[0] == "CPU" def rdma_dev(self): @@ -512,13 +492,10 @@ class HCQCompiled(Compiled, Generic[SignalType]): def finalize(self): try: self.synchronize() # Try to finalize device in any case. except RuntimeError as e: print(f"{self.device} synchronization failed before finalizing: {e}") - - # If the device has an interface, call its device_fini method to clean up resources. - if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini() + super().finalize() class HCQBuffer: - def __init__(self, va_addr:sint, size:int, meta:Any=None, _base:HCQBuffer|None=None, view:MMIOInterface|None=None, - owner:HCQCompiled|None=None): + def __init__(self, va_addr:sint, size:int, meta:Any=None, _base:HCQBuffer|None=None, view:MMIOInterface|None=None, owner:Any=None): self.va_addr, self.size, self.meta, self._base, self.view = va_addr, size, meta, _base, view self._devs, self.owner = ([owner] if owner is not None else []), owner self._mappings:dict[HCQCompiled, HCQBuffer] = {} # mapping to the other devices diff --git a/tinygrad_repo/tinygrad/runtime/support/hcq2.py b/tinygrad_repo/tinygrad/runtime/support/hcq2.py index 9e2429e30..5c8519b54 100644 --- a/tinygrad_repo/tinygrad/runtime/support/hcq2.py +++ b/tinygrad_repo/tinygrad/runtime/support/hcq2.py @@ -1,85 +1,62 @@ from __future__ import annotations -from typing import cast, Callable, TypeVar, Generic, Any, Sequence +from typing import cast, TypeVar, Generic, Any, Sequence, TYPE_CHECKING import struct, functools, time, collections, itertools, decimal, statistics -from dataclasses import replace, dataclass -from tinygrad.helpers import DEV, getenv, select_first_inited, select_by_name, suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE -from tinygrad.helpers import to_tuple, round_up, partition, data64_le, panic, ContextVar, perf_counter_us, Context +from dataclasses import replace, dataclass, field +from tinygrad.helpers import suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE, all_same, all_int +from tinygrad.helpers import to_tuple, ContextVar, perf_counter_us, Context, panic, partition, round_up, flatten, next_power2 from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator, MultiBuffer, DepsTracker from tinygrad.device import ProfileDeviceEvent, ProfileGraphEntry, ProfileGraphEvent from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp -from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const -from tinygrad.dtype import dtypes, truncate -from tinygrad.runtime.support.hcq import MMIOInterface -from tinygrad.runtime.support.memory import BumpAllocator +from tinygrad.uop.symbolic import symbolic +from tinygrad.dtype import dtypes, DType, AddrSpace +from tinygrad.runtime.support.memory import BumpAllocator, MMIOInterface from tinygrad.renderer import Renderer, Estimates -from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop -from tinygrad.engine.realize import pm_flatten_linear +from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop, pm_flatten_linear +from tinygrad.engine.realize import lower_and_compile + +if TYPE_CHECKING: from tinygrad.runtime.support.hcq import HCQBuffer # TODO: remove that # ***************** # 0. helpers HCQDeviceType = TypeVar('HCQDeviceType', bound='HCQ2Compiled') - HCQ_RUNTIME_DEV = ContextVar("HCQ_RUNTIME_DEV", "CPU") - -HCQ_DEVS = frozenset(("AMD",)) -HCQ_P2P_DEVS = HCQ_DEVS | frozenset(("CPU",)) -HCQ_CACHE_TAGS = frozenset(("program", "systems", "template")) +HCQ_DEVS = frozenset(("AMD", "CPU")) @dataclass(frozen=True) class HCQInfo: device:tuple[str, ...] estimates:Estimates = Estimates() - input_idxs:tuple[int, ...] = () # indexes into input_uops used by this call - inputs:int|None = None - prof:tuple[ProfileGraphEntry, ...] = () # st_id/en_id are timestamp signal slots until collect + kernels:tuple[tuple[tuple[str, ...], str, Estimates, tuple[int, ...], bytes], ...] = () + + args:tuple[tuple[UOp, UOp], ...] = () # placeholder -> the canonical body param it becomes, in call src order + table:Any = None # the inputs table placeholder (its src position after lower) + inputs:tuple[tuple[UOp, int, str], ...] = () # per table slot: (src, lane, device) the exec resolves to an address + vals:tuple[tuple[str, int], ...] = () # bound values of the body variables (the queue byte size, table slots) def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c -def unwrap_mstack(u): +def unwrap_mstack(u:UOp) -> tuple[UOp, ...]: if u.op is Ops.MSTACK: return tuple(x for s in u.src for x in unwrap_mstack(s)) - return unwrap_mstack(u.src[0]) if u.op in {Ops.MSELECT, Ops.SLICE} else (u,) + return unwrap_mstack(u.src[0]) if u.op is Ops.MSELECT else (u,) -def is_value_known_at_link(val:UOp) -> bool: - runtime_reads = [u for u in val.toposort() if u.op in (Ops.LOAD, Ops.INDEX)] - addressed_bufs = [b for g in val.toposort() if g.op is Ops.GETADDR for b in unwrap_mstack(g.buf_uop)] +def unwrap_view(v:UOp) -> tuple[UOp, int]: # look through views to (base, element offset) + return unwrap_view(v.src[0]) if v.op in (Ops.BITCAST, Ops.AFTER) else (v.src[0], v.src[1].val) if v.op is Ops.SHRINK else (v, 0) - # addr of input params is not known at link time - return not val.variables() and not runtime_reads and all(b.op is not Ops.PARAM or b.tag is not None for b in addressed_bufs) - -def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...]: - return tuple(buf.index(UOp(Ops.STACK, dtypes.int, tuple(UOp.const(off // buf.dtype.itemsize, dtypes.int) for off,_ in ps))) - .store(UOp(Ops.STACK, buf.dtype, tuple(val.cast(buf.dtype) for _,val in ps))).rtag(tag) - for ps, tag in zip(partition(patches, lambda p: is_value_known_at_link(p[1])), ("link", None)) if ps) - -def make_binary_patch(buf:UOp, blob:bytes) -> UOp: - data = UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype) - r = UOp.range(len(blob) // buf.dtype.itemsize, 0, dtype=dtypes.int, src=(buf, data)) - return buf.index(r).store(data.index(r).load()).end(r).rtag("link") - -def make_cmdbuf(lin, devs, buf:UOp|None=None): - blob, patches = bytearray(), [] - for s in (s for ins in lin.src for s in ins.src): - if s.op is not Ops.CONST: patches.append((len(blob), s)) - blob.extend(struct.pack(f'<{s.dtype.fmt}', s.val if s.op is Ops.CONST else 0x0)) - cmdbuf = buf if buf is not None else UOp.placeholder((len(blob) // 4,), dtypes.uint32, next(UOp.unique_num), device=devs).rtag("cmdbuf") - return cmdbuf.after(make_binary_patch(cmdbuf, bytes(blob)), *make_patches(cmdbuf, patches)) - -def make_signal(devs, slot:int=0, tag:str="signal") -> UOp: - return UOp.placeholder((1,), dtypes.uint64, slot, device=devs, volatile=True).rtag(tag) +def _lane(u:UOp, lane:int) -> UOp: return u.src[lane] if u.op is Ops.MSTACK else u.mselect(lane) if len(to_tuple(u.device)) > 1 else u def make_submit(*cmds, devs:str|tuple[str, ...], queue:str) -> UOp: return UOp.custom_function("submit_cmdbuf", UOp(Ops.LINEAR, src=tuple(cmds), arg=(to_tuple(devs), queue))) -def get_submit(ast:UOp) -> UOp: return next(u for u in ast.toposort() if u.op is Ops.CUSTOM_FUNCTION and u.arg == "submit_cmdbuf") +def get_submit(ast:UOp) -> UOp|None: + return next((u for u in ast.toposort() if u.op is Ops.CUSTOM_FUNCTION and u.arg == "submit_cmdbuf"), None) def make_call(name:str, body:UOp, info:HCQInfo) -> UOp: return UOp.custom_function("hcq", body).call(name=name, aux=info) -def encode_kernargs_clike(call:UOp, prg:UOp, devs:str|tuple[str, ...]) -> UOp: - data, info = prg.arg - buf = UOp.placeholder((data.kernargs_alloc_size // 4,), dtypes.uint32, next(UOp.unique_num), device=devs).rtag("kernargs") - words = [w for gi in info.globals for w in data64_le(get_call_arg_uops(call)[gi].getaddr(devs))] + list(info.vars) - return buf.after(*make_patches(buf, [(i * 4, w) for i, w in enumerate(words)])) +def hcq_size_var(cmdbuf:UOp) -> UOp: # the sealed byte count, bounded by the cmdbuf it walks so the submit copy loops stay in bounds + return UOp.variable("hcq_size", 0, cmdbuf.max_numel() * cmdbuf.dtype.itemsize, dtypes.uint32, param=True) + +def make_buf(devs, slot:int=0, tag:str="signal") -> UOp: return UOp.placeholder((1,), dtypes.uint64, slot, device=devs, volatile=True, tag=tag) # ***************** # 0.1. prep: replace buffers with params @@ -87,21 +64,56 @@ def encode_kernargs_clike(call:UOp, prg:UOp, devs:str|tuple[str, ...]) -> UOp: def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp|None: bufs, slots = ctx for s in call.src[1:]: - if s.op not in (Ops.PARAM, Ops.BIND) and slots.setdefault(s, len(bufs)) == len(bufs): bufs.append(s) - return call.replace(src=call.src[:1] + tuple(s if s.op in (Ops.PARAM, Ops.BIND) else s.param_like(slots[s]) for s in call.src[1:])) + if s.op is not Ops.PARAM and not s.is_bound_var and slots.setdefault(s, len(bufs)) == len(bufs): bufs.append(s) + return call.replace(src=call.src[:1] + tuple(s if s.op is Ops.PARAM or s.is_bound_var else s.param_like(slots[s]) for s in call.src[1:])) pm_replace_buffers = PatternMatcher([(UPat(Ops.CALL, name="call"), replace_call_buffers)]) # ***************** # 1.1. prep: staging copies -def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS) and not all_devices_in(b.device, HCQ_P2P_DEVS) +STAGING_SIZE, STAGING_SLOTS = 128 << 20, 2 + +@functools.cache +def _staging() -> Buffer: return Buffer("CPU", STAGING_SIZE, dtypes.uint8, preallocate=True) + +def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS - {"CPU"}) and not all_devices_in(b.device, HCQ_DEVS) + +def stage_copy_ext(call:UOp) -> UOp|None: + if (d:=next((d for b in call.src[1:] for d in to_tuple(b.device) if not d.startswith("CPU")), None)) is None: return None + return pm.rewrite(call) if (pm:=getattr(Device[d], "pm_stage_copy", None)) is not None else None def stage_copy(dst:UOp, src:UOp) -> UOp|None: if not (_need_staging(src, dst) or _need_staging(dst, src)): return None - stage = UOp.new_buffer("CPU", src.max_numel() * src.dtype.itemsize, dtypes.uint8) - return UOp(Ops.LINEAR, src=(src.copy_to_device("CPU").call(stage, src), stage.copy_to_device(dst.device).call(dst, stage))) -pm_insert_copy_staging = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), stage_copy)]) + assert src.dtype.itemsize == dst.dtype.itemsize, "staged copies must be dtype-size matched" + base, it, copies = UOp.from_buffer(_staging()), src.dtype.itemsize, [] + chunk = (STAGING_SIZE // STAGING_SLOTS) // it + for i, off in enumerate(range(0, src.max_numel(), chunk)): + stage = base[(so:=(i % STAGING_SLOTS) * chunk * it):so + (n:=min(chunk, src.max_numel() - off)) * it] + copies += [src[off:off+n].copy_to_device("CPU").call(stage, src[off:off+n]), stage.copy_to_device(dst.device).call(dst[off:off+n], stage)] + return UOp(Ops.LINEAR, src=tuple(copies)) + +# ***************** +# 1.2. prep: kernel copies + +def _get_enqueue_devs(call:UOp) -> Any|None: + if call.src[0].op not in (Ops.PROGRAM, Ops.COPY): return None # only these bodies can be enqueued + if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_DEVS) for b in bufs): return None + if call.src[0].op is Ops.COPY: bufs = bufs[::-1] # copies push from the src device: p2p writes are faster than reads + devs = min(bufs, key=lambda b: to_tuple(b.device)[0].startswith("CPU")).device # prio to enqueue on not CPU device + return devs if all_devices_in(devs, HCQ_DEVS) else None + +def copy_with_kernel(call:UOp, dst:UOp, src:UOp) -> UOp|None: + if (devs:=_get_enqueue_devs(call)) is None or Device[(dev:=to_tuple(devs)[0])].has_copy_queue: return None + d, s = (UOp.param(i, dst.dtype, n:=dst.max_numel(), device=devs) for i in range(2)) + ast = d.index(r:=UOp.range(n, 0)).store(s.index(r).load()).end(r).sink(arg=KernelInfo(name="copy"), tag=1) + return call.replace(src=(to_program(ast, Device[dev].renderer), dst, src)) + +pm_insert_copy_staging = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.COPY),), name="call", allow_any_len=True), stage_copy_ext), + (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), stage_copy), + (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src")), name="call"), copy_with_kernel) +]) # ***************** # 2. deps @@ -109,300 +121,340 @@ pm_insert_copy_staging = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.COPY), UP class HCQDepsTracker(DepsTracker): @staticmethod def _key(buf:Any) -> tuple[Any, int, int]: + if isinstance(buf, UOp) and buf.op is Ops.MSELECT: buf = buf.src[0] return (buf.arg.slot, 0, buf.max_numel() * buf.dtype.itemsize) if isinstance(buf, UOp) else DepsTracker._key(buf) -def _get_call_bufs_by_lane(call:UOp, devices:tuple[str, ...]) -> list[list[Any]]: - refs = get_call_arg_uops(call) - return [[b if b.op is Ops.PARAM else mb.bufs[lane] if isinstance(mb:=b.buffer, MultiBuffer) else mb for b in refs] for lane in range(len(devices))] +@dataclass(frozen=True) +class Dep: dev:str; queue:str; tag:int; lane:int # producer submit (dev, queue, tag) awaited by consumer lane # noqa: E702 -def _get_deps(ctx:DepsTracker, bufs_by_lane:list[list[Any]], write, key:tuple[tuple[str, ...], str, int]) -> list[tuple[tuple, int, int]]: - dep_lanes:list[tuple[tuple, int, int]] = [] +@dataclass +class BatchCtx: + batch:list[tuple[UOp, tuple[str, ...]]]; batch_info:list[tuple[tuple[str, ...], str]]; profile:bool # noqa: E702 + tracker:HCQDepsTracker = field(default_factory=HCQDepsTracker); signal_tags:set[int] = field(default_factory=set) # noqa: E702 + slots:dict[str, int] = field(default_factory=lambda: collections.defaultdict(lambda: next(UOp.unique_num))) + +def _get_call_bufs_by_lane(call:UOp, devices:tuple[str, ...]) -> list[list[Any]]: + def dep_buf(b:UOp) -> Any: return base if (base:=(b.src[0] if b.op is Ops.MSELECT else b).base).op is Ops.PARAM else b.buffer + return [[dep_buf(_lane(a, lane)) for a in get_call_arg_uops(call)] for lane in range(len(devices))] + +def _wait_ins(ctx:BatchCtx, bufs_by_lane:list[list[Any]], write, devices:tuple[str, ...], queue:str, tag:int) -> list[UOp]: + deps:list[Dep] = [] for lane, bufs in enumerate(bufs_by_lane): written = write if write is not None else list(range(len(bufs))) - dep_lanes += [(dep, dlane, lane) for dep, dlane in ctx.access_resources(bufs, written, (key, lane))] - return dep_lanes + deps += [Dep(d, q, t, lane) for d, q, t in ctx.tracker.access_resources(bufs, written, (devices[lane], queue, tag)) if t < tag] -def _build_wait_cmds(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int]], devices:tuple[str, ...], queue:str) -> tuple[list[UOp], set[int]]: - # opt1: same-queue ops are fifo-ordered - if devices[0].split(":")[0] in {"AMD", "QCOM"} or queue.startswith("COPY"): - dep_lanes = [(dep, dlane, lane) for dep, dlane, lane in dep_lanes if (dep[0][dlane], dep[1]) != (devices[lane], queue)] - - # opt2: keep latest dep per (dep device, queue, cur lane) - latest = {((dep[0][dlane], dep[1]), lane): (dep, dlane) for dep, dlane, lane in sorted(dep_lanes, key=lambda x: x[0][2])} - deps:dict[tuple, list[int|None]] = collections.defaultdict(lambda: [None]*len(devices)) - for (_, lane), (dep, dlane) in latest.items(): deps[dep][lane] = dlane + # same-queue submits are fifo-ordered, no wait needed + if devices[0].split(":")[0] in {"AMD", "QCOM", "CPU"} or queue.startswith("COPY"): + deps = [d for d in deps if (d.dev, d.queue) != (devices[d.lane], queue)] + latest = {(d.dev, d.queue, d.lane): d for d in sorted(deps, key=lambda d: d.tag)} + # keep only the latest signal + rows:dict[tuple[str, int], dict[int, list[str]]] = collections.defaultdict(lambda: collections.defaultdict(list)) + for d in latest.values(): rows[(d.queue, d.tag)][d.lane].append(d.dev) waits = [] - for (ddevs, dqueue, dtag), lanes in deps.items(): - sig = UOp.mstack(*[make_signal(d, tag="sentinel_signal") if dl is None else make_signal(ddevs[dl], slots[dqueue]) - for dl, d in zip(lanes, devices)]) - waits.append(UOp(Ops.INS, arg="wait", src=(sig, UOp.const(dtag + 1, dtypes.uint64)))) - return waits, {dtag for _, _, dtag in deps} + for (dqueue, dtag), by_lane in rows.items(): + for ds in itertools.zip_longest(*(by_lane[lane] for lane in range(len(devices)))): + sig = UOp.mstack(*[make_buf(d, tag="sentinel_signal") if dd is None else make_buf(dd, ctx.slots[dqueue]) for dd, d in zip(ds, devices)]) + waits.append(UOp(Ops.INS, arg=("wait", dtypes.void), src=(sig, UOp.const(dtag + 1, dtypes.uint64)))) + ctx.signal_tags |= {t for _, t in rows} + return waits -def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[tuple[tuple[str, ...], str]], - tracker:HCQDepsTracker, slots:dict[str, int]) -> tuple[list[UOp], list[UOp], set[int]]: +def _merge_submits(calls:list[UOp]) -> UOp: + if len(calls) == 1: return calls[0] + devs, queue = unwrap(get_submit(calls[0])).src[0].arg + body = make_submit(*[cmd for c in calls for cmd in unwrap(get_submit(c)).src[0].src], devs=devs, queue=queue).sink() + return make_call(f"submit {queue} ({len(calls)})", body, replace(calls[0].arg.aux, + estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) + +def _merge_queues(submits:list[UOp]) -> list[UOp]: + # CPU submits run inline and can block on another queue. Keep multi-queue CPU work in schedule order so every + # producer queue is submitted before a CPU wait; merging by queue can otherwise deadlock alternating dependencies. + keys = [unwrap(get_submit(call)).src[0].arg for call in submits] + if len(set(keys)) > 1 and any(any(d.split(":")[0] == "CPU" for d in devs) for devs, _ in keys): return submits + + merged:list[UOp] = [] + opened:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> hcq calls in submit order + limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) + for call in submits: + devs, queue = key = unwrap(get_submit(call)).src[0].arg + if (group:=opened.pop(key, None)) is None: + # first submit on this queue: close open groups on the same queue with shared devices, so submit order is kept + for k in [k for k in opened if k[1] == queue and set(k[0]) & set(devs)]: merged.append(_merge_submits(opened.pop(k))) + group = [] + elif limits[key] and len(group) >= limits[key]: merged, group, limits[key] = merged + [_merge_submits(group)], [], limits[key] * 2 + opened[key] = group + [call] + return merged + [_merge_submits(g) for g in opened.values()] + +def _make_finalizers(ctx:BatchCtx) -> tuple[list[UOp], list[UOp], list[UOp]]: # collect all buffers which belong to devices dev_bufs:dict[str, dict[int, Any]] = collections.defaultdict(dict) - for call, devices in batch: + for call, devices in ctx.batch: for b in itertools.chain.from_iterable(_get_call_bufs_by_lane(call, devices)): for bd in to_tuple(b.device): dev_bufs[bd][id(b)] = b - n, fences, fins, signal_tags = len(batch_info), [], [], set() + n, fences, resets, fins = len(ctx.batch_info), [], [], [] for _, devgroup in itertools.groupby(sorted(dev_bufs), key=lambda d: d.split(":")[0]): - devs = tuple(devgroup) + sched_epoch = make_buf(devs:=tuple(devgroup), next(UOp.unique_num), tag="epoch") + tl_signal, tl_value = make_buf(devs, tag="timeline_signal"), make_buf(devs, tag="timeline_value") - # to finalize the batch, sync all accesses from other devices to buffers that belong to this device - fin_deps = [dl for dl in _get_deps(tracker, [list(dev_bufs[d].values()) for d in devs], None, key=(devs, "COMPUTE:0", n)) if dl[0][2] < n] - waits, cur_signal_tags = _build_wait_cmds(slots, fin_deps, devs, "COMPUTE:0") - signal_tags |= cur_signal_tags + # fence: spin until the device timeline reaches this schedule's previous epoch + done = tl_signal.after(loop:=UOp.loop(0)).index(0).load() + fences.append(make_call("hcq_fence", UOp.sink(done.end(loop, done < sched_epoch.index(0).load())), HCQInfo(devs))) - # wait the syncs and signal the device epoch, then bump the timeline on the host - tl_signal, tl_value = make_signal(devs, tag="timeline_signal"), make_signal(devs, tag="timeline_value") - fin_submit = make_submit(*waits, UOp(Ops.INS, arg="store", src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") + # reset: queues of other groups wait on this group's signals, zero them only after every group reached its epoch + qs = dedup([qn for bdevs, qn in ctx.batch_info if set(bdevs) & set(devs)]) + rst:tuple[UOp, ...] = () + for q in qs: rst += (make_buf(devs, ctx.slots[q]).after(*rst[-1:]).index(0).store(0),) + if rst: resets.append(make_call("hcq_reset", UOp.sink(*rst), HCQInfo(devs))) + + # finalizer: the submit stores the host timeline into the device timeline signal, then the host bumps the + # timeline and remembers this schedule's epoch for the next fence + waits = _wait_ins(ctx, [list(dev_bufs[d].values()) for d in devs], None, devs, "COMPUTE:0", n) + fin_submit = make_submit(*waits, UOp(Ops.INS, arg=("store", dtypes.void), src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") epoch = (epoch_slot:=tl_value.after(fin_submit).index(0)).load() - - # fence once per device group on this schedule's previous epoch, then reset any queue signals used by the group - qs = dedup([qn for bdevs, qn in batch_info if set(bdevs) & set(devs)]) - sched_epoch = make_signal(devs, next(UOp.unique_num)) - - wait_device_epoch = (done:=tl_signal.after(loop:=UOp.loop(0)).index(0).load()).end(loop, done < sched_epoch.index(0).load()) - resets = [make_signal(devs, slots[q]).after(wait_device_epoch).index(0).store(0) for q in qs] - - fences.append(make_call("hcq_fence", UOp.sink(*(resets or [wait_device_epoch])), HCQInfo(devs))) fins.append(make_call("hcq_finalizer", UOp.sink(epoch_slot.store(epoch + 1), sched_epoch.after(fin_submit).index(0).store(epoch)), HCQInfo(devs))) - return fences, fins, signal_tags + return fences, resets, fins -def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> list[UOp]: - batch_info = [(devices, "COMPUTE:0" if call.src[0].op is Ops.PROGRAM else "COPY:0") for call, devices in batch] - - # schedule deps - signal_tags:set[int] = set() - slots:dict[str, int] = collections.defaultdict(lambda: next(UOp.unique_num)) - deps_tracker = HCQDepsTracker() - call_waits:list[list[UOp]] = [] - for tag, ((call, _), (devices, queue)) in enumerate(zip(batch, batch_info)): - deps = _get_deps(deps_tracker, _get_call_bufs_by_lane(call, devices), get_call_outs_ins(call)[0], key=(devices, queue, tag)) - cmds, cur_signal_tags = _build_wait_cmds(slots, deps, devices, queue) - call_waits.append(cmds) - signal_tags |= cur_signal_tags - - # build fences and finalizers - fences, finalizers, finalizer_signal_tags = _build_finalizers(batch, batch_info, deps_tracker, slots) - signal_tags |= finalizer_signal_tags - - src, prof = [], [] - for tag, ((call, _), (devices, queue), q) in enumerate(zip(batch, batch_info, call_waits)): +def _emit_submits(ctx:BatchCtx, call_waits:list[list[UOp]]) -> tuple[list[UOp], list[tuple]]: + # one submit per call: timeline sync on first queue use, timestamps, the call, and a signal if someone waits on it + src, kerns, seen_queues = [], [], set() + for tag, ((call, _), (devices, queue), q) in enumerate(zip(ctx.batch, ctx.batch_info, call_waits)): # first queue use, sync prior device work with the device timeline - if batch_info.index((devices, queue)) == tag: - epoch = make_signal(devices, tag="timeline_value").index(0) - 1 - q = [UOp(Ops.INS, arg="barrier", src=()), UOp(Ops.INS, arg="wait", src=(make_signal(devices, tag="timeline_signal"), epoch))] + q + if (devices, queue) not in seen_queues: + seen_queues.add((devices, queue)) + epoch = make_buf(devices, tag="timeline_value").index(0) - 1 + q = [UOp(Ops.INS, arg=("barrier", dtypes.void), src=()), + UOp(Ops.INS, arg=("wait", dtypes.void), src=(make_buf(devices, tag="timeline_signal"), epoch))] + q # and make hcq call name, info = get_call_name(call, get_call_arg_uops(call)), HCQInfo(devices, estimate_uop(call)) - ts_ids = [next(UOp.unique_num) for _ in range(2)] if profile else [] - prof += [ProfileGraphEntry(d, name, *ts_ids) for d in devices if ts_ids] + ts_ids = [next(UOp.unique_num) for _ in range(2)] if ctx.profile else [] + kerns.append((devices, name, info.estimates, tuple(ts_ids), make_call(name, call.src[0], info).key)) - ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_signal(devices, s),)) for s in ts_ids] + ts_ins = [UOp(Ops.INS, arg=("timestamp", dtypes.void), src=(make_buf(devices, s),)) for s in ts_ids] q += ts_ins[:1] + [call.replace(arg=replace(call.arg, aux=info))] + ts_ins[1:] # signal the queue if someone waits for us - if tag in signal_tags: q += [UOp(Ops.INS, arg="store", src=(make_signal(devices, slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] - src.append(make_call(name, make_submit(*q, devs=devices, queue=queue).sink(), info)) + if tag in ctx.signal_tags: + q += [UOp(Ops.INS, arg=("store", dtypes.void), src=(make_buf(devices, ctx.slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] + src.append(make_call(f"submit {name}", make_submit(*q, devs=devices, queue=queue).sink(), info)) + return src, kerns - # append batch timestamps to finalizers - finalizers = [f.replace(arg=replace(f.arg, aux=replace(a:=f.arg.aux, prof=tuple(e for e in prof if e.device in a.device)))) for f in finalizers] - return fences + src + finalizers +def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> list[UOp]: + ctx = BatchCtx(batch, [(devices, "COMPUTE:0" if call.src[0].op is Ops.PROGRAM else "COPY:0") for call, devices in batch], profile) -def sched_hcq_batches(l:UOp, profile:bool) -> UOp: + call_waits = [_wait_ins(ctx, _get_call_bufs_by_lane(call, devices), get_call_outs_ins(call)[0], devices, queue, tag) + for tag, ((call, _), (devices, queue)) in enumerate(zip(ctx.batch, ctx.batch_info))] + fences, resets, fins = _make_finalizers(ctx) + submits, kerns = _emit_submits(ctx, call_waits) + + # append batch kernels to the finalizers, their exec collects the profiles after everything is in flight + fins = [f.replace(arg=replace(f.arg, aux=replace(a:=f.arg.aux, kernels=tuple(x for x in kerns if set(x[0]) & set(a.device))))) for f in fins] + return fences + resets + _merge_queues(submits) + fins + +@rewrite_group(new_ctx=False) +def sched_batches(l:UOp, profile:bool) -> UOp: srcs:list[UOp] = [] batch:list[tuple[UOp, tuple[str, ...]]] = [] for call in l.src: - if (devs:=next((b.device for b in call.src[1:] if all_devices_in(b.device, HCQ_DEVS)), None)) is not None: batch.append((call, to_tuple(devs))) + if (devs:=_get_enqueue_devs(call)) is not None: batch.append((call, to_tuple(devs))) else: srcs, batch = srcs + _finalize_batch(batch, profile) + [call], [] return l.replace(src=tuple(srcs + _finalize_batch(batch, profile))) # ***************** -# 3. merge into queues +# 3. encode: the backend rewrites the ops of every submit into flat command words. a word is a const, a uop the +# link or the exec resolves to a value, or a getaddr of a nested LINEAR (an indirect blob like kernargs) -def _merged_hcq_call(calls:list[UOp]) -> UOp: # TODO: simplify? - if len(calls) == 1: return calls[0] - devs, queue = get_submit(calls[0]).src[0].arg - body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() - return make_call(f"submit {queue} ({len(calls)})", body, - replace(calls[0].arg.aux, estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()))) +class EncodeCtx: # devs/queue (and for the submit lowering, the sealed byte count) plus everything on the device + def __init__(self, dev, devs:tuple[str, ...], queue:str, nbytes:int=0): self.dev, self.devs, self.queue, self.nbytes = dev, devs, queue, nbytes + def __getattr__(self, name): return getattr(self.dev, name) -def merge_queues(linear:UOp) -> UOp: - new_src:list[UOp] = [] - opened_qs:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> list of hcq calls, kept in submit order - limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) - - for call in linear.src: - # non-hcq call, fence or finalizer: close all open queues - if not isinstance(call.arg.aux, HCQInfo) or (call.arg.name or "").startswith("hcq_"): - new_src += [_merged_hcq_call(opened_qs.pop(k)) for k in list(opened_qs)] + [call] - continue - - devs, queue = get_submit(call).src[0].arg - if (old:=opened_qs.pop(key:=(devs, queue), None)) is not None: - if limits[key] and len(old) >= limits[key]: new_src, old, limits[key] = new_src + [_merged_hcq_call(old)], [], limits[key] * 2 - new_rec = old + [call] - else: - # no such queue opened: close every open submit on this queue that shares a device, so submit order is kept - closing = [k for k in opened_qs if k[1] == queue and set(k[0]) & set(devs)] - new_src += [_merged_hcq_call(opened_qs.pop(k)) for k in closing] - new_rec = [call] - opened_qs[(devs, queue)] = new_rec - return linear.replace(src=tuple(new_src + [_merged_hcq_call(c) for c in opened_qs.values()])) - -pm_schedule_and_merge = PatternMatcher([(UPat(Ops.LINEAR, name="l"), - lambda ctx, l: merge_queues(sched_hcq_batches(l, ctx[1]).substitute(ctx[0], walk=True, enter_calls=True)))]) +def encode_call(call:UOp) -> UOp|None: + if (submit:=get_submit(call.src[0])) is None or (lin:=submit.src[0]).op is not Ops.LINEAR: return None + if not any(w.op in {Ops.INS, Ops.CALL} for w in lin.src): return None # already flat words + devs, queue = lin.arg + ctx = EncodeCtx(dev:=Device[devs[0]], devs, queue) + body = graph_rewrite(call.src[0], dev.pm_encode[queue.split(":")[0]] + pm_flatten_linear, ctx=ctx, name=f"encode {queue}") + return call.replace(src=(body, *call.src[1:])) if body is not call.src[0] else None +pm_encode = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), encode_call)]) # ***************** -# 4.2. hcq lowering: ops to ir +# 4. seal: pack the words into one blob per linear tag. consts bake into the blobs, link values fold in at link time, +# and the rest the body stores each call from the exec-filled inputs table and the link-filled offset table. +# the submit keeps only its buffer: custom_function(submit) -> placeholder -def encode_cmdbuf(submit:UOp, lin:UOp) -> UOp|None: - if (pm:=Device.get_class(lin.arg[0][0]).pm_lower) is None: return None - return graph_rewrite(submit, pm, name=f"encode {lin.arg[0]}", enter_calls=True) -pm_encode_cmdbufs = PatternMatcher([ - (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="lin"),), name="submit"), encode_cmdbuf)]) +def is_link_value(w:UOp) -> bool: # resolvable when the linear links: no variables or memory reads, no input-param or program addresses + if w.op is Ops.GETADDR: return all(b.op is not Ops.PARAM or b.tag is not None for b in unwrap_mstack(w.buf_uop)) + if w.op in {Ops.LOAD, Ops.INDEX} or w.is_variable or (w.op is Ops.PARAM and w.arg.addrspace is AddrSpace.ALU): return False + return all(is_link_value(s) for s in w.src) + +def blobify(ph:UOp, data:bytes, stores:Sequence[tuple[int, UOp]]=()) -> UOp: + binary = ph.store(UOp(Ops.BINARY, src=(), arg=data).bitcast(ph.dtype)) + return ph.after(binary, *[ph.shrink(((o, o + w.dtype.itemsize),)).bitcast(w.dtype).index(0).store(w) for o, w in stores]) + +def seal_call(call:UOp) -> UOp|None: + if (submit:=get_submit(call.src[0])) is None or (lin:=submit.src[0]).op is not Ops.LINEAR: return None + devs, queue = lin.arg + + # pack every linear into its own blob + patches:list[tuple[UOp, int, UOp]] = [] # (linear, local byte offset, word) + blobs:dict[UOp, bytearray] = {l: bytearray() for l in [lin] + [u for u in lin.toposort() if u.op is Ops.LINEAR and u is not lin]} + + for l, blob in blobs.items(): + for w in l.src: + if is_uop:=isinstance(c:=w.ssimplify(), UOp): patches.append((l, len(blob), w)) # the original w: simplify can rebuild nested linears + blob += (b"\xbe" * w.dtype.itemsize) if is_uop else cast(int, c).to_bytes(w.dtype.itemsize, 'little') + + # merge blobs based on tags + tags, offs, datas = {l: l.tag or ("cmdbuf", queue) for l in blobs}, {}, collections.defaultdict[Any, bytearray](bytearray) + for l, b in blobs.items(): + offs[l] = len(datas[tags[l]]) + datas[tags[l]] += b.ljust(round_up(len(b), 128), b"\xbf") + + bufs = {t: UOp.placeholder((next_power2(len(d)),), dtypes.uint8, next(UOp.unique_num), device=devs).rtag(t) for t, d in datas.items()} + views = {l: bufs[tags[l]][offs[l]:offs[l] + len(blobs[l])] for l in blobs} + + # place the words in the merged blobs, then split: link words fold at link time, runtime words the body stores every call + placed = UOp.sink(*[w for _, _, w in patches]).substitute(views).src + links, runtime = partition([(bufs[tags[l]], offs[l] + o, w) for (l, o, _), w in zip(patches, placed)], lambda p: is_link_value(p[2])) + + rt_sink = UOp.sink(*[w for _, _, w in runtime]) + rt_vars = {u: u.src[0] for u in rt_sink.toposort() if u.is_bound_var} + + # all getaddrs are one input table. the body walks it on the host, so it's a CPU buffer: an emulated runtime device has no memory of its own + gaddrs = dedup([g for g in rt_sink.toposort() if g.op is Ops.GETADDR]) + table_srcs = dedup([g.src[0].without_after for g in gaddrs]) + slots = {src: i * len(devs) for i, src in enumerate(table_srcs)} + table = UOp.placeholder((tsz:=next_power2(len(slots)*len(devs)),), dtypes.uint64, next(UOp.unique_num), device="CPU").rtag("inputs") + dvar = UOp.variable("_device_num", 0, len(devs) - 1, dtypes.int, param=True) if len(devs) > 1 else UOp.const(0, dtypes.int) + reads = {g: table.index(slots[g.src[0].without_after] + dvar).load() for g in gaddrs} + + # group rt-patches by target and uop + groups:dict[tuple[UOp, UOp], list[tuple[int, int]]] = collections.defaultdict(list) + for (buf, off, w), v in zip(runtime, rt_sink.substitute(reads | rt_vars).src): + if w.op is Ops.GETADDR: groups[(buf, table)].append((off, slots[w.src[0].without_after])) + else: groups[(buf, v)].append((off, 0)) + + stores, vals, base = [], [], UOp.const(0, dtypes.int) + offtbl = UOp.placeholder((osz:=next_power2(2 * len(runtime)),), dtypes.uint32, next(UOp.unique_num), device="CPU").rtag("offtbl") + for j, ((buf, v), grp) in enumerate(groups.items()): + n = UOp.variable(f"hcq_off_len{j}", 0, osz // 2, dtypes.uint32, param=True) + vals.append((n.arg.name, len(grp))) + + # the lens sum to the entry count, so both masks are no-ops. they just keep the table indices provably in bounds + r = UOp.range(n, 20 + j, dtype=dtypes.int, src=(buf,)) + ent = 2 * ((base + r) & (osz // 2 - 1)) + off = offtbl.index(ent).load().cast(dtypes.int) + val = table.index((offtbl.index(ent + 1).load().cast(dtypes.int) + dvar) & (tsz - 1)).load() if v is table else v # reindex table + stores.append(buf.shrink(((off, off + val.dtype.itemsize),)).bitcast(val.dtype).index(0).store(val).end(r)) + base = base + n.cast(dtypes.int) + + patched = bufs[tags[lin]].after(*stores) + body = call.src[0].substitute({submit: submit.replace(src=(patched,))}) + + # link-time patches are just stores + link_stores:dict[UOp, list[tuple[int, UOp]]] = collections.defaultdict(list) + for b, off, w in links: link_stores[b].append((off, w)) + + # blobs, blobs + data_blobs = [blobify(offtbl, struct.pack(f"<{2*len(runtime)}I", *flatten(flatten(groups.values()))))] if runtime else [] + link_blobs = [blobify(bufs[t], bytes(d), link_stores[bufs[t]]) for t, d in datas.items()] + prog_blobs = dedup([u for u in UOp.sink(*[w for _, _, w in links]).toposort() if u.op is Ops.AFTER]) + info = replace(call.arg.aux, table=table if table_srcs else call.arg.aux.table, vals=(("hcq_size", len(blobs[lin])), *vals), + inputs=call.arg.aux.inputs + tuple((src, lane, dev) for src in table_srcs for lane, dev in enumerate(devs))) + return call.replace(src=(body, *dedup([*call.src[1:], *link_blobs, *data_blobs, *prog_blobs])), arg=replace(call.arg, aux=info)) + +pm_seal = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), seal_call)]) # ***************** +# 5. lower submits: the backend's pm_lower turns custom_function(submit, cmdbuf) into the queue push. per-call values +# it needs (like the cmdbuf address) are written as plain uops: the body lowering routes them through the input table -def get_getaddrs(p:UOp) -> list[UOp]: return [u for u in p.toposort(gate=lambda u: u.op is not Ops.AFTER) if u.op is Ops.GETADDR] +def lower_submit_call(call:UOp) -> UOp|None: + if (submit:=get_submit(call.src[0])) is None: return None -def trim_link_patches(ctx:tuple[list[UOp], list[UOp]], a:UOp) -> UOp|None: - links, kept = partition(a.src[1:], lambda p: p.tag == "link") - ctx[0].extend(kept) + devs, queue = to_tuple((cmdbuf:=submit.src[0]).device), cmdbuf.without_after.tag[1] + ctx = EncodeCtx(Device[devs[0]], devs, queue, dict(call.arg.aux.vals)["hcq_size"]) + return call.replace(src=(call.src[0].substitute({submit: unwrap(ctx.dev.pm_lower[queue.split(":")[0]].rewrite(submit, ctx=ctx))}), *call.src[1:])) - # keep all patches from the link-time patches' subtrees in the C code - afters = [u for u in UOp.sink(*links).toposort() if u.op is Ops.AFTER] - ctx[1].extend(UOp.sink(*links).substitute({p: p.src[0] for p in afters}).src) - return a.src[0].after(*kept, *[d for p in afters for d in p.src[1:]]) if links else None -pm_trim_link_patches = PatternMatcher([(UPat(Ops.AFTER, src=(UPat((Ops.PARAM, Ops.MSTACK)),), allow_any_len=True, name="a"), trim_link_patches)]) - -def make_addr_table(call:UOp, gaddrs:list[UOp], name:str) -> tuple[UOp, dict[UOp, UOp], tuple[UOp, ...], dict[UOp, int]]: - bare = {g: g.replace(src=(g.src[0].without_after,)) for g in gaddrs} - - order = sorted(dedup(bare.values()), key=lambda g: ((b:=unwrap_mstack(g.buf_uop)[0]).arg.slot, repr(b.tag))) - slots = {g:i for i,g in enumerate(order)} - table = UOp.placeholder((len(order),), dtypes.uint64, next(UOp.unique_num), device=call.arg.aux.device).rtag(name) - - reads = {g: table.after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()).index(UOp.const(slots[bare[g]], dtypes.int)).load() for g in gaddrs} - fills = (table.after(*make_patches(table, [(i*table.dtype.itemsize, addr) for addr, i in slots.items()])),) if slots else () - return table, reads, fills, {g:slots[bare[g]] for g in gaddrs} - -def is_bare_addr(val:UOp) -> bool: return val.op is Ops.CAST and val.src[0].op in (Ops.AND, Ops.SHR) and val.src[0].src[0].op is Ops.GETADDR - -def make_scatter_loops(patches:list[UOp], inputs_table:tuple, lt_patches:list[UOp]) -> dict[UOp, UOp]: - table, _, _, slots = inputs_table - subs, by_dst = {}, collections.defaultdict(list) - for p in patches: by_dst[p.buf_uop].append(p) - for dst, patches in by_dst.items(): - data = [] - for p in patches: - words = [(off, val, get_getaddrs(val)) for off,val in zip(p.src[0].src[1].src, p.src[1].src)] - data += [(off.val, slots[gaddrs[0]]) for off,_,gaddrs in words if gaddrs][::2] - scalars = [(off.val*dst.dtype.itemsize, val) for off,val,gaddrs in words if not gaddrs] - subs[p] = UOp.group(*make_patches(dst, scalars)) if scalars else UOp(Ops.NOOP) - - word_table, slot_table = (UOp.placeholder((len(data),), dtypes.uint32, next(UOp.unique_num), device=dst.device).rtag("systems") for _ in range(2)) - ridx = UOp.range(len(data), next(UOp.unique_num), dtype=dtypes.int, src=(word_table, slot_table, dst)) - widx, slot = ((p.index(ridx).load() % bound).cast(dtypes.int) for p,bound in ((word_table, dst.max_numel()-1), (slot_table, table.max_numel()))) - loop = UOp.group(*[dst.index(widx+i).store((table.index(slot).load() >> 32*i).cast(dtypes.uint32)) for i in range(2)]).end(ridx) - lt_patches += [make_binary_patch(buf, struct.pack(f'<{len(data)}I', *vals)) for buf,vals in zip((word_table, slot_table), zip(*data))] - subs[patches[0]] = UOp.group(loop, subs[patches[0]]) - return subs - -def is_input_addr(g:UOp) -> bool: return all(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop)) - -def split_patches(call:UOp) -> UOp|None: - rt_patches:list[UOp] = [] - lt_patches:list[UOp] = [] - body = graph_rewrite(call.src[0], pm_trim_link_patches, ctx=(rt_patches, lt_patches), name=f"trim link-time patches ({call.arg.name})") - - # split patches - inputs, internals = partition(dedup(g for p in rt_patches for g in get_getaddrs(p)), is_input_addr) - runtimes, systems = partition(internals, lambda g: any(x.tag in {"program", "kernargs", "cmdbuf"} for x in unwrap_mstack(g.buf_uop))) - tables = [make_addr_table(call, gs, n) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] - reads, fills = {k:v for _,r,_,_ in tables for k,v in r.items()}, [f for t in tables[1:] for f in t[2]] # inputs table is filled by exec - input_patches = [p for p in rt_patches if (gs:=get_getaddrs(p)) and all(map(is_input_addr, gs)) - and all(is_bare_addr(v) for v in p.src[1].src if get_getaddrs(v))] - scatter = make_scatter_loops(input_patches, tables[0], lt_patches) - body = body.substitute({p:p.substitute(scatter | reads) for p in rt_patches}) - - if inputs: # fence inputs - fills.append((t:=tables[0][0]).after(make_binary_patch(t, bytes(t.max_numel() * 8)))) # zeroed at link, slot 0 is the host fence - body = body.replace(src=(UOp.sink(*body.src[0].src, t.after(*body.src[0].src).index(0).store(0)),)) # open it once consumed - - lt_srcs = collections.defaultdict(list) - for p in lt_patches: lt_srcs[p.buf_uop].append(p) - return call.replace(src=(body, *call.src[1:], *[b.after(*ps) for b,ps in lt_srcs.items()], *fills), - arg=replace(call.arg, aux=replace(call.arg.aux, input_idxs=tuple(sorted(dedup(b.arg.slot for g in inputs for b in unwrap_mstack(g.buf_uop))))))) -pm_split_patches = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), split_patches)]) +pm_lower_submit = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), lower_submit_call)]) # ***************** +# 7. lower the hcq bodies (submits, fences, finalizers) to plain programs: per-call addresses go through the input +# table, placeholders become canonically-sized params so one body program is shared across batches -def replace_params(call:UOp) -> UOp|None: - body, variables, param_ops = call.src[0], call.src[0].variables(), {Ops.PARAM, Ops.MSTACK} - args = dedup([s for u in body.toposort(gate=lambda u: u.op not in param_ops) for s in u.src if s.op in param_ops and s not in variables]) +def lower_hcq_call(call:UOp) -> UOp|None: + if call.arg.aux.args or get_submit(call.src[0]) is not None: return None # lowered already, or the submit isn't lowered yet - patched, refhold = partition(call.src[1:], lambda x: x.src[0] in args) - by_root = {p.src[0]: p for p in patched} - c_args = [by_root.get(a, a) for a in args] + variables = (body:=call.src[0]).variables() + tops = body.toposort(gate=lambda u: u.op not in {Ops.PARAM, Ops.MSTACK}) + placeholders = dedup([s for u in tops for s in u.src if s.op in {Ops.PARAM, Ops.MSTACK} and s not in variables]) - # keep buffers whose addresses become link-time constants alive and mapped - held = args + [r.without_after for r in refhold] - addrs = dedup([g.src[0].without_after for x in call.src for g in x.toposort() if g.op is Ops.GETADDR]) - refhold += [a for a in addrs if a not in held and all(b.op is not Ops.PARAM or b.tag is not None for b in unwrap_mstack(a))] + # args -> params + args = {b: UOp.param(i, b.dtype, shape=b.shape, device=HCQ_RUNTIME_DEV.value, volatile=any(x.arg.volatile for x in unwrap_mstack(b))) + for i, b in enumerate(placeholders)} - sub = {(b:=u.without_after): UOp.param(i, u.dtype, shape=b.shape, device=HCQ_RUNTIME_DEV.value, volatile=b.op is Ops.PARAM and b.arg.volatile) - for i,u in enumerate(c_args)} | {v: v.replace(arg=replace(v.arg, slot=-1)) for v in variables if v.op is Ops.PARAM} - info = replace(call.arg.aux, inputs=next((i for i,u in enumerate(c_args) if u.without_after.tag == "inputs"), None)) - return call.replace(src=(body.substitute(sub).replace(arg="hcq_args"), *c_args, *refhold), - arg=replace(call.arg, aux=info)) # TODO: call.after(*refhold)? -pm_replace_params = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), replace_params)]) + # vars slots aft args + base = max([len(args)] + [v.arg.slot + 1 for v in variables]) # above every existing slot, or a renumber can cycle + vrs = {v: v.replace(arg=replace(v.arg, slot=base + i)) for i, v in enumerate(sorted(variables, key=lambda v: (v.arg.name, v.arg.slot)))} + + # reenum ranges + rngs = {r: r.replace(arg=(i,)+r.arg[1:]) for i,r in enumerate(sorted([u for u in tops if u.op is Ops.RANGE], key=lambda r: r.arg))} + + sink = body.src[0].substitute(cast(dict[UOp, UOp], args) | vrs | rngs).replace(arg=KernelInfo("hcq_submit"), tag=1) + + # args the link writes into keep their seal after + patched = {s.without_after: s for s in call.src[1:] if s.op is Ops.AFTER} + arg_src = [patched.get(b, b) for b in placeholders] + src = (body.replace(src=(sink,)), *arg_src, *[s for s in call.src[1:] if s not in arg_src]) + + table = None if (t:=call.arg.aux.table) is None else next(i for i, x in enumerate(src) if x.without_after is t) + return call.replace(src=src, arg=replace(call.arg, aux=replace(call.arg.aux, table=table, args=tuple(args.items())))) + +pm_lower_hcq = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), lower_hcq_call)]) # ***************** +# 6. batch: adjacent hcq calls fold into one submitter on the host SUBMIT:0 queue: a submit whose cmds call the +# compiled piece programs, so the batch runs in fifo order and the python exec is one submitter call -def resolve_getaddr_slice(bv:UOp, g:UOp) -> UOp: - base = bv.src[0].after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()) - itemsize = bv.src[0].dtype.itemsize if bv.src[0].without_after.op in (Ops.BUFFER, Ops.SLICE, Ops.MSTACK, Ops.MSELECT) else bv.dtype.itemsize - return UOp(Ops.GETADDR, src=(base,), arg=g.arg) + UOp.const(bv.src[1].val * itemsize, dtypes.uint64) +def _lane_arg(a:UOp, lane:int) -> UOp: return a.mselect(lane) if len(to_tuple(a.device)) > 1 else a -pm_early_simplify = PatternMatcher([ - (UPat(Ops.GETADDR, src=(UPat.any(sl:=UPat(Ops.SLICE, name="bv"), sl.after(allow_any_len=True)),), name="g"), resolve_getaddr_slice), - (UPat(Ops.INDEX, src=(UPat(Ops.SLICE, name="bv"),), allow_any_len=True, name="x"), - lambda bv,x: x.replace(src=(bv.src[0], x.src[1] + bv.src[1].cast(x.src[1].dtype), *x.src[2:]))), -]) +def _batch_hcq_calls(calls:list[UOp]) -> UOp: + # flatten tables + views for each cmd + table = UOp.placeholder((next_power2(sum(len(c.arg.aux.inputs) for c in calls)),), dtypes.uint64, next(UOp.unique_num), + device=HCQ_RUNTIME_DEV.value).rtag("inputs") + offs = itertools.accumulate((len(c.arg.aux.inputs) for c in calls), initial=0) + views = {c: table[off:off + len(c.arg.aux.inputs)] for c, off in zip(calls, offs)} + + # build submitter + def cmd(c:UOp, j:int) -> UOp: + args = [views[c] if i == c.arg.aux.table else _lane_arg(c.src[i].without_after, j) for i in range(1, 1 + len(c.arg.aux.args))] + vals = [UOp.variable(n, 0, 0xffffffff).bind(v) for n, v in c.arg.aux.vals] + return c.src[0].src[0].call(*args, *vals, UOp.variable("_device_num", 0, 1 << 30).bind(j)) + cmds = [cmd(c, j) for c in calls for j in range(len(to_tuple(c.arg.aux.device)))] + body = UOp.custom_function("hcq", make_submit(*cmds, devs=HCQ_RUNTIME_DEV.value, queue="SUBMIT:0").sink()) + + # update info + estimates = sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify() + kernels = sum((c.arg.aux.kernels for c in calls), start=()) + inputs = sum((c.arg.aux.inputs for c in calls), start=()) + cargo = dedup([table] + [s for c in calls for s in c.src[1:] if s.without_after.tag != "inputs"]) + info = HCQInfo((HCQ_RUNTIME_DEV.value,), estimates, kernels=kernels, table=table, inputs=inputs) + return body.call(*cargo, name=f"hcq_submitter ({len(calls)})", aux=info) + +def batch_hcq_linear(linear:UOp) -> UOp: + def _key(c:UOp): return c.op is Ops.CALL and c.src[0].op is Ops.CUSTOM_FUNCTION and c.src[0].arg == "hcq" + return linear.replace(src=tuple(flatten([_batch_hcq_calls(list(g))] if k else g for k, g in itertools.groupby(linear.src, key=_key)))) # ***************** -# 5.3. pack placeholders buffers +# 8. hcq compile -def pack_hcq_placeholders(call:UOp) -> UOp|None: - bufs = [b for b in call.src[0].toposort() if b.op is Ops.PARAM and b.tag in {"scratch", "kernargs"}] - offs:dict[UOp, int] = {} - sizes:dict[Any, int] = {} - for b in bufs: - if b.tag == "scratch": sizes[b.tag] = max(sizes.get(b.tag, 0), b.max_numel()) - else: - offs[b] = round_up(sizes.get(b.tag, 0), 128 // b.dtype.itemsize) - sizes[b.tag] = offs[b] + b.max_numel() - counts = collections.Counter(b.tag for b in bufs) - bases = {b.tag:UOp.placeholder((sizes[b.tag],), b.dtype, next(UOp.unique_num), device=b.device).rtag(b.tag) for b in bufs if counts[b.tag] > 1} - subs = {b:UOp(Ops.SLICE, b.dtype, (bases[b.tag], UOp.const(offs.get(b, 0))), b.max_numel()) for b in bufs if b.tag in bases} - return call.replace(src=(call.src[0].substitute(subs, walk=True), *call.src[1:])) if subs else None -pm_pack_placeholders = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), pack_hcq_placeholders)]) - -# ***************** -# 8. callify hcq programs - -def callify_hcq(call:UOp, cf:UOp) -> UOp: - prg = to_program(cf.src[0].replace(arg=KernelInfo("hcq_submit"), tag=1), Device[HCQ_RUNTIME_DEV.value].renderer) - return call.replace(src=(cf.replace(src=(prg,), arg="hcq"), *call.src[1:])) -pm_callify_hcq = PatternMatcher([(UPat(Ops.CALL, src=( - UPat(Ops.CUSTOM_FUNCTION, arg="hcq_args", src=(UPat(Ops.SINK),), name="cf"),), name="call", allow_any_len=True), callify_hcq)]) +def hcq_lower(linear:UOp, name:str="lower") -> UOp: + linear = graph_rewrite(linear, pm_encode + pm_seal + pm_lower_submit + pm_lower_hcq, bottom_up=True, name=name) + with Context(EMULATED_DTYPES=""): return lower_and_compile(linear) hcq_compile_cache:dict[tuple[bytes, bool], UOp] = {} @@ -417,89 +469,90 @@ def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: linear = linear.substitute(back_map:={s.param_like(i): s for i,s in enumerate(input_uops)} if input_uops is not None else {}, walk=True) linear = graph_rewrite(linear, pm_insert_copy_staging+pm_flatten_linear, name="insert copy staging") - # schedule - linear = graph_rewrite(linear, pm_schedule_and_merge, ctx=({s:p for p,s in back_map.items()}, profile), walk=True, name="schedule and merge hcq") + # schedule on real buffers + linear = sched_batches(linear, profile).substitute({s:p for p,s in back_map.items()}, walk=True, enter_calls=True) - # lowering to hcq ir - linear = graph_rewrite(linear, pm_encode_cmdbufs+pm_pack_placeholders, walk=True, name="encode and pack", enter_calls=True) - - # patches and runtime uops - linear = graph_rewrite(linear, pm_early_simplify+symbolic+pm_fold_cast_const, bottom_up=False, name="simplify patches", enter_calls=True) - linear = graph_rewrite(linear, pm_split_patches, walk=True, name="split patches") - - # and compile it - linear = graph_rewrite(linear, pm_replace_params, name="replace params") - final_linear = hcq_compile_cache[cache_key] = graph_rewrite(linear, pm_callify_hcq, name="callify hcq", enter_calls=True) + # lower + linear = hcq_lower(linear) + final_linear = hcq_compile_cache[cache_key] = \ + hcq_lower(batch_hcq_linear(linear), name="lower c submitter") if HCQ_RUNTIME_DEV.value == "CPU" else linear return final_linear # ***************** -# 6. bufferize placeholders: replace placeholders with real buffers. +# 9. bufferize placeholders: replace placeholders with real buffers -def bufferize_buf(ctx:bool, buf:UOp) -> UOp|None: +def bufferize_buf(ctx:tuple[bool, list[UOp]], buf:UOp) -> UOp|None: if buf.tag is None: return None - return UOp.mstack(*(UOp.from_buffer((dv:=Device[dev]).pm_bufferize.rewrite(buf, ctx=(dv, ctx)), HCQ_RUNTIME_DEV.value) + return UOp.mstack(*(UOp.from_buffer((dv:=Device[dev]).pm_bufferize.rewrite(buf, ctx=(dv, ctx[0])), HCQ_RUNTIME_DEV.value) for dev in to_tuple(buf.device))) pm_bufferize = PatternMatcher([(UPat(Ops.PARAM, name="buf"), bufferize_buf)]) # ***************** -# 7. resolve patches +# 10. link: bufferize the placeholders, then the patch stores fold into plain memory writes -def push_stack(op, s): return UOp(Ops.STACK, - src=tuple(op.replace(dtype=op.dtype.scalar(), src=tuple(x if y is s else y for y in op.src)) for x in s.src)) +def push_stack(op:UOp) -> UOp|None: + if not (ns:=[s for s in op.src if s.op is Ops.STACK]) or not all_same([len(s.src) for s in ns]): return None + return UOp(Ops.STACK, src=tuple(op.replace(src=tuple(s.src[i] if s.op is Ops.STACK else s for s in op.src)) for i in range(len(ns[0].src)))) + +def _bufs(buf:UOp) -> list[Buffer]: # one Buffer per lane + if buf.op is Ops.MSTACK: return [cast(Buffer, x.buffer) for x in buf.src] + return list(m.bufs) if isinstance(m:=buf.buffer, MultiBuffer) else [m] def fold_binary(buf:UOp, blob:UOp) -> UOp: - for b in (m.bufs if isinstance(m:=buf.buffer, MultiBuffer) else (m,)): - b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[:len(blob.arg)] = blob.arg + for b in _bufs(buf): + if getattr(b, '_hcq_written', None) is not blob.arg: # programs are shared across linears, write them once + cast(Any, b.ensure_allocated())._hcq_written = blob.arg + b._buf.cpu_view().view(fmt='B')[:len(blob.arg)] = blob.arg return UOp(Ops.NOOP) -def fold_const_store(buf:UOp, off:UOp, val:UOp) -> UOp: - for off,val in zip(off.src, val.src): - for b,v in zip((bs:=mb.bufs if isinstance((mb:=buf.buffer), MultiBuffer) else (mb,)), val.src if val.op is Ops.STACK else (val,)*len(bs)): - data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype](v.val)) - b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[(bo:=off.val*buf.dtype.itemsize):bo+len(data)] = data +def fold_word_store(view:UOp, idx:UOp, val:UOp) -> UOp|None: + vals = [v.ssimplify() for v in (val.src if val.op is Ops.STACK else (val,))] + if not all_int(vals): return None + buf, start = unwrap_view(view) + width, bo = view.dtype.itemsize, start * buf.dtype.itemsize + idx.val * view.dtype.itemsize + for b, c in zip(_bufs(buf), itertools.cycle(vals)): # a single value broadcasts over the lanes + b.ensure_allocated()._buf.cpu_view().view(fmt='B')[bo:bo+width] = (c & (1 << 8 * width) - 1).to_bytes(width, 'little') return UOp(Ops.NOOP) -def resolve_getaddr(buf:UOp, g:UOp) -> UOp: - assert buf.op in (Ops.BUFFER, Ops.MSTACK, Ops.MSELECT), f"{buf.op}" - - devs, b = g.arg, buf.buffer - bufs = tuple(cast(Buffer, x.buffer) for x in buf.src) if buf.op is Ops.MSTACK else tuple(b.bufs if isinstance(b, MultiBuffer) else (b,)*len(devs)) +def resolve_getaddr(ctx:tuple[bool, list[UOp]], buf:UOp, g:UOp) -> UOp: + ctx[1].append(buf) # the address bakes into the blob, the linked linear refholds the buffer (amd scratch outlives its realloc) + devs, bufs = to_tuple(g.arg), _bufs(buf) + if len(bufs) == 1: bufs = bufs * len(devs) # one buffer shared by every lane assert len(bufs) == len(devs), f"can't resolve {len(bufs)} buffers on {len(devs)} devices" addrs = tuple(UOp.const(x.get_buf(d).va_addr, dtypes.uint64) for x, d in zip(bufs, devs)) return addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, src=addrs) +def resolve_getaddr_view(bv:UOp, g:UOp) -> UOp: + addr = UOp(Ops.GETADDR, src=(bv.src[0],), arg=g.arg) + return addr if bv.op is Ops.BITCAST else addr + UOp.const(bv.src[1].val * bv.dtype.itemsize, dtypes.uint64) + pm_resolve_patches = PatternMatcher([ # multi - (UPat(GroupOp.ALU, src=[UPat(Ops.STACK, name="s"), UPat(Ops.CONST)], name="op"), push_stack), - (UPat(Ops.CAST, src=(UPat(Ops.STACK, name="s"),), name="op"), push_stack), + (UPat(GroupOp.ALU | {Ops.CAST}, name="op"), push_stack), # getaddr - (UPat(Ops.GETADDR, src=(UPat(name="buf"),), name="g"), resolve_getaddr), + (UPat(Ops.GETADDR, src=(UPat(Ops.AFTER, name="a"),), name="g"), lambda a, g: g.replace(src=(a.src[0],))), + (UPat(Ops.GETADDR, src=(UPat((Ops.SHRINK, Ops.BITCAST), name="bv"),), name="g"), resolve_getaddr_view), + (UPat(Ops.GETADDR, src=(UPat((Ops.BUFFER, Ops.MSTACK, Ops.MSELECT), name="buf"),), name="g"), resolve_getaddr), # folders - (UPat(name="buf").index(UPat(Ops.RANGE), allow_any_len=True) - .store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast()).index(UPat(Ops.RANGE), allow_any_len=True).load()) - .end(UPat(Ops.RANGE)), fold_binary), - (UPat({Ops.BUFFER, Ops.SLICE, Ops.MSTACK}, name="buf").index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store), + (UPat(name="buf").store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast())), fold_binary), + (UPat((Ops.BITCAST, Ops.SHRINK, Ops.BUFFER, Ops.MSTACK), name="view") + .index(UPat(Ops.CONST, name="idx")).store(UPat(name="val")), fold_word_store), ]) pm_assert_no_afters = PatternMatcher([(UPat(Ops.AFTER, name="a"), lambda a: panic(RuntimeError, f"AFTER left at hcq_link: {a.src[0].op}"))]) -def link_buf_key(a:UOp): return a.key, to_tuple(a.device) -link_buf_cache:dict[tuple[bytes, tuple[str, ...]], UOp] = {} link_linear_cache:dict[bytes, UOp] = {} @rewrite_group(lambda _,cache,ret: f"HCQ Link {pluralize('Kernel', len(ret.src))}") def hcq_link(linear:UOp, cache=True) -> UOp: if (linked:=link_linear_cache.get(linear_key:=linear.key)) is not None: return linked - - bufs = {(j,i):a for j,c in enumerate(linear.src) for i,a in enumerate(c.src[1:], 1) - if a.op is Ops.AFTER and unwrap_mstack(a.src[0])[0].tag in HCQ_CACHE_TAGS} - linear = linear.substitute({x:link_buf_cache[k] for a in bufs.values() if (k:=link_buf_key(a)) in link_buf_cache for x in (a, a.src[0])}, walk=True) - linear = graph_rewrite(linear, pm_resolve_patches+symbolic+pm_fold_cast_const+pm_assert_no_afters, bpm=pm_bufferize, ctx=cache, bottom_up=False, + refs:list[UOp] = [] + linear = graph_rewrite(linear, pm_resolve_patches+symbolic+pm_assert_no_afters, bpm=pm_bufferize, ctx=(cache, refs), bottom_up=False, name="resolve patches") - for (j,i),a in bufs.items(): link_buf_cache.setdefault(link_buf_key(a), linear.src[j].src[i]) + if refs: linear = linear.replace(src=(linear.src[0].replace(src=linear.src[0].src + tuple(dedup(refs))), *linear.src[1:])) if cache: link_linear_cache[linear_key] = linear return linear @@ -508,23 +561,25 @@ def hcq_link(linear:UOp, cache=True) -> UOp: class HCQ2Compiled(Compiled): timestamp_divider: float = 1000.0 + wait_timeout_ms: float = 30000.0 + rt_nbytes: int = 64 << 20 # the pool every per-linear buffer is carved out of def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Renderer]], runtime, can_recover:bool=False, arch=None): - self.device_id:int = int(device.split(":")[1]) if ":" in device else 0 self.can_recover = can_recover self.pm_bufferize = PatternMatcher([ (UPat(Ops.PARAM, tag="sentinel_signal"), lambda ctx: ctx[0].signal("sentinel", (1 << 64) - 1)), (UPat(Ops.PARAM, tag="timeline_signal"), lambda ctx: ctx[0].signal("timeline")), - (UPat(Ops.PARAM, tag="timeline_value"), lambda ctx: ctx[0].signal("value", 1)), + (UPat(Ops.PARAM, tag="timeline_value"), lambda ctx: ctx[0].signal("value", 1, device="CPU")), + (UPat(Ops.PARAM, tag="epoch", name="b"), lambda ctx, b: ctx[0].signal(b.arg.slot, device="CPU")), (UPat(Ops.PARAM, tag="signal", name="b"), lambda ctx, b: ctx[0].signal(b.arg.slot)), (UPat(Ops.PARAM, name="b"), lambda ctx, b: None if b.tag is None else ctx[0].new_buffer(b, cache=ctx[1])) ]) super().__init__(device, allocator, compilers, runtime, None, arch=arch) - self.rt_buffer = Buffer(self.device, 64 << 20, dtypes.uint8, options=BufferSpec(uncached=True, cpu_access=True)) - self.rt_allocator = BumpAllocator(64 << 20) + self.rt_allocator = BumpAllocator(self.rt_nbytes) + self.prog_bufs:dict[UOp, Buffer] = {} self.prof_ents:dict[int, ProfileGraphEntry] = {} def collect_prof(self): @@ -547,52 +602,51 @@ class HCQ2Compiled(Compiled): tdiffs.append((st+perf_counter_us())/2 - gpu) Compiled.profile_events.append(ProfileDeviceEvent(self.device, statistics.median(tdiffs), self.device_props())) + @functools.cache + def rt_buffer(self, uncached:bool=True) -> Buffer: + return Buffer(self.device, self.rt_allocator.size, dtypes.uint8, options=BufferSpec(uncached=uncached, cpu_access=True), preallocate=True) + + def rt_view(self, nbytes:int, dtype:DType=dtypes.uint8, uncached:bool=True) -> Buffer: + return self.rt_buffer(uncached).view(nbytes // dtype.itemsize, dtype, self.rt_allocator.alloc(max(nbytes, 1), alignment=128)).ensure_allocated() + def new_buffer(self, b:UOp, cache:bool) -> Buffer: - if cache or b.tag in HCQ_CACHE_TAGS: - return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=True, cpu_access=True, nolru=True)) - return self.rt_buffer.view(b.max_numel(), b.dtype, self.rt_allocator.alloc(b.max_numel() * b.dtype.itemsize, alignment=128)) + if b.tag == "program": # program buffers are shared across linears, keyed on the placeholder + if (buf:=self.prog_bufs.get(b)) is None: + buf = self.prog_bufs[b] = Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(cpu_access=True, nolru=True)).ensure_allocated() + return buf + return self.rt_view(b.max_numel() * b.dtype.itemsize, b.dtype) @functools.cache - def signal(self, name:str|int, init_value:int=0) -> Buffer: - buf = Buffer(self.device, 1, dtypes.uint64, options=BufferSpec(host=True, uncached=True, cpu_access=True), preallocate=True) - buf.as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] = init_value + def signal(self, name:str|int, init_value:int=0, device:str|None=None) -> Buffer: + buf = Buffer(device or self.device, 1, dtypes.uint64, options=BufferSpec(host=True, uncached=True, cpu_access=True), preallocate=True) + buf._buf.cpu_view().view(fmt='Q')[0] = init_value return buf + def _wait_signal(self, sig:MMIOInterface|memoryview, value:int, timeout:int|None=None): + timeout = timeout if timeout is not None and self.can_recover else None + st, done = time.perf_counter(), sig[0] + while done < value: + if done != (done:=sig[0]): st = time.perf_counter() + elif time.perf_counter() - st > (timeout or self.wait_timeout_ms) / 1000: self.on_device_hang() + def synchronize(self, timeout:int|None=None): if HCQ_RUNTIME_DEV.value != self.device: Device[HCQ_RUNTIME_DEV.value].synchronize() - sig = self.signal("timeline").as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') - tl = self.signal("value", 1).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') - timeout = timeout if timeout is not None and self.can_recover else None - st = time.perf_counter() - while sig[0] < tl[0] - 1: - if time.perf_counter() - st > (timeout or 3000) / 1000: self.on_device_hang() + sig = self.signal("timeline")._buf.cpu_view().view(fmt='Q') + tl = self.signal("value", 1, device="CPU")._buf.cpu_view().view(fmt='Q') + self._wait_signal(sig, tl[0] - 1, timeout) if self.prof_ents: self.collect_prof() def on_device_hang(self): raise RuntimeError(f"{self.device} hang detected") def device_props(self) -> dict[str,Any]: return {} # to be overridden if needed. dict keys are backend dependent. - def count(self) -> int: return self.iface.count if hasattr(self, 'iface') else 1 - - def _select_iface(self): - assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \ - f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead" - assert hasattr(self, "ifaces"), "must have ifaces to select an iface" - t = DEV.target(dev:=type(self).__name__[:-6]) - filtered = select_by_name(self.ifaces, lambda i: i.__name__[:-5], t.interface, f"{dev} has no interface {t.interface!r}") - filtered = [i for i in filtered if t.interface.startswith("MOCK") or not i.__name__[:-5].startswith("MOCK")] # never fall back to mock ifaces - return select_first_inited([functools.partial(cast(Callable, iface), self, self.device_id) for iface in filtered], - f"No interface for {dev}:{self.device_id} is available") - def _is_cpu(self) -> bool: return hasattr(self, 'device') and self.device.split(":")[0] == "CPU" def finalize(self): try: self.synchronize() # try to finalize the device in any case except RuntimeError as e: print(f"{self.device} synchronization failed before finalizing: {e}") - - # if the device has an interface, call device_fini to clean up resources - if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini() + super().finalize() @dataclass class HCQ2Buffer: @@ -604,21 +658,23 @@ class HCQ2Buffer: return HCQ2Buffer(self.va_addr+offset, meta=self.meta, view=(self.view.view(offset=offset, size=size) if self.view is not None else None)) class HCQAllocator(LRUAllocator[HCQDeviceType], Generic[HCQDeviceType]): - def _as_buffer(self, buf:HCQ2Buffer) -> memoryview: + def _as_buffer(self, buf:HCQBuffer) -> memoryview: return unwrap(buf.view).mv - def _map(self, buf:HCQ2Buffer) -> HCQ2Buffer: + def _map(self, buf:HCQBuffer) -> HCQBuffer: if not hasattr(self, '_do_map'): raise NotImplementedError("map failed: no method implemented") return self._do_map(buf) + def _do_unmap(self, mb): self.dev.iface.free(mb) + @suppress_finalizing - def _free(self, buf:HCQ2Buffer, options:BufferSpec|None=None): + def _free(self, buf:HCQBuffer, options:BufferSpec|None=None): if options is not None and options.external_ptr is not None: return self.dev.synchronize() if hasattr(self, '_do_free'): self._do_free(buf, options) def _unmap(self, mb): self.dev.synchronize() - self.dev.iface.free(mb) + self._do_unmap(mb) - def _offset(self, buf, size:int, offset:int) -> HCQ2Buffer: return buf.offset(offset=offset, size=size) + def _offset(self, buf, size:int, offset:int) -> HCQBuffer: return buf.offset(offset=offset, size=size) diff --git a/tinygrad_repo/tinygrad/runtime/support/memory.py b/tinygrad_repo/tinygrad/runtime/support/memory.py index ac7239b76..317c2e212 100644 --- a/tinygrad_repo/tinygrad/runtime/support/memory.py +++ b/tinygrad_repo/tinygrad/runtime/support/memory.py @@ -1,6 +1,15 @@ -import collections, functools, dataclasses, enum +from __future__ import annotations +import collections, functools, dataclasses, enum, struct from typing import Any, ClassVar -from tinygrad.helpers import round_up, getenv +from tinygrad.helpers import round_up, getenv, to_mv + +class MMIOInterface: + def __init__(self, addr:int, nbytes:int, fmt='B'): self.mv, self.addr, self.nbytes, self.fmt = to_mv(addr, nbytes).cast(fmt), addr, nbytes, fmt + def __len__(self): return self.nbytes // struct.calcsize(self.fmt) + def __getitem__(self, k): return (self.mv[k] if self.fmt == 'B' else self.mv[k].tolist()) if isinstance(k, slice) else self.mv[k] + def __setitem__(self, k, v): self.mv[k] = v + def view(self, offset:int=0, size:int|None=None, fmt=None) -> MMIOInterface: + return MMIOInterface(self.addr+offset, (self.nbytes - offset) if size is None else size, fmt=fmt or self.fmt) class BumpAllocator: def __init__(self, size:int, base:int=0, wrap:bool=True): self.size, self.ptr, self.base, self.wrap = size, 0, base, wrap @@ -236,7 +245,7 @@ class MemoryManager: self.map_range(va:=self.alloc_vaddr(self.vram_size, self.vram_size), self.vram_size, [(0, self.vram_size)], AddrSpace.PHYS, uncached=uncached) return va - def valloc(self, size:int, align=0x1000, uncached=False, contiguous=False) -> VirtMapping: + def valloc(self, size:int, align=0x1000, uncached=False, contiguous=False, zero=False) -> VirtMapping: if not getenv("GMMU", 1): paddr = self.palloc(size:=round_up(size, 0x1000), align, zero=False) return VirtMapping(self.identity_va(uncached) + paddr, size, [(paddr, size)], aspace=AddrSpace.PHYS, uncached=uncached) @@ -251,7 +260,7 @@ class MemoryManager: while rem_size > 0: while self.palloc_ranges[nxt_range][0] > rem_size: nxt_range += 1 - try: paddrs += [(self.palloc(try_sz:=self.palloc_ranges[nxt_range][0], self.palloc_ranges[nxt_range][1], zero=False), try_sz)] + try: paddrs += [(self.palloc(try_sz:=self.palloc_ranges[nxt_range][0], self.palloc_ranges[nxt_range][1], zero=zero), try_sz)] except MemoryError: # Move to a smaller size and try again. nxt_range += 1 diff --git a/tinygrad_repo/tinygrad/runtime/support/mlx/mlxdev.py b/tinygrad_repo/tinygrad/runtime/support/mlx/mlxdev.py index a93319bf5..54049cdcf 100644 --- a/tinygrad_repo/tinygrad/runtime/support/mlx/mlxdev.py +++ b/tinygrad_repo/tinygrad/runtime/support/mlx/mlxdev.py @@ -1,8 +1,8 @@ from __future__ import annotations -import struct, random, socket, ctypes, functools, itertools +import struct, random, ctypes, functools, itertools from tinygrad.helpers import getenv, wait_cond, round_up, next_power2, ceildiv, DEBUG, hi32, lo32, to_be32, to_be64 from tinygrad.runtime.support.memory import BumpAllocator -from tinygrad.runtime.support.system import PCIDevice +from tinygrad.runtime.support.system import PCIDevice, ipv4_to_gid from tinygrad.runtime.autogen import mlx5, pci MLX_DEBUG = getenv("MLX_DEBUG", 0) @@ -11,8 +11,6 @@ MLX5_CMD_STRUCTS = {v: (getattr(mlx5, f"struct_mlx5_ifc_{n[12:].lower()}_in_bits getattr(mlx5, f"struct_mlx5_ifc_{n[12:].lower()}_out_bits", None)) for n, v in mlx5.__dict__.items() if n.startswith("MLX5_CMD_OP_")} MLX5_CMD_STRUCTS[mlx5.MLX5_CMD_OP_ACCESS_REG] = (mlx5.struct_mlx5_ifc_access_register_in_bits, mlx5.struct_mlx5_ifc_access_register_out_bits) -def ipv4_to_gid(ip): return bytes(10) + b'\xff\xff' + socket.inet_aton(ip) - def udp_sport(lqpn, rqpn): v = (lqpn * rqpn ^ ((lqpn * rqpn) >> 20) ^ ((lqpn * rqpn) >> 40)) & 0xFFFFF return ((v & 0x3FFF) ^ ((v & 0xFC000) >> 14)) | 0xC000 diff --git a/tinygrad_repo/tinygrad/runtime/support/nv/ip.py b/tinygrad_repo/tinygrad/runtime/support/nv/ip.py index 75a208d05..58885ac30 100644 --- a/tinygrad_repo/tinygrad/runtime/support/nv/ip.py +++ b/tinygrad_repo/tinygrad/runtime/support/nv/ip.py @@ -345,7 +345,7 @@ class NV_FLCN_COT(NV_IP): class NV_GSP(NV_IP): def init_sw(self): - self.handle_gen = itertools.count(0xcf000000) + self.handle_gen, self.chan_runlists = itertools.count(0xcf000000), {} self.init_rm_args() self.init_libos_args() self.init_wpr_meta() @@ -355,6 +355,7 @@ class NV_GSP(NV_IP): self.rpc_set_registry_table() self.gpfifo_class, self.compute_class, self.dma_class = nv_gpu.AMPERE_CHANNEL_GPFIFO_A, nv_gpu.AMPERE_COMPUTE_B, nv_gpu.AMPERE_DMA_COPY_B + self.viddec_class = {"AD":nv_gpu.NVC9B0_VIDEO_DECODER, "GB":nv_gpu.NVCFB0_VIDEO_DECODER}.get(self.nvdev.chip_name[:2]) # nvdec: ada and blackwell match self.nvdev.chip_name[:2]: case "AD": self.compute_class = nv_gpu.ADA_COMPUTE_A case "GB": @@ -453,8 +454,8 @@ class NV_GSP(NV_IP): self.wpr_meta, _, wpr_meta_addrs = self.nvdev._alloc_boot_mem(ctypes.sizeof(type(m)), data=bytes(m)) self.wpr_meta_sysmem = wpr_meta_addrs[0] - def promote_ctx(self, client:int, subdevice:int, obj:int, ctxbufs:dict[int, GRBufDesc], bufs=None, virt=None, phys=None): - res, prom = {}, nv_gpu.NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS(entryCount=len(ctxbufs), engineType=0x1, hChanClient=client, hObject=obj) + def promote_ctx(self, client:int, subdevice:int, obj:int, ctxbufs:dict[int, GRBufDesc], bufs=None, virt=None, phys=None, engine=0x1): + res, prom = {}, nv_gpu.NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS(entryCount=len(ctxbufs), engineType=engine, hChanClient=client, hObject=obj) for i,(buf,desc) in enumerate(ctxbufs.items()): use_v, use_p = (desc.virt if virt is None else virt), (desc.phys if phys is None else phys) x = (bufs or {}).get(buf, self.nvdev.mm.valloc(desc.size, contiguous=True)) # allocate buffers @@ -470,6 +471,9 @@ class NV_GSP(NV_IP): subdev = self.rpc_rm_alloc(hParent=dev, hClass=nv_gpu.NV20_SUBDEVICE_0, params=nv_gpu.NV2080_ALLOC_PARAMETERS()) vaspace = self.rpc_rm_alloc(hParent=dev, hClass=nv_gpu.FERMI_VASPACE_A, params=nv_gpu.NV_VASPACE_ALLOCATION_PARAMETERS()) + di = self.rpc_rm_control(subdev, nv_gpu.NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE, nv_gpu.NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS()) + self.runlists = {di.entries[i].engineData[2]: di.entries[i].engineData[3] for i in range(di.numEntries)} + # reserve 512MB for the reserved PDES res_va = self.nvdev.mm.alloc_vaddr(res_sz:=(512 << 20)) @@ -549,10 +553,16 @@ class NV_GSP(NV_IP): self.cmd_q.send_rpc(nv.NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, bytes(alloc_args) + (bytes(params) if params is not None else b'')) self.stat_q.wait_resp(nv.NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC) + if hClass == self.gpfifo_class: + self.chan_runlists[obj] = self.runlists.get((e:=params.engineType) + 10*(e >= nv_gpu.NV2080_ENGINE_TYPE_NVDEC0), 0) if hClass == nv_gpu.FERMI_VASPACE_A and client != self.priv_root: self.rpc_set_page_directory(device=hParent, hVASpace=obj, pdir_paddr=self.nvdev.mm.root_page_table.paddr, client=client) if hClass == nv_gpu.NV01_DEVICE_0 and client != self.priv_root: self.device = obj # save user device handle if hClass == nv_gpu.NV20_SUBDEVICE_0: self.subdevice = obj # save subdevice handle + if hClass == self.viddec_class and client != self.priv_root: + ctx, eng = {0: GRBufDesc(0x1000, phys=True, virt=True)}, nv_gpu.NV2080_ENGINE_TYPE_NVDEC0 + bufs = self.promote_ctx(client, self.subdevice, hParent, ctx, virt=False, engine=eng) + self.promote_ctx(client, self.subdevice, hParent, ctx, bufs, phys=False, engine=eng) if hClass == self.compute_class and client != self.priv_root: phys_gr_ctx = self.promote_ctx(client, self.subdevice, hParent, {k:v for k,v in self.grctx_bufs.items() if k in [0, 1, 2]}, virt=False) self.promote_ctx(client, self.subdevice, hParent, {k:v for k,v in self.grctx_bufs.items() if k in [0, 1, 2]}, phys_gr_ctx, phys=False) @@ -575,9 +585,10 @@ class NV_GSP(NV_IP): res = self.stat_q.wait_resp(nv.NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL) st = type(params).from_buffer_copy(res[len(bytes(control_args)):]) if params is not None else None - # NOTE: gb20x requires the enable bit for token submission. Patch workSubmitToken here to maintain userspace compatibility. - if self.nvdev.chip_name.startswith("GB2") and cmd == nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN: - cast(nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS, st).workSubmitToken |= (1 << 30) + # NOTE: gsp only fills in the channel id, the runlist id (and, on gb20x, the doorbell enable bit) are added by the driver. + if cmd == nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN: + cast(nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS, st).workSubmitToken |= (self.chan_runlists[hObject] << 16) | \ + ((1 << 30) if self.nvdev.chip_name.startswith("GB2") else 0) return st def rpc_set_page_directory(self, device:int, hVASpace:int, pdir_paddr:int, client=None, pasid=0xffffffff): diff --git a/tinygrad_repo/tinygrad/runtime/support/system.py b/tinygrad_repo/tinygrad/runtime/support/system.py index 44e560618..b6bd184aa 100644 --- a/tinygrad_repo/tinygrad/runtime/support/system.py +++ b/tinygrad_repo/tinygrad/runtime/support/system.py @@ -1,7 +1,8 @@ from __future__ import annotations -import os, mmap, array, functools, ctypes, select, contextlib, dataclasses, sys, itertools, struct, socket, subprocess, time, enum, atexit +import os, mmap, array, functools, ctypes, ctypes.util, select, contextlib, dataclasses, sys, itertools, struct, socket +import subprocess, time, enum, atexit from tinygrad.helpers import round_up, getenv, OSX, temp, ceildiv, unwrap, fetch, system, _ensure_downloads_dir, DEBUG, flatten, pluralize -from tinygrad.runtime.autogen import libc, pci, vfio, iokit, corefoundation +from tinygrad.runtime.autogen import libc, pci, vfio from tinygrad.runtime.support.hcq import FileIOInterface, MMIOInterface, HCQBuffer, hcq_filter_visible_devices from tinygrad.runtime.support.memory import VirtMapping, AddrSpace, BumpAllocator from tinygrad.runtime.support.usb import USB3, CustomASM24Controller, USBMMIOInterface @@ -9,6 +10,8 @@ from tinygrad.runtime.support.usb import USB3, CustomASM24Controller, USBMMIOInt MAP_FIXED, MAP_FIXED_NOREPLACE = 0x10, 0x100000 MAP_LOCKED, MAP_POPULATE, MAP_NORESERVE = 0 if OSX else 0x2000, getattr(mmap, "MAP_POPULATE", 0 if OSX else 0x008000), 0x400 +def ipv4_to_gid(ip:str) -> bytes: return bytes(10) + b'\xff\xff' + socket.inet_aton(ip) + class _System: def write_sysfs(self, path:str, value:str, msg:str, expected:str|None=None): if FileIOInterface(path, os.O_RDONLY).read().splitlines()[0] != (expected or value): @@ -55,6 +58,7 @@ class _System: def pci_scan_bus(self, vendor:int, devices:tuple[tuple[int, tuple[int, ...]], ...], base_class:int|None=None) -> list[str]: all_devs = [] if OSX: + from tinygrad.runtime.autogen import iokit, corefoundation def read_prop(svc, key) -> int: cfkey = corefoundation.CFStringCreateWithCString(None, key.encode(), corefoundation.kCFStringEncodingUTF8) cfdata = ctypes.cast(iokit.IORegistryEntryCreateCFProperty(svc, ctypes.cast(cfkey, iokit.CFStringRef), None, 0), corefoundation.CFDataRef) @@ -260,7 +264,7 @@ class PCIIfaceBase: self.dev_impl = dev_impl_t(self.pci_dev) self.dev, self.vram_bar, self.count = dev, vram_bar, len(hcq_filter_visible_devices(System.list_devices(vendor, devices, base_class), dn)) - def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, **kwargs) -> HCQBuffer: + def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, zero=False, **kwargs) -> HCQBuffer: should_use_sysmem = host or ((cpu_access if self.is_bar_small() else (uncached and cpu_access)) and not force_devmem) # Align size to huge pages for large allocations, otherwise the unaligned tail falls back to 4KB pages, increasing TLB pressure. @@ -272,7 +276,7 @@ class PCIIfaceBase: mapping = self.dev_impl.mm.map_range(vaddr, size, [(paddr, 0x1000) for paddr in paddrs], aspace=AddrSpace.SYS, snooped=True, uncached=True) return HCQBuffer(vaddr, size, meta=PCIAllocationMeta(mapping, has_cpu_mapping=True, hMemory=paddrs[0]), view=memview, owner=self.dev) - mapping = self.dev_impl.mm.valloc(size:=round_up(size, 0x1000), uncached=uncached, contiguous=cpu_access) + mapping = self.dev_impl.mm.valloc(size:=round_up(size, 0x1000), uncached=uncached, contiguous=cpu_access, zero=zero) barview = self.pci_dev.map_bar(bar=self.vram_bar, off=mapping.paddrs[0][0], size=mapping.size) if cpu_access else None return HCQBuffer(mapping.va_addr, size, view=barview, meta=PCIAllocationMeta(mapping, cpu_access, hMemory=mapping.paddrs[0][0]), owner=self.dev) diff --git a/tinygrad_repo/tinygrad/runtime/support/usb.py b/tinygrad_repo/tinygrad/runtime/support/usb.py index b7e8b8186..7fdb13099 100644 --- a/tinygrad_repo/tinygrad/runtime/support/usb.py +++ b/tinygrad_repo/tinygrad/runtime/support/usb.py @@ -1,6 +1,11 @@ import ctypes, struct, time, functools, itertools +from typing import Any, cast from tinygrad.runtime.autogen import libusb -from tinygrad.helpers import DEBUG, DEV, to_mv, round_up, ceildiv +from tinygrad.helpers import DEBUG, DEV, to_mv, from_mv, round_up, ceildiv, unwrap, to_tuple +from tinygrad.dtype import dtypes +from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher +from tinygrad.device import Buffer, BufferSpec, Device +from tinygrad.runtime.support.hcq2 import HCQInfo, make_buf, make_submit, HCQ_RUNTIME_DEV from tinygrad.runtime.support.hcq import MMIOInterface from tinygrad.runtime.support import c @@ -35,6 +40,11 @@ class USB3: self._tags, self._transferred = itertools.count(1), ctypes.c_int(0) self._bulk_buf, self._bulk_mv = alloc_cbuffer(4 << 20) self._ctrl_buf, self._ctrl_mv = alloc_cbuffer(0x1000) + # async bulk OUT state: tag -> (pooled transfer, keepalive payload mv); transfer errors latch into _async_err + self._async_seq, self._async_err = itertools.count(1), 0 + self._async_pending: dict = {} + self._async_pool: list = [] + self._async_cb = libusb.libusb_transfer_cb_fn(self._on_bulk_done) self.handle = c.init_c_var(c.POINTER[libusb.struct_libusb_device_handle], lambda x: checked(libusb.libusb_open)(dev, x)) @@ -73,6 +83,42 @@ class USB3: (self.handle, 0x02, self._bulk_buf, len(payload), self._transferred, timeout) assert self._transferred.value == len(payload), f"bulk OUT short write: {self._transferred.value}/{len(payload)} bytes" + def _on_bulk_done(self, xfer): # runs in libusb event handling; latch errors (exceptions here are unraisable) + exp = xfer.contents.length - 8 if xfer.contents.type == libusb.LIBUSB_TRANSFER_TYPE_CONTROL else xfer.contents.length + if xfer.contents.status != 0 or xfer.contents.actual_length != exp: self._async_err = xfer.contents.status or -1 + self._async_pool.append(self._async_pending.pop(int(xfer.contents.user_data or 0))[0]) + + def _submit_async(self, endpoint:int, xtype:int, payload:bytes|bytearray|memoryview, timeout:int) -> int: # payload kept alive till bulk_wait + tr = self._async_pool.pop() if self._async_pool else libusb.libusb_alloc_transfer(0) + tr.contents.dev_handle, tr.contents.endpoint, tr.contents.type = self.handle, endpoint, xtype + tr.contents.timeout, tr.contents.length = timeout, len(payload) + tr.contents.buffer = ctypes.cast(from_mv(memoryview(payload), ctypes.c_ubyte), ctypes.POINTER(ctypes.c_ubyte)) + tr.contents.callback, tr.contents.user_data = self._async_cb, (tag := next(self._async_seq)) + self._async_pending[tag] = (tr, payload) + checked(libusb.libusb_submit_transfer, "async submit failed")(tr) + return tag + + def bulk_write_async(self, payload:memoryview, timeout:int=10000) -> int: + """Queue a bulk OUT transfer without blocking; payload is kept alive until bulk_wait(tag).""" + return self._submit_async(0x02, libusb.LIBUSB_TRANSFER_TYPE_BULK, payload, timeout) + + def control_write_async(self, request:int, value:int=0, index:int=0, data:bytes=b"", timeout:int=1000) -> int: + """Queue a vendor control OUT without blocking; completes via bulk_wait(tag) like bulk_write_async.""" + setup = bytearray(struct.pack(' tuple[int, memoryview]: + """Queue a vendor control IN without blocking; the data lands in the returned buffer by bulk_wait(tag).""" + buf = bytearray(struct.pack(' memoryview: if length > len(self._bulk_mv): self._bulk_buf, self._bulk_mv = alloc_cbuffer(length) checked(libusb.libusb_bulk_transfer, "bulk IN 0x81 failed")(self.handle, 0x81, self._bulk_buf, length, self._transferred, timeout) @@ -160,13 +206,10 @@ class CustomASM24Controller: """Write to chip XDATA via vendor control OUT (bRequest=0xE5). wValue=addr, wIndex=val.""" for off, val in enumerate(data): self.usb.control_write(0xE5, value=base_addr + off, index=val) - def scsi_write(self, buf:bytes): + def scsi_write(self, buf:bytes, slot_start:int=0): """Write to SRAM via 0xF2 vendor command + bulk OUT.""" buf_padded = buf + b'\x00' * (round_up(len(buf), 512) - len(buf)) - sectors = len(buf_padded) // 512 - num_slots = ceildiv(len(buf_padded), 0x4000) # 16KB per slot - windex = (num_slots & 0xFF) << 8 - self.usb.control_write(0xF2, value=sectors, index=windex) + self.usb.control_write(0xF2, value=len(buf_padded) // 512, index=(slot_start & 0xFF) | (ceildiv(len(buf_padded), 0x4000) << 8)) self.usb.bulk_write(buf_padded) def scsi_read_arm(self, size:int): @@ -184,20 +227,92 @@ class USBMMIOInterface(MMIOInterface): return (index * self.el_sz, self.el_sz) def __getitem__(self, index): + Device[HCQ_RUNTIME_DEV.value].synchronize() # one driver on the link: drain the compiled submits before python touches it off, sz = self._off_from_index(index) if self.pcimem: assert sz % 4 == 0 and off % 4 == 0, f"pcie_mem_read requires 4-byte aligned access, got off={off}, sz={sz}" data = self.usb.pcie_mem_read(self.addr + off, sz) else: data = self.usb.scsi_read(sz) if self.addr == 0xf000 else self.usb.read(self.addr + off, sz) - return int.from_bytes(data, "little") if sz == self.el_sz else data + return data if isinstance(index, slice) else int.from_bytes(data, "little") def __setitem__(self, index, data): + Device[HCQ_RUNTIME_DEV.value].synchronize() off, _ = self._off_from_index(index) data = struct.pack(self.fmt, data) if isinstance(data, int) else bytes(data) if not self.pcimem: self.usb.scsi_write(data) if self.addr == 0xf000 else self.usb.write(self.addr + off, data) - else: self.usb.pcie_mem_write(self.addr+off, data) + else: + # writes are whole dwords + assert len(data) % 4 == 0 and off % 4 == 0, f"pcie_mem_write requires 4-byte aligned access, got off={off}, sz={len(data)}" + self.usb.pcie_mem_write(self.addr+off, data) def view(self, offset:int=0, size:int|None=None, fmt=None): return USBMMIOInterface(self.usb, self.addr+offset, self.nbytes-offset if size is None else size, fmt=fmt or self.fmt, pcimem=self.pcimem) +# ***************** + +def _libusb(devs, dep:tuple[UOp, ...], fn:str, *args) -> UOp: + return make_buf(devs, tag=f"func:{fn}").after(*dep).index(0).load().call(make_buf(devs, tag="usb_handle").index(0).load(), + *[UOp.const(a, dtypes.int) if isinstance(a, int) else a for a in args], ret_dtype=dtypes.void) + +def usb_bulk(devs, dep, endpoint:int, data:UOp, length, timeout:int=1000) -> UOp: # NULL actual_length out param + return _libusb(devs, dep, "libusb_bulk_transfer", endpoint, data, length, UOp.const(0, dtypes.uint64), timeout) + +def usb_stream(devs, dep:tuple[UOp, ...], addr:UOp, data:UOp, nbytes:int, write:bool) -> UOp: + hdr = UOp.placeholder((2,), dtypes.uint64, device=devs, tag="usb_scratch").after(*dep) + arm = _libusb(devs, (hdr.index(0).store(addr), hdr.index(1).store(UOp.const(nbytes // 4, dtypes.uint64))), "libusb_control_transfer", + 0x40, 0xF0, (0x60 if write else 0x20) | (0x0F << 8), 1 if write else 2, hdr.index(0), 12, 5000) + return usb_bulk(devs, (arm,), 0x02 if write else 0x81, data, nbytes) + +def usb_load(b:UOp, idx:UOp, dt) -> UOp: + got = UOp.placeholder((1,), dt, device=(devs:=to_tuple(b.device)), tag="usb_scratch") + addr = b.getaddr((HCQ_RUNTIME_DEV.value,)) + (idx*dt.itemsize).cast(dtypes.uint64) + return got.after(usb_stream(devs, b.src[1:] if b.op is Ops.AFTER else (), addr, got.index(0), dt.itemsize, False)).index(0).load() + +def usb_write(b:UOp, idx:UOp, v:UOp) -> UOp: + val = (s:=UOp.placeholder((1,), v.dtype, device=(devs:=to_tuple(b.device)), tag="usb_scratch")).after(s.index(0).store(v)) + addr = b.getaddr((HCQ_RUNTIME_DEV.value,)) + (idx*v.dtype.itemsize).cast(dtypes.uint64) + return usb_stream(devs, b.src[1:] if b.op is Ops.AFTER else (), addr, val.index(0), v.dtype.itemsize, True) + +def usb_idle(devs) -> UOp: + v = usb_load(make_buf(devs, tag="timeline_signal").after(loop:=UOp.loop(0)), UOp.const(0, dtypes.int), dtypes.uint64) + return v.end(loop, v + 1 < make_buf(devs, tag="timeline_value").index(0).load()) + +def usb_scsi(devs, read:bool, nbytes:int) -> UOp: + return _libusb(devs, (usb_idle(devs),), "libusb_control_transfer", 0x40, 0xF2, ceildiv(nbytes, 512) | (0x8000 if read else 0), + (ceildiv(nbytes, 0x4000) & 0xFF) << 8, UOp.const(0, dtypes.uint64), 0, 1000) + +def usb_stage_copy(dst:UOp, src:UOp) -> UOp|None: + if (cin:=to_tuple(src.device)[0].startswith("CPU")) == to_tuple(dst.device)[0].startswith("CPU"): return None + + total, ops, win = dst.nbytes(), [], cast(Any, Device[(devs:=to_tuple((dst if cin else src).device))[0]]).iface.usb_sram + for off in range(0, total, win.size): # off and nb are bytes, the two ends of the copy can have different dtypes + sram = UOp.from_buffer(win)[0:(nb:=min(win.size, total - off))] + s, d = src[off // src.dtype.itemsize:(off + nb) // src.dtype.itemsize], dst[off // dst.dtype.itemsize:(off + nb) // dst.dtype.itemsize] + if cin: + push = usb_bulk(devs, (usb_scsi(devs, False, nb),), 0x02, s.getaddr((HCQ_RUNTIME_DEV.value,)), round_up(nb, 512), 10000) + ops += [UOp.custom_function("hcq", push.sink()).call(sram, s, name="hcq_copyin", aux=HCQInfo(devs)), + sram.copy_to_device(d.device).call(d, sram)] + else: + pad = UOp.new_buffer("CPU", round_up(nb, 512), dtypes.uint8)[0:nb] + submit = make_submit(s.copy_to_device(sram.device).call(sram, s), devs=devs, queue="COPY:0") + pull = usb_bulk(devs, (submit,), 0x81, pad.getaddr((HCQ_RUNTIME_DEV.value,)), round_up(nb, 512), 10000) + ops += [UOp.custom_function("hcq", pull.sink()).call(pad, sram, s, name="hcq_copyout", aux=HCQInfo(devs)), + pad.copy_to_device("CPU").call(d, pad)] + return UOp(Ops.LINEAR, src=tuple(ops)) +pm_usb_stage = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), usb_stage_copy)]) + +USB_HOST_TAGS = {"signal", "timeline_signal"} +pm_usb_hostio = PatternMatcher([ + (UPat(Ops.LOAD, src=(UPat(Ops.INDEX, src=(UPat(Ops.PARAM, tag=USB_HOST_TAGS).or_after(name="b"), UPat(name="idx"))),), + name="ld"), lambda b, idx, ld: usb_load(b, idx, ld.dtype)), + (UPat(Ops.STORE, src=(UPat(Ops.INDEX, src=(UPat(Ops.PARAM, tag=USB_HOST_TAGS).or_after(name="b"), UPat(name="idx"))), UPat(name="v"))), usb_write)]) + +pm_usb_bufferize = PatternMatcher([ + (UPat(Ops.PARAM, tag={"systems", "runtime", "inputs", "usb_scratch"}, name="b"), + lambda ctx, b: Buffer("CPU", b.max_numel(), b.dtype, options=BufferSpec(nolru=True), preallocate=True)), + (UPat(Ops.PARAM, tag="usb_handle", name="b"), lambda ctx, b: ctx[0].signal(b.tag, ctx[0].iface.usb_handle, device="CPU")), + (UPat(Ops.PARAM, name="b"), lambda ctx, b: None if not isinstance(b.tag, str) or not b.tag.startswith("func:") else + ctx[0].signal(b.tag, unwrap(ctypes.cast(getattr(libusb.dll, b.tag[5:]), ctypes.c_void_p).value), device="CPU")), +]) + if DEV.interface.startswith("MOCK"): from test.mockgpu.usb import MockUSB3 as USB3 # type: ignore # noqa: F811 diff --git a/tinygrad_repo/tinygrad/schedule/__init__.py b/tinygrad_repo/tinygrad/schedule/__init__.py index a98e97b7c..b0543d4cb 100644 --- a/tinygrad_repo/tinygrad/schedule/__init__.py +++ b/tinygrad_repo/tinygrad/schedule/__init__.py @@ -8,14 +8,13 @@ from tinygrad.helpers import DEBUG, cpu_profile, TracingKey, SPEC, pluralize, SC # unwrap VIEW/CAST/etc to find the actual data source (kernel output, buffer, or multi-device op) def _unwrap_src(s: UOp) -> UOp: - while len(s.src) and s.op not in {Ops.AFTER, Ops.BUFFER, Ops.PARAM, Ops.MSELECT, Ops.MSTACK, Ops.BIND}: s = s.src[0] + while len(s.src) and s.op not in {Ops.AFTER, Ops.BUFFER, Ops.PARAM, Ops.MSELECT, Ops.MSTACK}: s = s.src[0] return s -# a buffer state is AFTER | BUFFER | PARAM. MSELECT/MSTACK join per-device states, BIND is not a buffer dependency +# a buffer state is AFTER | BUFFER | PARAM. MSELECT/MSTACK join per-device states def _states(s: UOp) -> list[UOp]: s = _unwrap_src(s) if s.op in {Ops.MSELECT, Ops.MSTACK}: return [st for ss in s.src for st in _states(ss)] - if s.op is Ops.BIND: return [] assert s.op in {Ops.AFTER, Ops.BUFFER, Ops.PARAM}, f"input to kernel must resolve to a buffer state, not {s.op}" return [s] @@ -71,7 +70,7 @@ def create_schedule(sched_sink:UOp) -> UOp: else: k = rk.src[0] if rk.op is Ops.END else rk assert k.op is Ops.CALL, f"unexpected op in queue: {k.op}" - buf_uops = tuple(_unwrap_src(s).buf_uop for s in k.src[1:] if s.op is not Ops.BIND) + buf_uops = tuple(_unwrap_src(s).buf_uop for s in k.src[1:] if not s.is_bound_var) linearized.append(k.src[0].call(*buf_uops)) for x in children.get(rk, []): in_degree[x] -= 1 @@ -81,6 +80,7 @@ def create_schedule(sched_sink:UOp) -> UOp: from tinygrad.schedule.memory import memory_plan_rewrite from tinygrad.engine.realize import capturing, pm_flatten_linear +from tinygrad.schedule.prepare import prepare_rangeify from tinygrad.schedule.rangeify import get_kernel_graph from tinygrad.helpers import CAPTURING from tinygrad.uop.ops import PatternMatcher, UPat, ParamArg @@ -94,14 +94,21 @@ pm_post_sched_cache = PatternMatcher([ # only resolve buffer PARAMs (slot>=0); ALU/shape vars use slot=-1 and must not be swapped for call args (UPat(Ops.PARAM, name="x"), lambda ctx,x: ctx[1][x.arg.slot] if x.arg.slot >= 0 else None), # create new BUFFERs - (UPat(Ops.BUFFER, src=(UPat(),), name="b"), lambda ctx,b: + (UPat(Ops.BUFFER, src=(), name="b"), lambda ctx,b: create_new_buffer(ctx, b) if isinstance(b.arg, ParamArg) and b.addrspace is AddrSpace.GLOBAL else None), ]) -def resolve_linear_call(linear_call:UOp): +def resolve_linear_call(linear_call:UOp, outer_binds:dict[str, UOp]|None=None): linear = graph_rewrite(linear_call.src[0], pm_post_sched_cache, ctx=({}, linear_call.src[1:]), walk=True, name="params to buffers") - binds = {f"p{i}":x.src[0] for i,x in enumerate(linear_call.src[1:]) if x.op is Ops.BIND} - return linear.substitute({v:binds[v.expr] for v in linear.variables() if v.expr in binds}, enter_calls=True, name="resolve scalar params") + # nested LINEAR calls are lexical scopes: their positional params shadow the enclosing scope, while calls without + # scalar args (e.g. precompiled allreduce) inherit it + binds = {**(outer_binds or {}), + **{f"p{i}":x.src[0].replace(op=Ops.PARAM) for i,x in enumerate(linear_call.src[1:]) if x.is_bound_var}} + def apply_binds(si:UOp) -> UOp: + if si.op is Ops.CALL and si.src[0].op is Ops.LINEAR: return resolve_linear_call(si, binds) + subs = {v:binds[v.expr] for v in si.variables() if v.expr in binds} + return si.replace(src=tuple(s.substitute(subs, name="resolve scalar params") for s in si.src)) + return linear.replace(src=tuple(apply_binds(si) for si in linear.src)) pm_resolve_linear_call = PatternMatcher([ # call LINEAR is resolved here @@ -117,7 +124,7 @@ def lower_sink_to_linear(function:UOp) -> UOp|None: if not SCACHE or (sc_ret:=schedule_cache.get(cache_key, None)) is None: if SPEC: type_verify(function, spec_tensor) # support recursive CALLs - linear = create_schedule(get_kernel_graph(function)) + linear = create_schedule(get_kernel_graph(prepare_rangeify(function))) if SCACHE: schedule_cache[cache_key] = linear else: # schedule cache hit @@ -150,7 +157,7 @@ def simplify_copy_kernel(call:UOp, ast:UOp, dst:UOp, src:UOp): # NOTE: this is a codegen for SDMA devices if dst.device == src.device and not (isinstance(dst.device, str) and dst.device.startswith("DISK")): return None from tinygrad.codegen.simplify import pm_flatten_range, pm_simplify_ranges - from tinygrad.schedule.rangeify import pm_mops + from tinygrad.schedule.prepare import pm_mops from tinygrad.uop.symbolic import sym sink = graph_rewrite(ast, sym+pm_mops+pm_flatten_range+pm_simplify_ranges, ctx={}, name="simplify ranges in copy") return call.replace(src=(sink,) + call.src[1:]) @@ -184,13 +191,13 @@ def create_linear_with_vars(big_sink:UOp) -> tuple[UOp, dict[str, int]]: # vars used in the schedule used_vars = set().union(*[{v.expr for v in si.src[0].variables()} for si in linear.src]) - # get var_vals + # get var_vals from the bound Variables in the call args var_vals: dict[str, int] = {} for b in big_sink.src[1:]: - if b.op is Ops.BIND: - nm = b.src[0].expr + if b.is_bound_var: + v, val = b.unbind() + nm = v.expr if nm not in used_vars: continue - val = b.src[1].val if var_vals.get(nm, val) != val: raise RuntimeError(f"bind mismatch on {nm}, {var_vals[nm]} != {val}") var_vals[nm] = val diff --git a/tinygrad_repo/tinygrad/schedule/indexing.py b/tinygrad_repo/tinygrad/schedule/indexing.py index b084b1fb5..26af302e4 100644 --- a/tinygrad_repo/tinygrad/schedule/indexing.py +++ b/tinygrad_repo/tinygrad/schedule/indexing.py @@ -5,15 +5,13 @@ from tinygrad.dtype import dtypes, AddrSpace from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, graph_rewrite, sint, AxisType, rewrite_group, broadcast_axes from tinygrad.uop.ops import gate_kernel_sink from tinygrad.uop.symbolic import symbolic, pm_simplify_valid, pm_drop_and_clauses -from tinygrad.helpers import argsort, all_same, cpu_profile, PCONTIG, colored, Context, SPEC +from tinygrad.helpers import argsort, all_same, cpu_profile, colored, Context, SPEC @dataclass class IndexingContext: realize_map: dict[UOp, None|list[int]] = field(default_factory=dict) non_removable: dict[UOp, None] = field(default_factory=dict) range_map: dict[UOp, tuple[tuple[UOp, ...], tuple[UOp, ...]]] = field(default_factory=dict) - # loads reachable from each UOp memoized across matches - buf_cache: dict[UOp, frozenset[UOp]] = field(default_factory=dict) # create ranges range_idx: Iterator[int] = field(default_factory=itertools.count) @@ -23,9 +21,9 @@ class IndexingContext: return UOp.range(s, next(self.range_idx), axistype) if resolve(s!=1) else UOp.const(0) -ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.AFTER, Ops.BUFFER, Ops.SLICE, - Ops.CONST, Ops.BIND, Ops.MSELECT, Ops.MSTACK, Ops.PARAM, - Ops.LOAD, Ops.CALL, Ops.FUNCTION} +ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.AFTER, Ops.BUFFER, + Ops.CONST, Ops.MSELECT, Ops.MSTACK, Ops.PARAM, + Ops.LOAD, Ops.CALL} def realize(ctx:IndexingContext, tr:UOp) -> None: ctx.realize_map[tr] = None @@ -34,12 +32,8 @@ def realize_srcs(ctx:IndexingContext, rb:UOp) -> None: if s.base.op not in ALWAYS_CONTIGUOUS: ctx.realize_map[s] = None def realize_store_after_src(ctx:IndexingContext, dest:UOp, src:UOp): - # don't realize SLICE when it's the direct source of STORE+AFTER — the target buffer is the output - if src.op is Ops.SLICE and src in ctx.realize_map \ - and not dest.op_in_backward_slice_with_self(Ops.SHRINK, Ops.PERMUTE, Ops.FLIP, Ops.PAD): - del ctx.realize_map[src] # you don't usually have to do this for assign unless there's a WAR hazard like TestAssign.test_assign_double_diamond_reduce - if dest.base in src.backward_slice_with_self: ctx.realize_map[src] = None + if dest.base in src.toposort(enter_calls=False): ctx.realize_map[src] = None def realize_custom_kernel_srcs(ctx:IndexingContext, c:UOp) -> None: for s in c.src[1:]: @@ -73,8 +67,10 @@ def broadcast_rngs(x:UOp, src:UOp, rngs:tuple[UOp, ...]) -> tuple[UOp, ...]: # TODO: srcs contain (real data srcs, something else, ranges) and the boundary is confusing. see range_start def data_srcs(op:Ops, src:tuple[UOp, ...]) -> tuple[UOp, ...]: - if op in {Ops.PARAM, Ops.BUFFER, Ops.RANGE, Ops.SPECIAL, Ops.BIND}: return () - if op in GroupOp.Movement|{Ops.INDEX, Ops.SLICE, Ops.STAGE, Ops.REDUCE, Ops.AFTER, Ops.END}: return src[:1] + if op in {Ops.PARAM, Ops.BUFFER, Ops.RANGE, Ops.SPECIAL}: return () + # the store of a bound Variable only carries the input value, it has no data srcs + if op is Ops.STORE and src[0].is_variable: return () + if op in GroupOp.Movement|{Ops.INDEX, Ops.STAGE, Ops.REDUCE, Ops.AFTER, Ops.END}: return src[:1] return src def create_bufferize_and_index_srcs(ctx:IndexingContext, x:UOp) -> list[UOp]: @@ -84,7 +80,7 @@ def create_bufferize_and_index_srcs(ctx:IndexingContext, x:UOp) -> list[UOp]: for i, s in enumerate(x.src): new_src = s src_rngs = broadcast_rngs(x, s, ctx.range_map[x][0]) if x in ctx.range_map else () - if s.op in {Ops.PARAM, Ops.BUFFER, Ops.SLICE, Ops.MSTACK, Ops.MSELECT, Ops.AFTER}: + if s.op in {Ops.PARAM, Ops.BUFFER, Ops.MSTACK, Ops.MSELECT, Ops.AFTER}: if x in ctx.range_map and i < data_src_count: new_src = new_src.index(*src_rngs) elif s in ctx.realize_map: realized_ranges = ctx.realize_map[s] @@ -189,7 +185,7 @@ def apply_movement_op(op:Ops, in_shape:tuple[sint,...], arg:tuple, rngs:tuple[UO return rngs @rewrite_group(new_ctx=False) -def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]: +def run_rangeify(tsink:UOp, debug:bool=False) -> UOp: if debug: print("**************************") rctx = IndexingContext() @@ -208,7 +204,7 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]: ending_ranges: dict[UOp, list[UOp]] = {} for x in reversed(tsink_toposort): # no ranges on kernels, they are internal - if x.op in {Ops.CALL, Ops.FUNCTION, Ops.LINEAR}: continue + if x.op in {Ops.CALL, Ops.LINEAR}: continue # AFTER doesn't have range if x.op is Ops.AFTER: continue @@ -252,13 +248,12 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]: local_rngs, valids = zip(*[(r.get_idx(), r.get_valid()) for r in valid_rngs]) rngs_valids.append((local_rngs, valids)) - # TODO: in RANGEIFY > 1 all_all_same isn't required all_all_same = all(all_same(local_rngs) for local_rngs,_ in rngs_valids) _out_rngs = [] _realize_axis = [] for i,(local_rngs,valids) in enumerate(rngs_valids): # we compare the ranges without their valids - if all_all_same or (PCONTIG and all_same(local_rngs)): + if all_all_same: # the new valid is the OR of all the children valids minimum_valid = UOp.const(False).usum(valids) _out_rngs.append(graph_rewrite(local_rngs[0].valid(minimum_valid), symbolic, name="minimum_valid")) @@ -272,15 +267,11 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]: # if this element is a reduce and there's ended ranges, we might have to end some other ranges if len(ending_ranges[x]) and x.op in GroupOp.Elementwise.union({Ops.REDUCE}): - _realize_axis = rctx.realize_map.get(x) or [] - for i,r in enumerate(out_rngs): - if i in _realize_axis: continue - if not (PCONTIG > 1) or any(any(rr.arg > e.arg for e in ending_ranges[x]) for rr in r.ranges): - _realize_axis.append(i) + _realize_axis = list(range(len(out_rngs))) ending_ranges[x] = [] if len(_realize_axis): rctx.realize_map[x] = _realize_axis - out_rngs = tuple([(rctx.new_range(x.shape[i]) if i in _realize_axis else r) for i,r in enumerate(out_rngs)]) + out_rngs = tuple(rctx.new_range(x.shape[i]) for i in range(len(out_rngs))) ending_ranges[x] += broadcast_ending_ranges # TODO: some ops don't have shape, enable this after the `.st` property is removed @@ -324,7 +315,7 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]: tsink = graph_rewrite(tsink, pm_apply_rangeify, ctx=rctx, bottom_up=True, name="apply rangeify") # if a deviceless value must materialize, place it on the sink device tsink = graph_rewrite(tsink, pm_fix_deviceless, ctx=tsink.device, name="add device to deviceless") - return tsink, rctx + return tsink def render_ranges(*rngs_list, realized) -> str: disp = [] diff --git a/tinygrad_repo/tinygrad/schedule/memory.py b/tinygrad_repo/tinygrad/schedule/memory.py index a16acb09f..90eb7e3f1 100644 --- a/tinygrad_repo/tinygrad/schedule/memory.py +++ b/tinygrad_repo/tinygrad/schedule/memory.py @@ -13,7 +13,7 @@ def _can_plan(b:UOp, held_bufs:set[UOp]) -> bool: if b in held_bufs: return False devs = (b.device,) if isinstance(b.device, str) else b.device # CL and WEBGPU do not support views, see explanation in contiguous_view_offset - return all(not d.startswith(("DISK", "TINYFS", "CL", "WEBGPU")) for d in devs) + return all(not d.startswith(("DISK", "CL", "WEBGPU")) for d in devs) LaneKey = tuple[str, int] @@ -52,11 +52,9 @@ def memory_plan_rewrite(linear:UOp, held_bufs:set[UOp]|None=None) -> UOp: peaks[_key(buf)] = (max(peaks[_key(buf)][0], offsets[buf] + buf.max_numel() * buf.dtype.itemsize), peaks[_key(buf)][1]) arena_sizes = {key: round_up(peak, block_size) for key, (peak, _) in peaks.items()} - # build replace_map: each buffer becomes a SLICE into a shared per-device-lane arena + # build replace_map: each buffer becomes a SHRINK/BITCAST into a shared per-device-lane arena arenas = {key: UOp.new_buffer(key[0], sz, dtypes.int8) for key, sz in arena_sizes.items()} - replace_map:dict[UOp, UOp] = {} - for buf_uop, offset in offsets.items(): - replace_map[buf_uop] = UOp(Ops.SLICE, buf_uop.dtype, (arenas[_key(buf_uop)], UOp.const(offset)), buf_uop.max_numel()) + replace_map = {buf_uop:arenas[_key(buf_uop)][offset:offset+buf_uop.nbytes()].bitcast(buf_uop.dtype) for buf_uop, offset in offsets.items()} if DEBUG >= 1 and (omem:=sum(nbytes.values()) / 1e6) != (nmem:=sum(arena_sizes.values()) / 1e6): print(f"memory reduced from {omem:.2f} MB -> {nmem:.2f} MB, {len(first_appearance)} -> {len(arenas)} bufs") diff --git a/tinygrad_repo/tinygrad/schedule/multi.py b/tinygrad_repo/tinygrad/schedule/multi.py index 14d412681..0cfff9952 100644 --- a/tinygrad_repo/tinygrad/schedule/multi.py +++ b/tinygrad_repo/tinygrad/schedule/multi.py @@ -15,29 +15,32 @@ def mstack_early_shrink(ms:UOp, shrink:UOp): ret:list[UOp] = [] for i, x in enumerate(ms.src): if x.op is Ops.COPY: - ret.append(_apply_shrink(shrink.marg, x.src[0], i).copy_to_device(x.device)) + src = _apply_shrink(shrink.marg, x.src[0], i) + ret.append(src.contiguous() if src.device == x.device else src.copy_to_device(x.device)) else: ret.append(_apply_shrink(shrink.marg, x, i).contiguous()) return ms.replace(src=tuple(ret)) def lower_broadcast_copy(c:UOp, x:UOp): if not (isinstance(c.device, tuple) and isinstance(x.device, str)): return None - if (sx:=x.simplify()).device is None and sx.base.op is Ops.CONST: return UOp(Ops.MSTACK, src=(sx,)*len(c.device)) + if (sx:=x.simplify()).device is None: return UOp(Ops.MSTACK, src=(sx,)*len(c.device)) return UOp(Ops.MSTACK, src=tuple(x.copy_to_device(d) for d in c.device)) replace_allreduce = PatternMatcher([ # BROADCAST: explicitly expand broadcast copies and combine with MSTACK - (UPat(Ops.COPY, name="c", src=(UPat(GroupOp.All-{Ops.CONST}, name="x"),)), lower_broadcast_copy), + (UPat(Ops.COPY, name="c", src=(UPat(name="x"),)), lower_broadcast_copy), # COPY_TO_ONE: if copying from multidevice to one, MSELECT the first (TODO: a little from each?) - (UPat(Ops.COPY, name="c", src=(UPat(GroupOp.All-{Ops.CONST}, name="x"),)), lambda c,x: + (UPat(Ops.COPY, name="c", src=(UPat(name="x"),)), lambda c,x: x.mselect(0).copy_to_device(c.device) if isinstance(c.device, str) and isinstance(x.device, tuple) else None), # MSELECT on MSTACK is replaced with nothing (UPat(Ops.MSELECT, src=(UPat(Ops.MSTACK, name="mstack"),), name="ms"), lambda mstack, ms: mstack.src[ms.arg]), # move shrink before MSTACK (UPat(Ops.SHRINK, src=(UPat(Ops.MSTACK, name="ms"),), allow_any_len=True, name="shrink"), mstack_early_shrink), - # move MSELECT before movement ops + # move MSELECT before movement/ALU ops (UPat(Ops.MSELECT, src=(UPat(GroupOp.Movement, src=(UPat.var("s"),), allow_any_len=True, name="v"),), name="ms"), lambda s,v,ms: v.replace(src=(s.mselect(ms.arg),)+v.src[1:])), + (UPat(Ops.MSELECT, src=(UPat(GroupOp.ALU, name="a"),), name="ms"), lambda a,ms: + a.replace(src=tuple(s.mselect(ms.arg) if isinstance(s.device, tuple) else s for s in a.src))), ]) _early_allreduce = PatternMatcher([ @@ -259,31 +262,22 @@ def store_dest_multi(root:UOp, multi:UOp): # (scalars arrive EXPANDed to the full shape by UOp.store's const_like, so they sub-view like everything else) srcs = [multi.src[0]] + [x.src[0] if x.op is Ops.UNSHARD else shard_subview(x, multi) if tuple(x.shape) == tuple(multi.shape) else x for x in root.src[1:]] - return UOp(root.op, root.dtype, tuple(srcs), root.arg) + return UOp(root.op, src=tuple(srcs), arg=root.arg) def passthrough_multi(root:UOp, multi:UOp): new_src = (multi.src[0],)+tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src[1:]) - return UOp(root.op, root.dtype, src=new_src, arg=root.arg).unshard(multi.arg, multi.src[1:]) + return UOp(root.op, src=new_src, arg=root.arg).unshard(multi.arg, multi.src[1:]) def rewrite_into_function(call:UOp): - if call.arg.precompile: return None + if call.arg is None or call.arg.precompile: return None + # the call body is a plain parametric program: multi rewrites it like anything else (the output PARAM dests sub-view per + # shard through the normal store rules), and all srcs (args and RETURNEDs) become their per-shard views new_body = graph_rewrite(call.src[0], multi_pm, name="subcall") - new_args = tuple(a.src[0] if a.op is Ops.UNSHARD else a for a in call.src[1:]) - # after multi resolution, TUPLE elements may be UNSHARD — strip UNSHARD from body, create per-shard FUNCTION, wrap each GETTUPLE in its own UNSHARD - assert new_body.op is Ops.TUPLE - if any(s.op is Ops.UNSHARD for s in new_body.src): - shard_call = call.replace(src=(UOp.maketuple(*[s.src[0] if s.op is Ops.UNSHARD else s for s in new_body.src]),)+new_args) - return UOp.maketuple(*[shard_call.gettuple(i).unshard(s.arg, s.src[1:]) if s.op is Ops.UNSHARD else shard_call.gettuple(i) - for i, s in enumerate(new_body.src)]) - return call.replace(src=(new_body,)+new_args) - -def param_to_multi(p:UOp): - if p.axis is None: return None - return UOp.param(p.arg.slot, p.dtype, p.shard_shape, p.device, p.arg.vmin_vmax, p.arg.multiple_of, p.arg.name, p.arg.addrspace).unshard(p.axis) + assert new_body.op is Ops.SINK + return call.replace(src=(new_body,) + tuple(a.src[0] if a.op is Ops.UNSHARD else a for a in call.src[1:])) # NOTE: this is the same pattern as unrolled ranges multi_pm = PatternMatcher([ - (UPat(Ops.PARAM, name="p"), param_to_multi), (UPat(GroupOp.ALU, name="root", custom_early_reject=set([Ops.UNSHARD])), alu_multi), (UPat(Ops.REDUCE, src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), reduce_multi), (UPat(Ops.RESHAPE, src=(UPat(Ops.UNSHARD, name="multi"), UPat()), name="root"), reshape_multi), @@ -299,17 +293,12 @@ multi_pm = PatternMatcher([ (UPat(Ops.ALLREDUCE, src=(UPat(Ops.UNSHARD, name="multi"),), name="red"), lambda multi,red: multi.src[0].allreduce(*red.arg).unshard(multi.arg, multi.src[1:])), - # resolve TUPLE+GETTUPLE (needed in multi) - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), - # GETTUPLE on UNSHARD: passthrough UNSHARD (e.g. when FUNCTION was replaced by UNSHARD(GETTUPLE(...))) - (UPat(Ops.GETTUPLE, src=(UPat(Ops.UNSHARD, name="multi"),), name="g"), - lambda g, multi: multi.src[0].gettuple(g.arg).unshard(multi.arg, multi.src[1:]) if multi.src[0].op in {Ops.FUNCTION, Ops.TUPLE} else multi), - # rewrite into FUNCTION calls explicitly for UNSHARD (value-producing) - (UPat(Ops.FUNCTION, name="call"), rewrite_into_function), - (UPat((Ops.CALL, Ops.FUNCTION, Ops.AFTER), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root", allow_any_len=True), passthrough_multi), - # just strip the UNSHARD from non-value-producing CALLs (custom kernels, etc.) — FUNCTION is handled by rewrite_into_function + # rewrite value-producing calls explicitly for UNSHARD + (UPat(Ops.CALL, name="call"), lambda call: rewrite_into_function(call) if call.num_returned else None), + (UPat((Ops.CALL, Ops.AFTER), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root", allow_any_len=True), passthrough_multi), + # just strip the UNSHARD from non-value-producing CALLs (custom kernels, etc.) — value-producing CALLs are handled by rewrite_into_function (UPat(Ops.CALL, dtype=dtypes.void, name="root", custom_early_reject=set([Ops.UNSHARD])), lambda root: - UOp(root.op, root.dtype, tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), root.arg)), + UOp(root.op, src=tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), arg=root.arg) if root.num_returned == 0 else None), (UPat((Ops.CAST, Ops.BITCAST, Ops.CONTIGUOUS, Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), passthrough_multi), # STORE of a sharded value into an unsharded dest (e.g. a fragment into a full output tile) diff --git a/tinygrad_repo/tinygrad/schedule/prepare.py b/tinygrad_repo/tinygrad/schedule/prepare.py new file mode 100644 index 000000000..b45e9d43c --- /dev/null +++ b/tinygrad_repo/tinygrad/schedule/prepare.py @@ -0,0 +1,221 @@ +import itertools +from tinygrad.dtype import dtypes, to_dtype +from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp +from tinygrad.uop.ops import graph_rewrite, rewrite_group, ParamArg, identity_element, resolve_returned_after +from tinygrad.uop.movement import mop_cleanup +from tinygrad.helpers import prod, getenv, all_int, DEBUG, SPLIT_REDUCEOP, OPENPILOT_HACKS, FLOAT16, argsort +from tinygrad.schedule.indexing import apply_movement_op +from tinygrad.schedule.allreduce import create_allreduce_function +from tinygrad.schedule.multi import multi_pm + +def walk_mop(u:UOp): + if u.op in GroupOp.Movement or u.op in {Ops.INDEX, Ops.UNSHARD, Ops.BITCAST}: return walk_mop(u.src[0]) + if u.op is Ops.AFTER and (b:=walk_mop(u.src[0])) is not u.src[0]: return b.after(*u.src[1:]) + return u + +def found_after(ctx:dict[UOp, UOp], after:UOp, src:UOp): + if (x:=src).op is Ops.CAST and x.dtype == dtypes.half and FLOAT16: x, after = x.src[0], after.cast(dtypes.float) + while True: + if x.op is Ops.PERMUTE: x, after = x.src[0], after.permute(argsort(x.marg)) + elif x.op is Ops.RESHAPE: x, after = x.src[0], after.reshape(x.src[0].shape) + elif x.op is Ops.WHERE and x.src[2].base.is_invalid and x.src[1].op is Ops.PAD: + x, after = x.src[1].src[0], after.shrink(tuple((o, s+o) for (o,_),s in zip(x.src[1].marg, x.src[1].src[0].shape))) + else: break + ctx[x] = after + +# *** fold moved AFTERs (hack for openpilot) *** +pm_fold_moved_after = PatternMatcher([ + (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE, src=(UPat(), UPat((*GroupOp.Movement,Ops.CAST,Ops.WHERE), name="src")))), name="after"), found_after), + # replace ALU sources with AFTER versions found above + (UPat(GroupOp.ALU, name="alu"), lambda ctx,alu: alu.replace(src=new_src) if (new_src:=tuple(ctx.get(s, s) for s in alu.src)) != alu.src else None), +]) + +# movement op on INDEX as a PatternMatcher +def _mop_index(r:UOp, idx:UOp): + idxs = idx.src[1:] + if len(idxs) == len(r.shape): + return r.src[0].index(*apply_movement_op(r.op, r.src[0].shape, r.marg, idxs), arg=idx.arg) + if r.op is Ops.RESHAPE: + src_prefix = len(r.src[0].shape) - len(r.shape[len(idxs):]) + if src_prefix >= 0 and r.src[0].shape[src_prefix:] == r.shape[len(idxs):]: + if src_prefix == 0: return r.src[0] + ret = r.src[0].index(*apply_movement_op(r.op, r.src[0].shape[:src_prefix], r.shape[:len(idxs)], idxs), arg=idx.arg) + return ret if ret.shape == idx.shape else None + +pm_mops = PatternMatcher([ + # handle movement ops on INDEX + (UPat(GroupOp.Movement, name="r").f(Ops.INDEX, allow_any_len=True, name="idx"), _mop_index), + # move movement ops and INDEX after AFTER + (UPat(GroupOp.Movement|{Ops.INDEX}, name="r").after(name="a", allow_any_len=True), + lambda r,a: UOp(r.op, src=(a.replace(src=(r.src[0],)+a.src[1:]),)+r.src[1:], arg=r.arg)), + (UPat(GroupOp.Movement, name="r").end(name="a", allow_any_len=True), lambda r,a: a.replace(src=(r.src[0],)+a.src[1:])), +]) + +# ***************** +# 0. do some cleanup rewrites, mostly copied from the old stuff + +def fix_store_hazard(target:UOp, src:UOp): + if (base:=target.base) not in src.toposort(enter_calls=False): return None + # PERMUTE and FLIP reorder indices, SHRINK can have overlapping regions when dest is also shrunk + unsafe = {Ops.PERMUTE, Ops.FLIP} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) + reaches_base: dict[UOp, bool] = {} + for s in src.toposort(gate=lambda s: s.op is not Ops.CONTIGUOUS): + reaches_base[s] = s is base or any(reaches_base.get(c) for c in s.src) + if reaches_base[s] and s.op in unsafe and not (s is target and s.op is Ops.SHRINK): return target.store(src.contiguous()) + +def split_reduceop(reduce:UOp, x:UOp): + if prod(reduce.shape) == 0: return None + if not SPLIT_REDUCEOP or not all_int(x.shape) or (prod(x.shape)//prod(reduce.shape))1) else 0 for i,s in enumerate(x.shape)]) + range_nums = [y.arg[0] for y in indexed.substitute({x.base:UOp(Ops.NOOP)}, extra_pm=pm_mops).ranges] + is_expanded = [i not in range_nums for i in range(len(x.shape))] + + if not (split_candidates:=[(i,d) for i in range(reduce.arg[1]) + for d in range(min(256,2**getenv("REDUCEOP_SPLIT_SIZE",22)//prod(reduce.shape)),8-1,-1) + if x.shape[i]%d==0 and not is_expanded[i]]): return None + dim_to_split, divisor = split_candidates[0] + splitted_shape = x.shape[:dim_to_split]+(divisor,)+(x.shape[dim_to_split]//divisor,)+x.shape[dim_to_split+1:] + splitted = x.reshape(splitted_shape).permute(tuple([d for d in range(len(splitted_shape)) if d!=dim_to_split]+[dim_to_split])) + if DEBUG >= 3: print(f"split {divisor}: {x.shape} -> {splitted.shape} -> {reduce.shape}") + # reduce original axes, then split + return splitted._rop(reduce.arg[0], tuple(range(reduce.arg[1]))).contiguous()._rop(reduce.arg[0], (len(reduce.shape),)).reshape(reduce.shape) + +pm_gather_params = PatternMatcher([ (UPat(Ops.PARAM, name="p"), lambda ctx, p: ctx.append(p) if p.arg.slot >= 0 else None), ]) +def resolve_function(c:UOp, allow_param_mismatch=True) -> UOp|None: + if c.arg.precompile: return None + params: list[UOp] = [] + graph_rewrite(c.src[0], pm_gather_params, bottom_up=True, ctx=params, name="gather params") + params = sorted(params, key=lambda x: x.arg.slot) + # the RETURNED inputs bind positionally to the output PARAMs, just like the args bind to the input PARAMs + args = c.src[1:] + + # NOTE: this isn't really needed. it's okay if there's unused args in the function + if not allow_param_mismatch: + if [x.arg.slot for x in params] != list(range(len(params))): raise RuntimeError(f"params not in order: {[x.arg.slot for x in params]}") + if len(params) != len(args): raise TypeError(f"expected {len(params)} args, got {len(args)}") + + # params have a flat storage size in the arg, the logical shape is a view (RESHAPE/SHRINK/UNSHARD) on top of it. + # substitute args by their flat max-shaped storage view so the movement views on the params stay valid + def flat_storage(a:UOp) -> tuple[int, UOp]: # returns (size, view of a as flat max-shaped storage) + shp = a.max_shard_shape if a.axis is not None and isinstance(a.device, tuple) else a.max_shape + return (n:=prod(shp)), a if a.shape == (n,) else a.pad_to(shp).reshape((n,)) + dict_map = {x:args[x.arg.slot] for x in params} + for i, (p, a) in enumerate(dict_map.items()): + if p.arg.size is not None: + n, flat = flat_storage(a) + if p.arg.size != n: raise TypeError(f"arg {i} shape mismatch: expected size {p.arg.size}, got {a.shape}") + dict_map[p] = flat + elif a.shape != (): + raise TypeError(f"arg {i} shape mismatch: expected scalar, got {a.shape}") + if p.dtype != a.dtype: raise TypeError(f"arg {i} dtype mismatch: expected {p.dtype}, got {a.dtype}") + return c.src[0].substitute(dict_map, walk=True) + +# shape-changing bitcast +def expand_bitcast(bc:UOp) -> UOp|None: + x = bc.src[0] + if (ns:=bc.dtype.itemsize) == (os:=x.dtype.itemsize) or (isinstance(x.device, str) and x.device.startswith("DISK")): return None + new_uint, tmp = to_dtype(f"uint{8*ns}"), x.bitcast(to_dtype(f"uint{8*os}")) + if ns > os: + tmp = tmp.reshape(x.shape[:-1] + (x.shape[-1]//(rate := ns//os), rate)) + parts = [tmp.shrink((None,)*(len(tmp.shape)-1) + ((i, i+1),)).cast(new_uint)<<8*i*os for i in range(rate)] + return parts[0].usum(*parts[1:]).squeeze(-1).bitcast(bc.dtype) + parts = [tmp>>8*i*ns for i in range(os//ns)] + return parts[0].stack(*parts[1:], dim=-1).flatten(-2).cast(new_uint).bitcast(bc.dtype) + +earliest_rewrites = mop_cleanup+PatternMatcher([ + # resolve calls with RETURNED inputs (inline the body) + (UPat(Ops.CALL, name="c"), lambda c: resolve_function(c) if c.num_returned else None), + + # resolve AFTER on RETURNED (call outputs) + (UPat(Ops.AFTER, src=(UPat(name="r"), UPat(Ops.SINK, name="t")), allow_any_len=True), resolve_returned_after), + + # resolve allreduce (must be bottom up) + (UPat(Ops.ALLREDUCE, src=(UPat.var("buf"),), name="red"), create_allreduce_function), + + # split_reduceop + (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), split_reduceop), + + # remove DETACH/CONTIGUOUS_BACKWARD (TODO: this is copied in allocations) + (UPat((Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), name="x"), lambda x: x.src[0]), + + # SINK only ever references the base + (UPat(Ops.SINK, name="x"), lambda x: x.replace(src=tuple(y.unsharded_base for y in x.src))), + + # ** copy rules ** + + # copy to same device is a no-op + (UPat(Ops.COPY, src=(UPat.var("x"),), name="copy"), lambda x,copy: x if x.device == copy.device else None), + + # copy on reshape is reshape on copy + (UPat(Ops.COPY, src=(UPat(Ops.RESHAPE, name="shp"),), name="cpy"), lambda shp,cpy: shp.src[0].copy_to_device(cpy.device).reshape(shp.shape)), + + # reshaping on STORE can be a NOOP + (UPat(Ops.STORE, src=(UPat(Ops.RESHAPE, src=(UPat.var("dst",),), allow_any_len=True), + UPat(Ops.RESHAPE, src=(UPat.var("src",),), allow_any_len=True))), + lambda dst,src: dst.store(src) if dst.shape == src.shape else None), + + # ** store rules ** + + # fix store hazard (dest is in used in src) by adding contiguous: TestAssign.test_post_flipped_assignment + (UPat(Ops.STORE, src=(UPat(name="target"), UPat(name="src"))), fix_store_hazard), + + # remove two STOREs that store the same thing to the same place: TestSchedule.test_dedup_Assign + (UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1").after(UPat.var("a1").store(UPat.var("src"))), lambda buf,src,a1:a1), + + # store a buffer's own current contents back into itself: TestAssign.test_nested_after_contiguous_store_no_init + (UPat.var("buf").after(UPat.var("buf").store(UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1"))), lambda buf,src,a1:a1), + + # move bitcast from store dest to source: TestAssign.test_assign_bitcast + (UPat(Ops.STORE, src=(UPat(Ops.BITCAST, src=(UPat(name="target"),)), UPat(name="src"))), + lambda target, src: target.store(src.bitcast(target.dtype))), + + (UPat(Ops.BITCAST, name="bc"), expand_bitcast), + + # ** size 0 ** + + # reduce of size 0 is the identity element + (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), + lambda reduce,x: reduce.const_like(identity_element(reduce.arg[0], reduce.dtype)) if 0 in x.shape and 0 not in reduce.shape else None), + # handle size 0 + (UPat(GroupOp.All-{Ops.SINK}, name="x"), lambda x: x.const_like(0).rtag(x.tag) if x._shape is not None and 0 in x.shape else None), + + # remove movement ops from SINK/AFTER. TODO: should be generic + (UPat(Ops.SINK, name="s"), lambda s: s.replace(src=tuple(walk_mop(u) for u in s.src if u.op is not Ops.NOOP))), + (UPat(Ops.AFTER, name="s"), lambda s: s.replace(src=(s.src[0],)+tuple(walk_mop(u) for u in s.src[1:] if u.op is not Ops.NOOP))), +]) + +def convert_copy_to_store(ctx, copy:UOp, existing_buf:UOp|None=None): + input_src = copy.src[0] + # if it's a COPY, we need to give the input buffer identity + if not input_src.has_buffer_identity(after_ok=True) and copy.op is Ops.COPY: input_src = input_src.contiguous() + input_src = input_src.flatten() + if existing_buf is not None: + # if the existing buffer is not a full buffer, we can't use it + if not existing_buf.has_buffer_identity(after_ok=True): return None + # if there's already a buffer, we just use it + return existing_buf.flatten().store(input_src) + # create the output buffer + buf = UOp(Ops.BUFFER, arg=ParamArg(next(ctx), copy.dtype, size=prod(input_src.max_shape), device=copy.device)) + # reshape back to input + return buf.reshape(input_src.max_shape).after(buf.store(input_src)).reshape(copy.shape) + +pm_copy_to_store = PatternMatcher([ + (UPat(name="existing_buf").store(UPat(Ops.COPY, name="copy")), convert_copy_to_store), + (UPat(Ops.COPY, name="copy"), convert_copy_to_store), +]) + +@rewrite_group(new_ctx=False) +def prepare_rangeify(sink:UOp) -> UOp: + # prepare for rangeify + tsink = graph_rewrite(sink, multi_pm, name="multi_pm") + if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, ctx={}, name="fold moved afters") + tsink = graph_rewrite(tsink, pm_mops+earliest_rewrites, bottom_up=True, name="earliest rewrites") + tsink = graph_rewrite(tsink, pm_copy_to_store, ctx=itertools.count(0), bottom_up=True, name="convert copy to store") + return tsink diff --git a/tinygrad_repo/tinygrad/schedule/rangeify.py b/tinygrad_repo/tinygrad/schedule/rangeify.py index 1dd683e3c..d2bbdf2eb 100644 --- a/tinygrad_repo/tinygrad/schedule/rangeify.py +++ b/tinygrad_repo/tinygrad/schedule/rangeify.py @@ -1,191 +1,21 @@ from dataclasses import dataclass, field, replace from typing import cast import itertools -from tinygrad.dtype import dtypes, AddrSpace, Invalid, to_dtype, strong_dtype -from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg, shape_to_shape_arg -from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group, identity_element -from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const -from tinygrad.uop.movement import mop_cleanup -from tinygrad.helpers import prod, getenv, dedup, all_int, DEBUG, SPLIT_REDUCEOP, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC -from tinygrad.helpers import PCONTIG, FLOAT16, OPENPILOT_HACKS, argsort, partition, get_single_element +from tinygrad.dtype import dtypes, AddrSpace, Invalid +from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg +from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group +from tinygrad.uop.symbolic import symbolic +from tinygrad.helpers import prod, dedup, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC +from tinygrad.helpers import get_single_element from tinygrad.codegen.simplify import pm_flatten_range, pm_reduce_simplify from tinygrad.codegen.opt import Opt -from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, IndexingContext, apply_movement_op -from tinygrad.schedule.multi import multi_pm -from tinygrad.schedule.allreduce import create_allreduce_function +from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, apply_movement_op +from tinygrad.schedule.prepare import pm_mops # creation can recurse a lot import sys sys.setrecursionlimit(10000) -def found_after(ctx:dict[UOp, UOp], after:UOp, src:UOp): - if (x:=src).op is Ops.CAST and x.dtype == dtypes.half and FLOAT16: x, after = x.src[0], after.cast(dtypes.float) - while True: - if x.op is Ops.PERMUTE: x, after = x.src[0], after.permute(argsort(x.marg)) - elif x.op is Ops.RESHAPE: x, after = x.src[0], after.reshape(x.src[0].shape) - elif x.op is Ops.WHERE and x.src[2].base.is_invalid and x.src[1].op is Ops.PAD: - x, after = x.src[1].src[0], after.shrink(tuple((o, s+o) for (o,_),s in zip(x.src[1].marg, x.src[1].src[0].shape))) - else: break - ctx[x] = after - -# *** fold moved AFTERs (hack for openpilot) *** -pm_fold_moved_after = PatternMatcher([ - (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE, src=(UPat(), UPat((*GroupOp.Movement,Ops.CAST,Ops.WHERE), name="src")))), name="after"), found_after), - # replace ALU sources with AFTER versions found above - (UPat(GroupOp.ALU, name="alu"), lambda ctx,alu: alu.replace(src=new_src) if (new_src:=tuple(ctx.get(s, s) for s in alu.src)) != alu.src else None), -]) - -# movement op on INDEX as a PatternMatcher -def _mop_index(r:UOp, idx:UOp): - idxs = idx.src[1:] - if len(idxs) == len(r.shape): - return r.src[0].index(*apply_movement_op(r.op, r.src[0].shape, r.marg, idxs), dtype=idx.dtype, arg=idx.arg) - if r.op is Ops.RESHAPE: - src_prefix = len(r.src[0].shape) - len(r.shape[len(idxs):]) - if src_prefix >= 0 and r.src[0].shape[src_prefix:] == r.shape[len(idxs):]: - if src_prefix == 0: return r.src[0] if r.src[0].dtype == idx.dtype else None - ret = r.src[0].index(*apply_movement_op(r.op, r.src[0].shape[:src_prefix], r.shape[:len(idxs)], idxs), dtype=idx.dtype, arg=idx.arg) - return ret if ret.shape == idx.shape else None - -pm_mops = PatternMatcher([ - # handle movement ops on INDEX - (UPat(GroupOp.Movement, name="r").f(Ops.INDEX, allow_any_len=True, name="idx"), _mop_index), - # move movement ops and INDEX after AFTER - (UPat(GroupOp.Movement|{Ops.INDEX}, name="r").after(name="a", allow_any_len=True), - lambda r,a: UOp(r.op, src=(a.replace(src=(r.src[0],)+a.src[1:]),)+r.src[1:], arg=r.arg)), - (UPat(GroupOp.Movement, name="r").end(name="a", allow_any_len=True), lambda r,a: a.replace(src=(r.src[0],)+a.src[1:])), -]) - -# ***************** -# 0. do some cleanup rewrites, mostly copied from the old stuff - -def fix_store_hazard(target:UOp, src:UOp): - if (base:=target.base) not in src.backward_slice_with_self: return None - # PERMUTE and FLIP reorder indices, SHRINK can have overlapping regions when dest is also shrunk - unsafe = {Ops.PERMUTE, Ops.FLIP} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) - reaches_base: dict[UOp, bool] = {} - for s in src.toposort(gate=lambda s: s.op is not Ops.CONTIGUOUS): - reaches_base[s] = s is base or any(reaches_base.get(c) for c in s.src) - if reaches_base[s] and s.op in unsafe and not (s is target and s.op is Ops.SHRINK): return target.store(src.contiguous()) - -def split_reduceop(reduce:UOp, x:UOp): - if prod(reduce.shape) == 0: return None - if not SPLIT_REDUCEOP or not all_int(x.shape) or (prod(x.shape)//prod(reduce.shape))1) else 0 for i,s in enumerate(x.shape)]) - range_nums = [y.arg[0] for y in indexed.substitute({x.base:UOp(Ops.NOOP, x.base.dtype)}, extra_pm=pm_mops).ranges] - is_expanded = [i not in range_nums for i in range(len(x.shape))] - - if not (split_candidates:=[(i,d) for i in range(reduce.arg[1]) - for d in range(min(256,2**getenv("REDUCEOP_SPLIT_SIZE",22)//prod(reduce.shape)),8-1,-1) - if x.shape[i]%d==0 and not is_expanded[i]]): return None - dim_to_split, divisor = split_candidates[0] - splitted_shape = x.shape[:dim_to_split]+(divisor,)+(x.shape[dim_to_split]//divisor,)+x.shape[dim_to_split+1:] - splitted = x.reshape(splitted_shape).permute(tuple([d for d in range(len(splitted_shape)) if d!=dim_to_split]+[dim_to_split])) - if DEBUG >= 3: print(f"split {divisor}: {x.shape} -> {splitted.shape} -> {reduce.shape}") - # reduce original axes, then split - return splitted._rop(reduce.arg[0], tuple(range(reduce.arg[1]))).contiguous()._rop(reduce.arg[0], (len(reduce.shape),)).reshape(reduce.shape) - -pm_gather_params = PatternMatcher([ (UPat(Ops.PARAM, name="p"), lambda ctx, p: ctx.append(p) if p.arg.slot >= 0 else None), ]) -def resolve_function(c:UOp, allow_param_mismatch=True) -> UOp|None: - if c.arg.precompile: return None - params: list[UOp] = [] - graph_rewrite(c.src[0], pm_gather_params, bottom_up=True, ctx=params, name="gather params") - params = sorted(params, key=lambda x: x.arg.slot) - args = c.src[1:] - - # NOTE: this isn't really needed. it's okay if there's unused args in the function - if not allow_param_mismatch: - if [x.arg.slot for x in params] != list(range(len(params))): raise RuntimeError(f"params not in order: {[x.arg.slot for x in params]}") - if len(params) != len(args): raise TypeError(f"expected {len(params)} args, got {len(args)}") - - dict_map = {x:args[x.arg.slot] for x in params} - for i, (p, a) in enumerate(dict_map.items()): - if p.axis != a.axis: raise TypeError(f"arg {i} axis mismatch: expected {p.axis}, got {a.axis}") - if p.max_shape != a.max_shape: raise TypeError(f"arg {i} shape mismatch: expected {p.shape}, got {a.shape}") - if p.dtype != a.dtype: raise TypeError(f"arg {i} dtype mismatch: expected {p.dtype}, got {a.dtype}") - return c.src[0].substitute(dict_map, walk=True) - -# shape-changing bitcast -def expand_bitcast(bc:UOp) -> UOp|None: - x = bc.src[0] - if (ns:=bc.dtype.itemsize) == (os:=x.dtype.itemsize) or (isinstance(x.device, str) and x.device.startswith(("DISK", "TINYFS"))): return None - new_uint, tmp = to_dtype(f"uint{8*ns}"), x.bitcast(to_dtype(f"uint{8*os}")) - if ns > os: - tmp = tmp.reshape(x.shape[:-1] + (x.shape[-1]//(rate := ns//os), rate)) - parts = [tmp.shrink((None,)*(len(tmp.shape)-1) + ((i, i+1),)).cast(new_uint)<<8*i*os for i in range(rate)] - return parts[0].usum(*parts[1:]).squeeze(-1).bitcast(bc.dtype) - parts = [tmp>>8*i*ns for i in range(os//ns)] - return parts[0].stack(*parts[1:], dim=-1).flatten(-2).cast(new_uint).bitcast(bc.dtype) - -earliest_rewrites = mop_cleanup+PatternMatcher([ - # resolve FUNCTION calls (inline the body) - (UPat(Ops.FUNCTION, name="c"), resolve_function), - - # resolve TUPLE+GETTUPLE - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), - - # resolve allreduce (must be bottom up) - (UPat(Ops.ALLREDUCE, src=(UPat.var("buf"),), name="red"), create_allreduce_function), - - # split_reduceop - (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), split_reduceop), - - # remove DETACH/CONTIGUOUS_BACKWARD (TODO: this is copied in allocations) - (UPat((Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), name="x"), lambda x: x.src[0]), - - # SINK only ever references the base - (UPat(Ops.SINK, name="x"), lambda x: x.replace(src=tuple(y.unsharded_base for y in x.src))), - - # ** copy rules ** - - # COPY transfers a contiguous range, so materialize a source that's resized (shrink/pad/expand) or reordered (permute/flip) - (UPat(Ops.COPY, src=(UPat(GroupOp.Movement, name="r"),), name="c"), - lambda c,r: c.replace(src=(r.contiguous(),)) if resolve(r.numel() != r.base.numel(), False) or r.contiguous_view_offset() is None else None), - - # copy to same device is a no-op - (UPat(Ops.COPY, src=(UPat.var("x"),), name="copy"), lambda x,copy: x if x.device == copy.device else None), - - # copy on reshape is reshape on copy - (UPat(Ops.COPY, src=(UPat(Ops.RESHAPE, name="shp"),), name="cpy"), lambda shp,cpy: shp.src[0].copy_to_device(cpy.device).reshape(shp.shape)), - - # reshaping on STORE can be a NOOP - (UPat(Ops.STORE, src=(UPat(Ops.RESHAPE, src=(UPat.var("dst",),), allow_any_len=True), - UPat(Ops.RESHAPE, src=(UPat.var("src",),), allow_any_len=True))), - lambda dst,src: dst.store(src) if dst.shape == src.shape else None), - - # ** store rules ** - - # fix store hazard (dest is in used in src) by adding contiguous: TestAssign.test_post_flipped_assignment - (UPat(Ops.STORE, src=(UPat(name="target"), UPat(name="src"))), fix_store_hazard), - - # remove two STOREs that store the same thing to the same place: TestSchedule.test_dedup_assign - (UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1").after(UPat.var("a1").store(UPat.var("src"))), lambda buf,src,a1:a1), - - # store a buffer's own current contents back into itself: TestAssign.test_nested_after_contiguous_store_no_init - (UPat.var("buf").after(UPat.var("buf").store(UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1"))), lambda buf,src,a1:a1), - - # move bitcast from store dest to source: TestAssign.test_assign_bitcast - (UPat(Ops.STORE, src=(UPat(Ops.BITCAST, src=(UPat(name="target"),)), UPat(name="src"))), - lambda target, src: target.store(src.bitcast(target.dtype))), - - (UPat(Ops.BITCAST, name="bc"), expand_bitcast), - - # ** size 0 ** - - # reduce of size 0 is the identity element - (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), - lambda reduce,x: reduce.const_like(identity_element(reduce.arg[0], reduce.dtype)) if 0 in x.shape and 0 not in reduce.shape else None), - # handle size 0 - (UPat(GroupOp.All-{Ops.SINK}, name="x"), lambda x: x.const_like(0).rtag(x.tag) if x._shape is not None and 0 in x.shape else None), -]) - # ***************** # 3.5 cleanups @@ -253,7 +83,7 @@ def remove_bufferize(src:UOp, buf:UOp, idx:UOp): accessed_buffers = dedup(accessed_buffers) # if this is generated from multiple buffers, don't remove this buffer - if len(accessed_buffers) > 3 and not (PCONTIG > 2): return None + if len(accessed_buffers) > 3: return None # if any reduces access a buffer, don't remove this buffer buffer_in_reduce = False @@ -264,22 +94,7 @@ def remove_bufferize(src:UOp, buf:UOp, idx:UOp): UOp.sink(*[x.src[0] for x in reduces]).toposort(gate=buf_gate) del buf_gate if buffer_in_reduce: - if PCONTIG > 2: - out_in_ratio = (prod(buf.shape)+1) / (sum([x.numel() for x in accessed_buffers])+1) - if out_in_ratio < 10: return None - # here we have to check the indexes, we might do a partial contig here - local_indexes = [x for x in indexes if x.src[0].op is Ops.STAGE and x.src[0].arg.addrspace == AddrSpace.LOCAL] - exclude_ranges = UOp.group(*[UOp.group(*x.src[1:]) for x in local_indexes]).ranges - subs = [(k,v) for k,v in zip(buf.src[1:], idx.src[1:]) if k.op is not Ops.CONST] - # if it's bufferized or a reduce, it's pcontig - is_pcontig, is_subs = partition(subs, lambda x: x[0] in exclude_ranges or any([r.arg[-1] == AxisType.REDUCE for r in x[1].ranges])) - if not len(is_subs): - return None - if len(is_pcontig): - ret = src.substitute(dict(is_subs), extra_pm=pm_gate_substitute) - return ret.bufferize(*[x[0] for x in is_pcontig], arg=BufferizeOpts(None, AddrSpace.LOCAL)).index(*[x[1] for x in is_pcontig]) - else: - return None + return None # if it makes it here, the bufferize is removed # this is the ranges replaced @@ -302,20 +117,16 @@ pm_const_buffer_folding = pm_mops+PatternMatcher([ (UPat(Ops.STAGE, name="b"), cleanup_dead_axes), # remove noop buffers. if we look at the next index we can remove even more of these (UPat(Ops.INDEX, name="idx").f(Ops.STAGE, allow_any_len=True, name="b2"), remove_noop_bufferize), - (UPat(Ops.INDEX, src=(UPat(Ops.STAGE),), allow_any_len=True, name="idx").f(Ops.NOOP).f(Ops.STAGE, allow_any_len=True, name="b2"), - remove_noop_bufferize), - # no buffers for const (ranges don't matter for const - it's the same value everywhere) - (UPat(Ops.CONST, name='c').f(Ops.STAGE, allow_any_len=True, name="b"), lambda c,b: b.const_like(c.val)), - # indexing a const is a const - (UPat(Ops.INDEX, src=(UPat(Ops.CONST, name="c"),),), lambda c: c), + # no buffers for a const, in either spelling + (UPat.cvar('c').or_casted().f(Ops.STAGE, allow_any_len=True, name="b"), lambda c,b: b.const_like(c.val)), + # indexing a const is the const + (UPat(Ops.INDEX, src=(UPat.cvar().or_casted("c"),),), lambda c: c), # indexing an after with all fully invalid stores is invalid (UPat(Ops.INDEX, src=(UPat(Ops.AFTER, name="after"),), allow_any_len=True, name="idx"), lambda idx,after: idx.const_like(Invalid) if after_all_invalid(after) else None), - # hack if a noop turned to a const - (UPat(Ops.NOOP, src=(UPat.cvar("c"),)), lambda c: c), - # mstack on CONST is CONST - (UPat(Ops.MSTACK, src=(UPat.var("s"),), allow_any_len=True).f(Ops.INDEX, allow_any_len=True), - lambda s: c if (c:=s.base).op is Ops.CONST else None), + # a deviceless MSTACK src is the same value on every device, so indexing the stack is just indexing that value + (UPat(Ops.MSTACK, src=(UPat.var("s"),), allow_any_len=True).f(Ops.INDEX, allow_any_len=True, name="idx"), + lambda s,idx: idx.replace(src=(s,)+idx.src[1:]) if s.device is None else None), ]) pm_remove_bufferize = PatternMatcher([ @@ -327,6 +138,9 @@ pm_remove_bufferize = PatternMatcher([ (UPat(Ops.END, src=(UPat(Ops.NOOP, name="x"),), allow_any_len=True), lambda x: x), ]) +def strip_zero_offset_shrink(x:UOp) -> UOp: + return x.src[0] if x.op is Ops.SHRINK and all(resolve(start == 0, False) for start,_ in x.marg) else x + def no_indexing_calls(u:UOp): new_srcs = [] for x in u.src: @@ -336,8 +150,9 @@ def no_indexing_calls(u:UOp): new_srcs.append(x.src[0]) elif x.op is Ops.SHRINK: # SHRINK with offset 0 is fine - # TODO: check offset - new_srcs.append(x.src[0]) + new_srcs.append(strip_zero_offset_shrink(x)) + elif x.op is Ops.MSTACK: + new_srcs.append(x.replace(src=tuple(strip_zero_offset_shrink(s) for s in x.src))) else: # everything else we pass through new_srcs.append(x) @@ -347,8 +162,19 @@ pm_no_indexing_calls = PatternMatcher([ (UPat(Ops.CALL, name="u"), no_indexing_calls), ]) +# the kernel graph is what gets executed: no shape views left in it, the storage of a value is just the storage +pm_no_views = PatternMatcher([ + (UPat((Ops.RESHAPE, Ops.SHRINK), name="v", src=(UPat((Ops.AFTER, Ops.PARAM, Ops.UNSHARD, Ops.MSTACK, Ops.BUFFER)),), allow_any_len=True), lambda v: + v.src[0]), +]) + DEVICE_MAX_BUFS = {"METAL": 31, "WEBGPU": 8, "CPU": 31} # TODO: get from device? -def limit_bufs(ctx:IndexingContext, root:UOp): +@dataclass +class LimitBufsContext: + buf_cache: dict[UOp, frozenset[UOp]] = field(default_factory=dict) + range_idx: itertools.count = field(default_factory=itertools.count) + +def _limit_bufs(ctx:LimitBufsContext, root:UOp): if (device:=root.device) is None: return None # no device, index related calculations device = device if isinstance(device, str) else device[0].split(":")[0] if not (MAX_BUFS:=MAX_KERNEL_BUFFERS.value or DEVICE_MAX_BUFS.get(device, 0)): return None @@ -370,7 +196,7 @@ def limit_bufs(ctx:IndexingContext, root:UOp): s = s.substitute(dict(zip(orig_ranges, end_ranges))).bufferize(*end_ranges, arg=BufferizeOpts(device=s.device)).index(*orig_ranges) srcs.append(s) return root.replace(src=tuple(srcs)) -pm_limit_bufs = PatternMatcher([(UPat(set.union(GroupOp.Binary, GroupOp.Ternary), name="root"), limit_bufs)]) +pm_limit_bufs = PatternMatcher([(UPat(set.union(GroupOp.Binary, GroupOp.Ternary), name="root"), _limit_bufs)]) # ***************** # 4. put in buffers for bufferize @@ -382,7 +208,7 @@ pm_limit_bufs = PatternMatcher([(UPat(set.union(GroupOp.Binary, GroupOp.Ternary) def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): size = prod(x.shape) - dtype = strong_dtype(x.dtype) # a BUFFER is never weak: store at the concrete dtype, the .cast(x.dtype) on the result keeps readers unchanged + dtype = x.commit_dtype() # a BUFFER is never weak: store at the committed dtype, the .cast(x.dtype) on the result keeps readers unchanged rngs = sorted(idx.ranges, key=lambda x: x.arg) assert size > 0 and isinstance(size, int), f"no zero sized or symbolic sized buffers {size}" @@ -403,7 +229,7 @@ def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): # NOTE: the local BUFFER needs to be disambiguated here if x.arg.addrspace == AddrSpace.GLOBAL: - buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg((size,)),), arg=ParamArg(next(ctx), dtype, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) + buf = UOp(Ops.BUFFER, arg=ParamArg(next(ctx), dtype, size=size, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) do_store = buf.index(idx).store(x.src[0].cast(dtype)).end(*rngs) return buf.after(do_store).cast(x.dtype) @@ -439,7 +265,7 @@ pm_add_buffers = pm_mops+pm_flatten_bufferize+PatternMatcher([ # INDEX of a buffer through the weak cast added above: index the buffer directly and cast the loaded value instead. # this must run in the same rewrite that adds the cast, or the expander expands the whole casted buffer into one big VECTORIZE (UPat(Ops.INDEX, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("buf"),)),), allow_any_len=True, name="u"), - lambda u,buf: u.replace(dtype=None, src=(buf,)+u.src[1:]).cast(u.dtype)), + lambda u,buf: u.replace(src=(buf,)+u.src[1:]).cast(u.dtype)), # move RESHAPEs through MSELECT/MSTACK (UPat((Ops.MSELECT, Ops.MSTACK), src=UPat(Ops.RESHAPE), name="m"), @@ -461,13 +287,13 @@ pm_add_buffers = pm_mops+pm_flatten_bufferize+PatternMatcher([ class LocalAddBufferContext: dg:int = 0 map:dict = field(default_factory=dict) - vars:dict = field(default_factory=dict) range:int = 0 opts:tuple|None = None def debuf(ctx:LocalAddBufferContext, buf:UOp): - param = UOp(Ops.PARAM, src=(UOp.const(prod(buf.max_shape)),), - arg=ParamArg(ctx.dg, buf.dtype, addrspace=buf.addrspace, device=buf.device)) + # Variables (ALU buffers with a value range) are scalar symbolic values, not real buffers: they become ALU params with no slot + if buf.is_variable: return buf.replace(op=Ops.PARAM) + param = UOp(Ops.PARAM, arg=ParamArg(ctx.dg, buf.dtype, prod(buf.max_shape), addrspace=buf.addrspace, device=buf.device)) ret = param.reshape(buf.max_shape) # if the buffer has symbolic shape, shrink the max-sized view to the actual shape if buf.max_shape != buf.shape: ret = ret.shrink(tuple((0, s) for s in buf.shape)) @@ -475,10 +301,6 @@ def debuf(ctx:LocalAddBufferContext, buf:UOp): ctx.dg += 1 return ret -def unbind_kernel(ctx:LocalAddBufferContext, b:UOp): - ctx.vars[b] = None - return b.src[0] - def handle_after(ctx:LocalAddBufferContext, after:UOp): if after.addrspace == AddrSpace.LOCAL: return None buf = after.buf_uop @@ -502,8 +324,7 @@ to_define_global = PatternMatcher([ (UPat(Ops.STORE, name="x"), find_bufs), (UPat((Ops.BUFFER, Ops.MSTACK, Ops.MSELECT), name="buf"), debuf), (UPat(Ops.PARAM, name="v"), lambda v: - UOp.variable(v.arg.name, v.arg.vmin_vmax[0], v.arg.vmin_vmax[1], v.dtype, multiple_of=v.arg.multiple_of) - if v.arg.name is not None and v.arg.vmin_vmax is not None else None), + v.replace(arg=replace(v.arg, slot=-1)) if v.arg.name is not None and v.arg.vmin_vmax is not None and v.arg.slot != -1 else None), # this renumbers the params (UPat(Ops.PARAM, name="buf"), lambda ctx, buf: @@ -512,7 +333,8 @@ to_define_global = PatternMatcher([ # ALU params are scalar symbolic values, not buffers. (UPat(Ops.INDEX, src=(UPat(Ops.PARAM, name="v"),)), lambda v: v if v.addrspace == AddrSpace.ALU else None), - (UPat(Ops.BIND, name="b"), unbind_kernel), + # bound Variables are stores into Variable buffers: strip the store, the buffer becomes an ALU param via debuf + (UPat(Ops.AFTER, name="b"), lambda b: b.src[0] if b.is_bound_var else None), (UPat(Ops.AFTER, name="after"), handle_after), # remove device from local BUFFERIZE @@ -528,10 +350,6 @@ def get_contiguous(ctx:LocalAddBufferContext, x:UOp): rangeify_codegen = PatternMatcher([ (UPat(Ops.CONTIGUOUS, name="x"), get_contiguous), - - # no NOOP in the kernel graph - # TODO: this can be moved into codegen? - (UPat(Ops.NOOP, name="x"), lambda x: x.src[0] if len(x.src) else None), ]) pm_add_param_range_tags = PatternMatcher([ @@ -541,53 +359,32 @@ pm_add_param_range_tags = PatternMatcher([ def split_store(x:UOp) -> UOp|None: # if we have any open ranges here, we don't split. open DEVICE ranges are fine, they are bound per device at launch if any(r.arg[-1] is not AxisType.DEVICE for r in x.ranges): return None + # the store of a bound Variable is an input value, not a kernel + st = x.src[0] if x.op is Ops.END else x + if st.op is Ops.STORE and st.src[0].is_variable: return None # local kernel rewrite lctx = LocalAddBufferContext() ret = graph_rewrite(x, to_define_global+pm_flatten_range+rangeify_codegen, ctx=lctx, name="kernel split", bottom_up=True) # create the Kernel. NOTE: buffers can be on different devices here now, they are compiled to SDMA copies later by schedule - return ret.sink(arg=KernelInfo(opts_to_apply=lctx.opts)).call(*lctx.map.values(), *lctx.vars.keys()) + return ret.sink(arg=KernelInfo(opts_to_apply=lctx.opts)).call(*lctx.map.values()) split_kernels = PatternMatcher([ (UPat((Ops.STORE, Ops.END), name="x"), split_store), ]) -def convert_copy_to_store(ctx, copy:UOp, existing_buf:UOp|None=None): - input_src = copy.src[0] - if not input_src.has_buffer_identity(after_ok=True): input_src = input_src.contiguous() - input_src = input_src.flatten() - if existing_buf is not None: - # if the existing buffer is not a full buffer, we can't use it - if not existing_buf.has_buffer_identity(after_ok=True): return None - # if there's already a buffer, we just use it - return existing_buf.flatten().store(input_src) - # create the output buffer - buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg(input_src.max_shape),), arg=ParamArg(next(ctx), copy.dtype, device=copy.device)) - # reshape back to input - return buf.after(buf.store(input_src)).reshape(copy.shape) - -pm_copy_to_store = PatternMatcher([ - (UPat(name="existing_buf").store(UPat(Ops.COPY, name="copy")), convert_copy_to_store), - (UPat(Ops.COPY, name="copy"), convert_copy_to_store), -]) - @rewrite_group(new_ctx=False) -def get_kernel_graph(sink:UOp) -> UOp: - tsink = graph_rewrite(sink, multi_pm, name="multi_pm") - if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, ctx={}, name="fold moved afters") - tsink = graph_rewrite(tsink, pm_mops+earliest_rewrites, bottom_up=True, name="earliest rewrites") - - tsink = graph_rewrite(tsink, pm_copy_to_store, ctx=itertools.count(0), bottom_up=True, name="convert copy to store") - +def get_kernel_graph(tsink:UOp) -> UOp: # convert movement ops to ranges - tsink, rctx = run_rangeify(tsink, bool(DEBUG_RANGEIFY)) + tsink = run_rangeify(tsink, bool(DEBUG_RANGEIFY)) + # cleanups for speed and runability tsink = graph_rewrite(tsink, - symbolic+pm_fold_cast_const+pm_reduce_simplify+pm_const_buffer_folding+pm_remove_bufferize+pm_no_indexing_calls, + symbolic+pm_reduce_simplify+pm_const_buffer_folding+pm_remove_bufferize, name="symbolic+reduce_collapse+debuf") - tsink = graph_rewrite(tsink, pm_limit_bufs, ctx=rctx, name="limit buffers") - + next_range = max((x.arg[0] for x in tsink.toposort() if x.op is Ops.RANGE), default=-1) + 1 + tsink = graph_rewrite(tsink, pm_limit_bufs, ctx=LimitBufsContext(range_idx=itertools.count(next_range)), name="limit buffers") if VIZ: graph_rewrite(tsink, PatternMatcher([]), name="View Rangeify") # bufferize -> store @@ -595,6 +392,8 @@ def get_kernel_graph(sink:UOp) -> UOp: paramarg_start: int = max([-1]+slots) + 1 tsink = graph_rewrite(tsink, pm_add_buffers+pm_add_param_range_tags, ctx=itertools.count(paramarg_start), bottom_up=True, name="stage to store") tsink = graph_rewrite(tsink, split_kernels, bottom_up=True, name="split kernels") + tsink = graph_rewrite(tsink, pm_no_indexing_calls, name="remove indexing from call args") + tsink = graph_rewrite(tsink, pm_no_views, name="remove views from the kernel graph") if VIZ: graph_rewrite(tsink, PatternMatcher([]), name="View Kernel Graph") if SPEC: diff --git a/tinygrad_repo/tinygrad/tensor.py b/tinygrad_repo/tinygrad/tensor.py index 304d1fe1a..be31d094e 100644 --- a/tinygrad_repo/tinygrad/tensor.py +++ b/tinygrad_repo/tinygrad/tensor.py @@ -4,13 +4,15 @@ import time, functools, sys, inspect, pathlib, hashlib, weakref from dataclasses import dataclass, field from typing import Any, Callable, cast, get_args, ParamSpec, TypeGuard, TypeVar, Generic, TYPE_CHECKING if TYPE_CHECKING: import numpy -from tinygrad.dtype import DType, DTypeLike, dtypes, ConstType, least_upper_dtype, to_dtype, strong_dtype, \ - _from_np_dtype, _to_np_dtype, PyConst, AddrSpace +from tinygrad.dtype import DType, DTypeLike, dtypes, ConstType, least_upper_dtype, to_dtype, _from_np_dtype, _to_np_dtype, PyConst, AddrSpace from tinygrad.helpers import all_int, getenv, fetch, Metadata, TRACEMETA, TracingKey -from tinygrad.helpers import cpu_profile, suppress_finalizing, disable_gc, VIZ, pluralize -from tinygrad.uop.ops import UOp, Ops, sint, all_metadata, Variable, ConstLike, UPat, PatternMatcher, GroupOp, ParamArg, graph_rewrite, rewrite_group +from tinygrad.helpers import cpu_profile, suppress_finalizing, disable_gc, VIZ, pluralize, SPEC +from tinygrad.uop.ops import UOp, Ops, sint, all_metadata, Variable, ConstLike, UPat, PatternMatcher, GroupOp, graph_rewrite, rewrite_group +from tinygrad.uop.ops import resolve_returned_after, remove_all_tags +from tinygrad.uop.spec import type_verify, spec_tensor from tinygrad.mixin.rand import RandMixin from tinygrad.schedule import create_linear_with_vars +from tinygrad.schedule.multi import multi_pm from tinygrad.device import Buffer, canonicalize_device from tinygrad.engine.realize import run_linear @@ -18,27 +20,25 @@ from tinygrad.engine.realize import run_linear @dataclass class AllocCtx: - uop_list: list[UOp] = field(default_factory=list) buffer_map: dict[UOp, UOp] = field(default_factory=dict) bases: set[UOp] = field(default_factory=set) - assigns: list[UOp] = field(default_factory=list) + stores: list[UOp] = field(default_factory=list) replacements: list[UOp] = field(default_factory=list) + views: set[UOp] = field(default_factory=set) -def tag_uop(ctx:AllocCtx, x:UOp): - if x.tag is not None: return None - ctx.uop_list.append(x) - return x.replace(tag=(len(ctx.uop_list)-1,)) +# a tag is the tuple of original pre-rewrite UOps a node provides storage for +def tag_uop(x:UOp): return None if x.tag is not None else x.replace(tag=(x,)) -def disk_like(u:UOp): return isinstance(u.device, str) and u.device.startswith(("DISK", "TINYFS")) +def on_disk(u:UOp): return isinstance(u.device, str) and u.device.startswith("DISK") +def is_creation_device(u:UOp): return isinstance(u.device, str) and u.device.startswith(("DISK", "NPY", "PYTHON")) def disk_copy_is_buffer(ctx:AllocCtx, u:UOp): # copies to disk are replaced with the disk buffer - if disk_like(u) and u.tag is None: + if on_disk(u) and u.tag is None: ctx.buffer_map[u] = u.empty_like() return u.rtag(()) # all copies from disk/numpy are realized into a real buffer - from_creation = isinstance(u.src[0].device, str) and u.src[0].device.startswith(("NPY", "DISK", "PYTHON", "TINYFS")) - if from_creation: return tag_uop(ctx, u) + if is_creation_device(u.src[0]): return tag_uop(u) # CONTIGUOUS and AFTER + parents are the only nodes that get updated add_tags = PatternMatcher([ @@ -47,7 +47,7 @@ add_tags = PatternMatcher([ (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE, src=(UPat(name="dest"), UPat(Ops.COPY, name="c")))), name="a"), lambda a,c,dest: a.replace(src=(a.src[0], a.src[1].replace(src=(dest, c.rtag(())))), tag=a.tag+c.tag) if a.tag and c.tag else None), (UPat((Ops.CONTIGUOUS, Ops.AFTER), name="x"), tag_uop), - (UPat(GroupOp.All, name="x"), lambda ctx,x: tag_uop(ctx,x) if x in ctx.bases else None), + (UPat(GroupOp.All, name="x"), lambda ctx,x: tag_uop(x) if x in ctx.bases else None), ]) def replace_contig_with_store_after(u:UOp): @@ -55,185 +55,164 @@ def replace_contig_with_store_after(u:UOp): if u.is_virtual: return None # if size is 0, remove the contig if 0 in u.shape: return u.src[0] - # no real contig for DISK/TINYFS tensors, they are left alone - if disk_like(u): return u.rtag(None) + # no real contig for DISK tensors, they are left alone + if on_disk(u): return u.rtag(None) buf = u.empty_like() return buf.after(buf.store(u.src[0])).rtag(u.tag) -def replace_store_after_with_contig(u:UOp, src:UOp): - assigned_to = u - while assigned_to.op in {Ops.BITCAST, Ops.AFTER, Ops.UNSHARD}: assigned_to = assigned_to.src[0].base - if assigned_to.op not in {Ops.BUFFER, Ops.SLICE}: return src.contiguous(tag=u.tag) +def wrap_tagged_in_contig(x:UOp): + if x.tag is None: return None # untouched + # empty tag from rtag(()): a COPY already handled via buffer_map or merged into a parent AFTER. + # () is falsy but not None, so it isn't re-tagged like a bare (tag=None) node would be; just strip it here + if not x.tag: return x.rtag(None) + return x.rtag(None).contiguous(tag=x.tag) # the tag moves onto the wrapping CONTIGUOUS -def _make_buffer_view(src:UOp) -> UOp|None: - """If movement ops on src collapse to a contiguous range, return SLICE. Otherwise None.""" - if (offset := src.contiguous_view_offset()) is None: return None +def contiguous_mops_to_view(ctx:AllocCtx, c:UOp, src:UOp): + """MOPS(BUFFER) → SHRINK when movement ops collapse to a contiguous range.""" buf = src.base - if buf.op is Ops.SLICE: - byte_offset = buf.src[1].val * buf.src[0].dtype.itemsize + offset * src.dtype.itemsize - buf = buf.src[0] - if byte_offset % buf.dtype.itemsize != 0: return None - offset = byte_offset // buf.dtype.itemsize - return UOp(Ops.SLICE, src.dtype, (buf, UOp.const(offset)), src.numel()) - -def contiguous_mops_to_view(c:UOp, src:UOp): - """MOPS(BUFFER) → SLICE when movement ops collapse to a contiguous range.""" - buf = src.base - if buf.op not in {Ops.BUFFER, Ops.SLICE, Ops.UNSHARD}: return None - if src.op is Ops.RESHAPE and src.src[0].op in {Ops.BUFFER, Ops.SLICE} and c.op is not Ops.BITCAST: return None - if c.op is not Ops.BITCAST and src.op is Ops.BUFFER: return None - + while buf.op is Ops.BITCAST: buf = buf.src[0].base # no symbolic shape - if not all_int(c.shape): return None + if buf.op not in {Ops.BUFFER, Ops.UNSHARD} or not all_int(c.shape): return None - if buf.op is not Ops.UNSHARD and (view := _make_buffer_view(src)) is not None: - view = (view.replace(dtype=c.dtype, arg=c.numel()) if c.op is Ops.BITCAST else view).reshape(c.shape) - return c.replace(src=(view,)) if c.op is Ops.COPY else view + # for UNSHARD tensors, use multi_pm to resolve per-shard movement ops, then view the resolved shard + unshard = None + if buf.op is Ops.UNSHARD: + if isinstance(c.device, str): return None + if (unshard := graph_rewrite(src, multi_pm, name="multi_buffer_view")).op is not Ops.UNSHARD: return None + src = unshard.src[0] - # for UNSHARD tensors, use multi_pm to resolve per-shard movement ops, then create SLICE on the resolved result - if not isinstance(c.device, str): - from tinygrad.schedule.multi import multi_pm - resolved = graph_rewrite(src, multi_pm, name="multi_buffer_view") - if resolved.op is not Ops.UNSHARD: return None - if (view := _make_buffer_view(resolved.src[0])) is None: return None - return view.reshape(resolved.src[0].shape).unshard(resolved.arg, resolved.src[1:]).contiguous(tag=c.tag) - - return None - -def _precompiled_output_redirect(s:UOp, t:UOp) -> UOp|None: - # how output s lands in the caller's buffer t, or None if it must be copied into t - # materialize straight into t - if s.op is Ops.CONTIGUOUS: return t.after(t.store(s.src[0])) - # rebind output storage to t - if s.op in {Ops.BUFFER, Ops.UNSHARD} and s.has_buffer_identity(): return t - return None + # offset the base buffer by the collapsed movement ops and view it + if (cv := src.contiguous_view()) is None or (buf := cv[0]).op is not Ops.BUFFER: return None + # NB: make offset a UOp.variable here to do the offset computation in the kernels + view = buf[cv[1]:cv[1] + src.max_numel() * src.element_size() // buf.element_size()].bitcast(src.dtype) + ctx.views.add(view) + if unshard is not None: return view.reshape(src.shape).unshard(unshard.arg, unshard.src[1:]) + view = view.reshape(c.shape) + return c.replace(src=(view,)+c.src[1:]) if c.op in {Ops.COPY, Ops.STORE} else view def transform_precompiled_call(c:UOp) -> UOp|None: - if not c.arg.precompile: return None - assert c.src[0].op is Ops.TUPLE, f"expected TUPLE body for precompiled FUNCTION, got {c.src[0].op}" - input_buffers = tuple(x.contiguous() if x.op not in {Ops.AFTER, Ops.BIND} else x for x in c.src[1:]) + if c.arg is None or not c.arg.precompile or c.num_returned == 0: return None + assert c.src[0].op is Ops.SINK, "precompiled call bodies are SINKs of stores into the output PARAMs" + # the RETURNED srcs are the call outputs (slots are src positions) + ret_pos = [p for p,a in enumerate(c.src[1:]) if a.unsharded_base.op is Ops.RETURNED] + srcs = tuple(st.src[1] for st in c.src[0].src if st.op is Ops.STORE) # add the outputs to the call - srcs = c.src[0].src - resolved = [c.gettuple(i) for i in range(len(srcs))] - outs = tuple(r.empty_like() for r in resolved) - targets = [o.param_like(len(c.src)-1+i).shrink_to(s.shape) for i,(o,s) in enumerate(zip(outs, srcs))] + outs = tuple(c.src[1+p].empty_like() for p in ret_pos) + targets = [o.param_like(p).shrink_to(s.shape) for p,o,s in zip(ret_pos, outs, srcs)] - subs:dict[UOp, UOp] = {} + # how each stored value lands in its output PARAM target: a CONTIGUOUS materializes straight into the target and + # a real buffer/UNSHARD rebinds its storage to the target (once per unique value); everything else is copied into it + placed:dict[UOp, UOp] = {} items:list[UOp] = [] for s, t in zip(srcs, targets): - after_deps:list[UOp] = [] + deps:list[UOp] = [] while s.op is Ops.AFTER: - after_deps.extend(s.src[1:]) + deps.extend(s.src[1:]) s = s.src[0] - if (placed := _precompiled_output_redirect(s, t)) is not None and s not in subs: - subs[s] = placed - items.append(s.after(*after_deps) if after_deps else s) - else: - items.append(t.after(t.store(s.after(*after_deps)))) - fxn = UOp.sink(*(x.substitute(subs) for x in items)) + if s not in placed: + if s.op is Ops.CONTIGUOUS: placed[s] = t.after(t.store(s.src[0])) + elif s.op in {Ops.BUFFER, Ops.UNSHARD} and s.has_buffer_identity(): placed[s] = t + if s in placed: + items.append(s.after(*deps)) + continue + items.append(t.after(t.store(s.after(*deps)))) + # swap every placed value for its target storage, also inside other stores' AFTER deps + fxn = UOp.sink(*(x.substitute(placed) for x in items)) - # body switches from TUPLE to SINK, so the node becomes an opaque CALL (not FUNCTION) - new_call = UOp(Ops.CALL, src=(fxn, *input_buffers, *outs), arg=c.arg) + # all bodies are SINKs now, the node just becomes an opaque CALL: outs take the RETURNEDs' places; afters on real + # buffers are the input storage, afters on RETURNED placeholders have no storage yet, materialize them + rmap = dict(zip(ret_pos, outs)) + new_call = UOp(Ops.CALL, src=(fxn, *[rmap.get(i, a if a.has_buffer_identity(after_ok=True) else a.contiguous()) + for i, a in enumerate(c.src[1:])]), arg=c.arg) rets = tuple(o.after(new_call) for o in outs) # if the CALL has symbolic shapes, shrink the max-sized output to the actual symbolic shape - # NOTE: must use resolved shapes from the FUNCTION (which substitutes PARAMs with external args), not raw body shapes - rets = tuple(r.shrink_to(rs.shape) for r,rs in zip(rets, resolved)) + # NOTE: must use the resolved shapes of the RETURNED placeholders (which substitute PARAMs with external args), not raw body shapes + rets = tuple(r.shrink_to(rs.shape) for r,rs in zip(rets, (c.src[1+p] for p in ret_pos))) - return UOp.maketuple(*rets) + # the AFTER outputs resolve against this: stores of each real output into its RETURNED placeholder + return UOp.sink(*[c.src[1+p].store(v) for p, v in zip(ret_pos, rets)]) # NOTE: adding rules to here is bad. these all need to run before the schedule cache pm_early_transform_tensor_graph = PatternMatcher([ - # transform precompiled FUNCTIONs into CALLs (body becomes SINK with stores) - (UPat(Ops.FUNCTION, name="c"), transform_precompiled_call), + # transform precompiled value-producing calls into opaque CALLs (outputs become real buffers) + (UPat(Ops.CALL, name="c"), transform_precompiled_call), - # resolve TUPLE+GETTUPLE (for precompiled calls) - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), + # resolve AFTER on RETURNED placeholders (for precompiled calls) + (UPat(Ops.AFTER, src=(UPat(name="r"), UPat(Ops.SINK, name="t")), allow_any_len=True), resolve_returned_after), - # fold MOPS+BITCAST over BUFFER/SLICE into SLICE when movement ops collapse to contiguous range - (UPat((Ops.BITCAST, Ops.COPY, Ops.CONTIGUOUS), src=(UPat(GroupOp.Movement|{Ops.BUFFER}, name="src"),), name="c"), contiguous_mops_to_view), + # fold MOPS+BITCAST over BUFFER into SHRINK when movement ops collapse to contiguous range + (UPat((Ops.COPY, Ops.CONTIGUOUS), src=(UPat(GroupOp.Movement|{Ops.BITCAST}, name="src"),), name="c"), contiguous_mops_to_view), + (UPat(Ops.STORE, src=(UPat(Ops.BITCAST, name="src"), UPat()), name="c", allow_any_len=True), contiguous_mops_to_view), # remove contiguous on movement ops before a copy on disk (UPat(GroupOp.Movement-{Ops.SHRINK, Ops.RESHAPE}, name="x").f(Ops.CONTIGUOUS).f(Ops.COPY, name="copy"), lambda x,copy: - copy.replace(src=(x,), tag=None) if isinstance(x.device, str) and x.device.startswith("DISK") else None), + copy.replace(src=(x,), tag=None) if on_disk(x) else None), # push copy past movement ops to disk (UPat(GroupOp.Movement-{Ops.SHRINK, Ops.RESHAPE}, name="x").f(Ops.COPY, name="copy"), lambda x,copy: - x.replace(src=(copy.replace(src=(x.src[0],), tag=None),)+x.src[1:]) \ - if isinstance(x.device, str) and x.device.startswith("DISK") else None), + x.replace(src=(copy.replace(src=(x.src[0],), tag=None),)+x.src[1:]) if on_disk(x) else None), # add CONTIGUOUS to tagged UOps - (UPat(GroupOp.All-{Ops.CONTIGUOUS, Ops.AFTER, Ops.STORE}, name="x"), - lambda x: None if x.tag is None else x.rtag(None).contiguous(tag=x.tag) if x.tag else x.replace(tag=None)), + (UPat(GroupOp.All-{Ops.CONTIGUOUS, Ops.AFTER, Ops.STORE}, name="x"), wrap_tagged_in_contig), # remove extra CONTIGUOUS on AFTER (only when target is contiguous) (UPat(Ops.CONTIGUOUS, src=(UPat(Ops.AFTER, name="a"),), name="c"), lambda a,c: a.replace(tag=(a.tag or ())+(c.tag or ())) if a.src[0].has_buffer_identity() else None), - # replace AFTER+STORE with CONTIGUOUS when target is not a buffer - (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE, src=(UPat(), UPat(name="src")))), name="u"), replace_store_after_with_contig), # replace CONTIGUOUS with STORE+AFTER (UPat(Ops.CONTIGUOUS, name="u"), replace_contig_with_store_after), # remove DETACH/CONTIGUOUS_BACKWARD (allows more contiguous removal) (UPat((Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), name="x"), lambda x: x.src[0]), ]) -def finalize_after(ctx:AllocCtx, x:UOp): - # untagged: record as an assign for the call body - if x.tag is None: - ctx.assigns.append(x) - return None - # tagged: untag and map each original pre-rewrite UOp to the stripped buffer; the untagged result is reprocessed as untagged - ret = x.replace(tag=None) - replace_uop = ret - # then, add views back - views:list[UOp] = [] - while replace_uop.op in GroupOp.Movement|{Ops.UNSHARD, Ops.BITCAST, Ops.AFTER}: - if replace_uop.op is not Ops.AFTER: views.append(replace_uop) - replace_uop = replace_uop.src[0] - for v in reversed(views): replace_uop = v.replace(src=(replace_uop,)+v.src[1:]) - for t in x.tag: - original_uop: UOp = ctx.uop_list[t] - ctx.buffer_map[original_uop] = replace_uop.shrink_to(original_uop.shape) - return ret +# a store's storage keeps the views and drops AFTERs (they only sequence stores) +pm_drop_after = PatternMatcher([(UPat(Ops.AFTER, name="a"), lambda a: a.src[0])]) def replace_input_buffer(ctx:AllocCtx, b:UOp): ctx.replacements.append(b) - if b.op is Ops.BIND: return b.param_like(len(ctx.replacements)-1) - return UOp.param(len(ctx.replacements)-1, b.dtype, b.shape, b.device, - addrspace=b.addrspace if b.addrspace is not None else AddrSpace.GLOBAL) - -pm_finalize_call = PatternMatcher([ - (UPat(Ops.AFTER, name="x"), finalize_after), - (UPat(Ops.COPY, name="x"), lambda ctx,x: ctx.assigns.append(x) if isinstance(x.device, str) and x.device.startswith(("DISK", "TINYFS")) else None), -]) + return b.param_like(len(ctx.replacements)-1) pm_replace_buf = PatternMatcher([ - # replace BUFFER with PARAM for cache key normalization - (UPat(Ops.BUFFER, src=(UPat(),), name="b"), lambda ctx,b: - replace_input_buffer(ctx, b) if isinstance(b.arg, ParamArg) and b.addrspace is AddrSpace.GLOBAL else None), - # replace SLICE with PARAM. this rewrite is bottom up so BUFFERs we don't need won't be in the input - (UPat(Ops.SLICE, src=(UPat(Ops.BUFFER), UPat(Ops.CONST, dtype=dtypes.weakint)), name="b"), replace_input_buffer), - # strip value from BIND for cache key normalization, so different values hit same cache - (UPat(Ops.BIND, src=(UPat(Ops.PARAM), UPat(Ops.CONST)), name="b"), replace_input_buffer), + # replace BUFFER with PARAM for cache key normalization (ALU addrspace buffers are Variables, they stay) + (UPat(Ops.BUFFER, src=(), name="b"), lambda ctx,b: replace_input_buffer(ctx, b) if b.addrspace is AddrSpace.GLOBAL else None), + # replace buffer views (SHRINK/BITCAST) with PARAM (only the views created by contiguous_mops_to_view) + (UPat((Ops.SHRINK, Ops.BITCAST), name="b"), lambda ctx,b: replace_input_buffer(ctx, b) if b in ctx.views else None), + # strip the stored value from bound Variables for cache key normalization, so different values hit same cache + (UPat(Ops.AFTER, name="b"), lambda ctx,b: replace_input_buffer(ctx, b) if b.is_bound_var else None), ]) @rewrite_group(lambda _,ret: f"Callify {pluralize('Buffer', len(ret[1]))}") def transform_to_call(big_sink:UOp) -> tuple[UOp, dict[UOp, UOp]]: if VIZ: graph_rewrite(big_sink, PatternMatcher([]), name="View Tensor Graph") - # uop list is a list in the original_sink graph and we can map to the tags later - # same predicate as Tensor.realize + if SPEC: type_verify(big_sink, spec_tensor) + # bases to realize: same predicate as Tensor.realize ctx = AllocCtx(bases={base for x in big_sink.src if not (base:=x.base).is_virtual and not base.has_buffer_identity() and base.op is not Ops.AFTER and base.addrspace is not AddrSpace.ALU}) # this rewrite is "read-only", it adds simple things to buffer_map and may sink things on big_sink, bottom_up # this is the only one where we have to be careful to not break the tensor graph - big_sink = graph_rewrite(big_sink, add_tags, ctx=ctx, bottom_up=True, name="number the uops") + big_sink = graph_rewrite(big_sink, add_tags, ctx=ctx, bottom_up=True, name="add tags") - # here we can break the tensor graph. this is the only place you need to maintain numbered tags - big_sink = graph_rewrite(big_sink, pm_early_transform_tensor_graph, name="early transform tensor graph") + # final outputs of value calls materialize with fresh storage + srcs:list[UOp] = [] + for u in big_sink.src: + if u.op is Ops.AFTER and u.src[0].unsharded_base.op is Ops.RETURNED: + # precompiled calls don't need this: transform_precompiled_call gives their outputs real buffers + call = u.src[1] + if not (call.op is Ops.CALL and call.arg is not None and call.arg.precompile and call.num_returned): + u = u.rtag(None).contiguous(tag=u.tag) + srcs.append(u) + big_sink = big_sink.replace(src=tuple(srcs)) - # here we construct the final buffer_map: as-built nodes -> their final storage. values are never keys - graph_rewrite(big_sink, pm_finalize_call, ctx=ctx, name="finalize call") - ret = graph_rewrite(UOp.sink(*ctx.assigns), pm_replace_buf, ctx=ctx, bottom_up=True, name="replace bufs").call(*ctx.replacements) + # here we can break the tensor graph. tags propagate through replaces so we can still find the original UOps + big_sink = graph_rewrite(big_sink, pm_early_transform_tensor_graph, ctx=ctx, name="early transform tensor graph") + + # collect the stores (never entering call bodies) and map tagged AFTERs to their storage; tags are stripped at the end + # copies to disk are stores to the disk buffer; bound Variables are call inputs and RETURNEDs are call outputs + for u in big_sink.toposort(enter_calls=False): + if (u.op is Ops.COPY and on_disk(u)) or (u.op is Ops.AFTER and not u.is_bound_var and u.src[0].unsharded_base.op is not Ops.RETURNED): + ctx.stores.append(u) + if u.tag: ctx.buffer_map.update({t:graph_rewrite(u.src[0], pm_drop_after).shrink_to(t.shape) for t in u.tag}) + ret = graph_rewrite(UOp.sink(*ctx.stores), pm_replace_buf+remove_all_tags, ctx=ctx, bottom_up=True, name="replace bufs").call(*ctx.replacements) assert not any(x in ctx.buffer_map for x in ctx.buffer_map.values()) if VIZ: graph_rewrite(ret, PatternMatcher([]), name="View Call") return ret, ctx.buffer_map @@ -257,6 +236,8 @@ def _apply_map_to_tensors(applied_map:dict[UOp, UOp], name:str) -> None: if s is ns: continue t.uop = ns +def _tensor_holds(u:UOp) -> bool: return any((t:=tref()) is not None and t.uop is u for tref in list(all_tensors)) + # **** Tensor helper functions **** def is_numpy_ndarray(x) -> "TypeGuard[numpy.ndarray]": return str(type(x)) == "" @@ -377,11 +358,11 @@ class Tensor(RandMixin): # ***** data handlers **** def as_param(self, slot:int): - return Tensor(UOp.param(slot, self.dtype, self.uop.shard_shape, self.device, axis=self.uop.axis)) + return Tensor(self.uop.param_like(slot)) def call(self, *lst:Tensor, fxn:Tensor|UOp, grad_fxn:Callable|None=None) -> Tensor: fret = fxn._uop.call(*[t.uop for t in (self,)+lst], grad_fxn=grad_fxn) - return Tensor(fret.gettuple(0)) + return Tensor(fret.returned_outputs[0]) def custom_kernel(self, *lst:Tensor, fxn:Callable, grad_fxn:Callable|None=None) -> list[Tensor]: """ @@ -431,7 +412,7 @@ class Tensor(RandMixin): def assign(self, x:Tensor|PyConst|list|tuple) -> Tensor: if self.dtype in dtypes.weaks: self.uop = self.uop.clone() - is_disk = isinstance(self.device, str) and self.device.startswith(("DISK", "TINYFS")) + is_disk = on_disk(self.uop) if not isinstance(x, Tensor): x = Tensor(x, device="CPU" if is_disk else self.device, dtype=self.dtype) if self.uop is x.uop: return self # a self assign is a NOOP # broadcast x (shape only, dtype must match) @@ -447,14 +428,23 @@ class Tensor(RandMixin): if is_disk: (b:=self._buffer()).copy_from(Buffer("PYTHON", b.size, b.dtype, opaque=x._data())) return self + # a STORE can only write into storage: the target must be backed by a BUFFER (possibly under views) + assigned_to = self.uop.storage_base + # assigning to a value (not storage-backed and not a CONTIGUOUS realization point) is initialization, + # not a write: a Tensor.assign always overwrites the whole tensor, so the pending value is dead + if assigned_to.op not in {Ops.BUFFER, Ops.CONTIGUOUS}: + # x is the new value: alias it if it materializes on its own (a CONTIGUOUS or a load from a creation device), + # otherwise give it a realization point so this tensor gets storage of its own + if x.uop.op is not Ops.CONTIGUOUS and not (x.uop.op is Ops.COPY and is_creation_device(x.uop.src[0])): x = x.contiguous() + self.uop = x.uop + return self # STORE+AFTER: STORE is the write effect (void), AFTER wraps the view for correct shape/ranging assign = self.uop.after(self.uop.store(x.uop)) - if (base := self.uop.base).op in {Ops.BUFFER, Ops.AFTER} and self.uop is not base and not self.uop.has_buffer_identity(): + ib = self.uop + while ib.op in GroupOp.Movement|{Ops.BITCAST, Ops.DETACH} and not (ib.has_buffer_identity() and _tensor_holds(ib)): ib = ib.src[0] + if ib is not self.uop and ib.has_buffer_identity(after_ok=True): # view assign: replace at the buffer-identity level (e.g. RESHAPE(BUFFER)) so @function's substitution catches it - ib = self.uop - while not ib.has_buffer_identity() and ib is not base: ib = ib.src[0] - assigned_ib = ib.after(assign) - _apply_map_to_tensors({ib: assigned_ib}, name="Embed View Assign") + _apply_map_to_tensors({ib: ib.after(assign)}, name="Embed View Assign") else: # simple assign self.uop = assign @@ -480,7 +470,7 @@ class Tensor(RandMixin): print(np.frombuffer(t.data(), dtype=np.int32)) ``` """ - if self.dtype in dtypes.weaks: return self.cast(strong_dtype(self.dtype)).data() + if self.dtype in dtypes.weaks: return self.cast(self.commit_dtype()).data() if 0 in self.shape: return memoryview(bytearray(0)).cast(self.dtype.fmt) # type: ignore[arg-type,return-value] assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" buf = self._buffer() @@ -521,7 +511,7 @@ class Tensor(RandMixin): print(repr(t.numpy())) ``` """ - if self.dtype in dtypes.weaks: return self.cast(strong_dtype(self.dtype)).numpy() + if self.dtype in dtypes.weaks: return self.cast(self.commit_dtype()).numpy() assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" import numpy as np if self.dtype in { dtypes.bfloat16, *dtypes.fp8s }: return self.float().numpy() @@ -662,13 +652,13 @@ class Tensor(RandMixin): ``` """ all_uops = self.uop.toposort() - # backward fills .grad for every in-scope non-CONST float tensor + # backward fills .grad for every in-scope float tensor with a device tensors_need_grad: list[Tensor] = [t for tref in all_tensors if (t:=tref()) is not None and \ - t.uop in all_uops and t.is_floating_point() and t.uop.op is not Ops.CONST] + t.uop in all_uops and t.is_floating_point() and t.device is not None] # clear contexts for t,g in zip(tensors_need_grad, self.gradient(*tensors_need_grad, gradient=gradient)): assert g.shape == t.shape, f"grad shape must match tensor shape, {g.shape!r} != {t.shape!r}" - if g.device is None and t.device is not None: g = g.clone(device=t.device) + if g.device is None: g = g.clone(device=t.device) if t.grad is None: t.grad = g else: t.grad.assign(t.grad + g.to(t.grad.device)) return self @@ -691,7 +681,7 @@ class Tensor(RandMixin): if (t:=tref()) is not None and t is not self and t.uop is not v_uop and t.uop not in v_bw): raise RuntimeError("can't setitem on a tensor with other uses") idx = [indices] if (isinstance(indices, list) and all_int(indices)) or not isinstance(indices, (tuple, list)) else list(indices) - is_disk = isinstance(self.device, str) and self.device.startswith("DISK") + is_disk = on_disk(self.uop) advanced = any(isinstance(i, (Tensor, list, tuple)) for i in idx) realized = is_disk or self.uop.base.op is Ops.BUFFER or self.uop._base_buffer_is_realized() if (not self.uop.base.is_realized and self.is_floating_point()) or not (advanced or realized): @@ -739,9 +729,9 @@ class Tensor(RandMixin): the reference frames (`ref_frames`). """ ref_frames = [x.contiguous() for x in ref_frames or []] - assert frame_pos.op is Ops.BIND, "frame_pos must be a bound Variable" + assert frame_pos.is_bound_var, "frame_pos must be a bound Variable" srcs = (out:=Tensor.empty(*shape, device=self.device, dtype=self.dtype), self.contiguous(), state.contiguous(), *ref_frames) - fn = UOp(Ops.CUSTOM_FUNCTION, src=(frame_pos.src[0], *[UOp.const(s, dtypes.int) for s in shape]), arg="encdec") + fn = UOp(Ops.CUSTOM_FUNCTION, src=(frame_pos.src[0], *[UOp.const(s) for s in shape]), arg="encdec") return Tensor(out.uop.after(fn.call(*[s.uop for s in srcs], frame_pos))) P = ParamSpec("P") diff --git a/tinygrad_repo/tinygrad/uop/__init__.py b/tinygrad_repo/tinygrad/uop/__init__.py index e39fc8b87..060d5671f 100644 --- a/tinygrad_repo/tinygrad/uop/__init__.py +++ b/tinygrad_repo/tinygrad/uop/__init__.py @@ -13,9 +13,6 @@ class FastEnum(IntEnum): class Ops(FastEnum): # ** 1 -- defines/special ** - # BIND pairs a symbolic PARAM with a concrete value - BIND = auto() - # this is a RANGE for GPU dimensions, similar to symbolic shapes but not exactly SPECIAL = auto() @@ -26,8 +23,8 @@ class Ops(FastEnum): # uops that aren't rendered NOOP = auto(); REWRITE_ERROR = auto() - # FUNCTION has a TUPLE body and is gradient-able; CALL is an opaque kernel invocation - PARAM = auto(); FUNCTION = auto(); CALL = auto() + # CALL is a kernel invocation; calls with RETURNED inputs are value-producing (and gradient-able), the rest are opaque + PARAM = auto(); CALL = auto() # renderer # LINEAR is a list of UOps, SOURCE has a str arg that's human readable, BINARY has bytes arg that's compiled @@ -40,8 +37,8 @@ class Ops(FastEnum): # vector creation / item selection STACK = auto() - # tuple/gettuple for function with multiple returns - TUPLE = auto(); GETTUPLE = auto() + # RETURNED is a placeholder for a buffer a call writes and returns: it's an input to the call and you AFTER on it + RETURNED = auto() # hcq specific GETADDR = auto() @@ -76,7 +73,7 @@ class Ops(FastEnum): # ** 5 -- control flow / consts / custom ** # control flow ops - BARRIER = auto(); RANGE = auto(); IF = auto(); END = auto(); ENDIF = auto(); WAIT = auto() + BARRIER = auto(); RANGE = auto(); IF = auto(); END = auto(); ENDIF = auto() # const. CONST = auto() @@ -93,7 +90,7 @@ class Ops(FastEnum): CONTIGUOUS = auto(); CONTIGUOUS_BACKWARD = auto(); DETACH = auto() # buffer ops - STAGE = auto(); COPY = auto(); SLICE = auto(); MSELECT = auto(); MSTACK = auto(); CUSTOM_FUNCTION = auto() + STAGE = auto(); COPY = auto(); MSELECT = auto(); MSTACK = auto(); CUSTOM_FUNCTION = auto() # the core 6 movement ops! these only exist in the tensor graph RESHAPE = auto(); PERMUTE = auto(); EXPAND = auto(); PAD = auto(); FLIP = auto() diff --git a/tinygrad_repo/tinygrad/uop/ops.py b/tinygrad_repo/tinygrad/uop/ops.py index 9ae599b98..b54879606 100644 --- a/tinygrad_repo/tinygrad/uop/ops.py +++ b/tinygrad_repo/tinygrad/uop/ops.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, replace from enum import Enum, auto from tinygrad.uop import Ops, GroupOp from tinygrad.dtype import ConstType, dtypes, DType, DTypeLike, truncate, least_upper_dtype, least_upper_float, Invalid, AddrSpace, strong_dtype -from tinygrad.dtype import PyConst, InvalidType, weak_dtype, bitcast +from tinygrad.dtype import PyConst, InvalidType, bitcast from tinygrad.device import Buffer, MultiBuffer, canonicalize_device, TinyELF from tinygrad.helpers import ContextVar, all_int, prod, getenv, all_same, Context, partition, temp, unwrap, T, argfix, Metadata, flatten, TRACEMETA from tinygrad.helpers import PROFILE, dedup, cdiv, cmod, floordiv, floormod, diskcache_put, to_function_name, cpu_profile, TracingKey @@ -17,36 +17,39 @@ if TYPE_CHECKING: class AxisType(Enum): def __repr__(self): return str(self) DEVICE = auto(); GLOBAL = auto(); WARP = auto(); LOCAL = auto(); WEAK = auto(); GROUP_REDUCE = auto(); REDUCE = auto(); UPCAST = auto() # noqa: E702 - UNROLL = auto(); THREAD = auto(); PLACEHOLDER = auto(); LOOP = auto() # noqa: E702 + UNROLL = auto(); PLACEHOLDER = auto(); LOOP = auto() # noqa: E702 @dataclass(frozen=True, order=True) class ParamArg: slot: int dtype: DType + # number of elements in the buffer. always a concrete int (never symbolic), None for scalars (shape ()) + size: int|None = None vmin_vmax: tuple[PyConst, PyConst]|None = None multiple_of: int|None = None name: str|None = None addrspace: AddrSpace|None = AddrSpace.GLOBAL - axis: int|None = None device: str|tuple[str, ...]|None = None volatile: bool = False + # (h, w) if this is an image2d buffer, then size == h*w*4 + image: tuple[int, int]|None = None def __repr__(self): - fields = (("vmin_vmax", None), ("multiple_of", None), ("name", None), ("addrspace", AddrSpace.GLOBAL), ("axis", None), ("device", None), - ("volatile", False)) - args = [repr(self.slot), repr(self.dtype)] + [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] + fields = (("vmin_vmax", None), ("multiple_of", None), ("name", None), ("addrspace", AddrSpace.GLOBAL), ("device", None), + ("volatile", False), ("image", None)) + args = [repr(self.slot), repr(self.dtype)] + ([repr(self.size)] if self.size is not None else []) + \ + [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] return f"ParamArg({', '.join(args)})" -axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.THREAD: "t", AxisType.LOCAL: "l", AxisType.WARP: "w", AxisType.WEAK: "L", +axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.LOCAL: "l", AxisType.WARP: "w", AxisType.WEAK: "L", AxisType.LOOP: "L", AxisType.UPCAST: "u", AxisType.GROUP_REDUCE: "G", AxisType.REDUCE: "R", AxisType.UNROLL: "r"} -axis_colors = {AxisType.DEVICE: "green", AxisType.GLOBAL: "blue", AxisType.THREAD: "BLUE", AxisType.LOCAL: "cyan", AxisType.WARP: "CYAN", +axis_colors = {AxisType.DEVICE: "green", AxisType.GLOBAL: "blue", AxisType.LOCAL: "cyan", AxisType.WARP: "CYAN", AxisType.WEAK: "WHITE", AxisType.LOOP: "WHITE", AxisType.UPCAST: "yellow", AxisType.GROUP_REDUCE: "RED", AxisType.REDUCE: "red", AxisType.UNROLL: "magenta"} # NOTE: LOCAL and GROUP_REDUCE have the same priority. the order here matters -axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.THREAD: 0, AxisType.GLOBAL: 0, AxisType.WARP: 1, +axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.GLOBAL: 0, AxisType.WARP: 1, AxisType.LOCAL: 2, AxisType.UPCAST: 3, AxisType.GROUP_REDUCE: 2, AxisType.REDUCE: 4, AxisType.UNROLL: 5} -range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.FUNCTION: 1, - Ops.SLICE: 2, Ops.LINEAR: 0} +range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.LINEAR: 0} # https://en.wikipedia.org/wiki/Identity_element def identity_element(op:Ops, dt:DType) -> PyConst: return dt.const({Ops.ADD:0, Ops.MUL:1, Ops.MAX:dt.min}[op]) @@ -95,11 +98,10 @@ def multirange_str(rngs:Iterable[UOp], color=False, pad=None) -> str: return ret def shape_to_shape_arg(arg:tuple[sint, ...]) -> UOp: - for x in arg: - if isinstance(x, UOp) and not dtypes.is_int(x.dtype): raise RuntimeError(f"shape must be int, got {x.dtype} in {arg}") - if len(arg) == 0: return UOp(Ops.STACK) - elif len(arg) == 1: return UOp.const(arg[0], dtypes.weakint) - else: return UOp(Ops.STACK, src=tuple(UOp.const(x) if isinstance(x, int) else x for x in arg)) + src = tuple(x if isinstance(x, UOp) else UOp.const(x) for x in arg) + for x in src: + if not dtypes.is_int(x.dtype): raise RuntimeError(f"shape must be int, got {x.dtype} in {arg}") + return src[0] if len(src) == 1 else UOp(Ops.STACK, src=src) def consumer_map_from_toposort(lst:Iterable[UOp]): ret: dict[UOp, dict[UOp, None]] = {} @@ -113,24 +115,24 @@ def promo_dtype(src:tuple[UOp,...]) -> DType: dts = [x.dtype for x in src] return dts[0] if all_same(dts) else least_upper_dtype(*dts) -def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: - # here are the dtype production rules, eventually this will go in UOp as a recursive property +def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: + # here are the dtype production rules, total over all Ops match op: case Ops.STORE | Ops.LINEAR | Ops.SINK | Ops.PROGRAM | Ops.SOURCE | \ - Ops.END | Ops.BARRIER | Ops.GROUP | Ops.IF | Ops.ENDIF | \ - Ops.TUPLE | Ops.FUNCTION | Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR: + Ops.END | Ops.BARRIER | Ops.GROUP | Ops.IF | Ops.ENDIF | Ops.NOOP | \ + Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR | Ops.PYLITERAL: # always void return dtypes.void case Ops.CALL: - # a CALL of an opaque body is void, a CALL of an address can return a value - return dtypes.void if src[0].dtype is dtypes.void else None - case Ops.CUSTOM | Ops.CUSTOMI | Ops.PYLITERAL: - return None + # a CALL of an opaque body (CallInfo arg) is void, a CALL of an address states its return dtype in the arg + return arg if isinstance(arg, DType) else dtypes.void + case Ops.CUSTOM | Ops.CUSTOMI: + assert isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[1], DType), f"CUSTOM/CUSTOMI arg must be (str, DType), got {arg}" + return arg[1] case Ops.INS: - return None - case Ops.NOOP: - # NOOP can be void or carry any dtype (e.g. x.f(Ops.NOOP) or substitute base with NOOP) - return None + # arg is (instruction, dtype), a queue command or an asm line is void + assert isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[1], DType), f"INS arg must be (instruction, DType), got {arg}" + return arg[1] case Ops.INDEX: # an image access is always float, no matter the storage dtype # TODO: should there be a CAST so src[0].dtype just work? @@ -144,36 +146,31 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: case Ops.CMPLT | Ops.CMPNE | Ops.CMPEQ: return dtypes.bool case Ops.SIN | Ops.LOG2 | Ops.EXP2 | Ops.SQRT | Ops.RECIPROCAL: - return least_upper_float(src[0].dtype) + return dtypes.bool if src[0].base.is_invalid else least_upper_float(src[0].dtype) case Ops.WHERE: if src[0].dtype != dtypes.bool: raise RuntimeError(f"where cond must be bool, got {src[0].dtype}") return promo_dtype(src[1:]) case Ops.STACK: if len(src) == 0: return dtypes.void return promo_dtype(src) - case Ops.BIND: - assert src[0].dtype == src[1].dtype, f"bind dtype mismatch {src[0].dtype} != {src[1].dtype}" - return src[0].dtype case Ops.WMMA: # WMMA output dtype is the accumulator dtype (src[2]) return src[2].dtype - case Ops.GETTUPLE: - # GETTUPLE extracts from a TUPLE (possibly through a FUNCTION) - in_tuple = src[0].src[0] if src[0].op is Ops.FUNCTION else src[0] - return in_tuple.src[arg].dtype case Ops.GETADDR: return dtypes.uint64 + case Ops.THREEFRY: + return dtypes.uint64 + case Ops.FDIV: + return least_upper_float(promo_dtype(src)) case Ops.SHL | Ops.SHR: - if not all(dtypes.is_int(x.dtype) for x in src): raise RuntimeError(f"shift operands must be int, got {[x.dtype for x in src]}") + if not all(dtypes.is_int(x.dtype) or x.base.is_invalid for x in src): + raise RuntimeError(f"shift operands must be int, got {[x.dtype for x in src]}") return src[0].dtype - case Ops.BUFFER | Ops.PARAM: - assert isinstance(arg, ParamArg), "BUFFER/PARAM must have ParamArg" + case Ops.BUFFER | Ops.PARAM | Ops.RETURNED: + assert isinstance(arg, ParamArg), f"{op} must have ParamArg" return arg.dtype case Ops.BINARY: return dtypes.uint8 - case Ops.SLICE: - # TODO: slice just shouldn't exist - return None case Ops.CAST | Ops.BITCAST: assert isinstance(arg, DType), f"CAST/BITCAST arg must be DType, got {arg}" return arg @@ -190,20 +187,18 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: if op in GroupOp.Movement: return src[0].dtype raise RuntimeError(f"no dtype for {op} with arg {arg}") +class _LegacyTupleValues: + """legacy compatibility shim: TUPLE is gone, a tuple-of-values just holds the values until they are called""" + def __init__(self, srcs:tuple[UOp, ...]): self.srcs = srcs + def call(self, *args:UOp, **kwargs) -> UOp: return UOp.call_outputs(self.srcs, *args, **kwargs) + class UOpMetaClass(type): ucache:dict[tuple, weakref.ReferenceType[UOp]] = {} - def __call__(cls, op:Ops, dtype:DType|None=None, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, + def __call__(cls, op:Ops, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, metadata:tuple[Metadata,...]|None=None, _buffer:Buffer|None=None): - if dtype is None: dtype = dtype_from_uop(op, src, arg) or dtypes.void - # CONST derives its dtype by value only when the constructor omits one - # TODO: delete this once the dtype field is removed, for now it just re-implements spec.py - # an INDEX presents its access dtype, which a still-weak source matches up to weakness - if SPEC == 2 and op is not Ops.CONST and \ - not any(s.base.is_invalid for s in src) and (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype and \ - not (op is Ops.INDEX and weak_dtype(expected_dtype) == weak_dtype(dtype)): - raise RuntimeError(f"bad dtype {dtype}, expected {expected_dtype} on {op}") - if (wret:=UOpMetaClass.ucache.get(key:=(op, dtype, src, arg, tag), None)) is not None and (ret:=wret()) is not None: return ret - UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(*key)) + # NOTE: the key must separate nodes of different dtype: a CONST's dtype is the type of its arg, and True == 1 as dict keys + if (wret:=UOpMetaClass.ucache.get(key:=(op, src, arg, tag, type(arg)), None)) is not None and (ret:=wret()) is not None: return ret + UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(op, src, arg, tag)) if metadata is not None: all_metadata[created] = metadata # NOTE: this value is set by pickle when pickling a realized tensor if _buffer is not None: @@ -221,7 +216,7 @@ class UOpMetaClass(type): return created # some uops map to other stuff -buffers:weakref.WeakKeyDictionary[UOp, Buffer|MultiBuffer] = weakref.WeakKeyDictionary() # this maps BUFFER/SLICE uops to their device Buffers +buffers:weakref.WeakKeyDictionary[UOp, Buffer|MultiBuffer] = weakref.WeakKeyDictionary() # this maps BUFFER/view uops to their device Buffers all_metadata:weakref.WeakKeyDictionary[UOp, tuple[Metadata, ...]] = weakref.WeakKeyDictionary() # TODO: should this be here? # recursive_property replaces functools.cached_property in recursive UOp functions to prevent RecursionError @@ -237,36 +232,38 @@ class recursive_property(property): return x.__dict__[self.nm] # we import this late so we can use resolve/smax in mixins -from tinygrad.mixin.op import OpMixin from tinygrad.mixin.rand import RandMixin # NOTE: this should be frozen, but frozen is slower @dataclass(eq=False, slots=True) class UOp(RandMixin, metaclass=UOpMetaClass): op:Ops - dtype:DType = dtypes.void src:tuple[UOp, ...] = tuple() arg:Any = None tag:Any = None + @recursive_property + def dtype(self) -> DType: return dtype_from_uop(self.op, self.src, self.arg) def __del__(self): - if Ops is not None and self.op is Ops.BUFFER and (buffer:=buffers.get(self)) is not None: buffer.ref(-1) - try: del UOpMetaClass.ucache[(self.op, self.dtype, self.src, self.arg, self.tag)] - except AttributeError: pass + # NOTE: getattr because this object may be partially constructed (e.g. if __init__ raised, like the BEAM timeout SIGALRM) + if Ops is not None and getattr(self, 'op', None) is Ops.BUFFER and (buffer:=buffers.get(self)) is not None: buffer.ref(-1) + try: del UOpMetaClass.ucache[(self.op, self.src, self.arg, self.tag, type(self.arg))] + except (AttributeError, KeyError): pass def __reduce__(self): - args = [self.op, self.dtype, self.src, self.arg, self.tag, self.metadata] + args = [self.op, self.src, self.arg, self.tag, self.metadata] if self.op is Ops.BUFFER and self.realized is not None: args.append(self.realized) return UOp, tuple(args) def replace(self, **kwargs) -> UOp: - new_args = (kwargs.pop("op", self.op), kwargs.pop("dtype", self.dtype), kwargs.pop("src", self.src), - kwargs.pop("arg", self.arg), kwargs.pop("tag", self.tag)) + new_args = (kwargs.pop("op", self.op), kwargs.pop("src", self.src), kwargs.pop("arg", self.arg), kwargs.pop("tag", self.tag)) assert len(kwargs) == 0, f"unused kwargs in replace {list(kwargs)}" - if (self.op, self.dtype, self.src, self.arg, self.tag) == new_args: return self + if (self.op, self.src, self.arg, self.tag) == new_args: return self return UOp(*new_args) def rtag(self, tag=True): return self.replace(tag=tag) @property def val(self): - assert self.op is Ops.CONST, f"val is only for CONST, got {self.op}" - return self.arg + if self.op is Ops.CONST: return self.arg + # a casted const CAST(dt, CONST(v)) is one const: .val reads the value through the CAST + assert self.op is Ops.CAST and self.src[0].op is Ops.CONST, f"val is only for consts, got {self.op}" + return self.src[0].val @property def is_invalid(self) -> bool: return self.op is Ops.CONST and self.val is Invalid @recursive_property @@ -280,8 +277,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): return repr(self.arg) def tagstr(self): return f", tag={self.tag}" if self.tag is not None else "" - def f(self, op, **kwargs): return UOp(op, dtype=kwargs.pop("dtype", self.dtype), src=(self,), **kwargs) - @functools.cached_property def backward_slice(self:UOp) -> dict[UOp, None]: res: dict[UOp, None] = self.toposort() @@ -309,7 +304,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if not visited: if gate is None or gate(node): stack.append((node, True)) # push node back on stack to process after its srcs - for s in reversed(node.src if enter_calls or node.op not in {Ops.CALL, Ops.FUNCTION} else node.src[1:]): + for s in reversed(node.src if enter_calls or node.op is not Ops.CALL else node.src[1:]): stack.append((s, False)) # push srcs on the stack else: cache[node] = None # second time i'm seeing this node, add it to returned toposort return cache @@ -328,7 +323,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @functools.cached_property def tuplize(self:UOp) -> tuple: - return (self.op.value, self.arg, self.dtype,)+tuple([x.tuplize for x in self.src]) + # arg goes through repr: args of different types (None, str, tuple) must stay mutually comparable for the sort + return (self.op.value, repr(self.arg), self.dtype,)+tuple([x.tuplize for x in self.src]) # *** uop shape stuff *** @@ -337,7 +333,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): match self.op: # late ops don't have shape case Ops.IF | Ops.BARRIER | Ops.SINK | Ops.REWRITE_ERROR | Ops.ENDIF | Ops.GROUP | \ - Ops.LINEAR | Ops.PROGRAM | Ops.SOURCE | Ops.TUPLE | Ops.FUNCTION: + Ops.LINEAR | Ops.PROGRAM | Ops.SOURCE: return None # a void CALL has no shape, the return value of a CALL has the shape of its dtype @@ -357,17 +353,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): case Ops.NOOP: return self.src[0]._shape if len(self.src) >= 1 else None - case Ops.GETTUPLE: - # GETTUPLE extracts from a TUPLE (possibly through a FUNCTION) - in_tuple = self.src[0].src[0] if self.src[0].op is Ops.FUNCTION else self.src[0] - assert in_tuple.op is Ops.TUPLE - inner_shape = in_tuple.src[self.arg]._shape - if inner_shape is None: return None - # if through a FUNCTION, substitute internal PARAMs in the shape with corresponding args - if self.src[0].op is Ops.FUNCTION: - return tuple(graph_rewrite(s, _pm_resolve_params, self.src[0].src[1:], walk=True) if isinstance(s, UOp) else s for s in inner_shape) - return inner_shape - case Ops.INDEX: shp:list[sint] = [] for s in self.src[1:]: shp.extend(list(s.shape)) @@ -381,15 +366,12 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # some ops init the shape case Ops.GETADDR: return () - case Ops.BIND | Ops.RANGE | Ops.SPECIAL: return () + case Ops.RANGE | Ops.SPECIAL: return () case Ops.BINARY: return (len(self.arg),) - case Ops.BUFFER: - if len(self.src): return self.src[0].as_shape - return () - case Ops.SLICE: - # HACK: SLICE is used inside kernels, so we set the shape to () if it's on an INDEX - if self.src[0].op is Ops.INDEX: return () - return (self.arg,) + case Ops.BUFFER | Ops.PARAM | Ops.RETURNED: + # these don't have a shape input, they have a size in the arg: int gives shape (size,), None gives () + if (img:=self.arg.image) is not None: return (img[0], img[1], 4) + return () if self.arg.size is None else (self.arg.size,) case Ops.CUSTOM | Ops.CUSTOMI: if self.dtype is dtypes.void: return None input_shapes = [x._shape for x in self.src if x._shape is not None] @@ -400,10 +382,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # STAGE adds the existing shape to the front, opposite of INDEX return tuple([int(r.vmax+1) for r in self.src[1:]])+self.src[0].shape - # param has shape as the only arg - case Ops.PARAM: - return self.src[0].as_shape - # wmma output shape = accumulator shape (src[2]) case Ops.WMMA: wmma_b = _broadcast_shape(self.src[0].shape[:-1], self.src[1].shape[:-1], self.src[2].shape[:-1]) @@ -433,7 +411,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): match self.op: case Ops.RESHAPE: if not all(x >= 0 for x in self.marg): raise ValueError(f"shape can't contain negative numbers {self.marg}") - if prod(ps) != prod(self.marg): raise ValueError(f"bad reshape: {ps} -> {self.marg}") + # with symbolic views prod equality can be true at runtime but unprovable, only reject provably unequal products + if resolve(prod(ps) != prod(self.marg), False): raise ValueError(f"bad reshape: {ps} -> {self.marg}") return self.marg case Ops.EXPAND: return tuple(self.marg) + ps @@ -481,10 +460,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if (ret:=self._shape) is None: raise RuntimeError(f"shape requested, but {self.op} doesn't have a shape") return ret - @property - def max_shape(self) -> tuple[int, ...]: return to_max_shape(self.shape) - def max_numel(self) -> int: return prod(self.max_shape) - @property def shard_shape(self) -> tuple[sint, ...]: if not isinstance(self.device, tuple) or self.axis is None: return self.shape @@ -528,10 +503,12 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op is Ops.CONST: return self if self.op is Ops.SINK and all(s.op is Ops.CONST or (s.op is Ops.STACK and len(s.src) == 0) for s in self.src): return self # late import! - from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const + from tinygrad.uop.symbolic import symbolic with Context(TRACK_MATCH_STATS=0 if not tracked else TRACK_MATCH_STATS.value): - return graph_rewrite(self, symbolic+pm_fold_cast_const, name="simplify") - def ssimplify(self) -> UOp|ConstType: return ret.val if (ret:=self.simplify()).op is Ops.CONST else ret + return graph_rewrite(self, symbolic, name="simplify") + def ssimplify(self) -> UOp|ConstType: + if (ret := self.simplify()).op is Ops.CAST and ret.src[0].op is Ops.CONST: return ret.dtype.const(ret.src[0].val) + return ret.val if ret.op is Ops.CONST else ret def _eval(self, dtype, expected_type:Type[T]) -> T: assert self.dtype in dtype, f"eval with wrong dtype {self}" vmin, vmax = (simple_self:=self.simplify())._min_max @@ -555,22 +532,41 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @recursive_property def trace_num(self): num = next(ucount) - uop_fields[num] = (self.op, self.dtype, tuple(s.trace_num for s in self.src), self.arg, self.tag)+((self.metadata,) if TRACEMETA>=2 else ()) + # tags can contain UOps (callify tags nodes with their originals): store them as trace_nums, same as srcs + tag = tuple(t.trace_num if isinstance(t, UOp) else t for t in self.tag) if isinstance(self.tag, tuple) else self.tag + uop_fields[num] = (self.op, tuple(s.trace_num for s in self.src), self.arg, tag)+((self.metadata,) if TRACEMETA>=2 else ()) return num # *** uop syntactic sugar *** def sink(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument return UOp(Ops.SINK, src=tuple([x for x in srcs if x is not None]), **kwargs) - def maketuple(*srcs:UOp): # pylint: disable=no-self-argument - return UOp(Ops.TUPLE, src=srcs) + @staticmethod + def returned(slot:int, dtype:DType, shape:tuple[sint, ...]|sint|None=None, device=None, axis:int|None=None) -> UOp: + """create a RETURNED placeholder for a buffer a call writes and returns: it's an input to the call and you AFTER on it + like a normal buffer. its slot is its position among the call's srcs, which is its identity (identical slots merge) + like PARAM, the arg only stores the concrete max size: a shape is a view (RESHAPE/SHRINK/UNSHARD) on the flat placeholder""" + if isinstance(shape, (int, UOp)): shape = (shape,) + # multi-device values have a per-shard sized storage wrapped in UNSHARD: the sharding lives in the graph, not the arg + if shape is None or len(shape) == 0: return UOp(Ops.RETURNED, arg=ParamArg(slot, dtype, None, device=device)) + shp = tuple(s//len(device) if (i == axis and isinstance(device, tuple)) else s for i,s in enumerate(shape)) + ret = UOp(Ops.RETURNED, arg=ParamArg(slot, dtype, prod(to_max_shape(shp)), device=device)) + return ret.view_as(shp, axis) + @property + def num_returned(self) -> int: return sum(x.unsharded_base.op is Ops.RETURNED for x in self.src[1:]) + @property + def returned_outputs(self) -> tuple[UOp, ...]: + """the outputs of a value-producing call: an AFTER on each RETURNED input, usable like a normal buffer""" + return tuple(x.after(self) for x in self.src[1:] if x.unsharded_base.op is Ops.RETURNED) + # legacy compatibility: TUPLE/GETTUPLE are gone. a tuple of values called is call_outputs, gettuple is returned_outputs[i] + @staticmethod + def maketuple(*srcs:UOp) -> _LegacyTupleValues: return _LegacyTupleValues(srcs) def gettuple(self, idx:int) -> UOp: - in_tuple = self.src[0] if self.op is Ops.FUNCTION else self - assert in_tuple.op is Ops.TUPLE, f"gettuple requires FUNCTION or TUPLE source, got {self.op}" - return UOp(Ops.GETTUPLE, src=(self,), arg=idx) - def group(*srcs:UOp|None): # pylint: disable=no-self-argument + assert self.op is Ops.CALL and self.num_returned, f"gettuple requires a CALL with RETURNED outputs, got {self.op}" + return self.returned_outputs[idx] + def group(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument if len(srcs) == 1 and isinstance(srcs[0], UOp): return srcs[0] - return UOp(Ops.GROUP, src=tuple([x for x in srcs if x is not None])) + return UOp(Ops.GROUP, src=tuple([x for x in srcs if x is not None]), **kwargs) def index(self, *srcs:UOp|int|None, **kwargs): new_srcs: list[UOp] = [UOp.const(x) if isinstance(x, int) else x for x in srcs if x is not None] if len(new_srcs) == 1 and new_srcs[0].op is Ops.CONST and self.op is Ops.STACK: return self.src[new_srcs[0].val] @@ -596,9 +592,9 @@ class UOp(RandMixin, metaclass=UOpMetaClass): def const_like(self, b:ConstLike, dtype:DType|None=None): ret = UOp.const(b, dtype or self.dtype) return ret._mop(Ops.EXPAND, arg=self._shape) if self._shape and ret._shape != self._shape else ret - def vconst_like(self, b:ConstLike, dtype:DType|None=None): + def vconst_like(self, b:ConstLike): # for use after movement ops have been removed - return UOp.const(b, dtype or self.dtype).broadcast(self.max_numel()) + return UOp.const(b, self.dtype).broadcast(self.max_numel()) def ufix(self, x): if isinstance(x, UOp): return x return UOp.const(x) @@ -614,7 +610,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @property def without_after(self) -> UOp: return self.src[0] if self.op is Ops.AFTER else self def barrier(self, *src:UOp): return UOp(Ops.BARRIER, src=(self,)+src) - def ins(self, arg, **kwargs): return UOp(Ops.INS, kwargs.pop("dtype", self.dtype), kwargs.pop("src", self.src), arg, kwargs.pop("tag", self.tag)) + def ins(self, arg, **kwargs): return UOp(Ops.INS, kwargs.pop("src", self.src), (arg, kwargs.pop("dtype", self.dtype)), kwargs.pop("tag", self.tag)) def contract(self, *rngs:UOp): assert all(x.arg[-1] == AxisType.UPCAST for x in rngs), "all contract ranges must be upcast" return UOp.stack(*[self.substitute(dict(zip(rngs, [r.const_like(i) for r,i in zip(rngs, idx)]))) @@ -626,14 +622,20 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if isinstance(b, UOp): return b.cast(dtype) # NOTE: it always has to be STACK now, even if they are all the same if isinstance(b, tuple): return UOp.stack(*[UOp.const(c, dtype) for c in b]) - return UOp(Ops.CONST, dtype, arg=dtype.const(b), src=()) + # .cast folds away at exactly the dtypes a CONST derives (bool/weakint/weakfloat): bare there, the pair everywhere else + return UOp(Ops.CONST, arg=dtype.const(b), src=()).cast(dtype) + # cast, except for CONST, in which case rebuild a new CONST at the dtype + def ccast(self, dtype:DType): return UOp.const(self.val, dtype) if self.op is Ops.CONST else self.cast(dtype) + # a forced CAST for bool: .cast(bool) folds, so UOp.const cannot state the width + @staticmethod + def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, src=(UOp.const(b),), arg=dtype) @staticmethod def range(end:sint, axis_id, axis_type=AxisType.WEAK, *arg, dtype=dtypes.weakint, src=(), **kwargs): return UOp(Ops.RANGE, src=(sint_to_uop(end, dtype),)+src, arg=(axis_id, axis_type)+arg, **kwargs) @staticmethod def loop(axis_id:int, *arg): return UOp(Ops.RANGE, src=(UOp(Ops.NOOP),), arg=(axis_id, AxisType.WEAK)+arg) @staticmethod - def special(end:sint, name:str, dtype=dtypes.weakint): return UOp(Ops.SPECIAL, src=(sint_to_uop(end, dtype),), arg=name) + def special(end:sint, name:str): return UOp(Ops.SPECIAL, src=(sint_to_uop(end),), arg=name) @staticmethod def wmma(a:UOp, b:UOp, acc:UOp, dims:tuple[int, int, int], device:str, threads:int, tc_upcast_axes=None): # dtype_in is stored in the arg (not derived from src[0].dtype) because bitcast rewrites change src dtypes @@ -708,11 +710,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op is Ops.UNSHARD: if len(self.arg) != 1: raise RuntimeError(f"UOp is sharded on multiple axes {self.arg}, use .sharding") return self.arg[0] - # GETTUPLE: axis comes from the specific TUPLE element, not src[0] - if self.op is Ops.GETTUPLE: - in_tuple = self.src[0].src[0] if self.src[0].op is Ops.FUNCTION else self.src[0] - return in_tuple.src[self.arg].axis if in_tuple.op is Ops.TUPLE else None - if self.op is Ops.PARAM: return self.arg.axis + if self.op is Ops.PARAM: return None # NOTE: they all have to share an axis, we always choose [-1]. src axes are right-aligned into the output shape if self.op in GroupOp.ALU.union({Ops.STACK}): return axes[-1] if (axes := dedup([x.axis+len(self.shape)-len(x.shape) for x in self.src if x.axis is not None])) else None @@ -781,6 +779,13 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op is Ops.UNSHARD: return self.src[0].base return self + # the storage this uop ultimately targets: base with UNSHARD, BITCAST and AFTER stripped + @property + def storage_base(self) -> UOp: + b = self.unsharded_base + while b.op in {Ops.BITCAST, Ops.AFTER, Ops.UNSHARD}: b = b.src[0].unsharded_base + return b + # cached property here makes external_uop_gc fail, why? @property def as_shape(self) -> tuple[sint, ...]: @@ -807,11 +812,9 @@ class UOp(RandMixin, metaclass=UOpMetaClass): case Ops.PAD | Ops.SHRINK: src_args = list(zip(*arg)) case Ops.PERMUTE | Ops.FLIP: src_args = [] case Ops.STACK: - # arg is the other srcs; all are cast to the promoted dtype, spec requires STACK srcs to match its dtype srcs = (self,)+tuple(arg) - dtype = cast(DType, dtype_from_uop(Ops.STACK, srcs, None)) - # TODO: why cast here? - return UOp(Ops.STACK, dtype, tuple(u if u.base.is_invalid else u.cast(dtype) for u in srcs)) + dtype = dtype_from_uop(Ops.STACK, srcs, None) + return UOp(Ops.STACK, src=tuple(u if u.base.is_invalid else u.ccast(dtype) for u in srcs)) case _: raise RuntimeError(f"{op} is not a MovementOp") usrcs = [shape_to_shape_arg(arg) for arg in src_args] if len(usrcs) == 0: return UOp(op, src=(self,), arg=arg) @@ -822,13 +825,14 @@ class UOp(RandMixin, metaclass=UOpMetaClass): unique_num = itertools.count(0) def getaddr(self, device=None) -> UOp: - if self.without_after.op not in {Ops.BUFFER, Ops.SLICE, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self + if self.without_after.op not in {Ops.BUFFER, Ops.SHRINK, Ops.BITCAST, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM, Ops.LINEAR}: return self return UOp(Ops.GETADDR, src=(self,), arg=device or to_tuple(self.device)[0]) @staticmethod def new_buffer(device:str|tuple[str, ...], size:int, dtype:DType, num=None): if dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {dtype}") + assert isinstance(size, int), f"new_buffer size must be a concrete int, got {size}" slot = next(UOp.unique_num) if num is None else num - return UOp(Ops.BUFFER, src=(shape_to_shape_arg((size,)),), arg=ParamArg(slot, dtype, device=device)) + return UOp(Ops.BUFFER, arg=ParamArg(slot, dtype, size=size, device=device)) @staticmethod def from_buffer(opaque:Buffer, device:str|tuple[str, ...]|None=None): if (uop:=UOp.new_buffer(device or opaque.device, opaque.size, opaque.dtype, num=-id(opaque))) not in buffers: buffers[uop] = opaque.ref(1) @@ -837,7 +841,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): def empty_like(self, dtype:DTypeLike|None=None, device:str|tuple[str, ...]|None=None) -> UOp: device = canonicalize_device(self.device if device is None else device) axis = self.axis if isinstance(device, tuple) else None - ret = UOp.empty(self.shard_shape if axis is not None else self.shape, dtype=strong_dtype(self.dtype) if dtype is None else dtype, device=device) + ret = UOp.empty(self.shard_shape if axis is not None else self.shape, dtype=self.commit_dtype() if dtype is None else dtype, device=device) return ret.unshard(axis) if axis is not None else ret @staticmethod def _frompy(x:list|tuple|bytes, dtype:DType, device:str|tuple[str, ...]|None=None) -> UOp: @@ -859,7 +863,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): return ret.after(ret.store(src.cast(ret.dtype))) @recursive_property def device(self) -> str|tuple[str, ...]|None: - if self.op is Ops.PARAM: return self.arg.device + if self.op in (Ops.PARAM, Ops.RETURNED): return self.arg.device if self.op is Ops.STAGE: return self.arg.device if self.op is Ops.AFTER: return self.src[0].device if self.op is Ops.MSELECT: @@ -879,7 +883,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): return self.device is None or self.dtype in dtypes.weaks @recursive_property def addrspace(self) -> AddrSpace|None: - if self.op is Ops.PARAM: return self.arg.addrspace + if self.op in (Ops.PARAM, Ops.RETURNED): return self.arg.addrspace if self.op is Ops.BUFFER: return self.arg.addrspace if self.op in {Ops.SPECIAL, Ops.RANGE}: return AddrSpace.ALU if self.op is Ops.LOAD: return AddrSpace.ALU # LOAD brings things into the ALU @@ -893,7 +897,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): return None @property def buf_uop(self) -> UOp: - if self.op in {Ops.BUFFER, Ops.PARAM}: return self + if self.op in {Ops.BUFFER, Ops.PARAM, Ops.RETURNED}: return self if self.op is Ops.MSELECT: return self.src[0].buf_uop.mselect(self.arg) if self.op is Ops.MSTACK: return UOp(Ops.MSTACK, src=tuple(x.buf_uop for x in self.src)) if self.base.op is Ops.AFTER: return self.base.src[0].buf_uop.base @@ -901,9 +905,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): while len(s.src) and s.op not in {Ops.BUFFER, Ops.PARAM, Ops.STAGE, Ops.MSTACK}: s = s.src[0] return s - def contiguous_view_offset(self) -> int|None: - """If movement ops on a BUFFER collapse to a contiguous range, return `offset` in elements. Otherwise None.""" - from tinygrad.schedule.rangeify import pm_mops + def contiguous_view(self) -> tuple[UOp, int]|None: + from tinygrad.schedule.prepare import pm_mops from tinygrad.uop.symbolic import symbolic # WEBGPU and CL do not support views. @@ -915,14 +918,17 @@ class UOp(RandMixin, metaclass=UOpMetaClass): idx = self.flatten().index(UOp.range(self.numel(), 0)) out = graph_rewrite(idx, pm_mops+symbolic+pm_contiguous_view_offset, ctx=self, name="contiguous_view_offset") - return out.val if out.op is Ops.CONST and isinstance(out.val, int) else None + if out.op is not Ops.INDEX or not (b:=out.src[0]).tag or (c:=out.src[1]).op is not Ops.CONST or not isinstance(c.val, int): return None + return b.rtag(None), c.val + + def contiguous_view_offset(self) -> int|None: return None if (view := self.contiguous_view()) is None else view[1] def has_buffer_identity(self, after_ok=False): """Check if this UOp has a concrete buffer identity in the graph (RESHAPE/UNSHARD -> BUFFER chain).""" # TODO: this is confusing because UOp.variable('v', 0, 1, dtypes.weakfloat) is True for jit to work, but it doesn't have a buffer if self.op in {Ops.RESHAPE, Ops.UNSHARD, Ops.MSELECT}: return self.src[0].has_buffer_identity(after_ok) if after_ok and self.op == Ops.AFTER: return self.src[0].has_buffer_identity(after_ok) - return self.op in {Ops.BUFFER, Ops.SLICE, Ops.PARAM} + return self.op in {Ops.BUFFER, Ops.PARAM} def _base_buffer_is_realized(self) -> bool: """Walk through AFTER chain to find if the underlying buffer is realized (has allocated memory).""" @@ -932,30 +938,19 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @property def buffer(self) -> Buffer|MultiBuffer: - if self.op in {Ops.CONTIGUOUS, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer + if self.op in {Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer # this buffer can process disk tensors and simple movement ops - if self is not self.base: - buf = self.base.buffer - assert isinstance(buf, Buffer), "must be a Buffer for movement ops" - offset = self.contiguous_view_offset() - if offset is None: raise RuntimeError(f"non-contiguous view is not supported for {buf.device} buffer") - return buf.view(prod(self.max_shape), self.dtype, offset*self.dtype.itemsize) - if self.op is Ops.BITCAST: - buf = self.src[0].buffer - assert isinstance(buf, Buffer), "must be a Buffer for BITCAST" - return buf.view(prod(self.max_shape), self.dtype, 0) - if self.op is Ops.SLICE: + if self is not self.base or self.op is Ops.BITCAST: if (cret:=buffers.get(self)) is not None: return cret - buf = self.src[0].buffer - offset = self.src[1].val + if (cv := self.contiguous_view()) is None: raise RuntimeError(f"non-contiguous view is not supported for {self.device} buffer") + buf, offset = (b:=cv[0]).base.buffer, cv[1] if isinstance(buf, MultiBuffer): mbuf = MultiBuffer.__new__(MultiBuffer) - mbuf.bufs = [b.view(self.arg, self.dtype, offset * self.src[0].dtype.itemsize) for b in buf.bufs] + mbuf.bufs = [x.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) for x in buf.bufs] buffers[self] = mbuf return mbuf - assert isinstance(buf, Buffer), "must be a Buffer for SLICE" - buffers[self] = bv = buf.view(self.arg, self.dtype, offset * self.src[0].dtype.itemsize) - return bv + buffers[self] = buf.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) + return buffers[self] if self.op is Ops.MSELECT: ret = self.src[0].buffer assert isinstance(ret, MultiBuffer) @@ -977,8 +972,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op is Ops.UNSHARD: return self.src[0].realized # only these can be realized if self.op not in (Ops.BUFFER, Ops.MSTACK): return None - # LOCAL/REG scratch buffers are never realized - if self.op is Ops.BUFFER and self.addrspace in (AddrSpace.LOCAL, AddrSpace.REG): return None + # LOCAL/REG scratch buffers are never realized, and Variables (ALU) have no real storage + if self.op is Ops.BUFFER and self.addrspace in (AddrSpace.LOCAL, AddrSpace.REG, AddrSpace.ALU): return None # an unbacked intermediate BUFFER (directly or as an MSTACK source) is not realized if any(b.op is Ops.BUFFER and buffers.get(b) is None for b in self.backward_slice_with_self): return None # NOTE: this is used by the JIT to determine which inputs we capture @@ -989,29 +984,42 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # *** uop Variable stuff *** @staticmethod - def variable(name:str, min_val:PyConst, max_val:PyConst, dtype:DType=dtypes.weakint, multiple_of:int=1) -> UOp: - return UOp(Ops.PARAM, src=(shape_to_shape_arg(()),), - arg=ParamArg(-1, dtype, name=name, vmin_vmax=(min_val, max_val), multiple_of=multiple_of, addrspace=AddrSpace.ALU)) + def variable(name:str, min_val:PyConst, max_val:PyConst, dtype:DType=dtypes.weakint, multiple_of:int=1, param:bool=False) -> UOp: + # a Variable is a 0-d BUFFER in the ALU addrspace; binding it is storing a CONST into it + # param=True creates the kernel-side form directly: an ALU PARAM (what the BUFFER becomes inside kernels) + arg = ParamArg(-1, dtype, name=name, vmin_vmax=(min_val, max_val), multiple_of=multiple_of, addrspace=AddrSpace.ALU) + return UOp(Ops.PARAM if param else Ops.BUFFER, arg=arg) + @property + def is_variable(self) -> bool: + # a Variable is a 0-d BUFFER in the ALU addrspace that carries a value range (it becomes a PARAM inside kernels) + return self.op is Ops.BUFFER and isinstance(self.arg, ParamArg) and \ + self.arg.vmin_vmax is not None and self.arg.addrspace is AddrSpace.ALU and self._shape == () + @property + def is_bound_var(self) -> bool: + # a bound Variable is bind()'s AFTER(var, STORE(var, CONST)) + return self.op is Ops.AFTER and self.src[0].is_variable and self.src[1].op is Ops.STORE and \ + self.src[1].src[0] is self.src[0] and self.src[1].src[1].op is Ops.CONST and len(self.src) == 2 @property def expr(self) -> str: - assert self.op is Ops.PARAM + assert self.op in {Ops.PARAM, Ops.BUFFER} return unwrap(self.arg.name) def bind(self, val:int|UOp): - assert self.op is Ops.PARAM and self.addrspace is AddrSpace.ALU, f"op is {self.op}, need PARAM" - uval = self.const_like(val) if isinstance(val, int) else val + assert self.is_variable, f"op is {self.op}, need Variable" + # the Variable states the width, so the bound value stays bare: is_bound_var tests for a CONST there, unbind reads .val + uval = UOp.const(val) if isinstance(val, int) else val assert self.vmin <= uval.vmin and uval.vmax <= self.vmax, f"bind {val} not in range [{self.vmin}, {self.vmax}]" assert uval.divides(self.arg.multiple_of) is not None, f"bind {val} not divisible by {self.arg.multiple_of}" - return UOp(Ops.BIND, src=(self, uval)) + return self.after(self.store(uval)) def unbind(self) -> tuple[Variable, int]: - assert self.op is Ops.BIND and self.src[0].op is Ops.PARAM and self.src[1].op is Ops.CONST, f"can't unbind {self}" - return self.src[0], self.src[1].val + assert self.is_bound_var, f"can't unbind {self}" + return self.src[0], self.src[1].src[1].val def unbind_all(self) -> tuple[UOp, dict[Variable, int]]: ret:dict[Variable, int] = {} return graph_rewrite(self, pm_unbind, ctx=ret), ret def variables(self) -> list[Variable]: - return sorted({x if x.op is Ops.PARAM else UOp.variable("_device_num", 0, x.vmax, dtype=x.dtype) - for x in self.backward_slice_with_self if (x.op is Ops.RANGE and x.arg[-1] is AxisType.DEVICE) or x.op is Ops.PARAM - and x.arg.addrspace is AddrSpace.ALU}, key=lambda v: v.expr) + return sorted({x if x.op in {Ops.PARAM, Ops.BUFFER} else UOp.variable("_device_num", 0, x.vmax, dtype=x.dtype, param=True) + for x in self.backward_slice_with_self if (x.op is Ops.RANGE and x.arg[-1] is AxisType.DEVICE) or + (x.op is Ops.PARAM and x.arg.addrspace is AddrSpace.ALU) or x.is_variable}, key=lambda v: v.expr) # *** uop symbolic stuff *** @@ -1022,7 +1030,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op is Ops.STACK: return math.gcd(*[x.const_factor() for x in self.src]) if self.op is Ops.ADD: return math.gcd(self.src[0].const_factor(), self.src[1].const_factor()) if self.op is Ops.MUL: return self.src[0].val if self.src[0].op is Ops.CONST else self.src[1].val if self.src[1].op is Ops.CONST else 1 - if self.op is Ops.PARAM and self.arg.multiple_of is not None: return self.arg.multiple_of + if self.op in (Ops.PARAM, Ops.BUFFER) and isinstance(self.arg, ParamArg) and self.arg.multiple_of is not None: return self.arg.multiple_of return 1 def divides(self, v:int) -> UOp|None: if v==1: return self @@ -1034,7 +1042,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op is Ops.MUL: if (d0:=self.src[0].divides(v)) is not None: return d0 * self.src[1] if (d1:=self.src[1].divides(v)) is not None: return self.src[0] * d1 - if self.op is Ops.PARAM and self.arg.multiple_of is not None: return self // v if self.arg.multiple_of%v == 0 else None + if self.op in (Ops.PARAM, Ops.BUFFER) and isinstance(self.arg, ParamArg) and self.arg.multiple_of is not None: + return self // v if self.arg.multiple_of%v == 0 else None return None # generic None if we aren't sure def pop_const(self, op=Ops.ADD) -> tuple[UOp, PyConst]: # NOTE: assume Invalid ALU is resolved return (self.src[0], self.src[1].val) if self.op is op and self.src[1].op is Ops.CONST else (self, identity_element(op, self.dtype)) @@ -1099,12 +1108,12 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # float has NAN issue and we use explicit NAN in transcendental if self.op is Ops.WHERE and dtypes.is_int(self.dtype): return min(self.src[1].vmin, self.src[2].vmin), max(self.src[1].vmax, self.src[2].vmax) # NOTE: returned UOp is assumed to be CONST - if self.op is Ops.PARAM and self.arg.vmin_vmax is not None: return self.arg.vmin_vmax + if self.op in (Ops.PARAM, Ops.BUFFER) and isinstance(self.arg, ParamArg) and self.arg.vmin_vmax is not None: return self.arg.vmin_vmax if self.op in (Ops.RANGE, Ops.SPECIAL) and self.dtype is not dtypes.void: return 0, (self.src[0]-1).vmax - if self.op is Ops.BIND: return self.src[0]._min_max # ignore the bound value if self.op is Ops.STACK: return min(x.vmin for x in self.src), max(x.vmax for x in self.src) if self.op is Ops.CONST and self.val is not Invalid: return self.val, self.val - if self.op is Ops.INDEX: return self.src[0]._min_max + if self.op is Ops.PAD: return min(self.src[0].vmin, 0), max(self.src[0].vmax, 0) # PAD adds zeros + if self.op in GroupOp.Movement|{Ops.INDEX, Ops.STAGE, Ops.AFTER, Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD}: return self.src[0]._min_max if self.op is Ops.CAST: # rounding is monotone (truncation toward zero into an int, to-nearest onto the value grid into a float) smin, smax = self.src[0]._min_max @@ -1118,7 +1127,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): def _sym_fxn(self): from tinygrad.uop.render import _render_with_splits, renderer_infer sself = self.simplify() - varnames = tuple(dedup(x.expr for x in sself.toposort() if x.op is Ops.PARAM and x.arg.addrspace == AddrSpace.ALU)) + varnames = tuple(dedup(x.expr for x in sself.toposort() if (x.op is Ops.PARAM and x.arg.addrspace == AddrSpace.ALU) or x.is_variable)) # TODO: sanitize varnames, or don't use naked eval while staying fast ret = _render_with_splits(list(sself.toposort()), renderer_infer, {sself}) lines = [f" {k}={v}" for k,v in ret.items() if k != "ast"] + [f" return {ret['ast']}"] @@ -1145,14 +1154,16 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # *** uop high level syntactic sugar *** @staticmethod - def placeholder(shape:tuple[int, ...], dtype:DType, slot:int, addrspace=AddrSpace.GLOBAL, device=None, volatile=False): + def placeholder(shape:tuple[int, ...], dtype:DType, slot:int|None=None, addrspace=AddrSpace.GLOBAL, device=None, volatile=False, tag=None): dtype = strong_dtype(dtype) # storage is never weak: a placeholder commits the width of what's put in it + if slot is None: slot = next(UOp.unique_num) if addrspace is AddrSpace.GLOBAL: - ret = UOp(Ops.PARAM, src=(shape_to_shape_arg((prod(shape),)),), arg=ParamArg(slot, dtype, addrspace=addrspace, device=device,volatile=volatile)) + ret = UOp(Ops.PARAM, arg=ParamArg(slot, dtype, size=prod(shape), addrspace=addrspace, device=device, volatile=volatile)) else: assert addrspace in (AddrSpace.LOCAL, AddrSpace.REG) assert device is None, "LOCAL and REG placeholders cannot have a device" - ret = UOp(Ops.BUFFER, src=(shape_to_shape_arg((prod(shape),)),), arg=ParamArg(slot, dtype, addrspace=addrspace)) + ret = UOp(Ops.BUFFER, arg=ParamArg(slot, dtype, size=prod(shape), addrspace=addrspace)) + if tag is not None: ret = ret.rtag(tag) if len(shape) > 1: ret = ret.reshape(shape) return ret def placeholder_like(self, slot:int, addrspace=AddrSpace.GLOBAL): @@ -1165,34 +1176,70 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # TODO: this should replace placeholder @staticmethod - def param(slot:int, dtype:DType, shape:tuple[sint, ...]|None=None, device=None, vmin_vmax:tuple[PyConst, PyConst]|None=None, - multiple_of:int|None=None, name=None, addrspace=AddrSpace.GLOBAL, axis:int|None=None, volatile:bool=False): + def param(slot:int, dtype:DType, shape:tuple[sint, ...]|sint|None=None, device=None, vmin_vmax:tuple[PyConst, PyConst]|None=None, + multiple_of:int|None=None, name=None, addrspace=AddrSpace.GLOBAL, volatile:bool=False): + """create a PARAM: a single sint or 1-d shape gives a flat param of that size, a None shape gives a scalar param. + the arg only stores the concrete max size (never symbolic): a multi-dim shape is a RESHAPE on top of the flat param, + a symbolic shape is a max-size param shrunk to the real shape""" if dtype in dtypes.weaks: raise RuntimeError(f"cannot create param for weak dtype {dtype}") - if shape is not None and axis is not None and isinstance(device, tuple): - shape = tuple(s*len(device) if i == axis else s for i,s in enumerate(shape)) - src: tuple[UOp, ...] = (UOp(Ops.NOOP) if shape is None else shape_to_shape_arg(shape),) - return UOp(Ops.PARAM, src=src, arg=ParamArg(slot, dtype, vmin_vmax, multiple_of, name, addrspace, axis, device, volatile)) + if isinstance(shape, (int, UOp)): shape = (shape,) + if shape is None or len(shape) == 0: + return UOp(Ops.PARAM, arg=ParamArg(slot, dtype, None, vmin_vmax, multiple_of, name, addrspace, device, volatile)) + max_shape = to_max_shape(shape) + ret = UOp(Ops.PARAM, arg=ParamArg(slot, dtype, prod(max_shape), vmin_vmax, multiple_of, name, addrspace, device, volatile)) + return ret.view_as(shape) def param_like(self, slot:int): - if self.op is Ops.BIND: return self.src[0].replace(arg=replace(self.src[0].arg, slot=slot, name=f"p{slot}")) + # Variables become ALU params in the call body; the stored value (if bound) stays in the call args + if self.is_bound_var or self.is_variable: + b = self.src[0] if self.op is Ops.AFTER else self + return UOp(Ops.PARAM, arg=replace(b.arg, slot=slot, name=f"p{slot}")) addrspace = self.addrspace if self.addrspace is not None else AddrSpace.GLOBAL - return UOp.param(slot, self.dtype, self.shard_shape if self.axis is not None else self._shape, self.device, addrspace=addrspace, axis=self.axis) + # multi-device values become a per-shard sized param wrapped in UNSHARD: the sharding lives in the graph, not the arg + if self.axis is not None and isinstance(self.device, tuple): + return UOp(Ops.PARAM, arg=ParamArg(slot, self.dtype, prod(to_max_shape(self.shard_shape)), + addrspace=addrspace, device=self.device)).view_as(self.shard_shape, self.axis) + return UOp.param(slot, self.dtype, self._shape, self.device, addrspace=addrspace) + def view_as(self:UOp, shape:tuple[sint, ...], axis:int|None=None) -> UOp: + """view flat storage as the given (possibly symbolic) shape, optionally sharded on axis, the UNSHARD gives back the multiplied shape""" + max_shape = to_max_shape(shape) + ret = self.reshape(max_shape) if len(shape) > 1 else self + if tuple(max_shape) != tuple(shape): ret = ret.shrink_to(shape) + return ret if axis is None else ret.unshard(axis) @staticmethod def custom_function(name:str, *src:UOp) -> UOp: return UOp(Ops.CUSTOM_FUNCTION, src=src, arg=name) - # opaque bodies stay as Ops.CALL; value-producing bodies become Ops.FUNCTION (wrapped in TUPLE) - _OPAQUE_CALL_BODIES = {Ops.SINK, Ops.PROGRAM, Ops.LINEAR, Ops.COPY, Ops.SLICE, Ops.CUSTOM_FUNCTION} + # opaque bodies are just CALLs; value-producing bodies become CALLs with RETURNED placeholders as extra inputs + _OPAQUE_CALL_BODIES = {Ops.SINK, Ops.PROGRAM, Ops.LINEAR, Ops.COPY, Ops.CUSTOM_FUNCTION} def call(self, *srcs:UOp, ret_dtype:DType|None=None, grad_fxn:Callable|None=None, name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None) -> UOp: - if ret_dtype is not None: return UOp(Ops.CALL, ret_dtype, src=(self,)+srcs) + if ret_dtype is not None: return UOp(Ops.CALL, src=(self,)+srcs, arg=ret_dtype) # calls are launched per device, so an open DEVICE range is allowed to cross the call boundary assert all(r.arg[-1] is AxisType.DEVICE for r in self.ranges), \ f"ranges {self.ranges} are leaking out of the call in {self.pyrender()}" if self.op in UOp._OPAQUE_CALL_BODIES: return UOp(Ops.CALL, src=(self,)+srcs, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) - # value-producing bodies are always wrapped in TUPLE so FUNCTION dtype is always void - body = self if self.op is Ops.TUPLE else UOp.maketuple(self) - return UOp(Ops.FUNCTION, src=(body,)+srcs, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) + # value-producing bodies delegate to call_outputs with a single output + return UOp.call_outputs((self,), *srcs, grad_fxn=grad_fxn, name=name, precompile=precompile, + precompile_backward=precompile_backward, aux=aux) + + @staticmethod + def call_outputs(values:tuple[UOp, ...], *srcs:UOp, grad_fxn:Callable|None=None, + name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None) -> UOp: + """call a body producing the given values: the body stores into output PARAMs, and the outputs are RETURNED + placeholders that are inputs to the call (you AFTER on them like normal buffers). the RETURNEDs are bound to the + output PARAMs positionally wherever the call is resolved, just like the args are bound to the input PARAMs""" + # the device defaults to the first device in the values or args, like srcs-based device resolution + default_dev = next((x.device for x in itertools.chain(values, srcs) if x.device is not None), None) + # the RETURNED storage has the resolved shape: substitute internal PARAMs in the shapes with corresponding args + def returned(o:UOp, i:int) -> UOp: + return UOp.returned(len(srcs)+i, o.dtype, None if (shp:=o._shape) is None else + tuple(graph_rewrite(s, _pm_resolve_params, srcs, walk=True) if isinstance(s, UOp) else s for s in shp), + dev if (dev:=o.device) is not None else default_dev, o.axis if isinstance(o.device, tuple) else None) + rets = tuple(returned(o, i) for i, o in enumerate(values)) + # the body only knows PARAMs: the output PARAMs get the slots right after the input PARAM slots + body = UOp.sink(*[v.param_like(len(srcs)+i).store(v) for i, v in enumerate(values)]) + return UOp(Ops.CALL, src=(body,)+srcs+rets, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) def custom_kernel(*srcs:UOp, fxn:Callable, grad_fxn:Callable|None=None) -> list[UOp]: placeholders = [UOp.placeholder_like(s, slot=i) for i,s in enumerate(srcs)] kernel = fxn(*placeholders).call(*srcs, grad_fxn=grad_fxn) @@ -1202,13 +1249,11 @@ class UOp(RandMixin, metaclass=UOpMetaClass): assert self.op is Ops.PROGRAM and isinstance(self.arg, ProgramInfo), "to_elf should only be called on a PROGRAM ast" sig = tuple((u.arg.name, u.arg.slot, u.dtype, u._shape) for u in tuple(filter(lambda u: u.op is Ops.PARAM and u.addrspace != AddrSpace.ALU, self.src[1].src)) + self.arg.vars) - return TinyELF(self.src[3].arg, self.arg.function_name, self.arg.target, sig) + return TinyELF(self.src[3].arg, self.arg.function_name, self.arg.target, sig, self.key) @dataclass(frozen=True) class KernelInfo: name: str = "test" # name of the kernel - axis_types: tuple[AxisType, ...] = tuple() - dont_use_locals: bool = False # don't use local indexing applied_opts: tuple = tuple() opts_to_apply: tuple|None = None estimates: Estimates|None = None @@ -1220,7 +1265,7 @@ class KernelInfo: class ProgramInfo: name: str = "test" global_size: tuple[int|float, ...] = (1, 1, 1) - local_size: tuple[int, ...]|None = None + local_size: tuple[int, ...] = (1, 1, 1) vars: tuple[UOp, ...] = () globals: tuple[int, ...] = () outs: tuple[int, ...] = () @@ -1230,16 +1275,13 @@ class ProgramInfo: @property def function_name(self): return to_function_name(self.name) - @property - def runtimevars(self) -> dict[str, int]: return {v.expr: i for i, v in enumerate(self.vars) if v.expr == 'core_id'} - - def launch_dims(self, var_vals:dict[str, int]) -> tuple[tuple[int, ...], tuple[int, ...]|None]: + def launch_dims(self, var_vals:dict[str, int]) -> tuple[tuple[int, ...], tuple[int, ...]]: global_size = tuple([sym_infer(sz, var_vals) for sz in self.global_size]) # type: ignore[arg-type] - local_size = tuple([sym_infer(sz, var_vals) for sz in self.local_size]) if self.local_size is not None else None + local_size = tuple([sym_infer(sz, var_vals) for sz in self.local_size]) return global_size, local_size - def vals(self, var_vals:dict[str, int]) -> tuple[int|None, ...]: - try: return tuple(var_vals[k.expr] if k.expr not in self.runtimevars else None for k in self.vars) + def vals(self, var_vals:dict[str, int]) -> tuple[int, ...]: + try: return tuple(var_vals[k.expr] for k in self.vars) except KeyError as e: raise RuntimeError(f"unbound Variable {e} used by {self.function_name}") from None @staticmethod @@ -1249,21 +1291,17 @@ class ProgramInfo: outs: list[int] = [] ins: list[int] = [] global_size: list[int] = [1, 1, 1] - local_size: list[int]|None = [1, 1, 1] + local_size: list[int] = [1, 1, 1] for u in sink.toposort(): if u.op is Ops.PARAM and u.addrspace == AddrSpace.ALU: _vars.append(u) if u.op is Ops.PARAM and u.addrspace != AddrSpace.ALU: _globals.append(u.arg.slot) if u.op in (Ops.STORE, Ops.LOAD): if (idx:=u.src[0]).op in (Ops.INDEX, Ops.SHRINK) or (u.src[0].op is Ops.CAST and (idx:=u.src[0].src[0]).op is Ops.INDEX): if (buf:=idx.src[0].buf_uop).op is Ops.PARAM: (outs if u.op is Ops.STORE else ins).append(buf.arg.slot) - if u.op is Ops.SPECIAL: - if u.arg[0] == 'i': local_size = None - special_size = local_size if u.arg[0] == 'l' else global_size - if special_size is not None: special_size[int(u.arg[-1])] = cast(int, u.src[0].ssimplify()) - if u.op is Ops.PARAM and u in _vars and u.expr == 'core_id': global_size[0] = int(u.vmax) + 1 - return ProgramInfo(sink.arg.name if isinstance(sink.arg, KernelInfo) else "test", tuple(global_size), - tuple(local_size) if local_size is not None else None, tuple(sorted(dedup(_vars), key=lambda v: v.arg.slot)), - tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), tuple(sorted(dedup(ins))), target) + if u.op is Ops.SPECIAL: (local_size if u.arg[0] == 'l' else global_size)[int(u.arg[-1])] = cast(int, u.src[0].ssimplify()) + return ProgramInfo(sink.arg.name if isinstance(sink.arg, KernelInfo) else "test", tuple(global_size), tuple(local_size), + tuple(sorted(dedup(_vars), key=lambda v: v.arg.slot)), tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), + tuple(sorted(dedup(ins))), target) @dataclass(frozen=True) class CallInfo: @@ -1319,7 +1357,7 @@ def get_location() -> tuple[str, int]: frm = frm.f_back return frm.f_code.co_filename, frm.f_lineno -class UPat(OpMixin): +class UPat(RandMixin): __slots__ = ("op", "match_dtype", "match_tag", "arg", "name", "src", "is_any") def __init__(self, op:Ops|tuple[Ops, ...]|set[Ops]|None=None, dtype:DType|tuple[DType, ...]|set[DType]|None=None, src:tuple[UPat, ...]|list[UPat]|UPat|None=None, arg:Any=None, @@ -1411,7 +1449,7 @@ class UPat(OpMixin): if self.is_any: return flatten([x.match(uop, store.copy()) for x in self.src[0]]) if (self.op is not None and uop.op not in self.op) or \ (self.name is not None and store.setdefault(self.name, uop) is not uop) or \ - (self.match_dtype is not None and uop.dtype not in self.match_dtype and uop.dtype.scalar() not in self.match_dtype) or \ + (self.match_dtype is not None and uop.dtype not in self.match_dtype) or \ (self.arg is not None and self.arg != uop.arg) or \ (self.match_tag is not None and uop.tag not in self.match_tag) or \ (len(uop.src) < self.required_len) or \ @@ -1427,9 +1465,9 @@ class UPat(OpMixin): return res def deconstruct_function(fxn:Callable) -> tuple: - new_globals = {k:v for k,v in fxn.__globals__.items() if k in fxn.__code__.co_names} - for co in fxn.__code__.co_consts: - if isinstance(co, types.CodeType): new_globals.update({k:v for k,v in fxn.__globals__.items() if k in co.co_names}) + # globals can be referenced from arbitrarily nested code objects (comprehensions/lambdas, pre PEP 709) + def names(co:types.CodeType) -> set: return set(co.co_names).union(*(names(c) for c in co.co_consts if isinstance(c, types.CodeType))) + new_globals = {k:v for k,v in fxn.__globals__.items() if k in names(fxn.__code__)} # NOTE: optional round trip through pickle! assert fxn.__closure__ is None, "closures are not supported in pattern matchers" ret = fxn.__code__, new_globals, fxn.__name__, fxn.__defaults__ @@ -1664,14 +1702,15 @@ class RewriteContext: continue # no rewrite, process children then come back to rebuild stack.append((n, True)) - if not self.enter_calls and (n.op is Ops.FUNCTION or (n.op is Ops.CALL and n.src[0].op in UOp._OPAQUE_CALL_BODIES)): + # calls with RETURNED inputs are always inlined into the enclosing graph, their bodies are never rewritten separately + if n.op is Ops.CALL and (n.num_returned or (not self.enter_calls and n.src[0].op in UOp._OPAQUE_CALL_BODIES)): self.replace[n.src[0]] = n.src[0] for x in reversed(n.src): if x not in self.replace: stack.append((x, False)) else: # rebuild node with rewritten srcs new_src = tuple(self.replace.get(x, x) for x in n.src) - new_n = UOp(n.op, _rebuild_dtype(n, new_src), new_src, n.arg, n.tag) if new_src != n.src else n + new_n = UOp(n.op, new_src, n.arg, n.tag) if new_src != n.src else n # top-down: try pm on rebuilt node, use result as-is (no re-traversal) if self.pm is not None and (rewritten:=self.pm_rewrite(new_n)) is not None: new_n = rewritten self.replace[n] = new_n @@ -1702,11 +1741,10 @@ class RewriteContext: if n in waitlist: stack.extend(waitlist.pop(n)) continue stack.append((n, 1, new_n)) - # NOTE: CALL/FUNCTION are handled as a special case. - # The function that is called is not included in the graph_rewrite. - # If you want to graph_rewrite a call, you can - # A CALL of an address is not a body, its srcs are regular dataflow - if not self.enter_calls and (new_n.op is Ops.FUNCTION or (new_n.op is Ops.CALL and new_n.src[0].op in UOp._OPAQUE_CALL_BODIES)): + # NOTE: CALLs are handled as a special case: the call body is not included in the graph_rewrite (a CALL of an + # address is not a body, its srcs are regular dataflow). calls with RETURNED inputs are always inlined into the + # enclosing graph, their bodies are never rewritten separately + if new_n.op is Ops.CALL and (new_n.num_returned or (not self.enter_calls and new_n.src[0].op in UOp._OPAQUE_CALL_BODIES)): self.replace[new_n.src[0]] = new_n.src[0] for x in reversed(new_n.src): if x in on_stack: continue @@ -1730,7 +1768,7 @@ class RewriteContext: continue else: # if srcs changed from rewrites, construct a new UOp with the new srcs - new_src_n = UOp(new_n.op, _rebuild_dtype(new_n, new_src), new_src, new_n.arg, new_n.tag) + new_src_n = UOp(new_n.op, new_src, new_n.arg, new_n.tag) # trigger a rewrite of new_src_n, then after that rewrite is done, link it back to n stack.append((n, 2, new_src_n)) stack.append((new_src_n, 0, new_src_n)) @@ -1750,16 +1788,19 @@ def graph_rewrite(sink:UOp, pm:PatternMatcher, ctx=None, bottom_up=False, name=N rewrite_ctx = RewriteContext(pm if not bottom_up else None, pm if bottom_up else bpm, ctx, enter_calls) return rewrite_ctx.walk_rewrite(sink) if walk else rewrite_ctx.unified_rewrite(sink) -def _rebuild_dtype(n:UOp, new_src:tuple[UOp,...]) -> DType: - # TODO: delete this once the dtype field is removed, every rebuild will re-derive - if all(a.dtype is b.dtype or b.base.is_invalid for a,b in zip(n.src, new_src)): return n.dtype - return dtype_from_uop(n.op, new_src, n.arg) or n.dtype def sint_to_uop(x:sint, dtype=dtypes.weakint) -> UOp: return UOp.const(x, dtype) def to_max_shape(shape:tuple[sint, ...]) -> tuple[int, ...]: return tuple(int(x.vmax) if isinstance(x, UOp) else x for x in shape) _substitute = PatternMatcher([(UPat(tuple(Ops), name="x"), lambda ctx,x: ctx.get(x,None))]) _pm_resolve_params = PatternMatcher([(UPat(Ops.PARAM, name="p"), lambda ctx,p: ctx[p.arg.slot])]) + +def resolve_returned_after(r:UOp, t:UOp) -> UOp|None: + """AFTER on a RETURNED placeholder extracts the call output value: the value of its matching store in a SINK body + (called from patterns that bind t to a SINK)""" + vals = [st.src[1] for st in t.src if st.op is Ops.STORE and st.src[0].unsharded_base is r.unsharded_base] \ + if r.unsharded_base.op is Ops.RETURNED else [] + return vals[0] if len(vals) == 1 else None remove_all_tags = PatternMatcher([(UPat(GroupOp.All, name="x"), lambda x: x.replace(tag=None) if x.tag is not None else None)]) def gate_kernel_sink(x:UOp) -> bool: @@ -1771,14 +1812,18 @@ def do_unbind(ctx:dict[Variable, int], x:UOp): v,i = x.unbind() ctx[v] = i return v -pm_unbind = PatternMatcher([(UPat(Ops.BIND, name="x"), do_unbind)]) +pm_unbind = PatternMatcher([(UPat(Ops.AFTER, name="x"), lambda ctx,x: do_unbind(ctx,x) if x.is_bound_var else None)]) # ctx is source UOp for which we are finding a contiguous view for. used in contiguous_view_offset pm_contiguous_view_offset = PatternMatcher([ - (UPat(Ops.INDEX, src=(UPat(),)), lambda: UOp.const(0)), - (UPat(Ops.INDEX, src=(UPat(), UPat(Ops.RANGE))), lambda: UOp.const(0)), - (UPat(Ops.INDEX, src=(UPat(), UPat(Ops.RANGE)+UPat.cvar('c'))), lambda c: c), - (UPat(Ops.INDEX, src=(UPat(), UPat.cvar('c'))), lambda ctx, c: c if resolve(ctx.numel() == 1, False) else None), + # normalize to 1d bitcasts + (UPat(Ops.BITCAST, name="b"), lambda b: b.src[0].flatten().bitcast(b.dtype).reshape(b.shape) if len(b.shape) != 1 else None), + (UPat(Ops.BITCAST, name="b").index(UPat.cvar("c")), lambda ctx, b, c: + b.src[0].flatten().index(UOp.range(ctx.numel() * (osz:=b.element_size())//(isz:=b.src[0].element_size()), 0) + (c * osz//isz)) if b.tag else None), + (UPat(Ops.INDEX, src=(UPat.var("b"),)), lambda b: b.rtag().index(0)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat(Ops.RANGE))), lambda b: b.rtag().index(0)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat(Ops.RANGE)+UPat.cvar('c'))), lambda ctx, b, c: b.rtag().index(c)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat.cvar('c'))), lambda ctx, b, c: b.rtag().index(c) if resolve(ctx.numel() == 1, False) else None), ]) # *** what was symbolic.py *** diff --git a/tinygrad_repo/tinygrad/uop/render.py b/tinygrad_repo/tinygrad/uop/render.py index 03133e1ce..c4d6db56b 100644 --- a/tinygrad_repo/tinygrad/uop/render.py +++ b/tinygrad_repo/tinygrad/uop/render.py @@ -1,6 +1,6 @@ from tinygrad.dtype import AddrSpace, dtypes from tinygrad.uop import Ops, GroupOp -from tinygrad.uop.ops import ParamArg, UOp, PatternMatcher, UPat, multirange_str, range_str, consumer_map_from_toposort +from tinygrad.uop.ops import ParamArg, UOp, PatternMatcher, UPat, multirange_str, range_str, consumer_map_from_toposort, sint from tinygrad.helpers import strip_parens def pretty_print(x:UOp, cache=None, d=0)->str: @@ -11,7 +11,7 @@ def pretty_print(x:UOp, cache=None, d=0)->str: if cache is None: dfs(x, cache:={}) if (cx:=cache.setdefault(x, [0,0,False]))[2]: return f"{' '*d}x{cx[0]}" cx[2], srcs = True, (''.join(f'\n{pretty_print(s, cache, d+2)},' for s in x.src)) - return f"{' '*d}{f'x{cx[0]}:=' * (cx[1]>1)}{type(x).__name__}({x.op}, {x.dtype}, arg={x.argstr()}{x.tagstr()}, src=({srcs}))" + return f"{' '*d}{f'x{cx[0]}:=' * (cx[1]>1)}{type(x).__name__}({x.op}, arg={x.argstr()}{x.tagstr()}, src=({srcs}))" # ***** uop helpers ***** @@ -33,12 +33,15 @@ def strip_binary_parens(x:UOp, left:str, right:str, code_for_op) -> str: renderer = PatternMatcher([ (UPat(Ops.PARAM, name="x"), lambda x: x.arg.name if x.arg.name is not None else f"p{x.arg.slot}"), + (UPat(Ops.BUFFER, name="x"), lambda x: x.arg.name if isinstance(x.arg, ParamArg) and x.arg.name is not None else f"b{x.arg.slot}"), + (UPat(Ops.AFTER, name="x"), lambda ctx,x: ctx[x.src[0]]), (UPat((Ops.SPECIAL), name="x"), lambda x: x.arg), (UPat(Ops.RANGE, dtypes.void, name="x"), lambda x: f"loop{x.arg[0]}"), (UPat(Ops.RANGE, name="x"), lambda x: f"r{range_str(x)}"), (UPat(Ops.CONST, name="x"), lambda x: str(x.val)), + # CAST states the width, the weak CONST carries the value + (UPat.cvar("c").cast(), lambda c: str(c.val)), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({str(x.dtype)[7:]})({ctx[x.src[0]]})"), - (UPat(Ops.BIND, name="x"), lambda ctx,x: ctx[x.src[0]]), (UPat(Ops.NEG, name="x"), lambda ctx,x: f"(-{ctx[x.src[0]]})"), (UPat(Ops.RECIPROCAL, name="x"), lambda ctx,x: f"(1/{ctx[x.src[0]]})"), (UPat(Ops.MAX, name="x"), lambda ctx,x: f"max({ctx[x.src[0]]}, {ctx[x.src[1]]})"), @@ -54,8 +57,6 @@ renderer = PatternMatcher([ ]) renderer_infer = PatternMatcher([ - (UPat(Ops.CMOD, name="x"), lambda ctx,x: f"cmod({ctx[x.src[0]]}, {ctx[x.src[1]]})"), - (UPat(Ops.CDIV, name="x"), lambda ctx,x: f"cdiv({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.FLOORMOD, name="x"), lambda ctx,x: f"floormod({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.FLOORDIV, name="x"), lambda ctx,x: f"floordiv({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.CAST, name="x"), @@ -66,23 +67,24 @@ renderer_infer = PatternMatcher([ # *** pyrender *** def srcs(ctx, src): return f"({ctx[src[0]]},)" if len(src) == 1 else f"({', '.join([ctx[x] for x in src])})" +# marg is ssimplify'd, so a bound can be a node this graph never contained +def marg_str(ctx, a:sint) -> str: return str(a) if not isinstance(a, UOp) else ctx[a] if a in ctx else a.render() + def render_marg(ctx,x:UOp): if x.op is Ops.PERMUTE: return str(x.marg) if x.op is Ops.FLIP: return str(tuple([i for i,x in enumerate(x.marg) if x])) pieces = [] - if x.op in {Ops.RESHAPE, Ops.EXPAND}: - pieces = [f"{ctx[a] if isinstance(a, UOp) else str(a)}" for a in x.marg] - if x.op in {Ops.PAD, Ops.SHRINK}: - pieces = [f"({ctx[a[0]] if isinstance(a[0], UOp) else str(a[0])}, {ctx[a[1]] if isinstance(a[1], UOp) else str(a[1])})" for a in x.marg] + if x.op in {Ops.RESHAPE, Ops.EXPAND}: pieces = [marg_str(ctx, a) for a in x.marg] + if x.op in {Ops.PAD, Ops.SHRINK}: pieces = [f"({marg_str(ctx, a[0])}, {marg_str(ctx, a[1])})" for a in x.marg] return f"({','.join(pieces)})" if len(pieces) != 1 else f"({pieces[0]},)" sugar = {Ops.SINK, Ops.END, Ops.STORE, Ops.LOAD, Ops.SQRT, Ops.INDEX, Ops.REDUCE, Ops.AFTER, Ops.THREEFRY, Ops.RECIPROCAL, Ops.EXP2, Ops.LOG2, Ops.SIN, Ops.CONTIGUOUS, Ops.BARRIER, Ops.DETACH} pm_pyrender_extra = PatternMatcher([ - (UPat(Ops.CONST, src=(), name="x"), lambda x: f"UOp.const({x.val}, {x.dtype})"), - (UPat((Ops.CAST, Ops.BITCAST), name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({x.dtype})"), - (UPat(Ops.SPECIAL, src=(UPat(Ops.CONST),), name="x"), lambda x: f"UOp.special({x.src[0].val}, {repr(x.arg)}, dtype={x.dtype})"), - (UPat(Ops.BUFFER, src=(UPat(),), name="x"), lambda x: + (UPat(Ops.CONST, src=(), name="x"), lambda x: f"UOp.const({x.val})"), + (UPat((Ops.CAST, Ops.BITCAST), name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({x.dtype})" if x.dtype != x.src[0].dtype else None), + (UPat(Ops.SPECIAL, src=(UPat(Ops.CONST),), name="x"), lambda x: f"UOp.special({x.src[0].val}, {repr(x.arg)})"), + (UPat(Ops.BUFFER, src=(), name="x"), lambda x: f"UOp.new_buffer({repr(x.arg.device)}, {x.max_numel()}, {x.dtype}, {x.arg.slot})" if isinstance(x.arg, ParamArg) and x.addrspace is AddrSpace.GLOBAL else None), (UPat(Ops.COPY, src=(UPat(name="x"),), name="copy"), lambda ctx,x,copy: f"{ctx[x]}.copy_to_device({repr(copy.arg)})"), @@ -91,11 +93,7 @@ pm_pyrender_extra = PatternMatcher([ # NOTE: range has srcs sometimes after control flow (UPat(Ops.RANGE, src=(UPat(Ops.CONST, name="c"),), allow_any_len=True, name="x"), lambda ctx,x,c: "UOp.range("+', '.join([str(c.val)] + [repr(y) for y in x.arg])+ - (f', src={srcs(ctx, x.src[1:])}' if len(x.src) > 1 else '')+(', dtype='+str(x.dtype) if x.dtype is not dtypes.weakint else '')+")"), - # TODO: index shouldn't mismatch dtype - (UPat(Ops.INDEX, src=(UPat(), UPat()), allow_any_len=True, name="x"), lambda ctx,x: - f"{ctx[x.src[0]]}.index({ctx[x.src[1]]}, "+''.join([f"{ctx[xx]}, " for xx in x.src[2:]])+ - f"dtype={x.dtype})" if x.src[0].dtype != x.dtype else None), + (f', src={srcs(ctx, x.src[1:])}' if len(x.src) > 1 else '')+")"), # TODO: movement ops simplify stuff, this can break SPEC=2 #(UPat(GroupOp.Movement, name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({render_marg(ctx,x)})"), # NOTE: CMPNE doesn't work cause there's no __rne__ @@ -108,6 +106,11 @@ pm_pyrender_extra = PatternMatcher([ (UPat(set(syms.keys())-{Ops.SUB, Ops.CDIV, Ops.CMOD}, name="x"), lambda ctx,x: strip_binary_parens(x, ctx[x.src[0]], ctx[x.src[1]], lambda a,b: f"({a}{syms[x.op]}{b})") if x.src[0]._broadcasted(x.src[1]) == x.src else f"{ctx[x.src[0]]}.alu({x.op}, {ctx[x.src[1]]})"), + # `.contiguous` is a no-op for weak dtypes, CONTIGUOUS, deviceless or buffer-backed inputs: + # the sugar would change the graph for those, so render them via .alu() instead + (UPat(Ops.CONTIGUOUS, name="x"), lambda ctx,x: + f"{ctx[x.src[0]]}.alu(Ops.CONTIGUOUS)" if x.src[0].dtype in dtypes.weaks or x.src[0].op is Ops.CONTIGUOUS + or x.src[0].device is None or x.src[0].has_buffer_identity() else None), (UPat(sugar, src=(), name="x"), lambda x: f"UOp.{x.op.name.lower()}("+', '.join(([f'arg={repr(x.arg)}'] if x.arg is not None else []))+")"), (UPat(sugar, name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}("+', '.join([ctx[y] for y in x.src[1:]] + \ ([f'arg={repr(x.arg)}'] if x.arg is not None else []))+")"), @@ -115,7 +118,7 @@ pm_pyrender_extra = PatternMatcher([ # NOTE: you can remove pm_pyrender_extra and it'll still be correct pm_pyrender = pm_pyrender_extra+PatternMatcher([ - (UPat(GroupOp.All, name="u"), lambda ctx,u: f"UOp({u.op}, {u.dtype}, {srcs(ctx,u.src)}"+(f", {repr(u.arg)})" if u.arg is not None else ")")), + (UPat(GroupOp.All, name="u"), lambda ctx,u: f"UOp({u.op}, {srcs(ctx,u.src)}"+(f", {repr(u.arg)})" if u.arg is not None else ")")), ]) def _render_with_splits(lst:list[UOp], pm:PatternMatcher, to_render:set[UOp], split_depth:int=100) -> dict[str, str]: @@ -142,7 +145,7 @@ def pyrender(ast:UOp) -> str: cmap = consumer_map_from_toposort(lst) not_rendered = {Ops.CONST} always_rendered = {Ops.PARAM, Ops.LOAD, Ops.SPECIAL, Ops.RANGE, Ops.CONTIGUOUS, Ops.STACK, - Ops.BUFFER, Ops.COPY, Ops.CALL, Ops.FUNCTION, Ops.WHERE, Ops.END} + Ops.BUFFER, Ops.COPY, Ops.CALL, Ops.WHERE, Ops.END} to_render: set[UOp] = {ast} for u in lst: @@ -150,7 +153,7 @@ def pyrender(ast:UOp) -> str: for s in u.src: to_render.add(s) if u.op is Ops.STORE: to_render.add(u.src[1]) if u.op is Ops.REDUCE: to_render.add(u.src[0]) - if u.op is Ops.FUNCTION or (u.op is Ops.CALL and u.src[0].dtype is dtypes.void): raise NotImplementedError("call can't be pyrendered") + if u.op is Ops.CALL and u.src[0].dtype is dtypes.void: raise NotImplementedError("call can't be pyrendered") if u.op in not_rendered: continue # checking the consumers is not enough, you have to make sure it's not used twice by the one consumer if len(cmap[u]) == 1 and len([x for x in list(cmap[u].keys())[0].src if x is u]) == 1 and u.op not in always_rendered: continue diff --git a/tinygrad_repo/tinygrad/uop/spec.py b/tinygrad_repo/tinygrad/uop/spec.py index 654f491cd..5b872cf09 100644 --- a/tinygrad_repo/tinygrad/uop/spec.py +++ b/tinygrad_repo/tinygrad/uop/spec.py @@ -1,4 +1,4 @@ -import math +import math, functools from typing import Any from tinygrad.uop.ops import PatternMatcher, UPat, GroupOp, Ops, UOp, AxisType, KernelInfo, ParamArg from tinygrad.uop.render import print_uops, pyrender @@ -54,7 +54,7 @@ spec_shared = PatternMatcher([ (UPat(Ops.NOOP), lambda: True), # CONST is everywhere; Invalid is a bool const - (UPat(Ops.CONST, src=(), name="x"), lambda x: x.dtype is dtypes.bool if x.is_invalid else type(x.val) is type(x.dtype.const(x.val))), + (UPat(Ops.CONST, src=(), name="x"), lambda x: x.is_invalid or type(x.val) is type(x.dtype.const(x.val))), # STACK is everywhere too (UPat(Ops.STACK, dtype=dtypes.void, src=()), lambda: True), @@ -68,16 +68,17 @@ spec_shared = PatternMatcher([ (UPat(GroupOp.Comparison, dtype=dtypes.bool, src=(UPat.var("x"), UPat.var("y"))), lambda x,y: matches_dtype(x, y.dtype) or matches_dtype(y, x.dtype) or x.dtype in dtypes.weaks or y.dtype in dtypes.weaks), (UPat((Ops.AND, Ops.OR, Ops.XOR, Ops.SHL, Ops.SHR), name="x"), lambda x: False if any(dtypes.is_float(s.dtype) for s in x.src) else None), - (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat(dtype=dtypes.uint)), name="a"), lambda a,x: matches_dtype(x, a.dtype) or None), - (UPat((Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD), name="x"), lambda x: None if dtypes.is_int(x.dtype) else False), + (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat.var("c")), name="a"), lambda a,x,c: + matches_dtype(c, a.dtype) or c.dtype in (dtypes.uint, dtypes.weakint) or x.base.is_invalid), + (UPat((Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD), name="x"), + lambda x: None if dtypes.is_int(x.dtype) or any(s.base.is_invalid for s in x.src) else False), (UPat(GroupOp.ALU, name="x"), lambda x: all(matches_dtype(y, x.dtype) or y.dtype in dtypes.weaks for y in x.src)), # CAST (UPat((Ops.BITCAST, Ops.CAST), src=(UPat(),), name="x"), lambda x: isinstance(x.arg, DType)), # RANGE can be in the big graph now. a void RANGE is a bound-less loop header, the arg is an axis id like RANGE - (UPat(Ops.RANGE, src=(UPat.var("x"),), allow_any_len=True, name="rng"), lambda rng,x: - matches_dtype(x, rng.dtype) and isinstance(rng.arg, tuple) and len(rng.arg) >= 2 and \ + (UPat(Ops.RANGE, src=(UPat(),), allow_any_len=True, name="rng"), lambda rng: isinstance(rng.arg, tuple) and len(rng.arg) >= 2 and \ all(isinstance(ra, int) for ra in rng.arg[0:-1]) and isinstance(rng.arg[-1], AxisType)), (UPat(Ops.INDEX, name="x"), lambda x: len(x.src)>0 and all(dtypes.is_int(y.dtype) or y.base.is_invalid for y in x.src[1:]) or None), # END closes RANGEs @@ -85,25 +86,25 @@ spec_shared = PatternMatcher([ # a loop-ended END requires a trailing bool condition for the backedge (loop again while true) (UPat(Ops.END, src=(UPat(), UPat(Ops.RANGE, dtypes.void), UPat(dtype=dtypes.bool))), lambda: True), - # PARAM - (UPat(Ops.PARAM, name="x"), lambda x: isinstance(x.arg, ParamArg)), - (UPat(Ops.BUFFER, src=(UPat(),), name="x"), lambda x: - isinstance(x.arg, ParamArg) and x.addrspace in (AddrSpace.REG, AddrSpace.LOCAL)), + # PARAM/BUFFER have a size in the arg, no shape input + (UPat(Ops.PARAM, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg)), + (UPat(Ops.BUFFER, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg) and x.addrspace in (AddrSpace.REG, AddrSpace.LOCAL)), # GROUP of stores (or groups, or NOOPs) (UPat(Ops.GROUP, dtypes.void, src=UPat((Ops.GROUP, Ops.STORE, Ops.NOOP, Ops.INS, Ops.END))), lambda: True), - # AFTER on Movement Op, PARAM, BUFFER, CONTIGUOUS, or another AFTER + # AFTER on Movement Op, PARAM, BUFFER, CONTIGUOUS, RETURNED, or another AFTER (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement.union({Ops.PARAM, Ops.BUFFER, Ops.CONTIGUOUS, Ops.INDEX, - Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS})),), - allow_any_len=True, name="x"), lambda x: matches_dtype(x.src[0], x.dtype)), + Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS, Ops.RETURNED})),), + allow_any_len=True), lambda: True), - # CUSTOM (inline and non inline) - (UPat((Ops.CUSTOMI, Ops.CUSTOM)), lambda: True), + # CUSTOM (inline and non inline): the arg is the source string and the dtype it produces, void for a bare statement + (UPat((Ops.CUSTOMI, Ops.CUSTOM), name="x"), + lambda x: isinstance(x.arg, tuple) and len(x.arg) == 2 and isinstance(x.arg[0], str) and isinstance(x.arg[1], DType)), # CALL of an external function (UPat(Ops.CALL, src=(UPat(),), allow_any_len=True, name="x"), - lambda x: matches_dtype(x.src[0], dtypes.uint64) if x.src[0].dtype is not dtypes.void else None), + lambda x: matches_dtype(x.src[0], dtypes.uint64) and isinstance(x.arg, DType) if x.src[0].dtype is not dtypes.void else None), # pattern compiler IR ops (not in tensor/program graphs, but spec-compliant) (UPat(Ops.PYLITERAL), lambda: True), @@ -112,7 +113,7 @@ spec_shared = PatternMatcher([ (UPat(Ops.BARRIER, dtypes.void), lambda: True), # assembly instruction - (UPat(Ops.INS), lambda: True), + (UPat(Ops.INS, name="x"), lambda x: isinstance(x.arg, tuple) and len(x.arg) == 2 and isinstance(x.arg[1], DType)), # LOAD(idx) / STORE(idx, val) with gates on the LOAD/STORE (UPat((Ops.INDEX, Ops.SHRINK), name="uidx").or_casted().load(), validate_index), @@ -121,8 +122,10 @@ spec_shared = PatternMatcher([ (UPat((Ops.INDEX, Ops.SHRINK), name="uidx").or_casted().store(UPat()), validate_index), (UPat((Ops.INDEX, Ops.SHRINK), name="uidx").or_casted().store(UPat(), UPat.var("gate", dtype=dtypes.bool)), validate_index), - # STORE in tensor graph: store a value into a target - (UPat(Ops.STORE, dtypes.void, (UPat(name="x"), UPat())), lambda x: True), + # STORE: the target must be storage or a CONTIGUOUS realization point (or an AFTER/BITCAST/view of one); + # CONTIGUOUS targets are written into the buffer the CONTIGUOUS creates. INDEX stores are checked above + (UPat(Ops.STORE, dtypes.void, (UPat(name="x"), UPat())), lambda x: + True if (b:=x.storage_base).op in {Ops.BUFFER, Ops.PARAM, Ops.RETURNED, Ops.CONTIGUOUS} else None if b.op is Ops.INDEX else False), # WMMA has a (UPat(Ops.WMMA, src=(UPat(), UPat(), UPat()), name="x"), lambda x: isinstance(x.arg, tuple) and len(x.arg) == 5), @@ -130,20 +133,18 @@ spec_shared = PatternMatcher([ def is_device(d): return isinstance(d, str) or (isinstance(d, tuple) and all(isinstance(s, str) for s in d)) -def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < len(t.src) and matches_dtype(t.src[g.arg], g.dtype) - # these ops can exist in tensor but not programs. example: movement spec_tensor = PatternMatcher([ - (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL), src=(UPat(),), name="u"), lambda u: dtypes.is_float(u.dtype)), + (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL), src=(UPat(),), name="u"), + lambda u: dtypes.is_float(u.dtype) or u.src[0].base.is_invalid), # BUFFER - (UPat(Ops.BUFFER, src=(UPat(),), name="buf"), lambda buf: - (isinstance(buf.dtype, DType) and matches_dtype(buf.src[0], dtypes.weakint) and is_device(buf.arg.device)) + (UPat(Ops.BUFFER, src=(), name="buf"), lambda buf: + (isinstance(buf.dtype, DType) and isinstance(buf.arg.size, int) and is_device(buf.arg.device)) if isinstance(buf.arg, ParamArg) and buf.addrspace is AddrSpace.GLOBAL else None), - # Tensor variable bindings - (UPat(Ops.BIND, (dtypes.int, dtypes.long, dtypes.weakint,), (UPat(Ops.PARAM), UPat.cvar(dtype=(dtypes.int,dtypes.long,dtypes.weakint,))), arg=None), - lambda: True), + # a Variable is a 0-d ALU BUFFER with a value range and no device + (UPat(Ops.BUFFER, src=(), name="buf"), lambda buf: buf.arg.device is None if buf.is_variable else None), # custom function (UPat(Ops.CUSTOM_FUNCTION, name="x"), lambda x: isinstance(x.arg, str)), @@ -151,14 +152,11 @@ spec_tensor = PatternMatcher([ # CALL (UPat(Ops.CALL, dtypes.void, src=(UPat((Ops.SINK, Ops.LINEAR, Ops.PROGRAM, Ops.COPY, Ops.CUSTOM_FUNCTION)),), allow_any_len=True), lambda: True), - # FUNCTION + TUPLE must have void dtype, GETTUPLE can only appear on FUNCTION or TUPLE - (UPat(Ops.FUNCTION, dtypes.void, src=(UPat(Ops.TUPLE),), allow_any_len=True), lambda: True), - (UPat(Ops.TUPLE, dtypes.void), lambda: True), - (UPat(Ops.GETTUPLE, src=(UPat(Ops.FUNCTION, src=(UPat(Ops.TUPLE, name="t"),), allow_any_len=True),), name="g"), valid_gettuple), - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), valid_gettuple), + # RETURNED is a placeholder for a buffer a call writes and returns: it has a size in the arg, no shape input + (UPat(Ops.RETURNED, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg)), # SPECIAL is index before index lowering. custom_kernel currently has this - (UPat(Ops.SPECIAL, src=(UPat.var("x", dtypes.weakint),), name="s"), lambda s,x: matches_dtype(x, s.dtype) and isinstance(s.arg, str)), + (UPat(Ops.SPECIAL, src=(UPat(dtype=dtypes.weakint),), name="s"), lambda s: isinstance(s.arg, str)), # movement ops (UPat((Ops.RESHAPE, Ops.EXPAND), src=(UPat(), UPat())), lambda: True), @@ -171,20 +169,19 @@ spec_tensor = PatternMatcher([ and isinstance(x.arg[1], int) and all(y.dtype in (dtypes.weakint, dtypes.int) for y in x.src[1:])), # COPY - (UPat(Ops.COPY, name="copy", src=(UPat.var("x"),)), lambda copy,x: matches_dtype(x, copy.dtype) and is_device(copy.arg)), - (UPat(Ops.ALLREDUCE, name="red", src=(UPat.var("x"),)), lambda red,x: matches_dtype(x, red.dtype) and isinstance(red.arg, tuple) and - len(red.arg) == 2 and red.arg[0] in GroupOp.Reduce and is_device(red.arg[1])), + (UPat(Ops.COPY, name="copy", src=(UPat(),)), lambda copy: is_device(copy.arg)), + (UPat(Ops.ALLREDUCE, name="red", src=(UPat(),)), + lambda red: isinstance(red.arg, tuple) and len(red.arg) == 2 and red.arg[0] in GroupOp.Reduce and is_device(red.arg[1])), # UNSHARD/MSELECT/MSTACK # an UNSHARD carries the value and one sharding range per sharded axis (usually a DEVICE RANGE, but can be a derived expression) - (UPat(Ops.UNSHARD, name="multi"), lambda multi: len(multi.src) == 1+len(multi.arg) and matches_dtype(multi.src[0], multi.dtype) + (UPat(Ops.UNSHARD, name="multi"), lambda multi: len(multi.src) == 1+len(multi.arg) and all(isinstance(a, int) for a in multi.arg) and all(r.dtype in dtypes.weaks for r in multi.src[1:])), (UPat(Ops.MSELECT, name="x"), lambda x: isinstance(x.src[0].device, tuple) and x.arg < len(x.src[0].device)), (UPat(Ops.MSTACK, name="x"), lambda x: all(isinstance(s.device, str) for s in x.src) or (all_same(x.src) and x.src[0].device is None)), # CONTIGUOUS ensures the source UOp realizes - (UPat((Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD), name="root", src=(UPat.var("x"),), arg=None), - lambda root,x: matches_dtype(x, root.dtype)), + (UPat((Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD), src=(UPat(),), arg=None), lambda: True), # TODO: this should not be here. STAGE is transformed to BUFFER later (UPat(Ops.STAGE, src=(UPat(),), allow_any_len=True), lambda: True), @@ -201,11 +198,12 @@ spec_tensor = PatternMatcher([ # these ops can exist in programs but not the tensor spec. example: LOAD spec_program = PatternMatcher([ - # index and weak dtypes are not allowed in programs - (UPat(GroupOp.All, (dtypes.weakint, dtypes.weakfloat)), lambda: False), + # every width in a program is stated: a CONST appears only under the CAST stating its width, and is the only weak node + (UPat(GroupOp.All, name="x"), lambda x: False if x.op is not Ops.CAST and any(s.op is Ops.CONST for s in x.src) else None), + (UPat(GroupOp.All-{Ops.CONST}, dtypes.weaks), lambda: False), # allow special SHRINK - (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.AFTER)), UPat(), UPat(Ops.CONST))), lambda: True), + (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.AFTER)), UPat(), UPat(Ops.CONST).or_casted())), lambda: True), # movement ops are not allowed in programs (UPat(GroupOp.Movement), lambda: False), @@ -221,11 +219,13 @@ spec_program = PatternMatcher([ (UPat(Ops.ENDIF, dtype=dtypes.void, src=(UPat(Ops.IF),)), lambda: True), # SPECIAL is int32 after index lowering - (UPat(Ops.SPECIAL, src=(UPat.var("x", dtypes.int32),), name="s"), lambda s,x: matches_dtype(x, s.dtype) and isinstance(s.arg, str)), + (UPat(Ops.SPECIAL, src=(UPat(dtype=dtypes.int32),), name="s"), lambda s: isinstance(s.arg, str)), ])+spec_shared spec_hcq = PatternMatcher([ - (UPat(Ops.GETADDR, dtypes.uint64, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),), name="x"), lambda x: is_device(x.arg)), + (UPat(Ops.GETADDR, dtypes.uint64, name="x", + src=(UPat((Ops.BUFFER, Ops.PARAM, Ops.SHRINK, Ops.BITCAST, Ops.MSTACK, Ops.MSELECT, Ops.LINEAR)).or_after(),)), + lambda x: is_device(x.arg)), (UPat(Ops.PROGRAM, dtypes.void, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),)), lambda: True), ])+spec_shared @@ -233,13 +233,6 @@ spec_hcq = PatternMatcher([ spec_full = PatternMatcher([ (UPat(Ops.REWRITE_ERROR, dtypes.void, name="x"), lambda x: isinstance(x.arg, str)), - # SLICE on BUFFER is allowed if BUFFER is - (UPat(Ops.SLICE, src=(UPat(GroupOp.Movement.union({Ops.BUFFER, Ops.PARAM, Ops.STAGE, Ops.AFTER})), - UPat(Ops.CONST, dtype=dtypes.weakint)), allow_any_len=True, name="bv"), - lambda bv: isinstance(bv.arg, int)), - - (UPat(Ops.CALL, dtypes.void, src=(UPat((Ops.SLICE,)),), allow_any_len=True), lambda: True), - # codegen may end ranges after gpudims has replaced RANGE with SPECIAL. (UPat(Ops.END, src=(UPat(), UPat()), allow_any_len=True), lambda: True), @@ -248,9 +241,6 @@ spec_full = PatternMatcher([ # all loads/stores (UPat((Ops.LOAD, Ops.STORE)), lambda: True), - - # while BIND is being casted - (UPat(Ops.BIND, (dtypes.int, dtypes.weakint), (UPat(), UPat()), arg=None), lambda: True), ])+spec_tensor+spec_program+spec_hcq # ***** kernel graph spec ***** @@ -258,19 +248,18 @@ spec_full = PatternMatcher([ spec_kernel_graph = PatternMatcher([ # sink (UPat(Ops.SINK, dtypes.void), lambda: True), - # bind - (UPat(Ops.BIND), lambda: True), - # const + stack to make vconsts + # the store of a bound Variable binds it: AFTER(BUFFER, STORE(BUFFER, CONST)) in call args + (UPat(Ops.STORE, dtypes.void, (UPat(Ops.BUFFER, name="b"), UPat(Ops.CONST))), lambda b: b.is_variable), + # const + stack to make vconsts and shape args. a 0-size/bound reduce keeps its const casted (UPat(Ops.CONST, src=()), lambda: True), - (UPat(Ops.STACK, src=()), lambda: True), - (UPat(Ops.STACK, src=UPat((Ops.CONST, Ops.BIND, Ops.PARAM))), lambda: True), + (UPat(Ops.CAST, src=(UPat(Ops.CONST, src=()),)), lambda: True), + (UPat(Ops.STACK, name="s"), lambda s: all(x.op in (Ops.CONST, Ops.PARAM) or x.is_variable or x.is_bound_var for x in s.src) or None), # linear for more kernels (TODO: we should enter non sink calls) #(UPat(Ops.LINEAR), lambda: True), - # param is outside buffer, buffer is local buffer - (UPat(Ops.PARAM, name="x"), lambda x: isinstance(x.arg, ParamArg)), - (UPat(Ops.BUFFER, name="x"), lambda x: isinstance(x.arg, ParamArg) and x.addrspace == AddrSpace.GLOBAL), - # RESHAPE/BITCAST are NOOPs in the kernel graph (do we need them?) - (UPat((Ops.RESHAPE, Ops.BITCAST)), lambda: True), + # param is outside buffer, buffer is local buffer. params have a size in the arg, no shape input + (UPat(Ops.PARAM, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg)), + (UPat(Ops.BUFFER, name="x"), lambda x: isinstance(x.arg, ParamArg) and x.addrspace in (AddrSpace.GLOBAL, AddrSpace.ALU)), + (UPat(Ops.BITCAST), lambda: True), # mstack/mselect (UPat(Ops.MSTACK, name="x"), lambda x: all(isinstance(s.device, str) for s in x.src) or (all_same(x.src) and x.src[0].device is None)), (UPat(Ops.MSELECT, name="x"), lambda x: isinstance(x.src[0].device, tuple) and x.arg < len(x.src[0].device)), @@ -278,22 +267,24 @@ spec_kernel_graph = PatternMatcher([ (UPat(Ops.CALL, src=(UPat((Ops.SINK, Ops.LINEAR, Ops.PROGRAM, Ops.CUSTOM_FUNCTION)),), allow_any_len=True), lambda: True), # after on PARAM or AFTER (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement.union({Ops.PARAM, Ops.AFTER, Ops.BUFFER, Ops.MSTACK, Ops.MSELECT, Ops.BITCAST, Ops.RESHAPE})),), - allow_any_len=True, name="x"), lambda x: matches_dtype(x.src[0], x.dtype)), + allow_any_len=True), lambda: True), ]) # **** pyrender (move this) **** -# late imports to avoid circular import -from tinygrad.codegen.opt import Opt, OptOps -from tinygrad.schedule.rangeify import BufferizeOpts -from tinygrad.renderer import Estimates -glbls:dict[str, Any] = {"inf": math.inf, "nan": math.nan, "KernelInfo": KernelInfo, "Metadata": Metadata, - "UOp": UOp, "dtypes": dtypes, "Ops": Ops, "AxisType": AxisType, "Invalid": Invalid, - "Opt": Opt, "OptOps": OptOps, "BufferizeOpts": BufferizeOpts, "AddrSpace": AddrSpace, "panic": panic, - "ConstFloat": ConstFloat, "ParamArg": ParamArg, "Estimates": Estimates} +# circular-import-safe eval globals for pyrender round-tripping (lazy: codegen/schedule/renderer are heavy) +@functools.cache +def pyrender_globals() -> dict[str, Any]: + from tinygrad.codegen.opt import Opt, OptOps + from tinygrad.schedule.rangeify import BufferizeOpts + from tinygrad.renderer import Estimates + return {"inf": math.inf, "nan": math.nan, "KernelInfo": KernelInfo, "Metadata": Metadata, + "UOp": UOp, "dtypes": dtypes, "Ops": Ops, "AxisType": AxisType, "Invalid": Invalid, + "Opt": Opt, "OptOps": OptOps, "BufferizeOpts": BufferizeOpts, "AddrSpace": AddrSpace, "panic": panic, + "ConstFloat": ConstFloat, "ParamArg": ParamArg, "Estimates": Estimates} def eval_pyrender(code:str) -> UOp: lcls:dict[str, Any] = {} - exec(code, glbls, lcls) + exec(code, pyrender_globals(), lcls) return lcls['ast'] def test_pyrender(test_ast:UOp, assert_parents=True): diff --git a/tinygrad_repo/tinygrad/uop/symbolic.py b/tinygrad_repo/tinygrad/uop/symbolic.py index 1eda87523..e5b401020 100644 --- a/tinygrad_repo/tinygrad/uop/symbolic.py +++ b/tinygrad_repo/tinygrad/uop/symbolic.py @@ -1,11 +1,12 @@ # all of symbolic lives here now import math from collections import defaultdict -from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu -from tinygrad.dtype import PyConst, ConstType, dtypes, can_lossless_cast, Invalid, bitcast +from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu, promo_dtype +from tinygrad.dtype import PyConst, dtypes, can_lossless_cast, Invalid, bitcast, truncate from tinygrad.helpers import partition, all_same, prod, flatten, unwrap, IMAGE, dedup from tinygrad.uop.divandmod import div_and_mod_symbolic from tinygrad.uop.movement import mop_cleanup +from tinygrad.uop.weak import pm_uncast_const # TODO: symbolic shouldn't be importing from codegen from tinygrad.codegen.decomp.transcendental import xpow @@ -21,16 +22,11 @@ def simplify_pow(x:UOp, c:UOp) -> UOp|None: def fold_bitcast(root:UOp, c:UOp) -> UOp|None: if c.dtype.fmt is None or root.dtype.fmt is None or c.dtype.itemsize != root.dtype.itemsize: return None - return root.const_like(bitcast(c.val, c.dtype, root.dtype)) + # the value is mathematical and may not fit: reading it as bits is the emission that pins it to the stated width + return root.const_like(bitcast(truncate[c.dtype](c.val), c.dtype, root.dtype)) -def const_arg(u:UOp) -> ConstType|tuple[ConstType, ...]|None: - if u.op is Ops.CONST: return u.val - if u.op is Ops.STACK and all(s.op is Ops.CONST for s in u.src): return tuple(s.val for s in u.src) - return None - -def fold_const_alu(a:UOp) -> UOp|None: - vals = [const_arg(s) for s in a.src] - return None if any(v is None for v in vals) else a.const_like(exec_alu(a.op, a.dtype, vals, False)) +# no truncate: ints stay mathematical past the fold (emission truncates); floats re-round in the mint +def fold_const_alu(a:UOp) -> UOp: return a.const_like(exec_alu(a.op, a.dtype, [const_arg(s) for s in a.src], False)) def _quotient_base(q:UOp, base:UOp, div:int) -> UOp|None: # the B with q == B//div and B%div == base%div, or None. only such congruence is needed to recombine, and canonicalization @@ -66,6 +62,20 @@ def fold_add_divmod_recombine(x:UOp) -> UOp|None: # this needs to be before symbolic so that 0*something_that_might_be_invalid doesnt become 0 invalid_pat = UPat(Ops.CONST, arg=Invalid, name="i") invalid_gate = UPat.var("cond").where(UPat.var("x"), invalid_pat) + +# the two const spellings: Invalid carries no width, so it rides bare inside either +bare_const = UPat.any(UPat(Ops.CONST), UPat(Ops.STACK, src=UPat(Ops.CONST))) +casted_const = UPat.any(p:=UPat(Ops.CAST, src=(UPat(Ops.CONST),)), UPat(Ops.STACK, src=UPat.any(p, UPat(Ops.CONST, arg=Invalid)))) +def const_arg(u:UOp): + return tuple(const_arg(s) for s in u.src) if u.op is Ops.STACK else u.val + +def lift_reduce_gate(red:UOp, cond:UOp, x:UOp, i:UOp) -> UOp|None: + # a REDUCE moves inside the gate clauses without its ranges: they invalidate every lane at once, so that gate lifts out + if red.arg[1] != 0: return None + keep, lift = partition(cond.split_uop(Ops.AND), lambda c: any(rr in c.ranges for r in red.src[1:] for rr in r.ranges)) + inner = keep[0].uprod(*keep[1:]).where(x, i) if keep else x + return lift[0].uprod(*lift[1:]).where(red.replace(src=(inner,)+red.src[1:]), i) if lift else None + pm_data_invalid = PatternMatcher([ (invalid_pat.broadcast(), lambda i: i), (UPat(GroupOp.Unary|{Ops.CAST, Ops.BITCAST}, src=(invalid_pat,)), lambda i: i), @@ -75,6 +85,7 @@ pm_data_invalid = PatternMatcher([ (UPat(GroupOp.Binary, src=(invalid_gate, UPat.var("y")), name="alu"), lambda cond,x,y,alu,i: cond.where(x.alu(alu.op,y), i)), (UPat(GroupOp.Binary, src=(UPat.var("y"), invalid_gate), name="alu"), lambda cond,x,y,alu,i: cond.where(y.alu(alu.op,x), i)), (UPat(GroupOp.Binary-GroupOp.Comparison, src=[invalid_pat, UPat()]), lambda i: i), + (invalid_gate.reduce(allow_any_len=True, name="red"), lift_reduce_gate), # an Invalid condition poisons the whole where; a gated Invalid condition lifts the gate out (invalid_pat.where(UPat(), UPat()), lambda i: i), (invalid_gate.where(UPat.var("a"), UPat.var("b")), lambda cond,x,i,a,b: cond.where(x.where(a,b), i)), @@ -96,15 +107,16 @@ pm_remove_invalid = PatternMatcher([ if any(x.is_invalid for x in s.src) else None), ]) -# the one rule that collapses the pair CAST(dt, CONST(v)) into a typed CONST -# TODO: delete this once CONST has no dtype -pm_fold_cast_const = PatternMatcher([(UPat(Ops.CAST, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.val))]) +def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: + # folding a strong dtype WHERE to a weak const branch keeps the strong dtype + ret = c0 if gate.val else c1 + return ret.ccast(w.dtype) if ret.op is Ops.CONST and ret.dtype in dtypes.weaks and w.dtype not in dtypes.weaks else ret symbolic_simple = pm_data_invalid + PatternMatcher([ # ** self folding ** - (UPat.var("x") + 0, lambda x: x), # x+0 -> x + (UPat({Ops.ADD, Ops.XOR, Ops.OR}, src=[UPat.var("x"), UPat.const(0)]), lambda x: x), # x+0 / x^0 / x|0 -> x + (UPat({Ops.SHL, Ops.SHR}, src=(UPat.var("x"), UPat.const(0))), lambda x: x), # x<<0 / x>>0 -> x (UPat.var("x") * 1, lambda x: x), # x*1 -> x - (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) ^ 0, lambda x: x), # x^0 -> x (UPat.var("x") // UPat.var("x"), lambda x: x.const_like(1)), # x//x -> 1 (UPat.var("x") // 1, lambda x: x), # x//1 -> x (UPat.var("x") // -1, lambda x: -x), # x//-1 -> -x @@ -136,17 +148,24 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) != UPat.var("x"), lambda x: x.const_like(False, dtypes.bool)), # x != x -> False (only ints) # ** constant folding ** - (UPat(GroupOp.Unary, src=(UPat((Ops.CONST, Ops.STACK)),), name="a"), fold_const_alu), + # canonicalize casted CONST + (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.val)), + # collapse committed const conversions when the target has a native constant format. fmt-less targets are emulated and would re-expand this pair. + (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat(Ops.CAST, dtypes.all, src=(UPat(Ops.CONST, name="c"),)),)), + lambda root,c: root.const_like(c.val) if root.dtype.fmt is not None else None), + # one rule per spelling: bare has no width, a pair evaluates at its stated width, mixed commits to the promotion # NOTE: THREEFRY(const,const) folds via its decomposition - (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=(UPat((Ops.CONST, Ops.STACK)),)*2, name="a"), fold_const_alu), - (UPat(GroupOp.Ternary, src=(UPat((Ops.CONST, Ops.STACK)),)*3, name="a"), fold_const_alu), + (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=bare_const, name="a"), fold_const_alu), + (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=casted_const, name="a"), fold_const_alu), + (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=[casted_const, bare_const], name="a"), lambda a: + a.replace(src=tuple(s.ccast(dt) if s.dtype in dtypes.weaks else s for s in a.src)) + if (dt:=promo_dtype(a.src)) not in dtypes.weaks else None), # bool MUL is AND, ADD/MAX is OR. prevents other rules to rewrite bool ADD/MUL incorrectly (UPat.var('x', dtype=dtypes.bool) * UPat.var('y', dtype=dtypes.bool), lambda x,y: x&y), (UPat.var('x', dtype=dtypes.bool) + UPat.var('y', dtype=dtypes.bool), lambda x,y: x|y), (UPat.var('x', dtype=dtypes.bool).maximum(UPat.var('y', dtype=dtypes.bool)), lambda x,y: x|y), # *** div rules *** (UPat.cvar('x', arg=0) / 0, lambda x: x.const_like(float('nan'))), # 0/0 -> nan - ((UPat.var("x") * 0) / 0, lambda x: x.const_like(float('nan'))), # (x*0)/0 -> nan # can be wrong if x or x2 is 0 (UPat.var("x") / UPat.var("x"), lambda x: x.const_like(1)), # x/x -> 1 ((UPat.var("x") * UPat.var("x2")) / UPat.var("x2"), lambda x,x2: x), # (x*x2)/x2 -> x @@ -157,7 +176,9 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ and isinstance(x.val, float) and (math.isnan(x.val) or math.isinf(x.val)) else 0)), # *** cast/bitcast *** (UPat((Ops.CAST, Ops.BITCAST), name="root"), lambda root: root.src[0] if root.dtype == root.src[0].dtype else None), - (UPat(Ops.BITCAST, name="root", src=(UPat.cvar("c"),)), fold_bitcast), + # a BITCAST reads its operand at the width it states, so a weak const is nonsense here: the bare arm is bool only + (UPat(Ops.BITCAST, name="root", src=(UPat.any(UPat(Ops.CONST, dtypes.bool, name="c"), UPat(Ops.CAST, src=(UPat(Ops.CONST),), name="c")),)), + fold_bitcast), # b.cast(a).cast(b) -> b if a preserves all values in b (UPat.var('x').cast(name="a").cast(name="b"), lambda x,a,b: x if x.dtype == b.dtype and can_lossless_cast(b.dtype, a.dtype) else None), # bitcast twice @@ -174,7 +195,8 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ # ** simple where folding ** # a conditional with the same results either way is a noop, also fold const conditionals (UPat.var().where(UPat.var("val"), UPat.var("val")), lambda val: val), - (UPat.cvar("gate").where(UPat.var("c0"), UPat.var("c1")), lambda gate, c0, c1: c0 if gate.val else c1), + (UPat.cvar("gate").where(UPat.var("c0"), UPat.var("c1")).named("w"), fold_const_where), + (UPat.var("gate").where(UPat.var("x"), 0) != 0, lambda gate,x: gate & (x != 0)), # a.where(b.where(c, d), d) -> (a & b).where(c, d) (UPat.var("a").where(UPat.var("b").where(UPat.var("c"), UPat.var("d")), UPat.var("d")), lambda a,b,c,d: (a&b).where(c,d)), # a.where(c, b.where(c, d)) -> (a | b).where(c, d) @@ -245,10 +267,12 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ # complementary zero branches under the same condition select directly (UPat.var("c").where(UPat.var("t"), 0) + UPat.var("c").where(0, UPat.var("f")), lambda c,t,f: c.where(t, f)), # ALU/variable min==max -> CONST - (UPat({Ops.CMPLT, Ops.CMPNE, Ops.FLOORDIV, Ops.FLOORMOD, Ops.PARAM, Ops.BIND, Ops.SPECIAL}, name="x"), + (UPat({Ops.CMPLT, Ops.CMPNE, Ops.FLOORDIV, Ops.FLOORMOD, Ops.PARAM, Ops.AFTER, Ops.SPECIAL}, name="x"), lambda x: x.const_like(x.vmin) if x.vmin == x.vmax else None), (UPat(Ops.RANGE, src=(UPat(Ops.CONST,)), name="x"), lambda x: x.const_like(x.vmin) if x.vmin == x.vmax else None), # max folding + ((UPat.cvar("a") < UPat.var("b")).where(UPat.var("b"), UPat.cvar("c")), lambda a,b,c: UOp.maximum(a,b) if a.val == c.val else None), + ((UPat.var("a") < UPat.cvar("b")).where(UPat.cvar("c"), UPat.var("a")), lambda a,b,c: UOp.maximum(a,b) if b.val == c.val else None), (UPat.maximum(UPat.var("x"), UPat.var("y")), lambda x,y: x if x.vmin >= y.vmax else y if x.vmax <= y.vmin else None), # TODO: why does this rule break beautiful_mnist? #((UPat.var("x")+UPat.var("z")).maximum(UPat.var("y")+UPat.var("z")), lambda x,y,z: x.maximum(y) + z), @@ -282,20 +306,23 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ # if the intermediate cast doesnt narrow we can do it in one cast (UPat.var('x').cast(name="a").cast(name="b"), lambda x,a,b: x.cast(b.dtype) if can_lossless_cast(x.dtype, a.dtype) else None), (UPat.var('x', dtypes.ints+(dtypes.weakint,)).cast(dtypes.ints+(dtypes.weakint,), name="a").cast(name="b"), - lambda x,a,b: x.cast(b.dtype) if a.dtype.min<=x.vmin and x.vmax<=a.dtype.max else None), + lambda x,a,b: x.ccast(b.dtype) if not x.overflows(a.dtype) else None), # try to do math in int instead of long, keep weak const weak - (UPat(GroupOp.Binary, src=(UPat.var("x", dtypes.long), UPat.var("y", dtypes.long)), name="u"), lambda u,x,y: + (UPat(GroupOp.Binary, src=(UPat.var("x", (dtypes.long, dtypes.weakint)), UPat.var("y", (dtypes.long, dtypes.weakint))), name="u"), lambda u,x,y: (UOp.const(x.val) if x.op is Ops.CONST else x.cast(dtypes.int)).alu(u.op, UOp.const(y.val) if y.op is Ops.CONST else y.cast(dtypes.int)).cast(u.dtype) - if not any(v.overflows(dtypes.int) for v in (u,x,y)) else None), + if dtypes.long in (x.dtype, y.dtype) and not any(v.overflows(dtypes.int) for v in (u,x,y)) else None), ((UPat.var("x", dtypes.weakint) + UPat.cvar("c")).cast(dtypes.sints, name="cast"), lambda x,c,cast:x.cast(cast.dtype)+cast.const_like(c.val)), # only RANGE/IF/STORE/KERNEL have side effects (UPat(Ops.AFTER, name="x"), lambda x: x.replace(src=(x.src[0],)+ - tuple(dedup(flatten([(y,) if y.op in {Ops.RANGE, Ops.STORE, Ops.CALL, Ops.FUNCTION, Ops.BARRIER, Ops.END, Ops.LINEAR, Ops.STAGE} + tuple(dedup(flatten([(y,) if y.op in {Ops.RANGE, Ops.STORE, Ops.CALL, Ops.BARRIER, Ops.END, Ops.LINEAR, Ops.STAGE} else y.src for y in x.src[1:]]))))), - # after with 1 src is just src[0] - (UPat(Ops.AFTER, src=(UPat.var("s"),)), lambda s: s), -])+div_and_mod_symbolic + # after/end with 1 src is just src[0] + (UPat((Ops.AFTER, Ops.END), src=(UPat.var("s"),)), lambda s: s), + # ranges can be subbed for CONSTs, remove them from ENDs while preserving a constant bool backedge + (UPat(Ops.END, name="x"), lambda x: x.replace(src=(x.src[0],)+tuple(r for r in x.src[1:] if r.op is not Ops.CONST or r.dtype is dtypes.bool))), + # the rules above key on bare CONSTs, so a redundantly committed const has to be uncast in the same fixpoint +])+div_and_mod_symbolic+pm_uncast_const # ******** we take a small aside to "simplify_valid" to rewrite valids ******** @@ -328,14 +355,14 @@ def uop_given_valid(valid:UOp, uop:UOp, try_simplex=True) -> UOp: for i,(expr,v) in enumerate(bounds.items()): v0, v1 = (expr.vmin if v[0] is None else v[0], expr.vmax if v[1] is None else v[1]) # try checking the whole clause - all_candidates.append((expr, UOp.variable(f"fake{i}", v0, v1, expr.dtype))) + all_candidates.append((expr, UOp.variable(f"fake{i}", v0, v1, expr.dtype, param=True))) if try_simplex: # every candidate is a set of constrained UOp based on valid, and if every item in a set simplifies the uop into a same output, we rewrite uop candidates = [[all_candidates[-1]]] if expr.op is Ops.ADD and v0 == 1 and all(u.op in GroupOp.Irreducible for u in expr.split_uop(Ops.ADD)): # if the constraint is a simplex: X0 + X1 + ... > 0, we can check if all Xi > 0 simplify into the same output - candidates.append([(Xi, UOp.variable(f"fake{i}", 1, Xi.vmax, Xi.dtype)) for Xi in expr.split_uop(Ops.ADD)]) + candidates.append([(Xi, UOp.variable(f"fake{i}", 1, Xi.vmax, Xi.dtype, param=True)) for Xi in expr.split_uop(Ops.ADD)]) for candidate in candidates: # if every branch in candidate gives the same simplified uop, we can rewrite the uop @@ -369,7 +396,6 @@ def simplify_valid(valid:UOp) -> UOp|None: def reduce_mul_chain(r:UOp) -> UOp|None: if r.arg[0] not in {Ops.ADD, Ops.MAX}: return None - if r.dtype != r.src[0].dtype: return None inside, outside = [], [] for m in r.src[0].split_uop(Ops.MUL): m_parents = m.backward_slice @@ -405,7 +431,8 @@ pm_move_where_on_load = PatternMatcher([ ]) def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: - if x.dtype is not dtypes.weakint: return None + # pure index math only: a LOAD in x executes even where cond is false, so its INDEX valid must survive the assumption + if x.dtype is not dtypes.weakint or x.op_in_backward_slice_with_self(Ops.INDEX): return None # Skip if x contains DIV/MOD AND IMAGE mode is enabled -> image index e.g. openpilot if IMAGE.value > 0 and x.op_in_backward_slice_with_self(Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD): return None return cond.where(uop_given_valid(cond, x, try_simplex=False), i) @@ -427,12 +454,10 @@ pm_clean_up_group_sink = PatternMatcher([ ]) sym = symbolic+pm_simplify_valid+PatternMatcher([ - # reorder ALU/VECTORIZE - (UPat(GroupOp.ALU, src=(UPat(Ops.STACK, src=UPat(name='x')), UPat(Ops.STACK, src=UPat(name='y'))), name='alu'), - lambda x,y,alu: UOp(Ops.STACK, src=(UOp(alu.op, src=(x,y)),))), # ** where ** # push cast to branches - (UPat.var("s").where(UPat.var("a"), UPat.var("b")).cast().named("cast"), lambda s,a,b,cast: s.where(a.cast(cast.dtype), b.cast(cast.dtype))), + (UPat.var("s").where(UPat.var("a"), UPat.var("b")).cast().named("cast"), + lambda s,a,b,cast: s.where(a.ccast(cast.dtype), b.ccast(cast.dtype))), # ** pow ** ((UPat(Ops.POW, name="p"), lambda p: xpow(*p.src))), # ** load/store folding ** @@ -451,8 +476,6 @@ sym = symbolic+pm_simplify_valid+PatternMatcher([ (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")), lambda x,d: 1-d), # x*/(1+x) -> 1-1/(1+x) (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")*UPat.var("y")), lambda x,y,d: y*(1-d)), (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")+UPat.var("y")), lambda x,y,d: (1-d)+x*y), - # move const multiply after REDUCE (NOTE: the mul chain can do this, but only if it's a same dtype reduce) - ((UPat.var("x")*UPat.cvar("c")).reduce(arg=Ops.ADD, name="r", allow_any_len=True), lambda x,c,r: r.replace(src=(x,)+r.src[1:])*c.val), # reduce mul chain, move muls after the reduce (UPat(Ops.MUL).reduce(name="r", allow_any_len=True), reduce_mul_chain), # ** combine terms (opinionated) ** diff --git a/tinygrad_repo/tinygrad/uop/upat.py b/tinygrad_repo/tinygrad/uop/upat.py index 9bbcab452..55ab01eae 100644 --- a/tinygrad_repo/tinygrad/uop/upat.py +++ b/tinygrad_repo/tinygrad/uop/upat.py @@ -2,6 +2,7 @@ from typing import Any, Callable import itertools, inspect, functools, types from tinygrad.helpers import partition, dedup, Context from tinygrad.uop.ops import UPat, UOp, Ops, PatternMatcher, graph_rewrite, deconstruct_function +from tinygrad.dtype import dtypes class UPatCompileError(Exception): pass @@ -18,40 +19,42 @@ def _get_clause(self:UPat, base:UOp, depth=0) -> UOp: # build the and_clause for acceptance and_clause:list[UOp] = [] if self.op is not None: - if len(self.op) > 1: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(int(x) for x in self.op))), arg="{0}.op in {1}")) - else: and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg="{0}.op == "+str(self.op[0].value))) + if len(self.op) > 1: + and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(int(x) for x in self.op))), arg=("{0}.op in {1}", dtypes.void))) + else: and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg=("{0}.op == "+str(self.op[0].value), dtypes.void))) if self.arg is not None: - if isinstance(self.arg, int): and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg="{0}.arg == "+str(int(self.arg)))) - else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.arg)), arg="{0}.arg == {1}")) + if isinstance(self.arg, int): and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg=("{0}.arg == "+str(int(self.arg)), dtypes.void))) + else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.arg)), arg=("{0}.arg == {1}", dtypes.void))) if self.strict_length or self.required_len > 0: - and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg=("len({0}.src)"+(" == " if self.strict_length else " >= ")+str(self.required_len)))) - if self.name is not None: and_clause.append(UOp(Ops.STORE, src=(UOp(Ops.CUSTOMI, arg=self.name), base))) + and_clause.append(UOp(Ops.CUSTOM, src=(base,), + arg=("len({0}.src)"+(" == " if self.strict_length else " >= ")+str(self.required_len), dtypes.void))) + if self.name is not None: and_clause.append(UOp(Ops.STORE, src=(UOp(Ops.CUSTOMI, arg=(self.name, dtypes.void)), base))) if self.match_dtype is not None: if len(self.match_dtype) > 1: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(self.match_dtype))), - arg="{0}.dtype in {1}")) + arg=("{0}.dtype in {1}", dtypes.void))) else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_dtype[0])), - arg="{0}.dtype == {1}")) + arg=("{0}.dtype == {1}", dtypes.void))) if self.match_tag is not None: if len(self.match_tag) > 1: - and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(self.match_tag))), arg="{0}.tag in {1}")) - else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_tag[0])), arg="{0}.tag == {1}")) + and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(self.match_tag))), arg=("{0}.tag in {1}", dtypes.void))) + else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_tag[0])), arg=("{0}.tag == {1}", dtypes.void))) if self.src is not None: # single match if len(self.src) == 1 and isinstance(self.src[0], tuple): and_clause += [_get_clause(s, base.index(i), depth) for i,s in enumerate(self.src[0])] # repeat match elif len(self.src) == 1 and isinstance(self.src[0], itertools.repeat): - it = UOp(Ops.CUSTOMI, arg=f"ituop{depth}") + it = UOp(Ops.CUSTOMI, arg=(f"ituop{depth}", dtypes.void)) match = _get_clause(next(self.src[0]), it, depth+1) - and_clause.append(UOp(Ops.CUSTOM, src=(match, it, base), arg="all([{0} for {1} in {2}.src])")) + and_clause.append(UOp(Ops.CUSTOM, src=(match, it, base), arg=("all([{0} for {1} in {2}.src])", dtypes.void))) # multi match (fork) elif len(self.src) > 1 and all(isinstance(x, tuple) for x in self.src): fork_cond = [UOp(Ops.AND, src=tuple([_get_clause(s, base.index(i), depth) for i,s in enumerate(ss)])) for ss in self.src] and_clause.append(UOp(Ops.OR, src=tuple(fork_cond))) else: raise RuntimeError("broken") - return UOp(Ops.AND, src=tuple(and_clause)) if and_clause else UOp(Ops.CUSTOMI, arg="True") + return UOp(Ops.AND, src=tuple(and_clause)) if and_clause else UOp(Ops.CUSTOMI, arg=("True", dtypes.void)) # *** pattern matcher *** @@ -91,7 +94,7 @@ def do_process_and(a:UOp) -> UOp|None: for store in stores: if store.src[0] in dict_stores: # duplicate store is an identity compare - new_src.append(UOp(Ops.CUSTOM, src=(dict_stores[store.src[0]], store.src[1]), arg="{0} is {1}")) + new_src.append(UOp(Ops.CUSTOM, src=(dict_stores[store.src[0]], store.src[1]), arg=("{0} is {1}", dtypes.void))) found = True else: dict_stores[store.src[0]] = store.src[1] @@ -108,17 +111,18 @@ pm_proc = PatternMatcher([(UPat(Ops.AND, name="a"), do_process_and)], compiled=F # renderer def wrap(ctx, x) -> UOp: ctx[ret:=f"a{len(ctx)}"] = x.arg - return UOp(Ops.CUSTOMI, arg=ret) + return UOp(Ops.CUSTOMI, arg=(ret, dtypes.void)) pm_renderer = PatternMatcher([ (UPat(Ops.PYLITERAL, name="x"), wrap), # AND of CUSTOMI fragments inside a CUSTOM becomes a single CUSTOMI (joined with " and ") (UPat(Ops.CUSTOM, src=(UPat(Ops.AND, src=UPat(Ops.CUSTOMI), name="x"), UPat(), UPat()), name="r"), - lambda r,x: r.replace(src=(UOp(Ops.CUSTOMI, arg="(" + ' and '.join(y.arg for y in x.src) + ")"),)+r.src[1:])), + lambda r,x: r.replace(src=(UOp(Ops.CUSTOMI, arg=("(" + ' and '.join(y.arg[0] for y in x.src) + ")", dtypes.void)),)+r.src[1:])), - (UPat(Ops.CUSTOM, src=UPat(Ops.CUSTOMI), name="x"), lambda x: UOp(Ops.CUSTOMI, arg=x.arg.format(*[y.arg for y in x.src]))), - (UPat(Ops.INDEX, src=(UPat(Ops.CUSTOMI, name="x"), UPat(Ops.CONST, name="c")), name="g"), lambda x,c,g: x.replace(arg=x.arg+f".src[{c.val}]")) + (UPat(Ops.CUSTOM, src=UPat(Ops.CUSTOMI), name="x"), lambda x: UOp(Ops.CUSTOMI, arg=(x.arg[0].format(*[y.arg[0] for y in x.src]), dtypes.void))), + (UPat(Ops.INDEX, src=(UPat(Ops.CUSTOMI, name="x"), UPat(Ops.CONST, name="c")), name="g"), + lambda x,c,g: x.replace(arg=(x.arg[0]+f".src[{c.val}]", dtypes.void))) ], compiled=False) def _final_render(x:UOp, has_ctx:bool, depth=1) -> list[str]: @@ -131,8 +135,8 @@ def _final_render(x:UOp, has_ctx:bool, depth=1) -> list[str]: for ss in s.src: or_pieces.extend(_final_render(ss, has_ctx, depth+1)) elif s.op is Ops.STORE: assert s.src[0].op is Ops.CUSTOMI and s.src[1].op is Ops.CUSTOMI - store_pieces.append(f"{s.src[0].arg}={s.src[1].arg}") - elif s.op is Ops.CUSTOMI: and_pieces.append(s.arg) + store_pieces.append(f"{s.src[0].arg[0]}={s.src[1].arg[0]}") + elif s.op is Ops.CUSTOMI: and_pieces.append(s.arg[0]) else: raise UPatCompileError(f"can't compile this {s}") # if we have an or, render it if len(or_pieces): @@ -145,7 +149,7 @@ def _final_render(x:UOp, has_ctx:bool, depth=1) -> list[str]: return [f"{' '*depth}if {and_clause}: return _ret"] def _get_code(self:UPat, has_ctx:bool): - ret = _get_clause(self, UOp(Ops.CUSTOMI, arg="uop")) + ret = _get_clause(self, UOp(Ops.CUSTOMI, arg=("uop", dtypes.void))) try: # TODO: this should be tracked in a "system" rewrite, not untracked or tracked with kernel with Context(TRACK_MATCH_STATS=0): diff --git a/tinygrad_repo/tinygrad/uop/validate.py b/tinygrad_repo/tinygrad/uop/validate.py index d87fce5a3..a408c28e4 100644 --- a/tinygrad_repo/tinygrad/uop/validate.py +++ b/tinygrad_repo/tinygrad/uop/validate.py @@ -37,6 +37,7 @@ z3_renderer = PatternMatcher([ # variables (UPat(Ops.SPECIAL, name="x"), lambda x,ctx: create_bounded(x.arg, 0, ctx[1][x.src[0]]-1, ctx[0])), (UPat(Ops.PARAM, name="x"), lambda x,ctx: create_bounded(x.arg.name, x.vmin, x.vmax, ctx[0])), + (UPat(Ops.BUFFER, name="x"), lambda x,ctx: create_bounded(x.arg.name, x.vmin, x.vmax, ctx[0]) if x.is_variable else None), (UPat(Ops.RANGE, name="x"), lambda x,ctx: create_bounded(x.render(simplify=False), 0, ctx[1][x.src[0]]-1, ctx[0])), # loads are variables bounded by the min/max of the dtype. non-pointer INDEX is also a LOAD (UPat((Ops.LOAD, Ops.INDEX), dtypes.ints+(dtypes.weakint,), name="x"), lambda x,ctx: @@ -44,13 +45,15 @@ z3_renderer = PatternMatcher([ (UPat((Ops.LOAD, Ops.INDEX), dtypes.bool), lambda ctx: (z3.Bool(f"load{len(ctx[1])}", ctx=ctx[0]), None)), # constants (UPat(Ops.CONST, arg=Invalid), lambda ctx: (z3.Int("Invalid", ctx=ctx[0]), None)), - (UPat(Ops.CONST, dtypes.ints+(dtypes.weakint,), name="x"), lambda x,ctx: (z3.IntVal(x.val, ctx=ctx[0]), None)), + (UPat(Ops.CONST, dtypes.weakint, name="x"), lambda x,ctx: (z3.IntVal(x.val, ctx=ctx[0]), None)), (UPat(Ops.CONST, dtypes.bool, name="x"), lambda x,ctx: (z3.BoolVal(x.val, ctx=ctx[0]), None)), # casts from floats create new variables (UPat(Ops.CAST, dtypes.ints+(dtypes.weakint,), src=(UPat(dtype=dtypes.floats),), name="x"), lambda x,ctx: create_bounded(f"cast{len(ctx[1])}", x.dtype.min, x.dtype.max, ctx[0])), # A comparison between floats introduces a new bool variable (UPat(GroupOp.Comparison, src=UPat(dtype=dtypes.floats)), lambda ctx: (z3.Bool(f"float_cmp{len(ctx[1])}", ctx=ctx[0]), None)), + # a same-dtype cast states a width, which z3 does not model: identity. must precede the rules below (bool->bool) + (UPat(Ops.CAST, name="x"), lambda x,ctx: (ctx[1][x.src[0]], None) if x.dtype == x.src[0].dtype else None), # casts from bool/int to int/bool (UPat(Ops.CAST, dtypes.ints+(dtypes.weakint,),src=(UPat.var("x", dtypes.bool),)), lambda x,ctx: (z3.If(ctx[1][x], 1, 0), None)), (UPat(Ops.CAST, dtypes.ints+(dtypes.weakint,), src=(UPat.var("x", dtypes.ints+(dtypes.weakint,)),)), lambda x,ctx: (ctx[1][x], None)), @@ -60,7 +63,7 @@ z3_renderer = PatternMatcher([ def uops_to_z3(solver:z3.Solver, *uops: UOp) -> list[z3.ExprRef]: # gate on upstream memory addressing, but keep INDEX as an unknown LOAD - lst = list(UOp.sink(*uops).toposort(gate=lambda x: x.op not in {Ops.AFTER, Ops.BUFFER, Ops.SHRINK} and \ + lst = list(UOp.sink(*uops).toposort(gate=lambda x: x.op not in {Ops.AFTER, Ops.SHRINK} and (x.op is not Ops.BUFFER or x.is_variable) and \ (x.dtype in dtypes.ints+(dtypes.bool, dtypes.weakint) or x.op is Ops.SINK)))[:-1] z3map: dict[UOp, z3.ExprRef] = {} for u in lst: diff --git a/tinygrad_repo/tinygrad/uop/weak.py b/tinygrad_repo/tinygrad/uop/weak.py index 0b0e578ca..c4c9bd485 100644 --- a/tinygrad_repo/tinygrad/uop/weak.py +++ b/tinygrad_repo/tinygrad/uop/weak.py @@ -1,78 +1,96 @@ from dataclasses import replace from tinygrad.dtype import dtypes, DType, AddrSpace, Invalid, least_upper_dtype, strong_dtype, weak_dtype -from tinygrad.helpers import unwrap -from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, graph_rewrite, dtype_from_uop -def select_dtype(u:UOp): - if u.dtype is dtypes.weakfloat: return dtypes.default_float - return dtypes.long if u.overflows(dtypes.int32) else dtypes.int +from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, dtype_from_uop, promo_dtype -def lower_weak_node(u:UOp) -> UOp|None: - start, src = (1 if u.op is Ops.WHERE else 0), tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) - if src == u.src or any(s.dtype in dtypes.weaks for s in src[start:]): return None - dt = strong_dtype(least_upper_dtype(select_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary - else unwrap(dtype_from_uop(u.op, src, u.arg))) - return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid else s.cast(dt) for s in src[start:])).cast(u.dtype) +# the decomps and float emulation commit bare consts at a dtype another src already states +def commit_weak_consts(u:UOp, dt:DType|None) -> UOp|None: + return None if dt is None else u.replace(src=tuple(s.ccast(dt) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in u.src)) -pm_lower_weak = PatternMatcher([ - (UPat(Ops.CONST, dtype=dtypes.weaks, name="u"), lambda u: UOp.const(u.val, select_dtype(u)).cast(u.dtype)), - # two stacked weak casts are a weakint value used as weakfloat (or vice versa): resolve the inner one at the outer kind's default. - # a SINGLE weak cast is never rewritten here, each consumer absorbs it on its own edge (see lower_weak_srcs) - (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), - lambda u,x: x.cast(select_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), - # Binary can widen from the bounds, all other nodes derive from the lowered sources. - # a weakfloat Unary (sin/exp2/...) must resolve here, before the transcendental decomposition - (UPat(GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL}, name="u"), lower_weak_node), - (UPat(Ops.PARAM, dtype=dtypes.weakint, name="u"), - lambda u: u.replace(dtype=None, arg=replace(u.arg, dtype=select_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), -]) +# the concrete dtypes u commits its srcs at: the operands' meet and u's own derived dtype, None if either is weak +def derived_dtypes(u:UOp, src:tuple[UOp, ...]) -> tuple[DType, DType]|None: + if u.op not in GroupOp.Broadcastable or (meet:=promo_dtype(src)) in dtypes.weaks \ + or (result:=dtype_from_uop(u.op, src, u.arg)) in dtypes.weaks: return None + return meet, result -def lower_weak_srcs(ctx:dict[UOp, UOp]|None, u:UOp) -> UOp|None: - if ctx is None: ctx = {} - def lower(s:UOp) -> UOp: - if (r:=ctx.get(s)) is None: - r = graph_rewrite(s, pm_lower_weak) - # the consumer absorbs the cast on its own edge - ctx[s] = r = r.src[0] if r.op is Ops.CAST and r.dtype in dtypes.weaks else r - return r - # a comparison demands a common operand width: lower it whole so the Binary rule unifies its operands - ret = lower(u) if u.op in GroupOp.Comparison else u.replace(src=tuple(lower(s) if s.dtype in dtypes.weaks else s for s in u.src)) - return None if ret is u else ret - -def commit_weak(s:UOp, dt:DType) -> UOp: - # a bare weak CONST commits directly (the value stays mathematical, emission truncates), a weak non-const src takes the demand cast - return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) +def commit_srcs_at(u:UOp, dt:DType) -> UOp|None: + # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too + bare = derived_dtypes(u, u.src) is not None + src = tuple(s if s.dtype not in dtypes.weaks else UOp.const(dt.const(s.val)) if bare and s.op is Ops.CONST else s.ccast(dt) for s in u.src) + return None if (ret := u.replace(src=src)) is u else ret def commit_weak_srcs(u:UOp) -> UOp|None: - if not any(s.dtype in dtypes.weaks for s in u.src): return None - if (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None - # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too - return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) - -# runs in index lowering and in the decomps: a rule that mints a weak const commits it in the same rewrite, so none reaches the renderer -pm_commit_weak = PatternMatcher([ - (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), - # demand from the destination: a STORE's weak value commits at the destination's dtype - (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), - lambda u: u.replace(src=(u.src[0], commit_weak(u.src[1], u.src[0].dtype), *u.src[2:]))), -]) + if not any(s.dtype in dtypes.weaks for s in u.src) or (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None + return commit_srcs_at(u, dt) # a concrete CAST over a weak node states the width the value will live at. that width is a floor, never a narrowing def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: + # only within the kind: an int cast of a weakfloat node is a value conversion, not a statement about the node's width if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None - dt = least_upper_dtype(c.dtype, select_dtype(u)) - return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)).cast(c.dtype) + # every weak src commits at the one width: the node's own bounds and each src's, none of them narrowed + dt = least_upper_dtype(c.dtype, u.commit_dtype(dtypes.int), *(s.commit_dtype(dtypes.int) for s in u.src if s.dtype in dtypes.weaks)) + return None if (ret:=commit_srcs_at(u, dt)) is None else ret.cast(c.dtype) -pm_cast_weak = PatternMatcher([ +# rides every round that can mint a weak const, and must reach fixpoint before pm_lower_weak below defaults one +pm_commit_weak = PatternMatcher([ + (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), + (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), + lambda u: u.replace(src=(u.src[0], u.src[1].ccast(u.src[0].dtype), *u.src[2:]))), + # no CONST arm: a concrete CAST over a weak CONST is already committed, minted that way by UOp.const (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), ]) -pm_lower_index_dtype = pm_commit_weak+pm_cast_weak+PatternMatcher([ - (UPat(GroupOp.All, name="u"), - lambda ctx,u: lower_weak_srcs(ctx, u) if u.dtype not in dtypes.weaks and any(s.dtype in dtypes.weaks for s in u.src) else None), - # a valid index into an n-element buffer lives in [0,n): a gated long index narrows when n-1 fits int32 (out-of-gate wraps, discarded) - # TODO: more generic +# consumers absorb the weak CAST off their srcs and default underivable consts; dtype-producing ops settle here. +# a weakfloat Unary (sin/exp2/...) must resolve before the transcendental decomposition. +_lower_weak_ops = GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL} +def lower_weak_node(u:UOp) -> UOp|None: + if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return None # a committed const, not a consumer + src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) + if derived_dtypes(u, src) is None: + src = tuple(s.ccast(s.commit_dtype(dtypes.int)) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in src) + if src == u.src: return None + start = 1 if u.op is Ops.WHERE else 0 # WHERE's cond is bool, never part of the width unification + if u.op not in _lower_weak_ops or any(s.dtype in dtypes.weaks and s.op is not Ops.CONST for s in src[start:]): return u.replace(src=src) + # resolve whole once every weak expression lowered: a Binary widens from its own bounds too, derivable consts wait + dt = strong_dtype(least_upper_dtype(u.commit_dtype(dtypes.int), *(s.dtype for s in src)) if u.op in GroupOp.Binary else + dtype_from_uop(u.op, src, u.arg)) + src = src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else s.ccast(dt) for s in src[start:]) + return u.replace(src=src).cast(u.dtype) + +pm_lower_weak = PatternMatcher([ + # a gated long index into a small buffer narrows; its out-of-gate value is discarded (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("gate").where(UPat.var("idx", dtypes.long), UPat(Ops.CONST, arg=Invalid))), allow_any_len=True, name="u"), lambda u,buf,gate,idx: u.replace(src=(buf, idx.cast(dtypes.int).valid(gate))+u.src[2:]) if buf.max_numel()-1 <= dtypes.int32.max else None), + # two stacked weak casts are two kind conversions: each resolves at its own kind's default + (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), + lambda u,x: x.cast(u.src[0].commit_dtype(dtypes.int)).cast(u.commit_dtype(dtypes.int)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), + (UPat((Ops.PARAM, Ops.BUFFER), dtype=dtypes.weakint, name="u"), + lambda u: u.replace(arg=replace(u.arg, dtype=u.commit_dtype(dtypes.int))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), + (UPat(GroupOp.All, name="u"), lower_weak_node), ]) + +# drop the CAST off a committed const where the consumer re-derives it anyway, so bare-CONST rules keep matching. +# the drop must change nothing the consumer derives: neither the operands' meet nor the node's own dtype +def uncast_const(u:UOp) -> UOp|None: + # a weak CAST over a const is not a commit, it is still resolving + src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype not in dtypes.weaks and s.src[0].op is Ops.CONST + and s.src[0].dtype in dtypes.weaks else s for s in u.src) + if src == u.src or (dts:=derived_dtypes(u, src)) is None or dts[0] != promo_dtype(u.src) or dts[1] is not u.dtype: return None + return u.replace(src=src) + +pm_uncast_const = PatternMatcher([(UPat(GroupOp.Broadcastable, name="u"), uncast_const)]) + +def cast_const(u:UOp, s:UOp) -> UOp: + if s.op is not Ops.CONST or s.is_invalid: return s # Invalid never commits + # bool is the one strong bare dtype: cconst, since .cast(bool) would fold at construction + if s.dtype is dtypes.bool: return UOp.cconst(s.val, s.dtype) + # commit at the dtype its consumer derives + return s.ccast(dts[0]) if (dts:=derived_dtypes(u, u.src)) is not None else s + +# commit every remaining bare const, keyed on the consumer: "bare" is a property of the edge +def cast_consts(u:UOp) -> UOp|None: + if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return None # a committed const's CONST is its value, not an edge + return None if (src:=tuple(cast_const(u, s) for s in u.src)) == u.src else u.replace(src=src) + +pm_cast_const = PatternMatcher([(UPat(GroupOp.All, name="u"), cast_consts)]) diff --git a/tinygrad_repo/tinygrad/viz/index.html b/tinygrad_repo/tinygrad/viz/index.html index ea208358d..055ccd36e 100644 --- a/tinygrad_repo/tinygrad/viz/index.html +++ b/tinygrad_repo/tinygrad/viz/index.html @@ -380,9 +380,6 @@ tr.main-row > td { padding: 8px 12px; } - td.Instruction { - font-family: monospace; - } thead { position: sticky; top: 0; diff --git a/tinygrad_repo/tinygrad/viz/js/index.js b/tinygrad_repo/tinygrad/viz/js/index.js index 8ea2ef384..b5ba7a381 100644 --- a/tinygrad_repo/tinygrad/viz/js/index.js +++ b/tinygrad_repo/tinygrad/viz/js/index.js @@ -230,7 +230,7 @@ const waveColor = (op) => { }; const colorScheme = {TINY:new Map([["Schedule","#1b5745"],["precompile","#1d2e62"],["compile","#63b0cd"],["DEFAULT","#354f52"]]), DEFAULT:["#2b2e39", "#2c2f3a", "#31343f", "#323544", "#2d303a", "#2e313c", "#343746", "#353847", "#3c4050", "#404459", "#444862", "#4a4e65"], - BUFFER:["#342483", "#3E2E94", "#4938A4", "#5442B4", "#5E4CC2", "#674FCA"], SIMD:new Map([["OCC", "#101725"], ["INST", "#0A2042"]]), + BUFFER:["#342483", "#3E2E94", "#4938A4", "#5442B4", "#5E4CC2", "#674FCA"], GPC:new Map([["NONE","#1a7a2e"],["MEMORY_DEPENDENCY","#8b1a00"],["EXEC_DEPENDENCY","#006b6b"],["INST_FETCH","#7a7a00"],["SYNC","#6b006b"], ["PIPE_BUSY","#7a4a00"],["MEMORY_THROTTLE","#5c0000"],["CONSTANT_MEMORY","#1a3d7a"],["NOT_SELECTED","#2e2e3a"],["OTHER","#4a4a55"], ["SLEEPING","#1a1a2a"],["DEFAULT","#3a3a45"]]), WAVE:waveColor, VMEMEXEC:waveColor, ALUEXEC:waveColor} @@ -487,13 +487,6 @@ async function renderProfiler(path, opts) { for (let si=start; si k in ns && state[k] !== ns[k])) saveToHistory(state); const { ctx:prevCtx, step:prevStep } = select(state.currentCtx, state.currentStep); const prevRewrite = state.currentRewrite; Object.assign(state, ns); @@ -999,15 +992,6 @@ async function main() { } if (!ckey.startsWith("/graph")) { if (!(ckey in cache)) cache[ckey] = ret = await fetchValue(ckey); - if (ret.steps?.length > 0) { - const el = select(state.currentCtx, state.currentStep); - if (el.step.querySelectorAll("ul").length === ret.steps.length) return; - // re render the list with new items - ctx.steps.push(...ret.steps); - while (el.ctx.children.length > 1) el.ctx.children[1].remove(); - appendSteps(el.ctx, state.currentCtx, ctx.steps); - return setState({ currentStep:state.currentStep+1, expandSteps:true }); - } // timeline with cycles on the x axis if (ret instanceof ArrayBuffer) { const pkts = step.query.includes("sqtt"); @@ -1050,10 +1034,6 @@ async function main() { } return table; } - if (ret.ref != null) { - const disasmIdx = ctxs[ret.ref+1].steps.findIndex(s => s.name === "View Disassembly") - metadata.appendChild(d3.create("a").text("View Disassembly").on("click", () => switchCtx(ret.ref, disasmIdx)).node()); - } if (ret.cols != null) renderTable(root, ret); else if (ret.src != null) root.append(() => codeBlock(ret.src, ret.lang)); return document.querySelector("#custom").replaceChildren(root.node()); diff --git a/tinygrad_repo/tinygrad/viz/js/worker.js b/tinygrad_repo/tinygrad/viz/js/worker.js index 00de46c5f..55e2e2fc5 100644 --- a/tinygrad_repo/tinygrad/viz/js/worker.js +++ b/tinygrad_repo/tinygrad/viz/js/worker.js @@ -55,7 +55,7 @@ const layoutUOp = (g, { graph, change }, opts) => { height += lineHeight; } const op = label.split("\n", 1)[0]; - const callNode = op === "CALL" || op === "FUNCTION", programNode = op === "PROGRAM"; + const callNode = op === "CALL", programNode = op === "PROGRAM"; const collapsePorts = callNode ? [0] : programNode ? [0, 1] : null; if (callNode) callCount++; g.setNode(k, {...rectDims(width, height), label, labelX:0, ref, id:k, color, tag, callNode, collapsePorts, exclude, addrspace, diff --git a/tinygrad_repo/tinygrad/viz/serve.py b/tinygrad_repo/tinygrad/viz/serve.py index 506fcac00..98abf551c 100755 --- a/tinygrad_repo/tinygrad/viz/serve.py +++ b/tinygrad_repo/tinygrad/viz/serve.py @@ -50,8 +50,8 @@ uops_colors = {Ops.LOAD: "#ffc0c0", Ops.STORE: "#87CEEB", Ops.CONST: "#e0e0e0", Ops.INDEX: "#CEF9B7", Ops.STACK: "#D8F9E4", Ops.WMMA: "#efefc0", Ops.UNSHARD: "#f6ccff", Ops.INS: "#eec4ff", **{x:"#D8F9E4" for x in GroupOp.Movement}, **{x:"#ffffc0" for x in GroupOp.ALU}, Ops.THREEFRY:"#ffff80", - Ops.SLICE: "#E5EAFF", Ops.BUFFER: "#B0BDFF", Ops.GETADDR: "#9DB1F0", Ops.COPY: "#a040a0", Ops.CUSTOM_FUNCTION: "#bf71b6", - Ops.CALL: "#00B7C8", Ops.FUNCTION: "#C07788", Ops.PARAM: "#14686F", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", + Ops.BUFFER: "#B0BDFF", Ops.GETADDR: "#9DB1F0", Ops.COPY: "#a040a0", Ops.CUSTOM_FUNCTION: "#bf71b6", + Ops.CALL: "#00B7C8", Ops.PARAM: "#14686F", Ops.RETURNED: "#C07788", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", Ops.LINEAR: "#7DF4FF", Ops.ALLREDUCE: "#ff40a0", Ops.MSELECT: "#d040a0", Ops.MSTACK: "#d040a0", Ops.CONTIGUOUS: "#FFC14D", Ops.STAGE: "#AC640D", Ops.REWRITE_ERROR: "#1a1b26", Ops.AFTER: "#8A7866", Ops.END: "#524C46"} @@ -145,7 +145,7 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: label += f"\n({multirange_str(rngs, color=True)})" if u._shape is not None: label += f"\n{shape_to_str(u.shape)}" - if u.op in {Ops.CALL, Ops.FUNCTION}: + if u.op is Ops.CALL: label += f"\n{u.src[0].key.hex()[:8]}\n{u.src[0].op}" if u.op in {Ops.INDEX, Ops.STAGE}: label += f"\n{u.render()}" if sum(len(s.toposort()) for s in u.src[1:]) < 30 else "\nINDEX TOO LARGE" @@ -156,10 +156,10 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: label += "\n"+' '.join([f"{range_str(s, color=True)}({s.vmax+1})" for s in trngs]) except Exception: label += "\n" - ref = data.ref_map.get(canonicalize_ast(u.src[0])) if u.op in {Ops.CALL, Ops.FUNCTION} else None + ref = data.ref_map.get(canonicalize_ast(u.src[0])) if u.op is Ops.CALL else None if ref is not None: label += f"\ncodegen@{fmt_colored(data.ctxs[ref]['name'])}" # NOTE: kernel already has metadata in arg - if TRACEMETA >= 2 and u.metadata is not None and u.op not in {Ops.CALL, Ops.FUNCTION}: label += "\n"+str(u.metadata) + if TRACEMETA >= 2 and u.metadata is not None and u.op is not Ops.CALL: label += "\n"+str(u.metadata) # limit SOURCE labels line count if u.op is Ops.SOURCE and len(lines:=label.split("\n")) > 40: label = "\n".join(lines[:30]) + "\n..." @@ -171,9 +171,9 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: def _reconstruct(data:VizData, a:int, depth:int|None=None): if depth is None and a in data.all_uops: return data.all_uops[a] - op, dtype, src, arg, *rest = data.trace.uop_fields[a] - if depth is not None and depth <= 0: return UOp(op, dtype, (), arg, *rest) - ret = UOp(op, dtype, tuple(_reconstruct(data, s, None if depth is None else depth-1) for s in src), arg, *rest) + op, src, arg, *rest = data.trace.uop_fields[a] + if depth is not None and depth <= 0: return UOp(op, (), arg, *rest) + ret = UOp(op, tuple(_reconstruct(data, s, None if depth is None else depth-1) for s in src), arg, *rest) if depth is None: data.all_uops[a] = ret return ret @@ -231,10 +231,11 @@ def timeline_layout(data:VizData, dev_events:list[tuple[int, int, float, DevEven ei:ProfilePointEvent|None = None for st,et,dur,e in dev_events: if isinstance(e, ProfilePointEvent) and e.name == "exec": ei = e - if dur == 0: continue + # only visualize range events with an end timestamp + if dur == 0 or isinstance(e, ProfilePointEvent): continue name, key = e.name, None fmt:dict = {} - if (ref:=data.ref_map.get(name)) is not None and ref < len(data.ctxs): + if (ref:=data.ref_map.get(e.profile_key)) is not None and ref < len(data.ctxs): name = data.ctxs[ref]["name"] if (ki:=data.ctxs[ref].get("ki")) is not None and ki.estimates is not None and ei is not None: for est_key,est_val in (("FLOPS", ki.estimates.ops), ("B/s mem", ki.estimates.mem), ("B/s lds", ki.estimates.lds)): @@ -333,14 +334,14 @@ def unpack_pmc(e) -> dict: def load_amd_counters(data:VizData, profile:list) -> None: counter_events:dict[tuple[int, int], dict] = {} - durations:dict[str, list[float]] = {} + durations:dict[bytes|str, list[float]] = {} prg_events:dict[int, ProfileProgramEvent] = {} arch = "" for e in profile: if type(e).__name__ in {"ProfilePMCEvent", "ProfileSQTTEvent"}: counter_events.setdefault((e.kern, e.exec_tag), {}).setdefault(type(e).__name__, []).append(e) - if isinstance(e, ProfileRangeEvent) and e.device.startswith("AMD") and e.en is not None: - durations.setdefault(str(e.name), []).append(float(e.en-e.st)) + if isinstance(e, ProfileRangeEvent) and e.device.startswith("AMD") and e.en is not None and e.profile_key is not None: + durations.setdefault(e.profile_key, []).append(float(e.en-e.st)) if isinstance(e, ProfileProgramEvent) and e.device.startswith("AMD") and e.tag is not None: prg_events[e.tag] = e if isinstance(e, ProfileDeviceEvent) and e.device.startswith("AMD"): arch = f"gfx{unwrap(e.props)['gfx_target_version']//1000}" if len(counter_events) == 0: return None @@ -348,21 +349,16 @@ def load_amd_counters(data:VizData, profile:list) -> None: run_number = {n:0 for n,_ in counter_events} for (k, tag),v in counter_events.items(): # use the colored name if it exists - name = data.ctxs[r]["ki"].name if (r:=data.ref_map.get(pname:=prg_events[k].name)) is not None else pname + name = data.ctxs[r]["ki"].name if (r:=data.ref_map.get(unwrap(prg_events[k].profile_key))) is not None else prg_events[k].name run_number[k] += 1 steps:list[dict] = [] if (pmc:=v.get("ProfilePMCEvent")): steps.append(create_step("PMC", ("/prg-pmc", len(data.ctxs), len(steps)), pmc[0])) - all_counters[(name, run_number[k], pname)] = pmc[0] + all_counters[(name, run_number[k], unwrap(prg_events[k].profile_key))] = pmc[0] # to decode a SQTT trace, we need the raw stream, program binary and device properties if (sqtt:=v.get("ProfileSQTTEvent")): for e in sqtt: if e.itrace: steps.append(create_step(f"SE:{e.se} PKTS", (f"/sqtt-{e.se}",len(data.ctxs),len(steps)), data=(e.blob,prg_events[k].lib,arch))) - try: - with Context(DEBUG=0): from extra.sqtt.roc import unpack_occ - steps.append(create_step("OCC", ("/amd-sqtt-occ", len(data.ctxs), len(steps)), - data={"fxn":unpack_occ, "args":((k, tag), sqtt, prg_events[k], arch)})) - except Exception: pass data.ctxs.append({"name":f"SQTT {name}"+(f" n{run_number[k]}" if run_number[k] > 1 else ""), "steps":steps}) wave_colors = {"WMMA": "#1F7857", **{x:"#ffffc0" for x in ["VALU", "VINTERP"]}, "SALU": "#cef263", "SMEM": "#ffc0c0", "STORE": "#4fa3cc", @@ -469,11 +465,12 @@ def get_profile(data:VizData, profile:list[ProfileEvent], sort_fn:Callable[[str] start_ts:int|None = None end_ts:int|None = None for ts,en,e in flatten_events(profile, device_ts_diffs): - dev_events.setdefault(e.device,[]).append((st:=int(ts), et:=int(en), float(en-ts), e)) - if start_ts is None or st < start_ts: start_ts = st - if end_ts is None or et > end_ts: end_ts = et - if isinstance(e, ProfilePointEvent) and e.name == "marker": markers.append(e) if isinstance(e, ProfilePointEvent) and e.name == "JSON": ext_data[e.key] = e.arg + else: + dev_events.setdefault(e.device,[]).append((st:=int(ts), et:=int(en), float(en-ts), e)) + if start_ts is None or st < start_ts: start_ts = st + if end_ts is None or et > end_ts: end_ts = et + if isinstance(e, ProfilePointEvent) and e.name == "marker": markers.append(e) if start_ts is None: return None # return layout of per device events layout:dict[str, bytes|None] = {} @@ -496,10 +493,10 @@ def get_profile(data:VizData, profile:list[ProfileEvent], sort_fn:Callable[[str] def load_nv_counters(data:VizData, profile:list) -> None: steps:list[dict] = [] sm_version = {e.device:e.props.get("sm_version", 0x800) for e in profile if isinstance(e, ProfileDeviceEvent) and e.props is not None} - run_number:dict[str, int] = {} + run_number:dict[bytes, int] = {} for e in profile: if type(e).__name__ == "ProfilePMAEvent": - run_number[e.kern] = run_num = run_number.get(e.kern, 0)+1 + run_number[profile_key] = run_num = run_number.get(profile_key:=unwrap(e.profile_key), 0)+1 steps.append(create_step(f"PMA {e.kern}"+(f"n{run_num}" if run_num>1 else ""), ("/prg-pma-pkts", len(data.ctxs), len(steps)), data=(e.blob, sm_version[e.device]))) if steps: data.ctxs.append({"name":"All Counters", "steps":steps}) @@ -649,9 +646,6 @@ def get_render(viz_data:VizData, query:str, **kwargs) -> dict: ret = {"value":events, "content_type":"application/octet-stream"} else: ret = {"src":"No SQTT trace on this SE."} return ret - # viewers for the amd decoder in extra - if fmt.startswith("amd-sqtt"): return data["fxn"](viz_data, i, j, *data["args"]) - if fmt == "cu-sqtt": return {"value":get_profile(viz_data, data, sort_fn=row_tuple), "content_type":"application/octet-stream"} if fmt == "prg-pma-pkts": ret = {} with soft_err(lambda err:ret.update(err)):