diff --git a/openpilot/selfdrive/modeld/modeld.py b/openpilot/selfdrive/modeld/modeld.py index d78249bba3..07320b82e1 100755 --- a/openpilot/selfdrive/modeld/modeld.py +++ b/openpilot/selfdrive/modeld/modeld.py @@ -167,6 +167,13 @@ def input_view(buffer: Buffer, shape: tuple[int, ...], dtype: DType, offset: int return Tensor(UOp.from_buffer(view, dtype)).reshape(shape) +def compile_copy(tensor: Tensor) -> Callable[[], Tensor]: + from tinygrad.engine.jit import CapturedJit + from tinygrad.engine.realize import compile_linear + jit = CapturedJit(tensor, compile_linear(tensor.schedule_linear()), [], []) + return lambda: jit([], {}) + + class ModelState: lat_delay: float frame_drop_ratio: float @@ -200,6 +207,7 @@ class ModelState: state = self.input_queues[name] assert all_int(state.shape) self.outputs[next_name] = input_view(state._buffer(), state.shape, state.dtype, 0) + self.copy_outputs = None if selected_legacy() else compile_copy(self.outputs['outputs'].to('CPU')) self.parser = Parser() def pack_inputs(self) -> None: @@ -210,8 +218,15 @@ class ModelState: if name not in self.state_pairs and name != 'new_img'} npy_size = sum(round_up(math.prod(shape) * 4, 128) for shape in shapes.values()) self.packed_input = np.zeros(npy_size + 2 * self.frame_copy_size, dtype=np.uint8) - self.input_host = Tensor(self.packed_input, device='NPY')._buffer() - self.input_device = Tensor(self.packed_input, device=self.model_device)._buffer() + if selected_legacy(): + self.input_host = Tensor(self.packed_input, device='NPY')._buffer() + self.input_device = Tensor(self.packed_input, device=self.model_device)._buffer() + else: + host = Tensor(self.packed_input, device='NPY') + device = host.to(self.model_device) + self.copy_inputs = compile_copy(device) + self.copy_inputs() + self.input_host, self.input_device = host._buffer(), device._buffer() self.npy = {} offset = 0 for name, shape in shapes.items(): @@ -237,7 +252,10 @@ class ModelState: self.npy['traffic_convention'][:] = inputs['traffic_convention'] self.npy['action_t'][:] = inputs['action_t'] - self.input_device.copy_from(self.input_host) + if selected_legacy(): + self.input_device.copy_from(self.input_host) + else: + self.copy_inputs() self.input_queues['new_img'] = self.run_warp(**self.warp_inputs) self.run_model(output_buffers=self.outputs, **self.input_queues) if (self.chestnut and not selected_legacy() and @@ -245,7 +263,11 @@ class ModelState: raise RuntimeError(f"chestnut failed with {err}") if after_enqueue is not None: after_enqueue() - model_output = self.outputs['outputs'].numpy()[0] + if selected_legacy(): + model_output = self.outputs['outputs'].numpy()[0] + else: + assert self.copy_outputs is not None + model_output = self.copy_outputs().numpy()[0] if self.chestnut and not np.all(np.isfinite(model_output)): raise RuntimeError("model output not finite") outputs_dict = self.parser.parse_outputs(self.slice_outputs(model_output, self.output_slices)) diff --git a/openpilot/starpilot/flm/producer_clock_equivalence.py b/openpilot/starpilot/flm/producer_clock_equivalence.py index 87d09ee04d..ba8614e4b1 100644 --- a/openpilot/starpilot/flm/producer_clock_equivalence.py +++ b/openpilot/starpilot/flm/producer_clock_equivalence.py @@ -18,7 +18,7 @@ PRODUCER_SOURCE_HASHES = ( ('openpilot', 'system/camerad/cameras/spectra.cc', '4fa26c17f0fb792a4ab6815657beccddee1d641faa2fd02e5aa7393f717a2f3a'), ('openpilot', 'system/camerad/cameras/camera_qcom2.cc', 'b34c24c9984a0ac086f7b961cb7ff09d0501231da50aaad0e676bb3dc3f2edd9'), ('openpilot', 'system/camerad/cameras/camera_common.cc', 'bd9a416a6ee21047b4deec6df04276d8717cd92fe14de43f3e9daff86b62bb8f'), - ('openpilot', 'selfdrive/modeld/modeld.py', '8d0fec445484b325e55cf4ee3bee95b7061d99b7e125611358eef407aa10e871'), + ('openpilot', 'selfdrive/modeld/modeld.py', '4ffe0193c64acab99f47b594ed98164e7ab6285f04f18205525d1b9931b9a086'), ('openpilot', 'selfdrive/modeld/fill_model_msg.py', 'b3916c34d1073d0c26a7a8adc51e7b148f63a1ac68b388226170fd0f8b8b56b4'), ('openpilot', 'system/sensord/sensord.py', '7c79ab07d775622a43001e5a9b9ccc9a7bafb065940185849b33eefc2fd26420'), ('openpilot', 'system/sensord/sensors/lsm6ds3_gyro.py', '3e561f3d8572e1477c9b4b810c7570837d51e81c5672fe21ec67a68cae00dd92'), diff --git a/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py b/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py index 25fac9414d..f81ed2a852 100644 --- a/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py +++ b/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py @@ -9,7 +9,7 @@ from openpilot.starpilot.flm import producer_clock_equivalence as clock APPROVED = { - 'selfdrive/modeld/modeld.py': '8d0fec445484b325e55cf4ee3bee95b7061d99b7e125611358eef407aa10e871', + 'selfdrive/modeld/modeld.py': '4ffe0193c64acab99f47b594ed98164e7ab6285f04f18205525d1b9931b9a086', 'visionipc/visionipc_py.cc': '6c131c4f5eccb9f822f1aaae63733f26b1427c52340b61c9cdaf6cd4aee2f89d', 'visionipc/visionipc_server.cc': 'a4325a5171eacb931eb271d06a992d2f27216040df113d92099f62cdfa67d81a', 'visionipc/visionipc_client.cc': 'd142cd60eedabfc581e9770fcd6f0ca81b762ba9dbc608b54dd20a27a317a52d', diff --git a/tinygrad_repo/.github/actions/setup-tinygrad/action.yml b/tinygrad_repo/.github/actions/setup-tinygrad/action.yml index 7bf8884cf4..5a14bcacf0 100644 --- a/tinygrad_repo/.github/actions/setup-tinygrad/action.yml +++ b/tinygrad_repo/.github/actions/setup-tinygrad/action.yml @@ -63,15 +63,11 @@ runs: echo "OMP_NUM_THREADS=1" >> "$GITHUB_ENV" # no buffers should be over 300MB in CI echo "MAX_BUFFER_SIZE=300000000" >> "$GITHUB_ENV" - if [[ "$RUNNER_OS" != "Windows" ]]; then - if [[ "$RUNNER_OS" == "macOS" ]]; then - sudo install -d -o "$USER" /opt/venv /opt/python - fi - echo "VIRTUAL_ENV=/opt/venv/${{ inputs.python-version }}" >> "$GITHUB_ENV" - echo "UV_PYTHON_INSTALL_DIR=/opt/python" >> "$GITHUB_ENV" - else - echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> "$GITHUB_ENV" + if [[ "$RUNNER_OS" == "macOS" ]]; then + sudo install -d -o "$USER" /opt/venv /opt/python fi + echo "VIRTUAL_ENV=/opt/venv/${{ inputs.python-version }}" >> "$GITHUB_ENV" + echo "UV_PYTHON_INSTALL_DIR=/opt/python" >> "$GITHUB_ENV" - name: Set up uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b @@ -131,12 +127,7 @@ runs: run: uv cache prune --ci - name: Configure venv shell: bash - run: | - if [[ "$RUNNER_OS" == "Windows" ]]; then - echo "$VIRTUAL_ENV/Scripts" >> "$GITHUB_PATH" - else - echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH" - fi + run: echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH" # ******************* apt ******************* - name: Setup apt diff --git a/tinygrad_repo/.github/workflows/autogen.yml b/tinygrad_repo/.github/workflows/autogen.yml index 7573ee6c27..262d44d971 100644 --- a/tinygrad_repo/.github/workflows/autogen.yml +++ b/tinygrad_repo/.github/workflows/autogen.yml @@ -41,7 +41,7 @@ jobs: run: | find tinygrad/runtime/autogen -type f -name "*.py" -not -path "*/amd/*" -not -name "__init__.py" -not -name "metal.py" -not -name "iokit.py" -not -name "corefoundation.py" -not -name "libclang.py" -delete python3 -c "from tinygrad.runtime.autogen import opencl" - python3 -c "from tinygrad.runtime.autogen import cuda, nvrtc, nvjitlink, nv_570, nv_580, nv_610, nv" + python3 -c "from tinygrad.runtime.autogen import cuda, nvrtc, nvjitlink, nv_570, nv_580, nv_610, nv_615, nv" python3 -c "from tinygrad.runtime.autogen import comgr, comgr_3, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd, amdgpu_drm" python3 -c "from tinygrad.runtime.autogen.am import *" python3 -c "from tinygrad.runtime.autogen.nv_regs import *" diff --git a/tinygrad_repo/.github/workflows/benchmark.yml b/tinygrad_repo/.github/workflows/benchmark.yml index 3efa970f2f..c5ca9d179a 100644 --- a/tinygrad_repo/.github/workflows/benchmark.yml +++ b/tinygrad_repo/.github/workflows/benchmark.yml @@ -6,6 +6,8 @@ env: ASSERT_PROCESS_REPLAY: "0" PYTHONPATH: . GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INFLUXDB_HOST: ${{ secrets.INFLUXDB_HOST }} + INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }} on: push: @@ -81,7 +83,7 @@ jobs: llmbenchmark: name: Benchmark ${{ matrix.model }} (DEV=${{ matrix.dev }}) - runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + runs-on: "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && '7900xtx' || '4090' }}" strategy: fail-fast: false matrix: @@ -89,7 +91,7 @@ jobs: model: ['llama3.2:3b-f16', 'qwen3.8:27b', 'olmoe'] # qwen3.8:27b doesn't fit on mac exclude: [{ dev: 'METAL', model: 'qwen3.8:27b' }, { dev: 'AMD', model: 'olmoe' }, { dev: 'NV', model: 'olmoe' }] - timeout-minutes: 15 + timeout-minutes: 20 defaults: run: shell: bash -e -o pipefail {0} @@ -99,14 +101,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v6 - - name: Setup (AMD) - if: ${{ matrix.dev == 'AMD' }} - run: | - ./extra/hcq/hcq_smi.py amd rmmod --expect - ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: ./extra/hcq/hcq_smi.py nv kill_pids --sudoless + run: sudo modprobe -a nvidia nvidia_uvm + - name: Setup (AMD) + if: ${{ matrix.dev == 'AMD' }} + run: AM_RESET=1 python3 -c 'from tinygrad import Device; Device[Device.DEFAULT]' - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -123,7 +123,7 @@ jobs: cifarbenchmark: name: HLB-CIFAR10 (DEV=${{ matrix.dev }}) - runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + runs-on: "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && '7900xtx' || '4090' }}" strategy: fail-fast: false matrix: @@ -138,14 +138,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v6 - - name: Setup (AMD) - if: ${{ matrix.dev == 'AMD' }} - run: | - ./extra/hcq/hcq_smi.py amd rmmod --expect - ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: ./extra/hcq/hcq_smi.py nv kill_pids --sudoless + run: sudo modprobe -a nvidia nvidia_uvm + - name: Setup (AMD) + if: ${{ matrix.dev == 'AMD' }} + run: AM_RESET=1 python3 -c 'from tinygrad import Device; Device[Device.DEFAULT]' - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -170,7 +168,7 @@ jobs: mlperfbenchmark: name: MLPerf (${{ matrix.dev }}) - runs-on: [self-hosted, Linux, "${{ matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + runs-on: "${{ matrix.dev == 'AMD' && '7900xtx' || '4090' }}" strategy: fail-fast: false matrix: @@ -185,18 +183,16 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v6 - - name: Setup (AMD) - if: ${{ matrix.dev == 'AMD' }} - run: | - ./extra/hcq/hcq_smi.py amd rmmod --expect - ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: ./extra/hcq/hcq_smi.py nv kill_pids --sudoless + run: sudo modprobe -a nvidia nvidia_uvm + - name: Setup (AMD) + if: ${{ matrix.dev == 'AMD' }} + run: AM_RESET=1 python3 -c 'from tinygrad import Device; Device[Device.DEFAULT]' - name: Symlink models and datasets run: | mkdir -p extra/datasets - ln -s /raid/datasets/imagenet extra/datasets/imagenet + ln -s /datasets/imagenet extra/datasets/imagenet - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -211,7 +207,7 @@ jobs: sdbenchmark: name: Stable Diffusion (DEV=${{ matrix.dev }}) - runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + runs-on: "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && '7900xtx' || '4090' }}" strategy: fail-fast: false matrix: @@ -226,14 +222,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v6 - - name: Setup (AMD) - if: ${{ matrix.dev == 'AMD' }} - run: | - ./extra/hcq/hcq_smi.py amd rmmod --expect - ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: ./extra/hcq/hcq_smi.py nv kill_pids --sudoless + run: sudo modprobe -a nvidia nvidia_uvm + - name: Setup (AMD) + if: ${{ matrix.dev == 'AMD' }} + run: AM_RESET=1 python3 -c 'from tinygrad import Device; Device[Device.DEFAULT]' - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -255,7 +249,7 @@ jobs: multigpubenchmark: name: Multi-GPU Benchmarks (DEV=${{ matrix.dev }}) - runs-on: [self-hosted, "${{ matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + runs-on: "${{ matrix.dev == 'AMD' && '4x7900xtx' || '4x4090' }}" strategy: fail-fast: false matrix: @@ -273,17 +267,13 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/hcq/hcq_smi.py amd rmmod --expect - ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - - name: Setup (NV) - if: ${{ matrix.dev == 'NV' }} - run: ./extra/hcq/hcq_smi.py nv kill_pids --sudoless + AM_RESET=1 python3 -c 'from tinygrad import Device; [Device[f"AMD:{i}"] for i in range(4)]' - name: Symlink models and datasets run: | mkdir -p weights mkdir -p extra/datasets - ln -s /raid/weights/LLaMA-3 weights/LLaMA-3 - ln -s /raid/datasets/imagenet extra/datasets/imagenet + ln -s /weights/LLaMA-3 weights/LLaMA-3 + ln -s /datasets/imagenet extra/datasets/imagenet - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -293,26 +283,29 @@ jobs: run: python3 test/external/process_replay/reset.py - name: Run LLaMA-3 8B on 4 GPUs with BEAM run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 - - name: Run full CIFAR training steps w 6 GPUS - run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py + - name: Run full CIFAR training steps w 4 GPUS + run: time BENCHMARK_LOG=cifar_multigpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=525 BS=1024 GPUS=4 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py - name: Run MLPerf resnet eval on training data - run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py - - name: Run 10 MLPerf ResNet50 training steps (6 gpu) - run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py - - name: Run 10 MLPerf Bert training steps (6 gpu) + run: time BENCHMARK_LOG=resnet_eval BS=512 GPUS=4 MODEL=resnet python3 examples/mlperf/model_eval.py + - name: Run 10 MLPerf ResNet50 training steps (4 gpu) + run: BENCHMARK_LOG=resnet_10steps_multigpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1024 GPUS=4 MODEL=resnet python3 examples/mlperf/model_train.py + - name: Run 10 MLPerf Bert training steps (4 gpu) # TODO: remove BERT_LAYERS once scheduler is fast - run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py + run: BENCHMARK_LOG=bert_10steps_multigpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=48 GPUS=4 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py + - name: Test benchmark allreduce + if: ${{ matrix.dev == 'NV' }} + run: GPUS=4 python test/external/external_benchmark_multitensor_allreduce.py - name: Run process replay tests uses: ./.github/actions/process-replay tests: name: Tests (DEV=${{ matrix.dev }}) - runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + runs-on: "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && '7900xtx' || '4090' }}" strategy: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 11 + timeout-minutes: 14 defaults: run: shell: bash -e -o pipefail {0} @@ -322,14 +315,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v6 - - name: Setup (AMD) - if: ${{ matrix.dev == 'AMD' }} - run: | - ./extra/hcq/hcq_smi.py amd rmmod --expect - ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: ./extra/hcq/hcq_smi.py nv kill_pids --sudoless + run: sudo modprobe -a nvidia nvidia_uvm + - name: Setup (AMD) + if: ${{ matrix.dev == 'AMD' }} + run: AM_RESET=1 python3 -c 'from tinygrad import Device; Device[Device.DEFAULT]' - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -388,9 +379,6 @@ jobs: run: IGNORE_BEAM_CACHE=1 CCACHE=0 BEAM_DEBUG=1 DEBUG=1 python -m pytest -rA test/external/speed_v_theoretical.py --durations=20 - name: Train MNIST run: time TARGET_EVAL_ACC_PCT=96.0 python3 examples/beautiful_mnist.py - - name: Test benchmark allreduce - if: ${{ matrix.dev == 'NV' }} - run: python test/external/external_benchmark_multitensor_allreduce.py # TODO: HEVC decode timing test # - name: HEVC Decode Benchmark # if: ${{ matrix.dev == 'NV' }} @@ -513,10 +501,10 @@ jobs: uses: ./.github/actions/process-replay testcommausbgpubenchmark: - name: UsbGPU Benchmark (comma) + name: openpilot UsbGPU Benchmark env: LFS_SHA: "51b12d20a25b76fa8a5717c2e5af01f148e469e2" - runs-on: [self-hosted, Linux, comma4] + runs-on: usb3-9060 timeout-minutes: 14 defaults: run: @@ -530,18 +518,20 @@ jobs: run: | echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal + - name: reset chestnut + run: python3 extra/usbgpu/debug.py -rnw - name: openpilot compile big_driving_supercombo run: | - GMMU=0 DEV=USB+AMD:LLVM taskset -c 4-7 python3 examples/openpilot/compile_onnx.py "https://huggingface.co/commaai/openpilot-lfs/resolve/$LFS_SHA/openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" openpilot.pkl --out-of-band --retargetable - BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo GMMU=0 DEV=USB+AMD:LLVM ASSERT_MIN_STEP_TIME=25 taskset -c 4-7 python3 examples/openpilot/load_pickle.py --run openpilot.pkl --out-of-band --retarget + GMMU=0 DEV=USB+AMD:LLVM python3 examples/openpilot/compile_onnx.py "https://huggingface.co/commaai/openpilot-lfs/resolve/$LFS_SHA/openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" openpilot.pkl --out-of-band --retargetable + BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo GMMU=0 DEV=USB+AMD:LLVM ASSERT_MIN_STEP_TIME=25 python3 examples/openpilot/load_pickle.py --run openpilot.pkl --out-of-band --retarget - name: openpilot load_pickle big_driving_supercombo - run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo_load_pickle GMMU=0 DEV=USB+AMD ASSERT_MIN_LOAD_TIME=25 taskset -c 4-7 python3 examples/openpilot/load_pickle.py openpilot.pkl --out-of-band + run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo_load_pickle GMMU=0 DEV=USB+AMD ASSERT_MIN_LOAD_TIME=25 python3 examples/openpilot/load_pickle.py openpilot.pkl --out-of-band - name: compile driving warp - run: GMMU=0 DEV=USB+AMD:LLVM taskset -c 4-7 python3 examples/openpilot/compile_warp.py --frame 1928,1208,2048,1216,608,3735552 --warp-to 512x256 --layout yuv420 --frames 2 --output warp.pkl --retargetable + run: GMMU=0 DEV=USB+AMD:LLVM python3 examples/openpilot/compile_warp.py --frame 1928,1208,2048,1216,608,3735552 --warp-to 512x256 --layout yuv420 --frames 2 --output warp.pkl --retargetable - name: run warp pickle - run: BENCHMARK_LOG=usbgpu_openpilot_driving_warp GMMU=0 DEV=USB+AMD taskset -c 4-7 python3 examples/openpilot/load_pickle.py --run warp.pkl --retarget + run: BENCHMARK_LOG=usbgpu_openpilot_driving_warp GMMU=0 DEV=USB+AMD python3 examples/openpilot/load_pickle.py --run warp.pkl --retarget - name: Test copy speeds - run: SIZE=64000000 GMMU=0 DEV=USB+AMD taskset -c 4-7 python3 test/external/external_test_usb_asm24.py + run: SIZE=64000000 GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py drivertests: name: PCI Driver Tests (DEV=${{ matrix.dev }}) @@ -613,16 +603,3 @@ jobs: trap - EXIT - name: Run process replay tests uses: ./.github/actions/process-replay - - llvmspeed: - name: LLVM Speed - runs-on: [self-hosted, Linux, tinyboxrandom] - timeout-minutes: 10 - if: github.repository_owner == 'tinygrad' - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: Speed Test - run: DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py - - name: Speed Test (BEAM=2) - run: IGNORE_BEAM_CACHE=1 BEAM=2 DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py diff --git a/tinygrad_repo/.github/workflows/platform.yml b/tinygrad_repo/.github/workflows/platform.yml index a25cf2259f..5f9bf37733 100644 --- a/tinygrad_repo/.github/workflows/platform.yml +++ b/tinygrad_repo/.github/workflows/platform.yml @@ -144,39 +144,3 @@ jobs: run: python -m pytest -n=auto test/test_tiny.py test/device/$(python -c 'from tinygrad import Device; print(Device.DEFAULT.lower())') --durations=20 - name: Run process replay tests uses: ./.github/actions/process-replay - -# ****** Windows Tests ****** - - testwindows: - strategy: - fail-fast: false - matrix: - dev: - - 'CPU:CLANG' - - 'CPU:LLVM' - - 'CPU:X86' - - 'WEBGPU' - - name: Windows (DEV=${{ matrix.dev }}) - runs-on: windows-2025 - timeout-minutes: 15 - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: Setup Environment - uses: ./.github/actions/setup-tinygrad - with: - key: windows-${{ matrix.dev }}-minimal - deps: testing_minimal - pydeps: ${{ matrix.dev == 'WEBGPU' && 'dawn-python' || '' }} - - name: Set env - shell: bash - run: printf "DEV=${{ matrix.dev }}${{ matrix.dev == 'CPU:CLANG' && '\nCPU_COUNT=2' || '' }}" >> $GITHUB_ENV - - name: Check Device.DEFAULT and print some source - shell: bash - run: | - python -c "from tinygrad import Device; from tinygrad.helpers import Target; assert Device.DEFAULT == Target.parse('${{ matrix.dev }}').device" - DEBUG=4 python test/test_tiny.py TestTiny.test_plus - - name: Run test_tiny and device tests - shell: bash - run: python -m pytest -n=auto test/test_tiny.py test/device/$(python -c 'from tinygrad import Device; print(Device.DEFAULT.lower())') --durations=20 diff --git a/tinygrad_repo/.github/workflows/test.yml b/tinygrad_repo/.github/workflows/test.yml index 5ef1fff25c..b8542bcff0 100644 --- a/tinygrad_repo/.github/workflows/test.yml +++ b/tinygrad_repo/.github/workflows/test.yml @@ -228,7 +228,7 @@ jobs: - name: Run test as module run: DEV=CPU python test/null/test_device.py TestRunAsModule.test_module_runs - name: Run GC tests - run: python test/external/external_uop_gc.py + run: DEV=CPU python test/external/external_uop_gc.py - name: External Benchmark Schedule run: python3 test/external/external_benchmark_schedule.py - name: Run process replay tests @@ -289,7 +289,7 @@ jobs: opencl: 'true' - name: Test CL IMAGE=1 ops run: | - DEV=CL IMAGE=1 python -m pytest -n=auto test/runtime/test_ops.py --durations=20 + DEV=CL IMAGE=1 python -m pytest -n=auto test/runtime/test_ops.py test/runtime/test_assign.py --durations=20 DEV=CL IMAGE=1 python test/models/test_end2end.py TestEnd2End.test_linear_mnist - name: Run process replay tests uses: ./.github/actions/process-replay @@ -562,6 +562,14 @@ jobs: run: python -m pytest test/null/test_hcq2.py test/runtime/test_hcq2.py - name: Run disk copy tests on MOCKPCI run: python -m pytest test/runtime/test_disk_tensor.py -k test_copy_from_disk + - name: Run test_tiny on MOCKPCI over REMOTE + run: | + python3 extra/remote/serve.py 6482 & + trap 'kill $!' EXIT + sleep 1 + REMOTE=127.0.0.1:6482 python -m pytest test/test_tiny.py + - name: Run tests on MOCK+HIP + run: DEV=MOCK+HIP python -m pytest -n=auto test/test_tiny.py test/runtime/test_jit.py test/runtime/test_hcq2.py test/runtime/test_dtype.py testamd: strategy: diff --git a/tinygrad_repo/.pre-commit-config.yaml b/tinygrad_repo/.pre-commit-config.yaml index 2c2122da43..6377e0ae2e 100644 --- a/tinygrad_repo/.pre-commit-config.yaml +++ b/tinygrad_repo/.pre-commit-config.yaml @@ -1,4 +1,3 @@ -# on Windows -- $env:SKIP="tests,example" repos: - repo: local hooks: diff --git a/tinygrad_repo/docs/developer/am.md b/tinygrad_repo/docs/developer/am.md index 2c150f8e7c..188fbf1d2b 100644 --- a/tinygrad_repo/docs/developer/am.md +++ b/tinygrad_repo/docs/developer/am.md @@ -5,10 +5,7 @@ AM driver is a userspace driver targeting AMD's RDNA3/RDNA4. You only need tinyg ## How to run? Make sure that amdgpu module is unloaded and just run tinygrad with `DEV=AMD`! -Optional requirements: - -* System without IOMMU for P2P / SDMA support -* vfio-pci module for IRQ handling +If the IOMMU translates (not `iommu=pt`), tinygrad drives the GPU through vfio-pci (`sudo modprobe vfio-pci disable_idle_d3=1`). ## Environment Variables diff --git a/tinygrad_repo/examples/hlb_cifar10.py b/tinygrad_repo/examples/hlb_cifar10.py index e4dd09c61d..9fe9907516 100644 --- a/tinygrad_repo/examples/hlb_cifar10.py +++ b/tinygrad_repo/examples/hlb_cifar10.py @@ -261,7 +261,8 @@ def train_cifar(): net_ema_param.assign(net_param.numpy()) @TinyJit - def update(self, net, decay): + def update(self, net, step): + decay = projected_ema_decay_val*(Tensor(step)/STEPS)**hyp['ema']['decay_pow'] for net_ema_param, (param_name, net_param) in zip(get_state_dict(self.net_ema).values(), get_state_dict(net).items()): # batchnorm currently is not being tracked if not ("num_batches_tracked" in param_name) and not ("running" in param_name): @@ -320,6 +321,7 @@ def train_cifar(): lr_sched_non_bias = OneCycleLR(opt_non_bias, max_lr=hyp['opt']['non_bias_lr'], pct_start=pct_start, div_factor=initial_div_factor, final_div_factor=1./(initial_div_factor*final_lr_ratio), total_steps=STEPS) def train_step(model, optimizer, lr_scheduler, X, Y): + X, Y = X.shard(GPUS, axis=0), Y.shard(GPUS, axis=0) # copies to GPUS (and back to CPU on return) happen inside the jit out = model(X) loss_batchsize_scaler = 512/BS loss = cross_entropy(out, Y, reduction='none', label_smoothing=hyp['opt']['label_smoothing']).mul(hyp['opt']['loss_scale_scaler']*loss_batchsize_scaler).sum().div(hyp['opt']['loss_scale_scaler']) @@ -328,16 +330,17 @@ def train_cifar(): # index 0 for bias and 1 for non-bias optimizer.zero_grad() loss.backward() - return loss.realize(*optimizer.schedule_step(), *lr_scheduler[0].schedule_step(), *lr_scheduler[1].schedule_step()) - return loss.realize() + loss.realize(*optimizer.schedule_step(), *lr_scheduler[0].schedule_step(), *lr_scheduler[1].schedule_step()) + return loss.to("CPU"), optimizer[1].lr.to("CPU") train_step_jitted = TinyJit(train_step) def eval_step(model, X, Y): + X, Y = X.shard(GPUS, axis=0), Y.shard(GPUS, axis=0) out = model(X, training=False) loss = cross_entropy(out, Y, reduction='mean') correct = out.argmax(axis=1) == Y.argmax(axis=1) - return correct.realize(), loss.realize() + return correct.to("CPU"), loss.to("CPU") eval_step_jitted = TinyJit(eval_step) eval_step_ema_jitted = TinyJit(eval_step) @@ -365,10 +368,6 @@ def train_cifar(): losses = [] losses_ema = [] for Xt, Yt in fetch_batches(X_test, Y_test, BS=EVAL_BS, is_train=False): - if len(GPUS) > 1: - Xt.shard_(GPUS, axis=0) - Yt.shard_(GPUS, axis=0) - correct, loss = eval_step_jitted(model, Xt, Yt) losses.append(loss.numpy().tolist()) corrects.extend(correct.numpy().tolist()) @@ -392,25 +391,21 @@ def train_cifar(): with WallTimeEvent(BenchEvent.STEP): X, Y = next(batcher) - if len(GPUS) > 1: - X.shard_(GPUS, axis=0) - Y.shard_(GPUS, axis=0) - with Context(BEAM=getenv("LATEBEAM", BEAM.value), WINO=getenv("LATEWINO", WINO.value)): - loss = train_step_jitted(model, optim.OptimizerGroup(opt_bias, opt_non_bias), [lr_sched_bias, lr_sched_non_bias], X, Y) + loss, lr = train_step_jitted(model, optim.OptimizerGroup(opt_bias, opt_non_bias), [lr_sched_bias, lr_sched_non_bias], X, Y) et = time.monotonic() loss_cpu = loss.numpy() # EMA for network weights if getenv("EMA") and i > hyp['ema']['steps'] and (i+1) % hyp['ema']['every_n_steps'] == 0: if model_ema is None: model_ema = modelEMA(W, model) - model_ema.update(model, Tensor([projected_ema_decay_val*(i/STEPS)**hyp['ema']['decay_pow']])) + model_ema.update(model, Variable("step", 0, STEPS).bind(i)) cl = time.monotonic() step_times.append((cl-st)*1000.0) - device_str = loss.device if isinstance(loss.device, str) else f"{loss.device[0]} * {len(loss.device)}" + device_str = Device.DEFAULT if len(GPUS) == 1 else f"{Device.DEFAULT} * {len(GPUS)}" # 53 221.74 ms run, 2.22 ms python, 219.52 ms CL, 803.39 loss, 0.000807 LR, 4.66 GB used, 3042.49 GFLOPS, 674.65 GOPS - print(f"{i:3d} {(cl-st)*1000.0:7.2f} ms run, {(et-st)*1000.0:7.2f} ms python, {(cl-et)*1000.0:7.2f} ms {device_str}, {loss_cpu:7.2f} loss, {opt_non_bias.lr.numpy()[0]:.6f} LR, {GlobalCounters.mem_used/1e9:.2f} GB used, {GlobalCounters.global_ops*1e-9/(cl-st):9.2f} GFLOPS, {GlobalCounters.global_ops*1e-9:9.2f} GOPS") + print(f"{i:3d} {(cl-st)*1000.0:7.2f} ms run, {(et-st)*1000.0:7.2f} ms python, {(cl-et)*1000.0:7.2f} ms {device_str}, {loss_cpu:7.2f} loss, {lr.numpy()[0]:.6f} LR, {GlobalCounters.mem_used/1e9:.2f} GB used, {GlobalCounters.global_ops*1e-9/(cl-st):9.2f} GFLOPS, {GlobalCounters.global_ops*1e-9:9.2f} GOPS") st = cl i += 1 diff --git a/tinygrad_repo/examples/mlperf/model_train.py b/tinygrad_repo/examples/mlperf/model_train.py index 15ee8479e9..262aa206c5 100644 --- a/tinygrad_repo/examples/mlperf/model_train.py +++ b/tinygrad_repo/examples/mlperf/model_train.py @@ -257,6 +257,7 @@ def train_resnet(): min_time = min(step_times) assert min_time < assert_time, f"Speed regression, expected min step time of < {assert_time} ms but took: {min_time} ms" + if not INITMLPERF: batch_loader.close() if (TRAIN_BEAM or EVAL_BEAM) and e == start_epoch: break return if MLLOGGER and RUNMLPERF: diff --git a/tinygrad_repo/extra/amdpci/setup_vfio.sh b/tinygrad_repo/extra/amdpci/setup_vfio.sh deleted file mode 100755 index 6ca786edaa..0000000000 --- a/tinygrad_repo/extra/amdpci/setup_vfio.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -sudo modprobe vfio-pci disable_idle_d3=1 diff --git a/tinygrad_repo/extra/export_model.py b/tinygrad_repo/extra/export_model.py index 1d307ddfeb..2b0574977c 100644 --- a/tinygrad_repo/extra/export_model.py +++ b/tinygrad_repo/extra/export_model.py @@ -39,7 +39,7 @@ def compile_net(linear:UOp, output_bufs:List[Buffer]) -> Tuple[Dict[str,str], Li prg = to_program(call.src[0], Device[arg_uops[0].device].renderer) info = prg.arg functions[prg.src[0].arg.function_name] = prg.src[2].arg - cargs = [name_of(bu, i == 0) for i, bu in enumerate(arg_uops)] + list(info.vars) + cargs = [p if p.addrspace is AddrSpace.ALU else name_of(call.src[1+p.arg.slot], p.arg.slot == 0) for p in prg.kernel_params] statements.append((prg.src[0].arg.function_name, cargs, info.global_size, info.local_size)) return functions, statements, {name:(size, dtype, key) for name, size, dtype, key in bufs.values()}, bufs_to_save @@ -56,7 +56,7 @@ def jit_model(model, *args) -> Tuple[UOp, List[Buffer]]: # run twice to trigger JIT capture for _ in range(2): the_output = run(*args) assert run.captured is not None - return run.captured.linear, [o.uop.base.realized for o in the_output] + return run.captured._linears[()][0], [o.uop.base.realized for o in the_output] def export_model_clang(functions:Dict[str,str], statements:Dict[str,Tuple[str,int,int]], bufs:Dict[str,Tuple[str,int,int]], bufs_to_save:Dict[str,Tensor], input_names:List[str], output_names:List[str], weight_names={}, model_name="model", symbolic_vars={}, wasm=False) -> str: @@ -72,7 +72,7 @@ def export_model_clang(functions:Dict[str,str], statements:Dict[str,Tuple[str,in weight = ''.join(["\\x%02X"%x for x in cl.as_memoryview()]) cprog.append(f"unsigned char {name}_data[] = \"{weight}\";") cprog += [f"{dtype_map[dtype]} {name}[{len}];" if name not in bufs_to_save else f"{dtype_map[dtype]} *{name} = ({dtype_map[dtype]} *){name}_data;" for name,(len,dtype,_key) in bufs.items() if name not in input_names+output_names] - cprog += [f"void net({forward_args}) {{"] + [f"{name}({', '.join(args)});" for (name, args, _global_size, _local_size) in statements] + ["}"] + cprog += [f"void net({forward_args}) {{"] + [f"{name}(&(struct {name}_args){{{', '.join(args)}}});" for (name, args, _global_size, _local_size) in statements] + ["}"] return '\n'.join(headers + cprog) else: if bufs_to_save: @@ -88,7 +88,7 @@ def export_model_clang(functions:Dict[str,str], statements:Dict[str,Tuple[str,in cprog += [f"void net({forward_args})"] + ["{"] get_weight_ptr = lambda x: f"({dtype_map[bufs_to_save[x][1]]} *)bufs[{buf_to_name[x]['idx']}]" if x in bufs_to_save else x - cprog += [f" {name}({', '.join(map(get_weight_ptr, args))});" for (name, args, _global_size, _local_size) in statements] + ["}"] + cprog += [f" {name}(&(struct {name}_args){{{', '.join(map(get_weight_ptr, args))}}});" for (name, args, _global_size, _local_size) in statements] + ["}"] weightMapping = "" if not bufs_to_save else f"""\nconst weightNames = [{", ".join([f'"{weight_name}"' for weight_name in [v["name"] for v in buf_to_name.values()]])}]; const {model_name}_name_to_id = Object.fromEntries(weightNames.map((name, index) => [name, index]));\n""" top = f"""import {model_name}Module from './{model_name}.js'{weightMapping}""" diff --git a/tinygrad_repo/extra/gemm/moe_gemm.py b/tinygrad_repo/extra/gemm/moe_gemm.py index a9a611a75e..c3ca05e50d 100644 --- a/tinygrad_repo/extra/gemm/moe_gemm.py +++ b/tinygrad_repo/extra/gemm/moe_gemm.py @@ -101,11 +101,12 @@ def custom_grouped_mx_gemm_bw(gradient:UOp, kernel:UOp, w_stored:bool=False) -> def grouped_mx_gemm_backward(g:Tensor, aq:Tensor, ae8:Tensor, bq:Tensor, be8:Tensor, expert_off:Tensor, *, w_stored:bool=False, quantized_gradient:tuple[Tensor, Tensor]|None=None) -> tuple[Tensor, Tensor]: + from extra.gptoss_kernels.transpose_requantize import transpose_requantize_mxfp8 E, N = bq.shape[:2] g = g.reshape(aq.shape[0], N).cast(dtypes.bfloat16) x_phys = (aq.cast(dtypes.bfloat16) * _mx_block_scale(ae8).cast(dtypes.bfloat16)) - w_phys = (bq.cast(dtypes.bfloat16) * _mx_block_scale_3d(be8).cast(dtypes.bfloat16)) - grad_x = grouped_mx_gemm(g if quantized_gradient is None else quantized_gradient, w_phys.transpose(1, 2), expert_off) + w_transposed = transpose_requantize_mxfp8(bq, be8) + grad_x = grouped_mx_gemm(g if quantized_gradient is None else quantized_gradient, w_transposed, expert_off) grad_w = grouped_mx_wgrad(g, x_phys, expert_off, E) grad_xq = grad_x * _mx_block_scale(ae8).cast(dtypes.bfloat16) grad_wq = grad_w.contiguous() if w_stored else (grad_w * _mx_block_scale_3d(be8).cast(dtypes.bfloat16)).contiguous() @@ -113,14 +114,14 @@ def grouped_mx_gemm_backward(g:Tensor, aq:Tensor, ae8:Tensor, bq:Tensor, be8:Ten _grouped_bw_stored = functools.partial(custom_grouped_mx_gemm_bw, w_stored=True) -def grouped_mx_gemm(x:Tensor|tuple[Tensor, Tensor], w:Tensor|tuple[Tensor, Tensor], expert_off:Tensor) -> Tensor: +def grouped_mx_gemm(x:Tensor|tuple[Tensor, Tensor], w:Tensor|tuple[Tensor, Tensor]|tuple[Tensor, Tensor, Tensor], expert_off:Tensor) -> Tensor: if isinstance(x, tuple): x_q, x_e8 = x x_si, x = mx_pack(x_e8), x_q else: x_q, x_e8, x_si = quantize_mxfp8(x) if (pre_quantized := isinstance(w, tuple)): - w_q, w_e8 = w + w_q, w_e8 = w[:2] E, N, K2 = w_q.shape else: E, N, K2 = w.shape @@ -130,7 +131,7 @@ def grouped_mx_gemm(x:Tensor|tuple[Tensor, Tensor], w:Tensor|tuple[Tensor, Tenso assert M % 256 == 0 and N % 256 == 0 and K % 128 == 0, f"grouped mxfp8 needs M%256,N%256,K%128, got {M,K,N}" dname = (x.device[0] if isinstance(x.device, tuple) else x.device).split(":")[0] if not pre_quantized: w_q, w_e8, _ = quantize_mxfp8(w) - w_si = mx_pack_3d(w_e8) + w_si = w[2] if pre_quantized and len(w) == 3 else mx_pack_3d(w_e8) xe_in, out_shape = x_e8.reshape(M, K // 32), (M, N) if isinstance(x.device, tuple) and (row_axis := x.uop.axis) is not None: ndev = len(x.device) diff --git a/tinygrad_repo/extra/gemm/triton_nv_matmul.py b/tinygrad_repo/extra/gemm/triton_nv_matmul.py index 09bea97c8c..8b463b9e3f 100644 --- a/tinygrad_repo/extra/gemm/triton_nv_matmul.py +++ b/tinygrad_repo/extra/gemm/triton_nv_matmul.py @@ -100,7 +100,7 @@ if __name__ == "__main__": gsize, lsize = info.launch_dims({}) tflops = [] for i in range(5): - tm = rt(*[b._buf for b in prg_bufs], global_size=gsize, local_size=lsize, vals=info.vals({}), wait=True) + tm = rt(*[b._buf for b in prg_bufs], global_size=gsize, local_size=lsize, wait=True) tflops.append((2*M*K*N/tm)*1e-12) print(f"TFLOPS: {max(tflops):.2f}") diff --git a/tinygrad_repo/extra/gptoss_kernels/qkv_rope/__init__.py b/tinygrad_repo/extra/gptoss_kernels/qkv_rope/__init__.py index 3839a3e655..3aa8a5666a 100644 --- a/tinygrad_repo/extra/gptoss_kernels/qkv_rope/__init__.py +++ b/tinygrad_repo/extra/gptoss_kernels/qkv_rope/__init__.py @@ -15,14 +15,19 @@ def _forward(q:UOp, k:UOp, v:UOp, xqkv:UOp, freqs:UOp) -> UOp: @functools.cache def _backward_kernel(out:UOp, dq:UOp, dk:UOp, dv:UOp, freqs:UOp) -> UOp: - assert out.shape == (2, 8192, 5120) and dq.shape == (2, 64, 8192, 64) and dk.shape == dv.shape == (16, 8192, 8, 64) + assert out.shape == (2, 8192, 5120) and dq.shape == (2, 64, 8192, 64) and dk.shape == dv.shape + assert dk.shape[1:] == (8192, 8, 64) and dk.shape[0] in (2, 8) + partials = dk.shape[0] // 2 + q_heads_per_wg = 8 if partials == 4 else 2 sink = UOp.sink(out.base, dq.base, dk.base, dv.base, freqs.base, - UOp.special(256, "lidx0"), UOp.special(2, "gidx0"), UOp.special(128, "gidx1"), UOp.special(80, "gidx2"), + UOp.special(256, "lidx0"), UOp.special(2, "gidx0"), UOp.special(128, "gidx1"), + UOp.special(64 // q_heads_per_wg + 16, "gidx2"), arg=KernelInfo("fused_qkv_rope_backward_gptoss")) src = (pathlib.Path(__file__).parent/"backward.cpp").read_text() include = pathlib.Path(__file__).parents[2]/"thunder"/"amd"/"include" return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=src), - UOp(Ops.BINARY, arg=compile_hip(src, [f"-I{include}", "-DKITTENS_CDNA4", "-DHIP_ENABLE_WARP_SYNC_BUILTINS"])))) + UOp(Ops.BINARY, arg=compile_hip(src, [f"-I{include}", "-DKITTENS_CDNA4", "-DHIP_ENABLE_WARP_SYNC_BUILTINS", + f"-DKV_PARTIALS={partials}"])))) def _backward(dq:UOp, dk:UOp, dv:UOp, call:UOp) -> tuple: from extra.thunder.amd.fa import _fa_native_grads diff --git a/tinygrad_repo/extra/gptoss_kernels/qkv_rope/backward.cpp b/tinygrad_repo/extra/gptoss_kernels/qkv_rope/backward.cpp index 340ef2051c..eebb617e17 100644 --- a/tinygrad_repo/extra/gptoss_kernels/qkv_rope/backward.cpp +++ b/tinygrad_repo/extra/gptoss_kernels/qkv_rope/backward.cpp @@ -23,9 +23,11 @@ using namespace kittens; constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; constexpr int HALF_D = ATTN_D / 2; constexpr int PACKED_H = ATTN_H_KV * (GROUP_SIZE + 2); -// The current D=64 FA emits one dK/dV partial per query head in each GQA group. -constexpr int KV_PARTIALS = GROUP_SIZE; -constexpr int Q_HEADS_PER_WG = 1; +// Full-causal FA groups two query heads; sliding-window FA groups all eight. +#ifndef KV_PARTIALS +#define KV_PARTIALS 4 +#endif +constexpr int Q_HEADS_PER_WG = KV_PARTIALS == 4 ? 8 : 2; constexpr int Q_FIELDS = ATTN_H / Q_HEADS_PER_WG; constexpr int NUM_WARPS = 4; constexpr int TILE_N = 16; @@ -49,7 +51,7 @@ __device__ __forceinline__ void load_fa_shuffled(RT &dst, const GL &src, const C #pragma unroll for (int j = 0; j < dst.width; j++) { const float4 loaded = std::bit_cast(llvm_amdgcn_raw_buffer_load_b128( - std::bit_cast(br), (i * tile_row_stride + j * tile_stride + lane * 8) * sizeof(U), 0, 0)); + std::bit_cast(br), (i * tile_row_stride + j * tile_stride + lane * 8) * sizeof(U), 0, 2)); // non-temporal dQ loads const U2 *packed = reinterpret_cast(&loaded); #pragma unroll for (int k = 0; k < dst.packed_per_base_tile; k++) dst.tiles[i][j].data[k] = packed[k]; diff --git a/tinygrad_repo/extra/gptoss_kernels/transpose_requantize/__init__.py b/tinygrad_repo/extra/gptoss_kernels/transpose_requantize/__init__.py new file mode 100644 index 0000000000..5ec7b9f907 --- /dev/null +++ b/tinygrad_repo/extra/gptoss_kernels/transpose_requantize/__init__.py @@ -0,0 +1,27 @@ +import functools, pathlib +from tinygrad import Tensor, dtypes +from tinygrad.uop.ops import UOp, Ops, KernelInfo +from extra.llama_kernels import alloc_like, compile_hip + +@functools.cache +def _kernel(q:UOp, e8:UOp, si:UOp, src_q:UOp, src_e8:UOp) -> UOp: + E, M, N = src_q.shape + assert M % 128 == 0 and N % 128 == 0 + assert src_e8.shape == (E, M, N // 32) + assert q.shape == (E, N, M) and e8.shape == (E, N, M // 32) and si.shape == (E, M // 128, N) + sink = UOp.sink(q.base, e8.base, si.base, src_q.base, src_e8.base, + UOp.special(128, "lidx0"), UOp.special(E * (M // 32) * (N // 128), "gidx0"), + arg=KernelInfo(f"transpose_requantize_mxfp8_{E}_{M}_{N}")) + src = (pathlib.Path(__file__).parent/"transpose_requantize.cpp").read_text() + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=src), + UOp(Ops.BINARY, arg=compile_hip(src, [f"-DE_DIM={E}", f"-DM_DIM={M}", f"-DN_DIM={N}"])))) + +def transpose_requantize_mxfp8(q:Tensor, e8:Tensor) -> tuple[Tensor, Tensor, Tensor]: + E, M, N = q.shape + assert q.dtype == dtypes.fp8e4m3 and e8.dtype == dtypes.uint8 and e8.shape == (E, M, N // 32) + axis = q.uop.axis + out_axis = None if axis is None else (0 if axis == 0 else 3 - axis) + qt = alloc_like((E, N, M), q.dtype, q.device, out_axis) + et = alloc_like((E, N, M // 32), dtypes.uint8, q.device, out_axis) + si = alloc_like((E, M // 128, N), dtypes.uint32, q.device, axis) + return tuple(Tensor.custom_kernel(qt, et, si, q, e8, fxn=_kernel)[:3]) diff --git a/tinygrad_repo/extra/gptoss_kernels/transpose_requantize/transpose_requantize.cpp b/tinygrad_repo/extra/gptoss_kernels/transpose_requantize/transpose_requantize.cpp new file mode 100644 index 0000000000..abf5e006ee --- /dev/null +++ b/tinygrad_repo/extra/gptoss_kernels/transpose_requantize/transpose_requantize.cpp @@ -0,0 +1,58 @@ +#include +#include +#include + +constexpr int BLK = 32; +constexpr int TILE_M = BLK; +constexpr int TILE_N = 128; +constexpr int N_TILES_N = N_DIM / TILE_N; +constexpr int REQUANT_M_GROUPS = M_DIM / 128; +static_assert(M_DIM % 128 == 0 && N_DIM % TILE_N == 0); +typedef short short2v __attribute__((ext_vector_type(2))); + +// Source15's M32 schedule: dequantize with its BF16 rounding boundary, then transpose and requantize. +extern "C" __global__ __launch_bounds__(128) void +transpose_requantize_mxfp8(__hip_fp8_storage_t* __restrict__ q_out, + uint8_t* __restrict__ e8_out, + uint32_t* __restrict__ si_out, + const __hip_fp8_storage_t* __restrict__ q_in, + const uint8_t* __restrict__ e8_in) { + const int tid = threadIdx.x; + int tile = blockIdx.x; + const int tile_n = tile % N_TILES_N; + tile /= N_TILES_N; + const int tile_m = tile % (M_DIM / BLK); + const int expert = tile / (M_DIM / BLK); + const int m_group = tile_m / 4, sub = tile_m % 4; + const long long n = tile_n * TILE_N + tid; + float vals[TILE_M]; + float amax = 0.0f; + #pragma unroll + for (int mm = 0; mm < TILE_M; mm++) { + const long long m = tile_m * TILE_M + mm; + const long long src = ((long long)expert * M_DIM + m) * N_DIM + n; + const __hip_fp8_e4m3 fq = __builtin_bit_cast(__hip_fp8_e4m3, q_in[src]); + const long long src_e8 = ((long long)expert * M_DIM + m) * (N_DIM / BLK) + n / BLK; + const unsigned src_scale = e8_in[src_e8]; + const float dqscale = __builtin_bit_cast(float, src_scale ? src_scale << 23 : 0x00400000u); + const float v = (float)(__hip_bfloat16)((float)fq * dqscale); + vals[mm] = v; + amax = fmaxf(amax, fabsf(v)); + } + int e8 = (int)((__builtin_bit_cast(unsigned, amax) >> 23) & 0xFFu); + e8 = max(0, min(254, e8)); + const float scale_f = __builtin_bit_cast(float, (unsigned)e8 << 23); + __hip_fp8_storage_t out[TILE_M]; + #pragma unroll + for (int mm = 0; mm < TILE_M; mm += 2) { + short2v acc = {0, 0}; + acc = __builtin_amdgcn_cvt_scalef32_pk_fp8_f32(acc, vals[mm], vals[mm + 1], scale_f, false); + const unsigned char *rb = reinterpret_cast(&acc); + out[mm] = rb[0]; out[mm + 1] = rb[1]; + } + const long long obase = ((long long)expert * N_DIM + n) * M_DIM + (long long)tile_m * TILE_M; + *reinterpret_cast(&q_out[obase]) = *reinterpret_cast(&out[0]); + *reinterpret_cast(&q_out[obase + 16]) = *reinterpret_cast(&out[16]); + e8_out[((long long)expert * N_DIM + n) * (M_DIM / BLK) + tile_m] = (uint8_t)e8; + reinterpret_cast(si_out)[(((long long)expert * REQUANT_M_GROUPS + m_group) * N_DIM + n) * 4 + sub] = (uint8_t)e8; +} diff --git a/tinygrad_repo/extra/hcq1/graph.py b/tinygrad_repo/extra/hcq1/graph.py index 64c1670ae0..b9827a6f8f 100644 --- a/tinygrad_repo/extra/hcq1/graph.py +++ b/tinygrad_repo/extra/hcq1/graph.py @@ -135,7 +135,6 @@ class HCQGraph(GraphRunner): # When profiling allocate 2 signals for each jit item to measure speed. The jth jit item have signals at 2*j and 2*j+1. # TODO: This logic might allocate a few extra signals... self.prof_signals: list[HCQSignal] = [] - self.prof_graph_deps: list[list[int]] = [] self.prof_graph_entries: list[ProfileGraphEntry] = [] self.last_j: dict[HWQueue, int|None] = collections.defaultdict(lambda: None) @@ -205,7 +204,6 @@ class HCQGraph(GraphRunner): prof_name = enqueue_dev.device if runtime is not None else f"{enqueue_dev.device}:SDMA:{queue_idx}" self.prof_graph_entries.append(ProfileGraphEntry(prof_name, prof_ji_desc, sig_st, j * 2 + 1, runtime.profile_key if runtime is not None else None)) - self.prof_graph_deps.append([d - 1 for _, d in rdeps]) self.last_j[enqueue_queue] = j @@ -368,7 +366,7 @@ class HCQGraph(GraphRunner): def collect_timestamps(self): # NOTE: Append to any device is fine... - self.devices[0].profile_events += [ProfileGraphEvent(self.prof_graph_entries, self.prof_graph_deps, [s.timestamp for s in self.prof_signals])] + self.devices[0].profile_events += [ProfileGraphEvent(self.prof_graph_entries, [s.timestamp for s in self.prof_signals])] def dev_name(self, dev) -> str: return dev.device.replace(":", "_") diff --git a/tinygrad_repo/extra/llama_kernels/fused_ce/__init__.py b/tinygrad_repo/extra/llama_kernels/fused_ce/__init__.py index 6e4a8e404c..fe4a79cc5c 100644 --- a/tinygrad_repo/extra/llama_kernels/fused_ce/__init__.py +++ b/tinygrad_repo/extra/llama_kernels/fused_ce/__init__.py @@ -57,7 +57,7 @@ def _fused_ce_loss_bwd(gradient:UOp, kernel:UOp, label_smoothing:float): seq_per_dev = SEQ # NOTE: .mean() backward gives same grad per row (1/N), so broadcast is safe; take scalar scale = Tensor(gradient, device=device).float().reshape(-1)[0:1].contiguous() - logits_t = Tensor(logits_u.after(kernel), device=device) + logits_t = Tensor(logits_u, device=device) lse_t = Tensor(lse_u.after(kernel), device=device) targets_t = Tensor(targets_u, device=device) fxn = functools.partial(_custom_fused_ce_loss_bwd, vocab=VOCAB, rows=rows_per_dev, seq=seq_per_dev, label_smoothing=label_smoothing) diff --git a/tinygrad_repo/extra/perfetto/to_perfetto.py b/tinygrad_repo/extra/perfetto/to_perfetto.py index f5ff58050b..d73e97cbc1 100644 --- a/tinygrad_repo/extra/perfetto/to_perfetto.py +++ b/tinygrad_repo/extra/perfetto/to_perfetto.py @@ -12,16 +12,12 @@ def dev_ev_to_perfetto_json(ev:ProfileDeviceEvent): def range_ev_to_perfetto_json(ev:ProfileRangeEvent): name = ev.name.display_name if isinstance(ev.name, TracingKey) else ev.name return [{"name": name, "ph": "X", "ts": prep_ts(ev.device, ev.st), "dur": float(ev.en-ev.st), **dev_to_pid(ev.device)}] -def graph_ev_to_perfetto_json(ev:ProfileGraphEvent, reccnt): +def graph_ev_to_perfetto_json(ev:ProfileGraphEvent): ret = [] - for i,e in enumerate(ev.ents): + for e in ev.ents: st, en = ev.sigs[e.st_id], ev.sigs[e.en_id] name = e.name.display_name if isinstance(e.name, TracingKey) else e.name ret += [{"name": name, "ph": "X", "ts": prep_ts(e.device, st), "dur": float(en-st), **dev_to_pid(e.device)}] - for dep in ev.deps[i]: - d = ev.ents[dep] - ret += [{"ph": "s", **dev_to_pid(d.device), "id": reccnt+len(ret), "ts": prep_ts(d.device, ev.sigs[d.en_id]), "bp": "e"}] - ret += [{"ph": "f", **dev_to_pid(e.device), "id": reccnt+len(ret)-1, "ts": prep_ts(e.device, st), "bp": "e"}] return ret def to_perfetto(profile:list[ProfileEvent]): # Start json with devices. @@ -30,7 +26,7 @@ def to_perfetto(profile:list[ProfileEvent]): prof_json = [x for ev in profile if isinstance(ev, ProfileDeviceEvent) for x in dev_ev_to_perfetto_json(ev)] for ev in tqdm(profile, desc="preparing profile"): if isinstance(ev, ProfileRangeEvent): prof_json += range_ev_to_perfetto_json(ev) - elif isinstance(ev, ProfileGraphEvent): prof_json += graph_ev_to_perfetto_json(ev, reccnt=len(prof_json)) + elif isinstance(ev, ProfileGraphEvent): prof_json += graph_ev_to_perfetto_json(ev) return {"traceEvents": prof_json} if __name__ == "__main__": diff --git a/tinygrad_repo/extra/remote/serve.py b/tinygrad_repo/extra/remote/serve.py index 821dea7165..c3fd1b3745 100644 --- a/tinygrad_repo/extra/remote/serve.py +++ b/tinygrad_repo/extra/remote/serve.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import socket, struct, sys, signal +import socket, struct, sys, signal, ctypes, time, json from tinygrad.runtime.support.system import PCIDevice, RemoteCmd, System, REMOTE_REQ, REMOTE_RESP from tinygrad.runtime.support.am.amdev import AMMemoryManager from tinygrad.runtime.support.system import FileIOInterface @@ -12,6 +12,7 @@ def resp_err(msg): return resp(len(err:=msg.encode()), status=1) + err discovered_devices: list[tuple[type, str]] = [] opened_devices: dict[int, PCIDevice] = {} mapped_bars: dict[tuple[int, int], object] = {} +sysmem_iovas: dict[int, int] = {} programs: list = [] def handle(conn, cmd, dev_id, bar, arg0, arg1, arg2): @@ -66,6 +67,7 @@ def handle(conn, cmd, dev_id, bar, arg0, arg1, arg2): else: bar_view[arg0:arg0+arg1] = data elif cmd == RemoteCmd.MAP_SYSMEM: memview, paddrs = System.alloc_sysmem(arg0, vaddr=arg2, contiguous=bool(arg1)) + if System.vfio is not None: sysmem_iovas[memview.addr] = paddrs[0] paddrs_bytes = struct.pack(f'<{len(paddrs) + 1}Q', memview.addr, *paddrs) conn.sendall(resp(len(paddrs_bytes)) + paddrs_bytes) elif cmd == RemoteCmd.SYSMEM_READ: @@ -73,13 +75,19 @@ def handle(conn, cmd, dev_id, bar, arg0, arg1, arg2): elif cmd == RemoteCmd.SYSMEM_WRITE: to_mv(arg0, arg1)[:] = conn.recv(arg1, socket.MSG_WAITALL) elif cmd == RemoteCmd.UNMAP_SYSMEM: + if (iova:=sysmem_iovas.pop(arg0, None)) is not None: System.vfio_unmap(iova, arg1) FileIOInterface.munmap(arg0, arg1) conn.sendall(resp()) elif cmd == RemoteCmd.LOAD_PROG: - programs.append(Device["CPU"].runtime(TinyELF(conn.recv(arg0, socket.MSG_WAITALL), "hcq_submit", Target("CPU"), ()))) + lib = conn.recv(arg0, socket.MSG_WAITALL) + links = {n:programs[i] for n,i in json.loads(conn.recv(arg1, socket.MSG_WAITALL)).items()} + programs.append(Device["CPU"].runtime(TinyELF(lib, "hcq_submit", Target("CPU"), ()), links=links)) conn.sendall(resp(len(programs) - 1)) elif cmd == RemoteCmd.EXEC_PROG: - et = programs[arg0](*struct.unpack(f'<{arg1}Q', conn.recv(arg1 * 8, socket.MSG_WAITALL)), wait=bool(arg2)) + kernargs = ctypes.create_string_buffer(conn.recv(arg1, socket.MSG_WAITALL)) + st = time.perf_counter() + programs[arg0].fxn(kernargs) + et = time.perf_counter() - st if (mock:=sys.modules.get("test.mockgpu.mockgpu")) is not None: # native programs bypass the mock's memoryview hooks for d in mock.drivers: d._emulate_execute() if arg2: conn.sendall(resp(int(et * 1e9))) diff --git a/tinygrad_repo/extra/thunder/amd/fa.py b/tinygrad_repo/extra/thunder/amd/fa.py index 30e36967ab..c3f0cfb3a7 100644 --- a/tinygrad_repo/extra/thunder/amd/fa.py +++ b/tinygrad_repo/extra/thunder/amd/fa.py @@ -65,16 +65,20 @@ def custom_fused_qkv_rope_backward(dxqkv:UOp, dq:UOp, dk:UOp, dv:UOp, freqs_cis: def _fa_native_grads(dq:UOp, dk:UOp, dv:UOp) -> tuple[UOp, UOp, UOp]|None: def unwrap_partial(x:UOp) -> UOp|None: - expected = (Ops.CAST, Ops.REDUCE, Ops.PERMUTE, Ops.CAST, Ops.RESHAPE, Ops.AFTER) - for op in expected: - if x.op is not op: return None - if op is not Ops.AFTER: x = x.src[0] - return x + for expected in ((Ops.CAST, Ops.REDUCE, Ops.PERMUTE, Ops.CAST, Ops.RESHAPE, Ops.AFTER), + (Ops.CAST, Ops.RESHAPE, Ops.CAST, Ops.RESHAPE, Ops.AFTER)): + cur = x + for op in expected: + if cur.op is not op: break + if op is not Ops.AFTER: cur = cur.src[0] + else: return cur + return None dq_native, dk_partial, dv_partial = dq.base, unwrap_partial(dk), unwrap_partial(dv) if dq_native.op is not Ops.AFTER or dk_partial is None or dv_partial is None: return None B, N, H, D, H_KV = dq.shape[0], dq.shape[1], dq.shape[2], dq.shape[3], dk.shape[2] - heads_per_wg = 2 if D == 128 and (H // H_KV) % 2 == 0 else 1 - partials = (H // H_KV) // heads_per_wg + if dk_partial.shape[0] % B: return None + partials = dk_partial.shape[0] // B + if partials == 0 or (H // H_KV) % partials: return None if dq_native.shape != (B, H, N, D) or dk_partial.shape != (B * partials, N, H_KV, D) or dv_partial.shape != dk_partial.shape: return None return dq_native, dk_partial, dv_partial @@ -126,6 +130,12 @@ def fused_qkv_rope(xqkv:Tensor, freqs_cis:Tensor, n_heads:int, n_kv_heads:int, h def _sharded_empty_like(ref:Tensor, axis:int|None=None) -> Tensor: return _sharded_empty(ref.shape, ref, axis) +def _bwd_heads_per_wg(D:int, group_size:int, window:int) -> int: + if D == 64: + assert group_size == 8, "D=64 backward is specialized for GPT-OSS GQA8" + return 8 if window else 2 + return 2 if D == 128 and group_size % 2 == 0 else 1 + def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink, window=0): def grad(dou:UOp, ker:UOp) -> tuple: do = Tensor(dou, device=dou.device) @@ -137,7 +147,7 @@ def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, sha dq = _sharded_empty((B, H, N, D), xq, axis=shard_axis_t) GROUP_SIZE = H_local // H_KV_local - HEADS_PER_WG = 2 if D == 128 and GROUP_SIZE % 2 == 0 else 1 + HEADS_PER_WG = _bwd_heads_per_wg(D, GROUP_SIZE, window) dk_partial = _sharded_empty((B * GROUP_SIZE // HEADS_PER_WG, N, H_KV, D), xk, axis=shard_axis) dv_partial = _sharded_empty((B * GROUP_SIZE // HEADS_PER_WG, N, H_KV, D), xv, axis=shard_axis) @@ -260,7 +270,7 @@ def custom_fa_backward_pre(delta_vec:UOp, dq:UOp, o:UOp, do:UOp, device:str, arc if not getenv("NO_HIPCC"): lib = bytearray(lib) rodata_off = next(sh.header.sh_offset for sh in elf_loader(bytes(lib))[1] if sh.name == ".rodata") - struct.pack_into('0: sliding-window backward (query i sees keys in [i-WINDOW+1, i]) +#ifndef WINDOW +#define WINDOW 0 +#endif + +#ifndef NUM_WARPS +#define NUM_WARPS 4 +#endif +#define NUM_THREADS (kittens::WARP_THREADS * NUM_WARPS) + +#ifndef GPTOSS_FA_BWD_RELAX_DQ_SYNC +#define GPTOSS_FA_BWD_RELAX_DQ_SYNC 0 +#endif +#if GPTOSS_FA_BWD_RELAX_DQ_SYNC +#if ATTN_B != 2 || ATTN_N != 8192 || ATTN_H != 64 || ATTN_H_KV != 8 || ATTN_D != 64 || WINDOW != 0 || BWD_BLOCK_SIZE_KV != 256 || NUM_WARPS != 4 || BWD_HEADS_PER_WG != 2 +#error "Relaxed dQ synchronization is qualified only for the exact full-causal GPTOSS backward shape" +#endif +// The retained publication/reuse synchronization protects shared dS. +// This intermediate point only separates wave-local register work; retain all waits. +#define DQ_MID_SYNC() ((void)0) +#else +#define DQ_MID_SYNC() __builtin_amdgcn_s_barrier() +#endif + +#ifndef GPTOSS_FA_BWD_DOUBLE_ATTN +#define GPTOSS_FA_BWD_DOUBLE_ATTN 0 +#endif +#if GPTOSS_FA_BWD_DOUBLE_ATTN +#if !GPTOSS_FA_BWD_RELAX_DQ_SYNC +#error "Double dS scratch requires the exact qualified GPTOSS full-causal schedule" +#endif +// Alternate dS planes. The next slice's publication fence releases this plane +// before its next writer two slices later; all memory waits remain. +#define DQ_REUSE_SYNC() ((void)0) +#define DQ_ODD_PLANE_OFFSET 8192u +#else +#define DQ_REUSE_SYNC() __builtin_amdgcn_s_barrier() +#define DQ_ODD_PLANE_OFFSET 0u +#endif + +using G = kittens::group; + +using namespace kittens; + +template +__device__ __forceinline__ void reload_k(const RT &dst, const ST &src, uint32_t addr) { + if constexpr (WINDOW) load(dst, src, addr); +} + +template +__device__ __forceinline__ void global_store_dwordx2_sbase(bf16 *base, uint32_t byte_offset) { + static_assert(GPR_START >= 0 && GPR_START + 1 < 256); + asm volatile("global_store_dwordx2 %0, v[%1:%2], %3" + : : "v"(byte_offset), "n"(GPR_START), "n"(GPR_START + 1), "s"(base) : "memory"); +} + +// Store one wave's fixed-register 64x64 accumulator tile directly to the +// GPT-OSS partial dK/dV layout. The generic tile store rebuilds these same +// coordinates while carrying a full buffer resource through the kernel. +template +__device__ __forceinline__ void store_d64_warp64_global(bf16 *dst_ptr, const RT &src, + int partial_idx, int seq_idx, + int kv_head_idx, int uniform_warpid) { + static_assert(std::is_same_v && std::is_same_v); + static_assert(std::is_same_v && RT::height == 2 && RT::width == 2); + (void)src; + + const int lane_row = kittens::laneid() % RT::base_tile_rows; + const int lane_col = RT::base_tile_stride * (kittens::laneid() / RT::base_tile_rows); + bf16 *wave_ptr = dst_ptr + (((partial_idx * ATTN_N + seq_idx * BLOCK_SIZE_KV + + uniform_warpid * WARP_SIZE_KV) * ATTN_H_KV + kv_head_idx) * ATTN_D); + + auto store_tile = [&]() { + using tile_range = ducks::art::get_nth_range_t; + constexpr int R0 = tile_range::lo; + constexpr int R1 = tile_range::lo + 4; + constexpr int R2 = tile_range::lo + 8; + constexpr int R3 = tile_range::lo + 12; + constexpr int B0 = tile_range::lo; + constexpr int B1 = tile_range::lo + 2; + constexpr int B2 = tile_range::lo + 4; + constexpr int B3 = tile_range::lo + 6; + + macros::v_cvt_pk_bf16_f32(); + macros::v_cvt_pk_bf16_f32(); + macros::v_cvt_pk_bf16_f32(); + macros::v_cvt_pk_bf16_f32(); + macros::v_cvt_pk_bf16_f32(); + macros::v_cvt_pk_bf16_f32(); + macros::v_cvt_pk_bf16_f32(); + macros::v_cvt_pk_bf16_f32(); + + auto store_chunk = [&]() { + const int row = N * RT::base_tile_rows + lane_row; + const int col = M * RT::base_tile_cols + lane_col + COL_OFFSET; + const uint32_t byte_offset = (row * ATTN_H_KV * ATTN_D + col) * sizeof(bf16); + global_store_dwordx2_sbase(wave_ptr, byte_offset); + }; + store_chunk.template operator()<0, B0>(); + store_chunk.template operator()<8, B1>(); + store_chunk.template operator()<16, B2>(); + store_chunk.template operator()<24, B3>(); + }; + store_tile.template operator()<0, 0>(); + store_tile.template operator()<0, 1>(); + store_tile.template operator()<1, 0>(); + store_tile.template operator()<1, 1>(); +} + +using _gl_QdO = gl; +using _gl_KV = gl; +using _gl_dQ = gl; +using _gl_Lvec = gl; + + +template struct attn_bwd_combined_globals { + _gl_QdO Q; + _gl_KV K, V; + _gl_QdO dOg; + _gl_dQ dQg; + _gl_Lvec L_vec, delta_vec; + dim3 grid() { return dim3(ATTN_H / HEADS_PER_WG, (ATTN_N / BLOCK_SIZE_KV), ATTN_B); } + dim3 block() { return dim3(NUM_THREADS); } + size_t dynamic_shared_memory() { return MAX_SHARED_MEMORY; } +}; + +template __launch_bounds__(NUM_THREADS, 1) +__global__ __attribute__((amdgpu_num_vgpr(29))) void attend_bwd_combined_ker(bf16 *dQ_ptr, bf16 *dK_ptr, bf16 *dV_ptr, bf16 *dO_ptr, bf16 *Q_ptr, bf16 *K_ptr, bf16 *V_ptr, float *L_vec_ptr, float *delta_vec_ptr) { + + const int q_head_idx_fixed = blockIdx.x * HEADS_PER_WG; // First query head handled by this workgroup. + const int kv_head_idx = q_head_idx_fixed / GROUP_SIZE; + const int q_head_in_group = q_head_idx_fixed % GROUP_SIZE; + const int seq_idx = blockIdx.y; + const int batch_idx = blockIdx.z; + const int first_q_head = q_head_idx_fixed; + + const int warpid = kittens::warpid(); + // The warp id is lane-invariant, but deriving it from threadIdx makes LLVM + // conservatively keep k_pos in a VGPR. Mark the value uniform so the causal + // block tests use scalar control flow instead of saveexec/restoreexec paths. + const int uniform_warpid = __builtin_amdgcn_readfirstlane(warpid); + const int j = seq_idx * NUM_WARPS + uniform_warpid; + + // optimization on loops bounds + const int total_steps_per_head = ATTN_N / STEP_QO; + const int j_min = seq_idx * NUM_WARPS; + const int k_start_min = j_min * WARP_SIZE_KV; + // first Q step that can overlap this K_span: + const int first_step = max(0, k_start_min / STEP_QO); +#if WINDOW + // GPT-OSS WINDOW=128 needs exactly six Q steps per 256-key block. The old shared D=64/128 + // schedule padded this with two fully masked steps to flush dQ; the dedicated D=64 epilogue does not need them. + const int num_steps_per_head = min(total_steps_per_head - first_step, (BLOCK_SIZE_KV + WINDOW) / STEP_QO); +#else + const int num_steps_per_head = total_steps_per_head - first_step; +#endif + const int num_steps = num_steps_per_head * HEADS_PER_WG; + const int k_pos = j * WARP_SIZE_KV; + + constexpr float L_SCALE_FACTOR = 1.44269504089f; + constexpr float P_SCALE_FACTOR = 0.125f*1.44269504089f; + constexpr float dP_SCALE_FACTOR = 0.125f; + + // Shared tiles + extern __shared__ alignment_dummy __shm[]; + shared_allocator al((int*)&__shm[0]); + + st_bf (&K_j_smem) = al.allocate>(); + st_bf (&Q_i_smem)[2][2] = al.allocate, 2, 2>(); + st_bf (&dO_i_smem)[2][2] = al.allocate, 2, 2>(); + #if GPTOSS_FA_BWD_DOUBLE_ATTN + st_bf (&attn_i_smem_storage)[2] = al.allocate, 2>(); + auto &attn_i_smem = attn_i_smem_storage[0]; + static_assert(sizeof(attn_i_smem) == 8192); +#else + st_bf (&attn_i_smem) = al.allocate>(); +#endif + sv_fl (&L_smem)[2] = al.allocate, 2>(); + sv_fl (&delta_smem)[2] = al.allocate, 2>(); + + // Register tiles + using Q_ranges = ducks::art::split_many_t>, 4>; + using dO_ranges = ducks::art::split_many_t>, 4>; + using dO_col_ranges = ducks::art::split_many_t>, 4>; + using K_ranges = ducks::art::split_many_t, ducks::art::range<264, 271>, ducks::art::range<272, 279>, ducks::art::range<280, 287>>, 4>; + using V_ranges = ducks::art::split_many_t, ducks::art::range<296, 303>, ducks::art::range<304, 311>, ducks::art::range<312, 319>>, 4>; + using P_ranges = ducks::art::split_many_t>, 4>; // 16 registers - v[40:55] + using dP_ranges = ducks::art::split_many_t>, 4>; // 16 registers - v[56:71] + using P_bf16_ranges = ducks::art::split_many_t>, 2>; // 8 registers - v[116:123] + using dP_bf16_ranges = ducks::art::split_many_t>, 2>; // 8 registers - v[56:63] + using P_bf16_col_ranges = ducks::art::split_many_t>, 4>; // 8 registers + using dP_bf16_col_ranges = ducks::art::split_many_t>, 4>; // 8 registers + using dS_ranges = ducks::art::split_many_t>, 4>; // 32 registers - v[24:55] + using dQ_ranges = ducks::art::split_many_t>, 4>; + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + + + using dV_ranges = ducks::art::split_many_t>, 16>; + using dK_ranges = ducks::art::split_many_t>, 16>; + using Kcol_ranges = ducks::art::split_many_t>, 4>; + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + + art Q_i; // 16 registers + art dO_i; // 16 registers + art Q_i_col; // 16 registers + art dO_i_col; // 16 registers + art K_j; // 64 registers + art V_j; // 64 registers + constexpr int L_i = 126; + constexpr int delta_i = 127; + constexpr int neg_inf_v = 29; + // Move -inf to VGPR neg_inf_v + kittens::macros::clobber_gpr(); + kittens::macros::v_mov_b32_up2p(0xff800000); + + art P_ij; // 16 registers + art dP_ij; // 16 registers + art P_ij_bf16; // 8 registers + art dP_ij_bf16; // 8 registers + art> dP_ij_bf16_accum_row; // 8 registers + + art P_ij_bf16_col; // 8 registers + art dP_ij_bf16_col; // 8 registers + + art K_j_col; + art dP_ij_bf16_col_T; // 32 registers // for dq + + art dK_j_T; // 128 registers + art dV_j_T; // 128 registers + art dQ_i_T; + art> dQ_i; + + // This is used for both dK_j_T and dV_j_T + art> dV_j; + + // Construct gl objects with compile-time dims AFTER clobbers so compiler knows which VGPRs are taken + _gl_dQ dQg{dQ_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_QdO dOg{dO_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_QdO Q{Q_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_KV K{K_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_KV V{V_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_Lvec L_vec_gl{L_vec_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_Lvec delta_vec_gl{delta_vec_ptr, nullptr, nullptr, nullptr, nullptr}; + attn_bwd_combined_globals g{Q, K, V, dOg, dQg, L_vec_gl, delta_vec_gl}; + + // Swizzled offsets for Q and dO + constexpr int bytes_per_thread = st_16x32_s::template bytes_per_thread(); + constexpr int bytes_per_warp = bytes_per_thread * kittens::WARP_THREADS; + constexpr int memcpy_per_tile = BLOCK_SIZE_KV * DOT_SLICE_QO * sizeof(bf16) / (bytes_per_thread * NUM_THREADS); + static_assert(BLOCK_SIZE_KV * DOT_SLICE_QO * sizeof(bf16) >= bytes_per_warp, "shared tile must be at least 1024 bytes"); + uint32_t swizzled_offsets_Q_dO[memcpy_per_tile]; + G::prefill_swizzled_offsets<1, false>(Q_i_smem[0][0], g.Q, swizzled_offsets_Q_dO); + + int tic = 0, toc = 1; + + // Load K_j from HBM to shared memory + G::load<1, false>(K_j_smem, g.K, {batch_idx, seq_idx, kv_head_idx, 0}); + + // Load V_j from HBM to registers + load<1>(V_j, g.V, {batch_idx, 0, kv_head_idx, 0}, {0, j, 0, 0}); + + // Load Q, dO, L, delta for this specific query head + load(L_smem[tic], g.L_vec, {batch_idx, first_q_head, 0, first_step}); + load(delta_smem[tic], g.delta_vec, {batch_idx, first_q_head, 0, first_step}); + G::load<1, false>(Q_i_smem[tic][0], g.Q, {batch_idx, first_step * 2 + 0, first_q_head, 0}, swizzled_offsets_Q_dO); + G::load<1, false>(dO_i_smem[tic][0], g.dOg, {batch_idx, first_step * 2 + 0, first_q_head, 0}, swizzled_offsets_Q_dO); + G::load<1, false>(Q_i_smem[tic][1], g.Q, {batch_idx, first_step * 2 + 1, first_q_head, 0}, swizzled_offsets_Q_dO); + G::load<1, false>(dO_i_smem[tic][1], g.dOg, {batch_idx, first_step * 2 + 1, first_q_head, 0}, swizzled_offsets_Q_dO); + __builtin_amdgcn_s_waitcnt(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Addresses + const uint32_t K_j_addr = get_address(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + // Compute K_j_col_addr + // uint32_t K_j_col_addr = get_address(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + const uint32_t K_j_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(K_j_smem, {0, warpid}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 16) * 4; + const int col_offset = ((laneid % 4) * 4); + const int lane_byte_offset = (row_offset * 16 + col_offset) * sizeof(bf16); + const uint32_t addr = src_ptr + lane_byte_offset; + return addr; + }(); + + auto attn_i_smem_subtile = subtile_inplace(attn_i_smem, {warpid, 0}); + const uint32_t dP_ij_bf16_accum_row_addr = get_address(attn_i_smem_subtile, dP_ij_bf16_accum_row); + + uint32_t Q_i_addr; + uint32_t dO_i_addr; + uint32_t dO_i_col_addr; + uint32_t Q_i_col_addr; + + // Compute dP_ij_bf16_col_T_addr + // const uint32_t dP_ij_bf16_col_T_addr = [&] { + // const int laneid = kittens::laneid(); + // const uint32_t src_ptr = reinterpret_cast(&attn_i_smem.data[0]); + // const int row_offset = (laneid % 16) / 4 + (laneid / 16) * 4; + // const int col_offset = ((laneid % 4) * 4); + // const int lane_byte_offset = (row_offset * 16 + col_offset) * sizeof(bf16); + // const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 7) << 3); + // const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + // return addr; + // }(); + uint32_t dP_ij_bf16_col_T_addr = get_address(dP_ij_bf16_col_T, attn_i_smem); + // Invariant dQ K operand; fixed v192:v223 are not aliased by another tile. + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + + + if (num_steps > 1) { + // Prologue + { + const int q_head_idx = first_q_head; + const int q_seq_idx = first_step; + const int q_pos = q_seq_idx * STEP_QO; +#if WINDOW + const bool kv_active = static_cast(q_pos - k_pos) <= WINDOW; +#else + // Later waves in this 256-key block have one to three fully causal-masked + // query tiles at the triangular edge. Skip their score/dP work while all + // waves continue through the block-wide barriers and dQ reduction. + const bool kv_active = q_pos >= k_pos; +#endif + const int next_q_head_idx = first_q_head; + const int next_q_seq_idx = first_step + 1; + + // dot slice 0 + { + load(L_smem[toc], g.L_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + G::load<1, false>(Q_i_smem[toc][0], g.Q, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}); + + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load(subvec_inplace(L_smem[tic], 0)); + load(subvec_inplace(delta_smem[tic], 0)); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + load<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + load<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mul(L_SCALE_FACTOR); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + load<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 0 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 0] and set [0, 1:4] to -inf + make_causal<0, 0, neg_inf_v>(P_ij, P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + // window lower boundary, mirror of the causal edge + } else if (q_pos - k_pos == WINDOW) { + make_window<0, 0, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + // Full-causal keeps K resident across query steps. The normal active + // prologue fills its upper half while computing scores; an inactive + // triangular-edge wave must still initialize those registers for the + // first later query tile that becomes active. + if constexpr (!WINDOW) { + load<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + } + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 1 + load(subvec_inplace(L_smem[tic], 1)); + load(subvec_inplace(delta_smem[tic], 1)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + mul(L_SCALE_FACTOR); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(0) lgkmcnt(" DQ_L6 ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][0], g.dOg, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load(delta_smem[toc], g.delta_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 1 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 1 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 1] and set [0, 2:4] to -inf + make_causal<0, 1, neg_inf_v>(P_ij, P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + make_window<0, 1, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 2 + load(subvec_inplace(L_smem[tic], 2)); + load(subvec_inplace(delta_smem[tic], 2)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + // Scale at the deferred atomic consumer, after the MFMA result has naturally cleared its + // 16-cycle accumulator-to-VALU hazard. Scaling at the producer required an explicit s_nop 15. + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(Q_i_smem[toc][1], g.Q, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 2 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 2 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 2] and set [0, 3:4] to -inf + make_causal<0, 2, neg_inf_v>(P_ij, P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + make_window<0, 2, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 3 + load(subvec_inplace(L_smem[tic], 3)); + load(subvec_inplace(delta_smem[tic], 3)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 1, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][1], g.dOg, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 3 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 3 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 3] + make_causal<0, 3, neg_inf_v>(P_ij, P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + make_window<0, 3, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 0 - next iteration + load(subvec_inplace(L_smem[toc], 0)); + load(subvec_inplace(delta_smem[toc], 0)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 2, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + tic ^= 1; toc ^= 1; + } + + int last_q_head_idx = first_q_head; + int last_q_seq_idx = first_step; + int q_head_idx = first_q_head; + int q_seq_idx = first_step + 1; + int q_pos = q_seq_idx * STEP_QO; + int next_q_head_idx = first_q_head; + int next_q_seq_idx = first_step + 2; + const int q_seq_end = first_step + num_steps_per_head; + if (next_q_seq_idx == q_seq_end) { + next_q_seq_idx = first_step; + ++next_q_head_idx; + } + + // 9. for 1 <= i <= T_r (1024 / 32 = 32) + for (int i = 1; i < num_steps - 1; ++i, tic ^= 1, toc ^= 1) { +#if WINDOW + const bool kv_active = static_cast(q_pos - k_pos) <= WINDOW; +#else + const bool kv_active = q_pos >= k_pos; +#endif + + // dot slice 0 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + G::load<1, false>(Q_i_smem[toc][0], g.Q, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + load(L_smem[toc], g.L_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 0 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 0] and set [0, 1:4] to -inf + make_causal<0, 0, neg_inf_v>(P_ij, P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + // window lower boundary, mirror of the causal edge + } else if (q_pos - k_pos == WINDOW) { + make_window<0, 0, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 1 + load(subvec_inplace(L_smem[tic], 1)); + load(subvec_inplace(delta_smem[tic], 1)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, last_q_head_idx, last_q_seq_idx * 4 + 3, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][0], g.dOg, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load(delta_smem[toc], g.delta_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 1 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 1 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 1] and set [0, 2:4] to -inf + make_causal<0, 1, neg_inf_v>(P_ij, P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + make_window<0, 1, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 2 + load(subvec_inplace(L_smem[tic], 2)); + load(subvec_inplace(delta_smem[tic], 2)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 0, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(Q_i_smem[toc][1], g.Q, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 2 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 2 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 2] and set [0, 3:4] to -inf + make_causal<0, 2, neg_inf_v>(P_ij, P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + make_window<0, 2, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 3 + load(subvec_inplace(L_smem[tic], 3)); + load(subvec_inplace(delta_smem[tic], 3)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 1, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][1], g.dOg, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 3 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 3 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 3] + make_causal<0, 3, neg_inf_v>(P_ij, P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + make_window<0, 3, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 0 - next iteration + load(subvec_inplace(L_smem[toc], 0)); + load(subvec_inplace(delta_smem[toc], 0)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 2, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + last_q_head_idx = q_head_idx; + last_q_seq_idx = q_seq_idx; + q_head_idx = next_q_head_idx; + q_seq_idx = next_q_seq_idx; + q_pos = q_seq_idx * STEP_QO; + if (++next_q_seq_idx == q_seq_end) { + next_q_seq_idx = first_step; + ++next_q_head_idx; + } + } + } + + const int last_q_head_idx = first_q_head + HEADS_PER_WG - 1; + const int last_q_seq_idx = first_step + num_steps_per_head - 2; + + const int q_head_idx = first_q_head + HEADS_PER_WG - 1; + const int q_seq_idx = first_step + num_steps_per_head - 1; + const int q_pos = q_seq_idx * STEP_QO; +#if WINDOW + const bool kv_active = static_cast(q_pos - k_pos) <= WINDOW; +#else + // The epilogue handles the final query tile of the final head, which is + // always at or beyond every key wave in this workgroup. + constexpr bool kv_active = true; +#endif + // Epilogue + { + // dot slice 0 + { + + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 0 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 0] and set [0, 1:4] to -inf + make_causal<0, 0, neg_inf_v>(P_ij, P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + make_window<0, 0, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 1 + load(subvec_inplace(L_smem[tic], 1)); + load(subvec_inplace(delta_smem[tic], 1)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // The last main iteration has no following VM wait to finish its dO half1 prefetch. + // Complete it before this existing publication barrier, ahead of epilogue slice2. + asm volatile("s_waitcnt vmcnt(0) lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + if (num_steps > 1) { + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, last_q_head_idx, last_q_seq_idx * 4 + 3, 0}, warpid); + } + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + // K_j_col is invariant and resident in v192:v223. + if (num_steps > 1) { + } + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 1 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 1 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 1] and set [0, 2:4] to -inf + make_causal<0, 1, neg_inf_v>(P_ij, P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + make_window<0, 1, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 2 + load(subvec_inplace(L_smem[tic], 2)); + load(subvec_inplace(delta_smem[tic], 2)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 2 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 2 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 2] and set [0, 3:4] to -inf + make_causal<0, 2, neg_inf_v>(P_ij, P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + make_window<0, 2, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 3 + load(subvec_inplace(L_smem[tic], 3)); + load(subvec_inplace(delta_smem[tic], 3)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(6)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 1, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + DQ_REUSE_SYNC(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 3 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 3 + kittens::macros::v_mov_b32_up2p(0xff800000); + if constexpr (causal) { + // Only the diagonal key tile needs masking; fully future tiles were rejected by kv_active. + if (q_pos == k_pos) { + // Apply the causal mask to [0, 3] + make_causal<0, 3, neg_inf_v>(P_ij, P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + make_window<0, 3, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + // K_j_col is invariant and resident in v192:v223. + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 0 - next iteration + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, (dP_ij_bf16_accum_row_addr + DQ_ODD_PLANE_OFFSET)); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(4)"); // resident-Kcol: wait for Q_i_col + + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // K_j_col is invariant and resident in v192:v223. + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 2, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + // K_j_col is invariant and resident in v192:v223. + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, (dP_ij_bf16_col_T_addr + DQ_ODD_PLANE_OFFSET)); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + DQ_MID_SYNC(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // K_j_col is invariant and resident in v192:v223. + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // ds_read_b128 a[112:115] + // ds_read_b128 a[116:119] + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(0)"); // final dQ: no next-Q LDS suffix + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // ds_read_b128 a[120:123] + // ds_read_b128 a[124:127] + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // ds_read_b128 a[0:3] + // ds_read_b128 a[4:7] + // ds_read_b128 a[8:11] + // ds_read_b128 a[12:15] + // ds_read_b128 a[16:19] + // ds_read_b128 a[20:23] + // ds_read_b128 a[24:27] + // ds_read_b128 a[28:31] + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + } + + const int partial_idx = batch_idx * (GROUP_SIZE / HEADS_PER_WG) + q_head_in_group / HEADS_PER_WG; + store_d64_warp64_global(dV_ptr, dV_j, partial_idx, seq_idx, kv_head_idx, uniform_warpid); + __builtin_amdgcn_s_waitcnt(0); + __builtin_amdgcn_s_barrier(); + + // We first copy dV_j_T from accumulator GPRs to vector GPRs and then perform the store + accvgpr_read(dV_j_T, dK_j_T); + mul(dV_j_T, dV_j_T, dP_SCALE_FACTOR); + store_d64_warp64_global(dK_ptr, dV_j, partial_idx, seq_idx, kv_head_idx, uniform_warpid); + + // Write out final dQ_i slice + mul(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 3, 0}, warpid); +} + +template __global__ void attend_bwd_combined_ker(bf16*, bf16*, bf16*, bf16*, bf16*, bf16*, bf16*, float*, float*); diff --git a/tinygrad_repo/extra/thunder/amd/fa_bwd_causal_d64_window.cpp b/tinygrad_repo/extra/thunder/amd/fa_bwd_causal_d64_window.cpp new file mode 100644 index 0000000000..73a3a874ed --- /dev/null +++ b/tinygrad_repo/extra/thunder/amd/fa_bwd_causal_d64_window.cpp @@ -0,0 +1,2883 @@ +#if !defined(ATTN_D) || ATTN_D != 64 +#error "fa_bwd_causal_d64.cpp only supports ATTN_D=64" +#endif + +#include "kittens.cuh" +#include "utils.cpp" + +#define DQ_L4 "2" +#if WINDOW +#define DQ_L2 "1" +#else +#define DQ_L2 "0" +#endif +#define DQ_L6 "3" +#define DQ_L6_LOOP "3" + +#ifndef ATTN_B +constexpr int ATTN_B = 16; // batch size +#endif + +#ifndef ATTN_H +constexpr int ATTN_H = 64; // number of query heads +#endif + +#ifndef ATTN_H_KV +constexpr int ATTN_H_KV = 8; // number of key/value heads (for GQA) +#endif + +constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; // queries per KV head group +#ifndef BWD_HEADS_PER_WG +#define BWD_HEADS_PER_WG 1 +#endif +constexpr int HEADS_PER_WG = BWD_HEADS_PER_WG; + +#ifndef ATTN_N +constexpr int ATTN_N = 1024; // sequence length +#endif + +constexpr int STEP_QO = 64; // block size for QO +#ifndef BWD_BLOCK_SIZE_KV +#define BWD_BLOCK_SIZE_KV 256 +#endif +constexpr int BLOCK_SIZE_KV = BWD_BLOCK_SIZE_KV; // block size for KV +constexpr int WARP_SIZE_KV = 64; // warp size for KV +constexpr int SLICE_QO = 32; +constexpr int DOT_SLICE_QO = 16; +constexpr bool causal = true; +// WINDOW>0: sliding-window backward (query i sees keys in [i-WINDOW+1, i]) +#ifndef WINDOW +#define WINDOW 0 +#endif + +#ifndef NUM_WARPS +#define NUM_WARPS 4 +#endif +#define NUM_THREADS (kittens::WARP_THREADS * NUM_WARPS) + +using G = kittens::group; + +using namespace kittens; + +template +__device__ __forceinline__ void reload_k(const RT &dst, const ST &src, uint32_t addr) { + if constexpr (WINDOW) load(dst, src, addr); +} + +using _gl_QdO = gl; +using _gl_KV = gl; +using _gl_dQ = gl; +using _gl_dKV = gl; +using _gl_Lvec = gl; + + +template struct attn_bwd_combined_globals { + _gl_QdO Q; + _gl_KV K, V; + _gl_QdO dOg; + _gl_dQ dQg; + _gl_dKV dKg, dVg; + _gl_Lvec L_vec, delta_vec; + dim3 grid() { return dim3(ATTN_H / HEADS_PER_WG, (ATTN_N / BLOCK_SIZE_KV), ATTN_B); } + dim3 block() { return dim3(NUM_THREADS); } + size_t dynamic_shared_memory() { return MAX_SHARED_MEMORY; } +}; + +template __launch_bounds__(NUM_THREADS, 1) +__global__ void attend_bwd_combined_ker(bf16 *dQ_ptr, bf16 *dK_ptr, bf16 *dV_ptr, bf16 *dO_ptr, bf16 *Q_ptr, bf16 *K_ptr, bf16 *V_ptr, float *L_vec_ptr, float *delta_vec_ptr) { + + const int q_head_idx_fixed = blockIdx.x * HEADS_PER_WG; // First query head handled by this workgroup. + const int kv_head_idx = q_head_idx_fixed / GROUP_SIZE; + const int q_head_in_group = q_head_idx_fixed % GROUP_SIZE; + const int seq_idx = blockIdx.y; + const int batch_idx = blockIdx.z; + const int first_q_head = q_head_idx_fixed; + + const int warpid = kittens::warpid(); + // The warp id is lane-invariant, but deriving it from threadIdx makes LLVM + // conservatively keep k_pos in a VGPR. Mark the value uniform so the causal + // block tests use scalar control flow instead of saveexec/restoreexec paths. + const int uniform_warpid = __builtin_amdgcn_readfirstlane(warpid); + const int j = seq_idx * NUM_WARPS + uniform_warpid; + + // optimization on loops bounds + const int total_steps_per_head = ATTN_N / STEP_QO; + const int j_min = seq_idx * NUM_WARPS; + const int k_start_min = j_min * WARP_SIZE_KV; + // first Q step that can overlap this K_span: + const int first_step = max(0, k_start_min / STEP_QO); +#if WINDOW + // GPT-OSS WINDOW=128 needs exactly six Q steps per 256-key block. The old shared D=64/128 + // schedule padded this with two fully masked steps to flush dQ; the dedicated D=64 epilogue does not need them. + const int num_steps_per_head = min(total_steps_per_head - first_step, (BLOCK_SIZE_KV + WINDOW) / STEP_QO); +#else + const int num_steps_per_head = total_steps_per_head - first_step; +#endif + const int num_steps = num_steps_per_head * HEADS_PER_WG; + const int k_pos = j * WARP_SIZE_KV; + + constexpr float L_SCALE_FACTOR = 1.44269504089f; + constexpr float P_SCALE_FACTOR = 0.125f*1.44269504089f; + constexpr float dP_SCALE_FACTOR = 0.125f; + + // Shared tiles + extern __shared__ alignment_dummy __shm[]; + shared_allocator al((int*)&__shm[0]); + + st_bf (&K_j_smem) = al.allocate>(); + st_bf (&Q_i_smem)[2][2] = al.allocate, 2, 2>(); + st_bf (&dO_i_smem)[2][2] = al.allocate, 2, 2>(); + st_bf (&attn_i_smem) = al.allocate>(); + sv_fl (&L_smem)[2] = al.allocate, 2>(); + sv_fl (&delta_smem)[2] = al.allocate, 2>(); + + // Register tiles + using Q_ranges = ducks::art::split_many_t>, 4>; + using dO_ranges = ducks::art::split_many_t>, 4>; + using dO_col_ranges = ducks::art::split_many_t>, 4>; + using K_ranges = ducks::art::split_many_t, ducks::art::range<264, 271>, ducks::art::range<272, 279>, ducks::art::range<280, 287>>, 4>; + using V_ranges = ducks::art::split_many_t, ducks::art::range<296, 303>, ducks::art::range<304, 311>, ducks::art::range<312, 319>>, 4>; + using P_ranges = ducks::art::split_many_t>, 4>; // 16 registers - v[40:55] + using dP_ranges = ducks::art::split_many_t>, 4>; // 16 registers - v[56:71] + using P_bf16_ranges = ducks::art::split_many_t>, 2>; // 8 registers - v[116:123] + using dP_bf16_ranges = ducks::art::split_many_t>, 2>; // 8 registers - v[56:63] + using P_bf16_col_ranges = ducks::art::split_many_t>, 4>; // 8 registers + using dP_bf16_col_ranges = ducks::art::split_many_t>, 4>; // 8 registers + using dS_ranges = ducks::art::split_many_t>, 4>; // 32 registers - v[24:55] + using dQ_ranges = ducks::art::split_many_t>, 4>; + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + + + using dV_ranges = ducks::art::split_many_t>, 16>; + using dK_ranges = ducks::art::split_many_t>, 16>; + using Kcol_ranges = ducks::art::split_many_t>, 4>; + ducks::art::clobber(); + ducks::art::clobber(); + ducks::art::clobber(); + + art Q_i; // 16 registers + art dO_i; // 16 registers + art Q_i_col; // 16 registers + art dO_i_col; // 16 registers + art K_j; // 64 registers + art V_j; // 64 registers + constexpr int L_i = 126; + constexpr int delta_i = 127; + constexpr int neg_inf_v = 29; + // Move -inf to VGPR neg_inf_v + kittens::macros::clobber_gpr(); + kittens::macros::v_mov_b32_up2p(0xff800000); + + art P_ij; // 16 registers + art dP_ij; // 16 registers + art P_ij_bf16; // 8 registers + art dP_ij_bf16; // 8 registers + art> dP_ij_bf16_accum_row; // 8 registers + + art P_ij_bf16_col; // 8 registers + art dP_ij_bf16_col; // 8 registers + + art K_j_col; + art dP_ij_bf16_col_T; // 32 registers // for dq + + art dK_j_T; // 128 registers + art dV_j_T; // 128 registers + art dQ_i_T; + art> dQ_i; + + // This is used for both dK_j_T and dV_j_T + art> dV_j; + + // Construct gl objects with compile-time dims AFTER clobbers so compiler knows which VGPRs are taken + _gl_dQ dQg{dQ_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_dKV dKg{dK_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_dKV dVg{dV_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_QdO dOg{dO_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_QdO Q{Q_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_KV K{K_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_KV V{V_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_Lvec L_vec_gl{L_vec_ptr, nullptr, nullptr, nullptr, nullptr}; + _gl_Lvec delta_vec_gl{delta_vec_ptr, nullptr, nullptr, nullptr, nullptr}; + attn_bwd_combined_globals g{Q, K, V, dOg, dQg, dKg, dVg, L_vec_gl, delta_vec_gl}; + + // Swizzled offsets for Q and dO + constexpr int bytes_per_thread = st_16x32_s::template bytes_per_thread(); + constexpr int bytes_per_warp = bytes_per_thread * kittens::WARP_THREADS; + constexpr int memcpy_per_tile = BLOCK_SIZE_KV * DOT_SLICE_QO * sizeof(bf16) / (bytes_per_thread * NUM_THREADS); + static_assert(BLOCK_SIZE_KV * DOT_SLICE_QO * sizeof(bf16) >= bytes_per_warp, "shared tile must be at least 1024 bytes"); + uint32_t swizzled_offsets_Q_dO[memcpy_per_tile]; + G::prefill_swizzled_offsets<1, false>(Q_i_smem[0][0], g.Q, swizzled_offsets_Q_dO); + + int tic = 0, toc = 1; + + // Load K_j from HBM to shared memory + G::load<1, false>(K_j_smem, g.K, {batch_idx, seq_idx, kv_head_idx, 0}); + + // Load V_j from HBM to registers + load<1>(V_j, g.V, {batch_idx, 0, kv_head_idx, 0}, {0, j, 0, 0}); + + // Load Q, dO, L, delta for this specific query head + load(L_smem[tic], g.L_vec, {batch_idx, first_q_head, 0, first_step}); + load(delta_smem[tic], g.delta_vec, {batch_idx, first_q_head, 0, first_step}); + G::load<1, false>(Q_i_smem[tic][0], g.Q, {batch_idx, first_step * 2 + 0, first_q_head, 0}, swizzled_offsets_Q_dO); + G::load<1, false>(dO_i_smem[tic][0], g.dOg, {batch_idx, first_step * 2 + 0, first_q_head, 0}, swizzled_offsets_Q_dO); + G::load<1, false>(Q_i_smem[tic][1], g.Q, {batch_idx, first_step * 2 + 1, first_q_head, 0}, swizzled_offsets_Q_dO); + G::load<1, false>(dO_i_smem[tic][1], g.dOg, {batch_idx, first_step * 2 + 1, first_q_head, 0}, swizzled_offsets_Q_dO); + __builtin_amdgcn_s_waitcnt(0); + __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_sched_barrier(0); + + // Addresses + const uint32_t K_j_addr = get_address(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + // Compute K_j_col_addr + // uint32_t K_j_col_addr = get_address(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + const uint32_t K_j_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(K_j_smem, {0, warpid}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 16) * 4; + const int col_offset = ((laneid % 4) * 4); + const int lane_byte_offset = (row_offset * 16 + col_offset) * sizeof(bf16); + const uint32_t addr = src_ptr + lane_byte_offset; + return addr; + }(); + + auto attn_i_smem_subtile = subtile_inplace(attn_i_smem, {warpid, 0}); + const uint32_t dP_ij_bf16_accum_row_addr = get_address(attn_i_smem_subtile, dP_ij_bf16_accum_row); + + uint32_t Q_i_addr; + uint32_t dO_i_addr; + uint32_t dO_i_col_addr; + uint32_t Q_i_col_addr; + + // Compute dP_ij_bf16_col_T_addr + // const uint32_t dP_ij_bf16_col_T_addr = [&] { + // const int laneid = kittens::laneid(); + // const uint32_t src_ptr = reinterpret_cast(&attn_i_smem.data[0]); + // const int row_offset = (laneid % 16) / 4 + (laneid / 16) * 4; + // const int col_offset = ((laneid % 4) * 4); + // const int lane_byte_offset = (row_offset * 16 + col_offset) * sizeof(bf16); + // const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 7) << 3); + // const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + // return addr; + // }(); + uint32_t dP_ij_bf16_col_T_addr = get_address(dP_ij_bf16_col_T, attn_i_smem); + + if (num_steps > 1) { + // Prologue + { + const int q_head_idx = first_q_head; + const int q_seq_idx = first_step; + const int q_pos = q_seq_idx * STEP_QO; +#if WINDOW + const bool kv_active = static_cast(q_pos - k_pos) <= WINDOW; +#else + constexpr bool kv_active = true; +#endif + const int next_q_head_idx = first_q_head; + const int next_q_seq_idx = first_step + 1; + + // dot slice 0 + { + load(L_smem[toc], g.L_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + G::load<1, false>(Q_i_smem[toc][0], g.Q, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}); + + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load(subvec_inplace(L_smem[tic], 0)); + load(subvec_inplace(delta_smem[tic], 0)); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + load<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + load<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mul(L_SCALE_FACTOR); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + load<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + load<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 0 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 0] and set [0, 1:4] to -inf + make_causal<0, 0, neg_inf_v>(P_ij, P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + // window lower boundary, mirror of the causal edge + } else if (q_pos - k_pos == WINDOW) { + make_window<0, 0, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 1 + load(subvec_inplace(L_smem[tic], 1)); + load(subvec_inplace(delta_smem[tic], 1)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col); + mul(L_SCALE_FACTOR); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(0) lgkmcnt(" DQ_L6 ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][0], g.dOg, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load(delta_smem[toc], g.delta_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 1 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 1 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 1] and set [0, 2:4] to -inf + make_causal<0, 1, neg_inf_v>(P_ij, P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + make_window<0, 1, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 2 + load(subvec_inplace(L_smem[tic], 2)); + load(subvec_inplace(delta_smem[tic], 2)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + // Scale at the deferred atomic consumer, after the MFMA result has naturally cleared its + // 16-cycle accumulator-to-VALU hazard. Scaling at the producer required an explicit s_nop 15. + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(Q_i_smem[toc][1], g.Q, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 2 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 2 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 2] and set [0, 3:4] to -inf + make_causal<0, 2, neg_inf_v>(P_ij, P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + make_window<0, 2, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 3 + load(subvec_inplace(L_smem[tic], 3)); + load(subvec_inplace(delta_smem[tic], 3)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 1, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][1], g.dOg, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 3 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 3 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 3] + make_causal<0, 3, neg_inf_v>(P_ij, P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + make_window<0, 3, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 0 - next iteration + load(subvec_inplace(L_smem[toc], 0)); + load(subvec_inplace(delta_smem[toc], 0)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 2, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + tic ^= 1; toc ^= 1; + } + + int last_q_head_idx = first_q_head; + int last_q_seq_idx = first_step; + int q_head_idx = first_q_head; + int q_seq_idx = first_step + 1; + int q_pos = q_seq_idx * STEP_QO; + int next_q_head_idx = first_q_head; + int next_q_seq_idx = first_step + 2; + const int q_seq_end = first_step + num_steps_per_head; + if (next_q_seq_idx == q_seq_end) { + next_q_seq_idx = first_step; + ++next_q_head_idx; + } + + // 9. for 1 <= i <= T_r (1024 / 32 = 32) + for (int i = 1; i < num_steps - 1; ++i, tic ^= 1, toc ^= 1) { +#if WINDOW + const bool kv_active = static_cast(q_pos - k_pos) <= WINDOW; +#else + constexpr bool kv_active = true; +#endif + + // dot slice 0 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + G::load<1, false>(Q_i_smem[toc][0], g.Q, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + load(L_smem[toc], g.L_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 0 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 0] and set [0, 1:4] to -inf + make_causal<0, 0, neg_inf_v>(P_ij, P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + // window lower boundary, mirror of the causal edge + } else if (q_pos - k_pos == WINDOW) { + make_window<0, 0, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 1 + load(subvec_inplace(L_smem[tic], 1)); + load(subvec_inplace(delta_smem[tic], 1)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, last_q_head_idx, last_q_seq_idx * 4 + 3, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][0], g.dOg, {batch_idx, next_q_seq_idx * 2, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load(delta_smem[toc], g.delta_vec, {batch_idx, next_q_head_idx, 0, next_q_seq_idx}); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 1 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 1 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 1] and set [0, 2:4] to -inf + make_causal<0, 1, neg_inf_v>(P_ij, P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + make_window<0, 1, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 2 + load(subvec_inplace(L_smem[tic], 2)); + load(subvec_inplace(delta_smem[tic], 2)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 0, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(Q_i_smem[toc][1], g.Q, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 2 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 2 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 2] and set [0, 3:4] to -inf + make_causal<0, 2, neg_inf_v>(P_ij, P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + make_window<0, 2, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 3 + load(subvec_inplace(L_smem[tic], 3)); + load(subvec_inplace(delta_smem[tic], 3)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 1, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + G::load<1, false>(dO_i_smem[toc][1], g.dOg, {batch_idx, next_q_seq_idx * 2 + 1, next_q_head_idx, 0}, swizzled_offsets_Q_dO); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 3 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 3 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 3] + make_causal<0, 3, neg_inf_v>(P_ij, P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + make_window<0, 3, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 0 - next iteration + load(subvec_inplace(L_smem[toc], 0)); + load(subvec_inplace(delta_smem[toc], 0)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 2, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt vmcnt(4) lgkmcnt(" DQ_L6_LOOP ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[toc][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + last_q_head_idx = q_head_idx; + last_q_seq_idx = q_seq_idx; + q_head_idx = next_q_head_idx; + q_seq_idx = next_q_seq_idx; + q_pos = q_seq_idx * STEP_QO; + if (++next_q_seq_idx == q_seq_end) { + next_q_seq_idx = first_step; + ++next_q_head_idx; + } + } + } + + const int last_q_head_idx = first_q_head + HEADS_PER_WG - 1; + const int last_q_seq_idx = first_step + num_steps_per_head - 2; + + const int q_head_idx = first_q_head + HEADS_PER_WG - 1; + const int q_seq_idx = first_step + num_steps_per_head - 1; + const int q_pos = q_seq_idx * STEP_QO; +#if WINDOW + const bool kv_active = static_cast(q_pos - k_pos) <= WINDOW; +#else + constexpr bool kv_active = true; +#endif + // Epilogue + { + // dot slice 0 + { + + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 0 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 0] and set [0, 1:4] to -inf + make_causal<0, 0, neg_inf_v>(P_ij, P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + make_window<0, 0, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 1 + load(subvec_inplace(L_smem[tic], 1)); + load(subvec_inplace(delta_smem[tic], 1)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + if (num_steps > 1) { + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, last_q_head_idx, last_q_seq_idx * 4 + 3, 0}, warpid); + } + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + if (num_steps > 1) { + } + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][0], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 1 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][0], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 1 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 1] and set [0, 2:4] to -inf + make_causal<0, 1, neg_inf_v>(P_ij, P_ij); + mov<0, 2, neg_inf_v>(P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + make_window<0, 1, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][0], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 2 + load(subvec_inplace(L_smem[tic], 2)); + load(subvec_inplace(delta_smem[tic], 2)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {0, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 2 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {0, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 2 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 2] and set [0, 3:4] to -inf + make_causal<0, 2, neg_inf_v>(P_ij, P_ij); + mov<0, 3, neg_inf_v>(P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + make_window<0, 2, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {0, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 3 + load(subvec_inplace(L_smem[tic], 3)); + load(subvec_inplace(delta_smem[tic], 3)); + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul(L_SCALE_FACTOR); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 1, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load Q_i from shared memory to registers + // load(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_addr = get_address(Q_i, subtile_inplace(Q_i_smem[tic][1], {1, 0})); + load<0, 0>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + load<0, 1>(Q_i, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_addr); + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // Load K_j from shared memory to registers + // load(K_j, subtile_inplace(K_j_smem, {warpid, 0})); + reload_k<0, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<0, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<1, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + + // dot slice 3 + { + // 10. S_ij = Q_i K_j^T * scale + // 11. P_ij = exp2(S_ij - L_i) + // 13. dP_ij = dO_i @ V_j^T + // 14. dS_ij = P_ij o (dP_ij - delta_i) + // mma_ABt(P_ij, Q_i, K_j); + if (kv_active) { + mma_ABt<0, 0, 0>(P_ij, Q_i, K_j); + reload_k<2, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<2, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 0, 1>(P_ij, Q_i, K_j, P_ij); + mma_ABt<0, 1, 0>(P_ij, Q_i, K_j); + reload_k<3, 0>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + reload_k<3, 1>(K_j, subtile_inplace(K_j_smem, {warpid, 0}), K_j_addr); + mma_ABt<0, 1, 1>(P_ij, Q_i, K_j, P_ij); + mul<0, 0>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 2, 0>(P_ij, Q_i, K_j); + // Load dO_i from shared memory to registers + // load(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_addr = get_address(dO_i, subtile_inplace(dO_i_smem[tic][1], {1, 0})); + load<0, 0>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + load<0, 1>(dO_i, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_addr); + mma_ABt<0, 2, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 0, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(6)"); + mul<0, 1>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(6)"); + mma_ABt<0, 3, 0>(P_ij, Q_i, K_j); + // Load dO_i_col from shared memory to registers + // load(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + // Compute dO_i_col_addr + // uint32_t dO_i_col_addr = get_address(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0})); + dO_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(dO_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const uint32_t addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + load<0, 1>(dO_i_col, subtile_inplace(dO_i_smem[tic][0], {0, 0}), dO_i_col_addr); + mma_ABt<0, 3, 1>(P_ij, Q_i, K_j, P_ij); + sub_row<0, 1, L_i>(P_ij, P_ij); + asm volatile("s_waitcnt lgkmcnt(8)"); + // Dot slice 3 + kittens::macros::v_mov_b32_up2p(0xff800000); if constexpr (causal) { + // If the query position is less than the key position, set P_ij to -inf + if (q_pos < k_pos) { + mov(P_ij); + // If the query position is equal to the key position, we need to apply a causal mask + } else if (q_pos == k_pos) { + // Apply the causal mask to [0, 3] + make_causal<0, 3, neg_inf_v>(P_ij, P_ij); +#if WINDOW + } else if (q_pos - k_pos == WINDOW) { + mov<0, 0, neg_inf_v>(P_ij); + mov<0, 1, neg_inf_v>(P_ij); + mov<0, 2, neg_inf_v>(P_ij); + make_window<0, 3, neg_inf_v>(P_ij, P_ij); + } else if (q_pos - k_pos > WINDOW) { + mov(P_ij); +#endif + } + } + mul<0, 2>(P_ij, P_ij, P_SCALE_FACTOR); + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_ABt(dP_ij, dO_i, V_j); + mma_ABt<0, 0, 0>(dP_ij, dO_i, V_j); + sub_row<0, 2, L_i>(P_ij, P_ij); + mma_ABt<0, 0, 1>(dP_ij, dO_i, V_j, dP_ij); + exp2<0, 0>(P_ij, P_ij); + // Load Q_i_col from shared memory to registers + // load(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + // Compute Q_i_col_addr + // uint32_t Q_i_col_addr = get_address(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0})); + Q_i_col_addr = [&] { + const int laneid = kittens::laneid(); + const uint32_t src_ptr = reinterpret_cast(&subtile_inplace(Q_i_smem[tic][1], {1, 0}).data[0]); + const int row_offset = (laneid % 16) / 4 + (laneid / 32) * 8; + const int col_offset = ((laneid % 4) * 4) + 16*((laneid % 32)/16); + const int lane_byte_offset = (row_offset * 32 + col_offset) * sizeof(bf16); + const int swizzled_lane_byte_offset = lane_byte_offset ^ ((lane_byte_offset >> 9) << 5); + const int addr = src_ptr + swizzled_lane_byte_offset; + return addr; + }(); + load<0, 0>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + exp2<0, 1>(P_ij, P_ij); + mma_ABt<0, 1, 0>(dP_ij, dO_i, V_j); + load<0, 1>(Q_i_col, subtile_inplace(Q_i_smem[tic][0], {0, 0}), Q_i_col_addr); + mma_ABt<0, 1, 1>(dP_ij, dO_i, V_j, dP_ij); + mul<0, 3>(P_ij, P_ij, P_SCALE_FACTOR); + sub_row<0, 3, L_i>(P_ij, P_ij); + copy<0, 0>(P_ij_bf16, P_ij); + mma_ABt<0, 2, 0>(dP_ij, dO_i, V_j); + exp2<0, 2>(P_ij, P_ij); + mma_ABt<0, 2, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 1>(P_ij_bf16, P_ij); + exp2<0, 3>(P_ij, P_ij); + mma_ABt<0, 3, 0>(dP_ij, dO_i, V_j); + mma_ABt<0, 3, 1>(dP_ij, dO_i, V_j, dP_ij); + copy<0, 2>(P_ij_bf16, P_ij); + copy<0, 3>(P_ij_bf16, P_ij); + swap_layout_inplace(P_ij_bf16_col, P_ij_bf16); + } else { + zero(P_ij); + zero(P_ij_bf16_col); + zero(dP_ij); + zero(dO_i_col); + } + asm volatile("s_waitcnt lgkmcnt(8)"); + // mma_AtB(dV_j_T, dO_i_col, P_ij_bf16_col); + mma_AtB<0, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + // Load K_j_col from shared memory to registers + // load(K_j_col, subtile_inplace(K_j_smem, {0, warpid})); + load<0, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + sub_row<0, 0, delta_i>(dP_ij, dP_ij); + sub_row<0, 1, delta_i>(dP_ij, dP_ij); + mma_AtB<1, 0, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + load<1, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<1, 1, 0>(dV_j_T, dO_i_col, P_ij_bf16_col, dV_j_T); + mul<0, 0>(dP_ij, dP_ij, P_ij); + mul<0, 1>(dP_ij, dP_ij, P_ij); + copy<0, 0>(dP_ij_bf16, dP_ij); + copy<0, 1>(dP_ij_bf16, dP_ij); + sub_row<0, 2, delta_i>(dP_ij, dP_ij); + load<2, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + // 12. dV_j += P_ij^T @ dO_i + // 16. dK_j += dS_ij^T @ Q_i (128x64)=(128x16)x(16x64) + // Store dP_ij_bf16_accum_row to shared memory + // store(attn_i_smem_subtile, dP_ij_bf16_accum_row); + store<0, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<1, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + sub_row<0, 3, delta_i>(dP_ij, dP_ij); + mul<0, 2>(dP_ij, dP_ij, P_ij); + mul<0, 3>(dP_ij, dP_ij, P_ij); + copy<0, 2>(dP_ij_bf16, dP_ij); + copy<0, 3>(dP_ij_bf16, dP_ij); + + // dot slice 0 - next iteration + + store<2, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + store<3, 0>(attn_i_smem_subtile, dP_ij_bf16_accum_row, dP_ij_bf16_accum_row_addr); + swap_layout_inplace(dP_ij_bf16_col, dP_ij_bf16); + asm volatile("s_waitcnt lgkmcnt(12)"); + // mma_AtB(dK_j_T, Q_i_col, dP_ij_bf16_col); + mma_AtB<0, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + load<3, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<4, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + asm volatile("s_waitcnt lgkmcnt(0)"); + __builtin_amdgcn_s_barrier(); + mma_AtB<1, 0, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + // Load dP_ij_bf16_col_T from shared memory to registers + // load(dP_ij_bf16_col_T, attn_i_smem); + load<0, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<1, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<2, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<3, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + mma_AtB<1, 1, 0>(dK_j_T, Q_i_col, dP_ij_bf16_col, dK_j_T); + mul<0, 0, 0>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 1>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 2>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + mul<0, 0, 3>(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2, 0, 0>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 2, 0}, warpid); + load<4, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<5, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + load<6, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + load<7, 0>(dP_ij_bf16_col_T, attn_i_smem, dP_ij_bf16_col_T_addr); + asm volatile("s_waitcnt lgkmcnt(" DQ_L6 ")"); + __builtin_amdgcn_s_barrier(); + // 15. dQ_i += dS_ij @ K_j (32x16)=(32x256)x(256x16) + // mma_AtB(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + mma_AtB<0, 0, 0>(dQ_i_T, K_j_col, dP_ij_bf16_col_T); + load<6, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 1>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 2>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + load<7, 0>(K_j_col, subtile_inplace(K_j_smem, {0, warpid}), K_j_col_addr); + mma_AtB<0, 0, 3>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + mma_AtB<0, 0, 4>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // ds_read_b128 a[112:115] + // ds_read_b128 a[116:119] + mma_AtB<0, 0, 5>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + asm volatile("s_waitcnt lgkmcnt(" DQ_L4 ")"); + __builtin_amdgcn_s_barrier(); + mma_AtB<0, 0, 6>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // ds_read_b128 a[120:123] + // ds_read_b128 a[124:127] + mma_AtB<0, 0, 7>(dQ_i_T, K_j_col, dP_ij_bf16_col_T, dQ_i_T); + // ds_read_b128 a[0:3] + // ds_read_b128 a[4:7] + // ds_read_b128 a[8:11] + // ds_read_b128 a[12:15] + // ds_read_b128 a[16:19] + // ds_read_b128 a[20:23] + // ds_read_b128 a[24:27] + // ds_read_b128 a[28:31] + asm volatile("s_waitcnt lgkmcnt(" DQ_L2 ")"); + } + } + + store<1>(g.dVg, dV_j, {batch_idx * (GROUP_SIZE / HEADS_PER_WG) + q_head_in_group / HEADS_PER_WG, 0, kv_head_idx, 0}, {0, j, 0, 0}); + __builtin_amdgcn_s_waitcnt(0); + __builtin_amdgcn_s_barrier(); + + // We first copy dV_j_T from accumulator GPRs to vector GPRs and then perform the store + accvgpr_read(dV_j_T, dK_j_T); + mul(dV_j_T, dV_j_T, dP_SCALE_FACTOR); + store<1>(g.dKg, dV_j, {batch_idx * (GROUP_SIZE / HEADS_PER_WG) + q_head_in_group / HEADS_PER_WG, 0, kv_head_idx, 0}, {0, j, 0, 0}); + + // Write out final dQ_i slice + mul(dQ_i_T, dQ_i_T, dP_SCALE_FACTOR); + atomic_pk_add_bf16_with_warpid<2>(g.dQg, dQ_i, {batch_idx, q_head_idx, q_seq_idx * 4 + 3, 0}, warpid); +} + +template __global__ void attend_bwd_combined_ker(bf16*, bf16*, bf16*, bf16*, bf16*, bf16*, bf16*, float*, float*); diff --git a/tinygrad_repo/pyproject.toml b/tinygrad_repo/pyproject.toml index 328f2087db..4fea0b1399 100644 --- a/tinygrad_repo/pyproject.toml +++ b/tinygrad_repo/pyproject.toml @@ -30,7 +30,6 @@ packages = [ 'tinygrad.nn', 'tinygrad.renderer', 'tinygrad.renderer.amd', - 'tinygrad.renderer.isa', 'tinygrad.runtime', 'tinygrad.runtime.autogen', 'tinygrad.runtime.autogen.am', @@ -59,10 +58,10 @@ tinygrad = ["py.typed"] # triton = ["triton-nightly>=2.1.0.dev20231014192330"] linting = [ "pylint", - "mypy==1.19.1", + "mypy==2.3.1", "typing-extensions", "pre-commit", - "ruff==0.14.10", + "ruff==0.16.4", "numpy", "typeguard", ] @@ -76,7 +75,7 @@ testing_minimal = [ "pytest-xdist", "pytest-split", "hypothesis>=6.148.9", - "z3-solver<4.15.4", # 4.15.4 has a segfault when creating many z3.Context() + "z3-solver==5.1.0.0", ] testing_unit = ["tinygrad[testing_minimal]", "tqdm", "safetensors", "tabulate", "openai", "gguf>=0.18", "capstone"] testing = [ diff --git a/tinygrad_repo/spec/tinyspec.pdf b/tinygrad_repo/spec/tinyspec.pdf index 3e062de527..5f379c9f85 100644 Binary files a/tinygrad_repo/spec/tinyspec.pdf and b/tinygrad_repo/spec/tinyspec.pdf differ diff --git a/tinygrad_repo/spec/tinyspec.tex b/tinygrad_repo/spec/tinyspec.tex index 024e3011fb..467d7e89ab 100644 --- a/tinygrad_repo/spec/tinyspec.tex +++ b/tinygrad_repo/spec/tinyspec.tex @@ -59,7 +59,7 @@ All nodes in the tinygrad graph are \textbf{UOps}. A UOp is a tuple $(\mathrm{op \end{tabular} \smallskip -\texttt{ParamArg} contains slot, dtype, value bounds, alignment, name, addrspace, device, volatility, optional image shape, and an optional bound device buffer (absent for \op{Alloc}s). \textbf{addrspace} is \texttt{GLOBAL}, \texttt{LOCAL}, or \texttt{REG}. The flat size $n$ is stated as a concrete weakint \op{Const} in src (an empty \op{Stack} for a scalar), followed by a \texttt{DEVICE} \op{Range} for multi-device storage. +\texttt{ParamArg} contains slot, dtype, value bounds, name, addrspace, device, volatility, optional image shape, and an optional bound device buffer (absent for \op{Alloc}s). \textbf{addrspace} is \texttt{GLOBAL}, \texttt{LOCAL}, or \texttt{REG}. The flat size $n$ is stated as a concrete weakint \op{Const} in src (an empty \op{Stack} for a scalar), followed by a \texttt{DEVICE} \op{Range} for multi-device storage. %% ============================================================ \subsection*{{\color{callblue}Call Ops} \normalfont\small--- function abstraction, like the lambda calculus} @@ -158,22 +158,24 @@ Assign is \op{Store} followed by \op{After}: write the value, then return the bu \textbf{Arity} & \textbf{src} & \textbf{Op} & \textbf{Semantics} \\ \midrule Unary & $(T,)$ - & \op{Recip} + & \op{Cast} + & Convert to target dtype (specified in arg). \\ +& & \op{Recip} & $1/x$ \\ & & \op{Trunc} & $\mathrm{trunc}(x)$: round toward zero. \\ -& & \op{Cast} - & Convert to target dtype (specified in arg). \\ \\[4pt] Binary & $(A, B)$ & \op{Add}, \op{Mul}, \op{Max} & $a+b$, $a \cdot b$, $\max(a,b)$ \\ -& & \op{Floordiv}, \op{Floormod} +& & \op{Div}, \op{Mod} & $\lfloor a/b \rfloor$, $a-b\lfloor a/b \rfloor$ \\ -& & \op{CmpLt}, \op{CmpNe} - & $[a < b]$, $[a \ne b]$ \\ -& & \op{Xor}, \op{Or}, \op{And}, \op{Shr}, \op{Shl} - & $a \oplus b$, $a \mid b$, $a \mathbin{\&} b$, $a \gg b$, $a \ll b$ \\[4pt] +& & \op{CmpLt}, \op{CmpEq} + & $[a < b]$, $[a = b]$ \\ +& & \op{Xor}, \op{Or}, \op{And} + & $a \oplus b$, $a \mid b$, $a \mathbin{\&} b$ \\ +& & \op{Shl}, \op{Shr} + & $a \ll b$, $a \gg b$ \\[4pt] Ternary & $(P, A, B)$ & \op{Where} & $A[\mathbf{i}]$ if $P[\mathbf{i}] \ne 0$, else $B[\mathbf{i}]$ \\ @@ -182,6 +184,10 @@ Ternary & $(P, A, B)$ \end{tabular} \medskip +\op{CmpEq} is primitive elementwise equality, returning bool (as do \op{CmpLt} and \op{CmpNe}). For floating-point inputs, equality is false if either operand is NaN, and $+0 = -0$. \op{CmpNe} is its logical complement, including for NaNs. + +\medskip +\noindent\begin{minipage}{\linewidth} \textbf{Decomposed elementwise ops} --- defined in terms of the primitives above. \smallskip @@ -191,14 +197,14 @@ Ternary & $(P, A, B)$ \midrule \op{Neg} & \op{Mul}($A$, $-1$) & $-x$ \\ \op{Sub} & \op{Add}($A$, \op{Neg}($B$)) & $a - b$ \\ -\op{Div} & \op{Mul}($A$, \op{Recip}($B$)) & $a / b$ \\ -\op{Cdiv} & $q+c$ & Integer division toward zero. \\ -\op{Cmod} & $r-bc$ & Remainder with dividend's sign (or zero). \\ +\op{Fdiv} & \op{Mul}($A$, \op{Recip}($B$)) & Floating-point division $a / b$. \\ +\op{Truncdiv} & $\operatorname{sgn}(a)\operatorname{sgn}(b)\lfloor |a|/|b| \rfloor$ & Integer division toward zero. \\ +\op{Truncmod} & $a - b \cdot \op{Truncdiv}(a,b)$ & Remainder with dividend's sign (or zero). \\ \op{CmpGt} & \op{CmpLt}($B$, $A$) & $[a > b]$ \\ -\op{CmpGe} & \op{CmpNe}(\op{CmpLt}($A$, $B$),\, $1$) & $[a \ge b]$ \\ -\op{CmpLe} & \op{CmpNe}(\op{CmpLt}($B$, $A$),\, $1$) & $[a \le b]$ \\ -\op{CmpEq} & \op{CmpNe}(\op{CmpNe}($A$, $B$),\, $1$) & $[a = b]$ \\ -\op{Not} & \op{CmpNe}($A$, $1$) & $\lnot a$ \\[4pt] +\op{CmpGe} & \op{CmpEq}(\op{CmpLt}($A$, $B$),\, $0$) & $[a \ge b]$ \\ +\op{CmpLe} & \op{CmpEq}(\op{CmpLt}($B$, $A$),\, $0$) & $[a \le b]$ \\ +\op{CmpNe} & \op{CmpEq}(\op{CmpEq}($A$, $B$),\, $0$) & $[a \ne b]$ \\ +\op{Not} & \op{CmpEq}($A$, $0$) & $\lnot a$ \\[4pt] \op{Exp2} & polynomial approx + \op{Mul}, \op{Add} & $2^x$ \\ \op{Log2} & exponent extract + polynomial approx & $\log_2 x$ \\ \op{Sin} & argument reduction + polynomial approx & $\sin x$ \\ @@ -208,6 +214,7 @@ Ternary & $(P, A, B)$ \op{Threefry} & 5 rounds of add-rotate-xor (ARX) & Threefry 2x32 PRNG \\ \bottomrule \end{tabular} +\end{minipage} %% ============================================================ \subsection*{{\color{markerorange}Marker Ops} \normalfont\small--- identity on data} @@ -271,7 +278,7 @@ ALU unary & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0].\mathrm{shape}$ \op{Mul} & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\min,\max]$ of products \\ \op{Max} & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\max(a,b),\, \max(A,B)]$ \\ Other binary & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\ -\op{CmpLt}, \op{CmpNe} & bool & broadcast & $\mathrm{src}[0].\mathrm{device}$ & from intervals \\ +\shortstack[l]{\op{CmpLt}, \op{CmpEq},\\\op{CmpNe}} & bool & broadcast & $\mathrm{src}[0].\mathrm{device}$ & from intervals \\ \op{Where} & $\mathrm{src}[1].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\min(b,c),\, \max(B,C)]$ \\[3pt] \op{Call} & void & --- & first non-null src device & --- \\ \op{Range} & index & $()$ & \textsc{null} & $[0,\, n{-}1]$ \\ diff --git a/tinygrad_repo/test/amd/hw/helpers.py b/tinygrad_repo/test/amd/hw/helpers.py index 5dab3e9a2f..3477494544 100644 --- a/tinygrad_repo/test/amd/hw/helpers.py +++ b/tinygrad_repo/test/amd/hw/helpers.py @@ -210,64 +210,22 @@ def run_program_emu(instructions: list, n_lanes: int = 1) -> WaveState: def run_program_hw(instructions: list, n_lanes: int = 1) -> WaveState: """Run instructions on real AMD hardware via HIPCompiler and the AMD runtime.""" - from tinygrad.device import Device, TinyELF, Buffer - from tinygrad.runtime.support.compiler_amd import HIPCompiler - from tinygrad.helpers import Target + from tinygrad.device import Device, Buffer + from tinygrad.dtype import dtypes, AddrSpace + from tinygrad.uop.ops import UOp, Ops, KernelInfo + from tinygrad.engine.realize import run_linear dev = Device["AMD"] - compiler = HIPCompiler(dev.arch) # type: ignore[attr-defined] - prologue, epilogue = get_prologue_epilogue(n_lanes) - code = assemble(prologue + instructions + epilogue) - - byte_str = ', '.join(f'0x{b:02x}' for b in code) - asm_src = f""".text -.globl test -.p2align 8 -.type test,@function -test: -.byte {byte_str} - -.rodata -.p2align 6 -.amdhsa_kernel test - .amdhsa_next_free_vgpr 256 - .amdhsa_next_free_sgpr 96 - .amdhsa_wavefront_size32 1 - .amdhsa_user_sgpr_kernarg_segment_ptr 1 - .amdhsa_kernarg_size 8 - .amdhsa_group_segment_fixed_size 65536 - .amdhsa_private_segment_fixed_size 65536 - .amdhsa_enable_private_segment 1 -.end_amdhsa_kernel - -.amdgpu_metadata ---- -amdhsa.version: - - 1 - - 0 -amdhsa.kernels: - - .name: test - .symbol: test.kd - .kernarg_segment_size: 8 - .group_segment_fixed_size: 65536 - .private_segment_fixed_size: 65536 - .kernarg_segment_align: 8 - .wavefront_size: 32 - .sgpr_count: 96 - .vgpr_count: 256 - .max_flat_workgroup_size: 1024 -... -.end_amdgpu_metadata -""" - - lib = compiler.compile(asm_src) - prg = dev.runtime(TinyELF(lib, "test", Target("AMD", arch=dev.arch), ())) - buf_sz = _out_bytes(n_lanes) out_gpu = Buffer(dev.device, buf_sz, preallocate=True) assert out_gpu._buf % 16 == 0, f"buffer not 16-byte aligned: 0x{out_gpu._buf:x}" - prg(out_gpu._buf, global_size=(1, 1, 1), local_size=(n_lanes, 1, 1), wait=True) + lds = UOp.placeholder((65536,), dtypes.uint8, 0, AddrSpace.LOCAL) + scratch = UOp.alloc((512,), dtypes.uint8, name="scratch", addrspace=AddrSpace.GLOBAL) + sink = UOp.sink(UOp.param(0, dtypes.uint32, (buf_sz // 4,)), UOp.special(n_lanes, "lidx0"), lds, scratch, arg=KernelInfo("test")) + body = UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(x, dtypes.void)) for x in prologue + instructions + epilogue)) + prg = UOp(Ops.PROGRAM, src=(sink, body)) + run_linear(UOp(Ops.LINEAR, src=(prg.call(UOp.from_buffer(out_gpu, dtypes.uint32)),)), wait=True) out_buf = out_gpu.as_memoryview() diff --git a/tinygrad_repo/test/amd/hw/test_cdna_ds.py b/tinygrad_repo/test/amd/hw/test_cdna_ds.py new file mode 100644 index 0000000000..d6555f946a --- /dev/null +++ b/tinygrad_repo/test/amd/hw/test_cdna_ds.py @@ -0,0 +1,18 @@ +import unittest +import tinygrad.runtime.autogen.amd.cdna.ins as cdna +from test.amd.hw.test_cdna_vop3 import run_cdna_lanes + +class TestCDNADSSwizzle(unittest.TestCase): + def test_modes(self): + # the source lane of lanes 0-7 for each mode of the ISA pseudocode + for offset, src in {0x041f: [1, 0, 3, 2, 5, 4, 7, 6], # bitmask: xor 1 + 0x00a0: [5]*8, # bitmask: or 5, broadcast + 0x801b: [3, 2, 1, 0, 7, 6, 5, 4], # quad permute: reversed + 0xc020: [1, 2, 3, 4, 5, 6, 7, 8], # rotate by 1 + 0xe000: [0, 16, 8, 24, 4, 20, 12, 28]}.items(): # fft: bit reversed + with self.subTest(offset=hex(offset)): + out = run_cdna_lanes([cdna.ds_swizzle_b32(vdst=cdna.v[2], addr=cdna.v[255], offset0=offset & 255, offset1=offset >> 8), cdna.s_waitcnt(0)]) + self.assertEqual(out[:8], src) + +if __name__ == "__main__": + unittest.main() diff --git a/tinygrad_repo/test/amd/hw/test_cdna_sdwa.py b/tinygrad_repo/test/amd/hw/test_cdna_sdwa.py index 61b95a5ba3..7901ba8a0f 100644 --- a/tinygrad_repo/test/amd/hw/test_cdna_sdwa.py +++ b/tinygrad_repo/test/amd/hw/test_cdna_sdwa.py @@ -10,6 +10,7 @@ class TestCDNASDWA(unittest.TestCase): cdna.v_mov_b32_e32(cdna.v[0], cdna.s[0]), cdna.v_mov_b32_e32(cdna.v[13], 1), cdna.v_add_co_u32_e32(cdna.v[0], cdna.SDWA, cdna.v[13], vsrc0=cdna.v[0], dst_sel=6, src0_sel=6), + cdna.s_nop(1), cdna.v_mov_b32_e32(cdna.v[2], cdna.VCC_LO), cdna.v_lshlrev_b32_e32(cdna.v[2], 31, cdna.v[2]), cdna.v_or_b32_e32(cdna.v[2], cdna.v[2], cdna.v[0]), diff --git a/tinygrad_repo/test/amd/hw/test_cdna_vop3.py b/tinygrad_repo/test/amd/hw/test_cdna_vop3.py index 2ae131a51c..f96d16d81a 100644 --- a/tinygrad_repo/test/amd/hw/test_cdna_vop3.py +++ b/tinygrad_repo/test/amd/hw/test_cdna_vop3.py @@ -5,98 +5,59 @@ gfx950 hardware when USE_HW=1. """ import ctypes, struct, unittest import tinygrad.runtime.autogen.amd.cdna.ins as cdna -from tinygrad.helpers import Target from tinygrad.renderer.amd.dsl import NULL from test.amd.hw.helpers import USE_HW, assemble from test.mockgpu.amd.emu import run_asm -LANES = 1 - -def _code(instructions: list, out_reg: int = 2, out_addr: int | None = None) -> bytes: - load_out_addr = [ - cdna.s_mov_b32(cdna.s[92], out_addr & 0xffffffff), - cdna.s_mov_b32(cdna.s[93], out_addr >> 32), - ] if out_addr is not None else [ - cdna.s_load_dwordx2(cdna.s[92:93], cdna.s[80:81], 0, soffset=NULL), - cdna.s_waitcnt(0), - ] - return assemble([ +def _code(instructions: list, out_reg: int = 2) -> list: + return [ cdna.s_mov_b32(cdna.s[80], cdna.s[0]), cdna.s_mov_b32(cdna.s[81], cdna.s[1]), cdna.v_mov_b32_e32(cdna.v[255], cdna.v[0]), *instructions, - *load_out_addr, + cdna.s_load_dwordx2(cdna.s[92:93], cdna.s[80:81], 0, soffset=NULL, imm=1), + cdna.s_waitcnt(0), cdna.v_lshlrev_b32_e32(cdna.v[240], 2, cdna.v[255]), cdna.global_store_dword(addr=cdna.v[240], data=cdna.v[out_reg], saddr=cdna.s[92:93], offset=0), cdna.s_endpgm(), - ]) + ] -def _run_emu(instructions: list, out_reg: int = 2) -> int: - out_buf = (ctypes.c_uint32 * LANES)(*([0] * LANES)) +def _run_emu(instructions: list, out_reg: int = 2, lanes: int = 1) -> list[int]: + out_buf = (ctypes.c_uint32 * lanes)(*([0] * lanes)) args = (ctypes.c_uint64 * 1)(ctypes.addressof(out_buf)) - code = _code(instructions, out_reg) + code = assemble(_code(instructions, out_reg)) kernel_buf = (ctypes.c_char * len(code)).from_buffer_copy(code) - result = run_asm(ctypes.addressof(kernel_buf), len(code), 1, 1, 1, LANES, 1, 1, ctypes.addressof(args), + result = run_asm(ctypes.addressof(kernel_buf), len(code), 1, 1, 1, lanes, 1, 1, ctypes.addressof(args), 0x19c | (128 << 15), 0x10000, arch="cdna") assert result == 0, f"run_asm failed with {result}" - return out_buf[0] + return list(out_buf) -def _run_hw(instructions: list, out_reg: int = 2) -> int: - from tinygrad.device import Device, TinyELF, Buffer - from tinygrad.runtime.support.compiler_amd import HIPCompiler +def _run_hw(instructions: list, out_reg: int = 2, lanes: int = 1) -> list[int]: + from tinygrad.device import Device, Buffer + from tinygrad.dtype import dtypes, AddrSpace + from tinygrad.uop.ops import UOp, Ops, KernelInfo + from tinygrad.engine.realize import run_linear dev = Device["AMD"] if dev.arch != "gfx950": raise unittest.SkipTest("requires gfx950 hardware") - out_gpu = Buffer(dev.device, LANES * 4, preallocate=True) - code = _code(instructions, out_reg, out_gpu._buf) - byte_str = ", ".join(f"0x{b:02x}" for b in code) - asm_src = f""".text -.globl test -.p2align 8 -.type test,@function -test: -.byte {byte_str} - -.rodata -.p2align 6 -.amdhsa_kernel test - .amdhsa_next_free_vgpr 256 - .amdhsa_next_free_sgpr 96 - .amdhsa_accum_offset 256 - .amdhsa_kernarg_size 0 -.end_amdhsa_kernel - -.amdgpu_metadata ---- -amdhsa.version: - - 1 - - 0 -amdhsa.kernels: - - .name: test - .symbol: test.kd - .kernarg_segment_size: 0 - .group_segment_fixed_size: 0 - .private_segment_fixed_size: 0 - .kernarg_segment_align: 8 - .wavefront_size: 64 - .sgpr_count: 96 - .vgpr_count: 256 - .max_flat_workgroup_size: 1024 -... -.end_amdgpu_metadata -""" - prg = dev.runtime(TinyELF(HIPCompiler(dev.arch).compile(asm_src), "test", Target("AMD", arch=dev.arch), ())) - prg(global_size=(1, 1, 1), local_size=(LANES, 1, 1), wait=True) + out_gpu = Buffer(dev.device, lanes * 4, preallocate=True) + lds = UOp.placeholder((65536,), dtypes.uint8, 0, AddrSpace.LOCAL) + sink = UOp.sink(UOp.param(0, dtypes.uint32, (lanes,)), UOp.special(lanes, "lidx0"), lds, arg=KernelInfo("test")) + body = UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(x, dtypes.void)) for x in _code(instructions, out_reg))) + prg = UOp(Ops.PROGRAM, src=(sink, body)) + run_linear(UOp(Ops.LINEAR, src=(prg.call(UOp.from_buffer(out_gpu, dtypes.uint32)),)), wait=True) out = out_gpu.as_memoryview() - return struct.unpack(" int: - emu = _run_emu(instructions, out_reg) +def run_cdna_lanes(instructions: list, out_reg: int = 2, lanes: int = 64) -> list[int]: + emu = _run_emu(instructions, out_reg, lanes) if not USE_HW: return emu - hw = _run_hw(instructions, out_reg) - if emu != hw: raise AssertionError(f"Emulator vs Hardware mismatch: emu=0x{emu:08x} hw=0x{hw:08x}") + hw = _run_hw(instructions, out_reg, lanes) + if emu != hw: raise AssertionError(f"Emulator vs Hardware mismatch: emu={[hex(x) for x in emu]} hw={[hex(x) for x in hw]}") return hw +def run_cdna(instructions: list, out_reg: int = 2) -> int: return run_cdna_lanes(instructions, out_reg, 1)[0] + class TestCDNAVOP3(unittest.TestCase): def test_cvt_pk_fp8_f32_preserves_upper_half(self): """V_CVT_PK_FP8_F32 with OPSEL[3]=0 writes only D[15:0].""" @@ -126,3 +87,27 @@ class TestCDNAVOP3(unittest.TestCase): cdna.v_cvt_pk_bf8_f32(cdna.v[2], cdna.v[0], cdna.v[1]), ]) self.assertEqual(out, 0xdead3c00 | expected) + +# CDNA has no RDNA true16: v128+ in a 16-bit e32 operand is that register, not the hi half of v[n-128] +class TestCDNATrue16(unittest.TestCase): + def test_v_add_f16_e32(self): + out = run_cdna([ + cdna.v_mov_b32_e32(cdna.v[130], 0x4000), # 2.0 + cdna.v_mov_b32_e32(cdna.v[131], 0x4200), # 3.0 + cdna.v_add_f16_e32(cdna.v[132], cdna.v[130], cdna.v[131]), + cdna.v_mov_b32_e32(cdna.v[2], cdna.v[132]), + ]) + self.assertEqual(out & 0xffff, 0x4500) # 5.0 + + def test_v_cmp_lt_f16_e32(self): + out = run_cdna([ + cdna.v_mov_b32_e32(cdna.v[130], 0x4000), # 2.0 + cdna.v_mov_b32_e32(cdna.v[131], 0x4200), # 3.0 + cdna.v_cmp_lt_f16_e32(cdna.v[130], cdna.v[131]), + cdna.s_nop(1), + cdna.v_mov_b32_e32(cdna.v[2], cdna.VCC_LO), + ]) + self.assertEqual(out & 1, 1) + +if __name__ == "__main__": + unittest.main() diff --git a/tinygrad_repo/test/amd/hw/test_dpp.py b/tinygrad_repo/test/amd/hw/test_dpp.py index eb921e3dec..6b855230d9 100644 --- a/tinygrad_repo/test/amd/hw/test_dpp.py +++ b/tinygrad_repo/test/amd/hw/test_dpp.py @@ -6,14 +6,13 @@ when USE_HW=1. """ import ctypes, unittest from tinygrad.runtime.autogen.amd.rdna3.ins import * -from tinygrad.helpers import Target from test.amd.hw.helpers import USE_HW, assemble from test.mockgpu.amd.emu import run_asm WAVE64 = 64 -def _wave64_code(instructions: list, out_reg: int = 1) -> bytes: - return assemble([ +def _wave64_code(instructions: list, out_reg: int = 1) -> list: + return [ s_mov_b32(s[80], s[0]), s_mov_b32(s[81], s[1]), v_mov_b32_e32(v[255], v[0]), @@ -23,12 +22,12 @@ def _wave64_code(instructions: list, out_reg: int = 1) -> bytes: v_lshlrev_b32_e32(v[240], 2, v[255]), global_store_b32(addr=v[240], data=v[out_reg], saddr=s[92:93], offset=0), s_endpgm(), - ]) + ] def _run_wave64_emu(instructions: list, out_reg: int = 1) -> list[int]: out_buf = (ctypes.c_uint32 * WAVE64)(*([0] * WAVE64)) args = (ctypes.c_uint64 * 1)(ctypes.addressof(out_buf)) - code = _wave64_code(instructions, out_reg) + code = assemble(_wave64_code(instructions, out_reg)) kernel_buf = (ctypes.c_char * len(code)).from_buffer_copy(code) rsrc2 = 0x19c | (128 << 15) scratch_size = 0x10000 @@ -37,55 +36,19 @@ def _run_wave64_emu(instructions: list, out_reg: int = 1) -> list[int]: return list(out_buf) def _run_wave64_hw(instructions: list, out_reg: int = 1) -> list[int]: - from tinygrad.device import Device, TinyELF, Buffer - from tinygrad.runtime.support.compiler_amd import HIPCompiler + from tinygrad.device import Device, Buffer + from tinygrad.dtype import dtypes, AddrSpace + from tinygrad.uop.ops import UOp, Ops, KernelInfo, AxisType + from tinygrad.engine.realize import run_linear dev = Device["AMD"] - compiler = HIPCompiler(dev.arch) # type: ignore[attr-defined] - code = _wave64_code(instructions, out_reg) - byte_str = ', '.join(f'0x{b:02x}' for b in code) - asm_src = f""".text -.globl test -.p2align 8 -.type test,@function -test: -.byte {byte_str} - -.rodata -.p2align 6 -.amdhsa_kernel test - .amdhsa_next_free_vgpr 256 - .amdhsa_next_free_sgpr 96 - .amdhsa_user_sgpr_kernarg_segment_ptr 1 - .amdhsa_kernarg_size 8 - .amdhsa_group_segment_fixed_size 65536 - .amdhsa_private_segment_fixed_size 65536 - .amdhsa_enable_private_segment 1 -.end_amdhsa_kernel - -.amdgpu_metadata ---- -amdhsa.version: - - 1 - - 0 -amdhsa.kernels: - - .name: test - .symbol: test.kd - .kernarg_segment_size: 8 - .group_segment_fixed_size: 65536 - .private_segment_fixed_size: 65536 - .kernarg_segment_align: 8 - .wavefront_size: 64 - .sgpr_count: 96 - .vgpr_count: 256 - .max_flat_workgroup_size: 1024 -... -.end_amdgpu_metadata -""" - lib = compiler.compile(asm_src) - prg = dev.runtime(TinyELF(lib, "test", Target("AMD", arch=dev.arch), ())) out_gpu = Buffer(dev.device, WAVE64 * 4, preallocate=True) - prg(out_gpu._buf, global_size=(1, 1, 1), local_size=(WAVE64, 1, 1), wait=True) + lds = UOp.placeholder((65536,), dtypes.uint8, 0, AddrSpace.LOCAL) + sink = UOp.sink(UOp.param(0, dtypes.uint32, (WAVE64,)), UOp.special(WAVE64, "lidx0"), + lds.end(UOp.range(WAVE64, 0, AxisType.WARP)), arg=KernelInfo("test")) + body = UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(x, dtypes.void)) for x in _wave64_code(instructions, out_reg))) + prg = UOp(Ops.PROGRAM, src=(sink, body)) + run_linear(UOp(Ops.LINEAR, src=(prg.call(UOp.from_buffer(out_gpu, dtypes.uint32)),)), wait=True) out = out_gpu.as_memoryview() return [int.from_bytes(out[i*4:(i+1)*4], 'little') for i in range(WAVE64)] diff --git a/tinygrad_repo/test/amd/hw/test_ds.py b/tinygrad_repo/test/amd/hw/test_ds.py index 1b322e0710..6ec5cea5f5 100644 --- a/tinygrad_repo/test/amd/hw/test_ds.py +++ b/tinygrad_repo/test/amd/hw/test_ds.py @@ -607,6 +607,16 @@ class TestDSRegisterWidth(unittest.TestCase): self.assertEqual(st.vgpr[0][1], 0xDEADBEEF) self.assertEqual(st.vgpr[0][2], 0x11111111, "v2 should be untouched") + def test_ds_load_inactive_lane(self): + st = run_program([ + v_mov_b32_e32(v[0], 0x40000000), # set lds base addr to 1GB + v_mov_b32_e32(v[1], 99), + s_mov_b32(EXEC_LO, 0), # all lanes are exec 0 + ds_load_b64(addr=v[0], vdst=v[1:2]), + s_waitcnt_lgkmcnt(sdst=NULL, simm16=0), + s_mov_b32(EXEC_LO, 1), + ]) + self.assertEqual(st.vgpr[0][1], 99) class TestDS2AddrStride64(unittest.TestCase): """Tests for DS_*_2ADDR_STRIDE64 (offset * 256 for B32, offset * 512 for B64).""" @@ -812,7 +822,7 @@ class TestDsPermute(unittest.TestCase): def test_ds_bpermute_b32_identity(self): """DS_BPERMUTE_B32 with identity permutation (each lane reads from itself).""" instructions = [ - v_mov_b32_e32(v[0], 0), # addr = 0 (read from lane 0) + v_mov_b32_e32(v[0], 128), # addr = 128 / 4 % 32 = 0 (read from lane 0) v_mov_b32_e32(v[1], 0xCAFEBABE), # data in lane 0 ds_bpermute_b32(v[2], v[0], v[1]), s_waitcnt_lgkmcnt(sdst=NULL, simm16=0), diff --git a/tinygrad_repo/test/amd/hw/test_sop.py b/tinygrad_repo/test/amd/hw/test_sop.py index f6a5006f25..f1bda8059a 100644 --- a/tinygrad_repo/test/amd/hw/test_sop.py +++ b/tinygrad_repo/test/amd/hw/test_sop.py @@ -234,6 +234,7 @@ class TestBranch(unittest.TestCase): st = run_program(instructions, n_lanes=1) self.assertEqual(st.vgpr[0][0], 42, "Branch should NOT be taken (VCC_LO is 1)") + @unittest.skip("TODO: fix forward branch lifting") def test_cbranch_vccnz_branches_on_vcc_lo(self): """S_CBRANCH_VCCNZ branches when VCC_LO is non-zero.""" instructions = [ diff --git a/tinygrad_repo/test/amd/hw/test_vop1.py b/tinygrad_repo/test/amd/hw/test_vop1.py index ced25512e6..f1767b9509 100644 --- a/tinygrad_repo/test/amd/hw/test_vop1.py +++ b/tinygrad_repo/test/amd/hw/test_vop1.py @@ -15,6 +15,7 @@ class TestMov(unittest.TestCase): instructions = [ s_mov_b32(s[0], 42), v_mov_b32_e32(v[0], s[0]), + v_nop_e32() ] st = run_program(instructions, n_lanes=1) self.assertEqual(st.vgpr[0][0], 42) diff --git a/tinygrad_repo/test/amd/hw/test_vop3.py b/tinygrad_repo/test/amd/hw/test_vop3.py index 52ed3c73e5..8c2f172bf2 100644 --- a/tinygrad_repo/test/amd/hw/test_vop3.py +++ b/tinygrad_repo/test/amd/hw/test_vop3.py @@ -1184,7 +1184,7 @@ class TestF64Ops(unittest.TestCase): # run_program with USE_HW=1 will verify exact bit match with hardware st = run_program(instructions, n_lanes=1) result_bits = st.vgpr[0][6] | (st.vgpr[0][7] << 32) - self.assertNotEqual(result_bits, 0, "Result should not be zero") + self.assertAlmostEqual(i642f(result_bits), a*b+c, delta=math.ulp(a*b)) class TestMad64More(unittest.TestCase): diff --git a/tinygrad_repo/test/amd/test_asm_kernel.py b/tinygrad_repo/test/amd/test_asm_kernel.py index 5d3a29bb2d..500887ffad 100644 --- a/tinygrad_repo/test/amd/test_asm_kernel.py +++ b/tinygrad_repo/test/amd/test_asm_kernel.py @@ -14,6 +14,7 @@ import tinygrad.runtime.autogen.amd.rdna3.ins as r3 import tinygrad.runtime.autogen.amd.rdna4.ins as r4 from tinygrad.renderer.amd.dsl import s, v, NULL, Reg, Inst from test.amd.helpers import TARGET_TO_ARCH +from test.mockgpu.amd.emu import RELIFT from extra.gemm.amd_asm_matmul import Kernel # small pattern matcher converting CALL to INS @@ -204,14 +205,10 @@ def custom_data_deps(A:UOp) -> UOp: sink = UOp.sink(A.base, threads, arg=KernelInfo("custom_data_deps")) return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) -# import contextvar to use it -from test.mockgpu.amd.emu import ASM_CALL # noqa: F401 - @unittest.skipUnless(Device.DEFAULT == "AMD", "requires AMD device") class TestAsmKernel(unittest.TestCase): def setUp(self): self.arch = TARGET_TO_ARCH[Device["AMD"].arch] - self.enterContext(Context(ASM_CALL=1)) def test_simple(self): if self.arch != "rdna3": self.skipTest("only rdna3") @@ -256,8 +253,9 @@ class TestAsmKernel(unittest.TestCase): a.realize() self.assertTrue((a.numpy() == 6.0).all()) - @unittest.expectedFailure + @Context(RELIFT=1) def test_cfg_branch_diamond(self): + if self.arch != "rdna3": self.skipTest("only rdna3") def cfg_kernel(out:UOp): k = Kernel() k.emit(s_load_b64(s[0:1], s[0:1], soffset=NULL)) @@ -282,6 +280,7 @@ class TestAsmKernel(unittest.TestCase): self.assertListEqual(out.tolist(), [2.0, 1.0, 3.0]) def test_cfg_loop(self): + if self.arch != "rdna3": self.skipTest("only rdna3") def cfg_kernel(out:UOp): k = Kernel() k.emit(s_load_b64(s[0:1], s[0:1], soffset=NULL)) @@ -317,7 +316,7 @@ class TestAsmKernel(unittest.TestCase): def test_readlane(self): if self.arch != "rdna3": self.skipTest("only rdna3") - def readlane_kernel(out:UOp): + def readlane_kernel(out:UOp, threads:int): insts = [ s_load_b64(s[0:1], s[0:1], soffset=NULL), s_waitcnt_lgkmcnt(sdst=NULL, simm16=0), @@ -328,10 +327,11 @@ class TestAsmKernel(unittest.TestCase): global_store_b32(addr=v[0], data=v[1], saddr=s[0:1]), s_endpgm(), ] - sink = UOp.sink(out.base, UOp.special(64, "lidx0"), arg=KernelInfo("wave_sgprs")) + sink = UOp.sink(out.base, UOp.special(threads, "lidx0"), arg=KernelInfo(f"wave_sgprs_{threads}")) return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts)))) - out = Tensor(np.full(64, 0xFFFFFFFF, dtype=np.uint32)).realize().custom_kernel(fxn=readlane_kernel)[0] - self.assertListEqual(out.tolist(), [0]*32 + [32]*32) + for t in [31, 55, 64]: + out = Tensor(np.full(t, 0xFFFFFFFF, dtype=np.uint32)).realize().custom_kernel(fxn=functools.partial(readlane_kernel, threads=t))[0] + self.assertListEqual(out.tolist(), [i//32*32 for i in range(t)]) def test_plus_tensor(self): out = Tensor.arange(1, 4).clone() + Tensor.arange(4, 7).clone() @@ -344,5 +344,21 @@ class TestAsmKernel(unittest.TestCase): out = a@b self.assertEqual(out.tolist(), (a.numpy()@b.numpy()).tolist()) + def test_relift_simple(self): + if self.arch != "rdna3": self.skipTest("only rdna3") + def k(out:UOp): + insts = [ + s_load_b64(s[0:1], s[0:1], soffset=NULL), s_waitcnt_lgkmcnt(sdst=NULL, simm16=0), + s_mov_b32(s[2], 2), s_add_u32(s[2], s[2], 2), + v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], s[2]), + global_store_b32(addr=v[0], data=v[1], saddr=s[0:1]), s_endpgm(), + ] + sink = UOp.sink(out.base, arg=KernelInfo("k")) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts)))) + out = Tensor.empty(1, dtype=dtypes.uint32).custom_kernel(fxn=k)[0] + with Context(RELIFT=max(RELIFT.value, 1)): + out.realize() + self.assertListEqual(out.tolist(), [4]) + if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/amd/test_pdf.py b/tinygrad_repo/test/amd/test_pdf.py index 180f7a0927..71d4672c80 100644 --- a/tinygrad_repo/test/amd/test_pdf.py +++ b/tinygrad_repo/test/amd/test_pdf.py @@ -44,9 +44,9 @@ class TestPcodePDF(unittest.TestCase): "if SCC then\nD0.i32 = 32'I(signext(SIMM16.i16))\nendif") def test_swizzle_spans_blocks_and_pages(self): - for arch in ('rdna3', 'rdna4'): + for arch, opcode in (('rdna3', 53), ('rdna4', 53), ('cdna', 61)): with self.subTest(arch=arch): - code = self.pcode[arch][('DS_SWIZZLE_B32', 53)] + code = self.pcode[arch][('DS_SWIZZLE_B32', opcode)] self.assertIn('} elsif (offset >= 0xc000) {', code) self.assertIn('thread_out[i+3]', code) self.assertIn('xor_mask = offset[14:10];', code) diff --git a/tinygrad_repo/test/amd/test_sqtt_encoder.py b/tinygrad_repo/test/amd/test_sqtt_encoder.py deleted file mode 100644 index ce6805c95c..0000000000 --- a/tinygrad_repo/test/amd/test_sqtt_encoder.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for SQTT encoder: verifies the emulator produces correct SQTT traces for known kernels. - -Run with: DEV=MOCK+AMD python -m pytest test/amd/test_sqtt_encoder.py -v -""" -import ctypes, unittest -from tinygrad.helpers import Context -from tinygrad.renderer.amd.sqtt import decode, LAYOUT_HEADER, WAVESTART, WAVEEND, INST, IMMEDIATE, VALUINST, InstOp -from tinygrad.runtime.autogen.amd.rdna3.ins import * - -def _run_kernel(instructions: list, lx=1, ly=1, lz=1, gx=1, gy=1, gz=1, args_ptr=0) -> bytes: - """Assemble instructions, run on emulator with PROFILE=1, return the SQTT blob.""" - from test.mockgpu.amd.emu import run_asm, sqtt_traces - code = b''.join(inst.to_bytes() for inst in instructions) - buf = (ctypes.c_char * len(code))(*code) - lib = ctypes.addressof(buf) - sqtt_traces.clear() - with Context(PROFILE=1): - run_asm(lib, len(code), gx, gy, gz, lx, ly, lz, args_ptr) - assert len(sqtt_traces) == 1, f"expected 1 trace, got {len(sqtt_traces)}" - return sqtt_traces.pop() - -class TestSQTTEncoder(unittest.TestCase): - - def test_simple_salu(self): - """A simple s_mov + s_endpgm kernel emits SALU INST packet.""" - blob = _run_kernel([s_mov_b32(s[0], 42), s_endpgm()]) - packets = list(decode(blob)) - inst_pkts = [p for p in packets if isinstance(p, INST)] - self.assertEqual(len(inst_pkts), 1) - self.assertEqual(inst_pkts[0].op, InstOp.SALU) - - def test_valu_emits_valuinst(self): - """Regular VALU ops emit VALUINST packets.""" - blob = _run_kernel([v_mov_b32_e32(v[0], 0), v_add_f32_e32(v[1], v[0], v[0]), s_endpgm()]) - packets = list(decode(blob)) - valu_pkts = [p for p in packets if isinstance(p, VALUINST)] - self.assertEqual(len(valu_pkts), 2) - # no INST packets for regular VALU - self.assertEqual(len([p for p in packets if isinstance(p, INST)]), 0) - - def test_waitcnt_emits_immediate(self): - """s_waitcnt and s_nop emit IMMEDIATE packets.""" - blob = _run_kernel([s_nop(simm16=0), s_waitcnt(simm16=0), s_endpgm()]) - imm_pkts = [p for p in decode(blob) if isinstance(p, IMMEDIATE)] - self.assertEqual(len(imm_pkts), 2) # s_nop + s_waitcnt - - def test_endpgm_skipped(self): - """s_endpgm does not emit any packet.""" - blob = _run_kernel([s_endpgm()]) - packets = list(decode(blob)) - self.assertEqual(len([p for p in packets if isinstance(p, INST)]), 0) - self.assertEqual(len([p for p in packets if isinstance(p, IMMEDIATE)]), 0) - - def test_wave_lifecycle(self): - """Every WAVESTART has a matching WAVEEND.""" - blob = _run_kernel([s_mov_b32(s[0], 0), s_endpgm()]) - packets = list(decode(blob)) - self.assertEqual(sum(1 for p in packets if isinstance(p, WAVESTART)), sum(1 for p in packets if isinstance(p, WAVEEND))) - - def test_layout_header(self): - """First packet is LAYOUT_HEADER with layout=3.""" - blob = _run_kernel([s_endpgm()]) - packets = list(decode(blob)) - self.assertIsInstance(packets[0], LAYOUT_HEADER) - self.assertEqual(packets[0].layout, 3) - - def test_blob_32byte_aligned(self): - """SQTT blob is 32-byte aligned.""" - blob = _run_kernel([s_mov_b32(s[0], 0), s_mov_b32(s[1], 1), s_endpgm()]) - self.assertEqual(len(blob) % 32, 0) - - def test_multiple_waves(self): - """Multiple wavefronts each get their own WAVESTART/WAVEEND.""" - blob = _run_kernel([s_mov_b32(s[0], 0), s_endpgm()], lx=64) # 64 threads = 2 waves (WAVE_SIZE=32) - packets = list(decode(blob)) - self.assertEqual(sum(1 for p in packets if isinstance(p, WAVESTART)), 2) - self.assertEqual(sum(1 for p in packets if isinstance(p, WAVEEND)), 2) - - def test_branch_taken_and_not_taken(self): - """A loop with s_cbranch_scc1 emits JUMP when taken, JUMP_NO on final iteration.""" - # s[0] = 2; loop: s[0] -= 1; cmp s[0] != 0 (SCC=1 if true); cbranch_scc1 loop; endpgm - # iteration 1: s[0]=2→1, SCC=1 (1!=0), branch taken (JUMP) - # iteration 2: s[0]=1→0, SCC=0 (0==0), branch not taken (JUMP_NO) - blob = _run_kernel([s_mov_b32(s[0], 2), s_sub_u32(s[0], s[0], 1), s_cmp_lg_u32(s[0], 0), s_cbranch_scc1(simm16=-3), s_endpgm()]) - inst_pkts = [p for p in decode(blob) if isinstance(p, INST)] - ops = [p.op for p in inst_pkts] - self.assertIn(InstOp.JUMP, ops) - self.assertIn(InstOp.JUMP_NO, ops) - - def test_timestamps_monotonic(self): - """Timestamps are monotonically non-decreasing.""" - blob = _run_kernel([s_mov_b32(s[0], 0), s_mov_b32(s[1], 1), s_mov_b32(s[2], 2), s_endpgm()]) - times = [p._time for p in decode(blob)] - self.assertEqual(times, sorted(times)) - - def test_no_trace_without_profile(self): - """No SQTT trace is emitted when PROFILE=0.""" - from test.mockgpu.amd.emu import run_asm, sqtt_traces - code = s_endpgm().to_bytes() - buf = (ctypes.c_char * len(code))(*code) - sqtt_traces.clear() - with Context(PROFILE=0): - run_asm(ctypes.addressof(buf), len(code), 1, 1, 1, 1, 1, 1, 0) - self.assertEqual(len(sqtt_traces), 0) - -if __name__ == "__main__": - unittest.main() diff --git a/tinygrad_repo/test/device/amd/test_llm.py b/tinygrad_repo/test/device/amd/test_llm.py index 312a4a4681..5480c50bef 100644 --- a/tinygrad_repo/test/device/amd/test_llm.py +++ b/tinygrad_repo/test/device/amd/test_llm.py @@ -2,15 +2,16 @@ import unittest from unittest.mock import patch import numpy as np from tinygrad import Tensor, UOp, dtypes, nn, function, Device, TinyJit +from tinygrad.helpers import DEV from tinygrad.llm.kernels.amd import (Linear, amd_custom_kernels_supported, q8_quantize, flash_attention, gated_delta_prefill, - QUANT_SIZES, HALFWORD_QUANTS, _wmma_rdna4) + QUANT_SIZES, HALFWORD_QUANTS, SMALL_BLOCK_QUANTS, _is_rdna4, _is_cdna, Q4_K, IQ3_S, ExpertWeights) from tinygrad.llm.gguf import ggml_data_to_tensor from test.runtime.test_llm_quantized import QuantLinearMixin class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): def test_wmma_architecture_selection(self): if not amd_custom_kernels_supported(device:=Tensor.empty(1).device): self.skipTest("RDNA3/4 required") - self.assertEqual(_wmma_rdna4(device), Device[device].target[0] == 12) + self.assertEqual(_is_rdna4(device), Device[device].target[0] == 12) def test_values_and_scales(self): if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") @@ -58,6 +59,7 @@ class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): def test_q4_k_linear(self): self._test_quant_linear(12, 144) def test_iq4_linear(self): self._test_quant_linear(23, 136) def test_q5_linear(self): self._test_quant_linear(13, 176) + def test_q8_0_linear_small_blocks(self): self._test_quant_linear(8, 272, in_features=288, out_features=16, token_counts=(1, 16)) def test_q6_linear_odd_blocks(self): self._test_quant_linear(14, 210, in_features=768, out_features=3, token_counts=(1, 3, 16)) def test_q2_k_linear(self): self._test_quant_linear(10, 84, in_features=768, token_counts=(1, 3, 16)) def test_q3_k_linear(self): self._test_quant_linear(11, 110, in_features=768, token_counts=(1, 3, 16)) @@ -86,7 +88,7 @@ class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): for typ, size in QUANT_SIZES.items(): with self.subTest(ggml_type=typ): packed = rng.integers(0, 256, (48*3, size), dtype=np.uint8) - blocks = packed.reshape(-1, 18) if typ == 20 else packed + blocks = packed.reshape(-1, SMALL_BLOCK_QUANTS[typ]*2) if typ in SMALL_BLOCK_QUANTS else packed offset = size-2 if typ in (11, 14) else 80 if typ == 10 else 0 blocks[:, offset:offset+2] = np.resize(scales, len(blocks)).view(np.uint8).reshape(-1, 2) if typ in (10, 12, 13): @@ -220,6 +222,30 @@ class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): np.testing.assert_array_equal(out.numpy(), 32) np.testing.assert_array_equal(state.numpy(), 1) + def _test_moe_decode(self, typ:int): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3/4 required") + rng = np.random.default_rng(5) + E, T, K, IN, OUT, size = 4, 1, 2, 256, 16, QUANT_SIZES[typ] + blocks = rng.integers(0, 256, (E*OUT*IN//256, size), dtype=np.uint8) + blocks[:, :2] = np.array([0.001], np.float16).view(np.uint8) + blocks[:, 2:4] = np.array([0.0001], np.float16).view(np.uint8) + decoded = ggml_data_to_tensor(Tensor(blocks.flatten()).contiguous().realize(), E*OUT*IN, typ).reshape(E, OUT, IN) + w = decoded.numpy() + act = rng.standard_normal((1, T, K, IN)).astype(np.float32) + sel = np.stack([rng.choice(E, K, replace=False) for _ in range(T)])[None].astype(np.int32) + ew = ExpertWeights(E, IN, OUT) + ew.weight = decoded.half() + ew.set_quantized() + self.assertEqual(ew.ggml_type, typ) + got = ew(Tensor(sel), Tensor(act)).numpy()[0] + for t in range(T): + for j in range(K): + ref = w[sel[0, t, j]] @ act[0, t, j] + np.testing.assert_allclose(got[t, j], ref, rtol=2e-2, atol=2e-2*np.abs(ref).max()) + + def test_moe_decode(self): self._test_moe_decode(Q4_K) + def test_moe_decode_iq3_s(self): self._test_moe_decode(IQ3_S) + def test_gated_delta_random_scan(self): if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3/4 required") rng = np.random.default_rng(42) @@ -342,7 +368,7 @@ class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): np.testing.assert_allclose(run(q, kv, end).numpy(), expected, rtol=1e-3, atol=1e-3) def test_prefill_attention_random(self): - if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3/4 required") + if not amd_custom_kernels_supported(d:=Tensor.empty(1).device) or _is_cdna(d): self.skipTest("RDNA3/4 required") rng = np.random.default_rng(42) cases = ((2, 4, 2, 32, 64, 45, 64, False), (1, 3, 1, 64, 128, 83, 128, False), (1, 2, 1, 96, 256, 149, 192, False), (1, 4, 2, 17, 128, 128, 128, True), (1, 2, 1, 3, 64, 33, 64, True), @@ -410,7 +436,8 @@ class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): np.testing.assert_allclose(flash_attention(q, cache, valid_kv_len).numpy(), 5500, rtol=2e-3, atol=2e-3) def test_prefill_attention_unaligned_start(self): - if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + if not amd_custom_kernels_supported(d:=Tensor.empty(1).device): self.skipTest("RDNA3/4 or CDNA3/4 required") + if _is_cdna(d) and DEV.interface.startswith("MOCK"): self.skipTest("too slow on the CDNA emulator") rng = np.random.default_rng(42) start_pos = 1718 q = Tensor.zeros(1, 8, 32, 128) diff --git a/tinygrad_repo/test/device/cl/test_ocl.py b/tinygrad_repo/test/device/cl/test_ocl.py index 8f17648808..08d9013e46 100644 --- a/tinygrad_repo/test/device/cl/test_ocl.py +++ b/tinygrad_repo/test/device/cl/test_ocl.py @@ -1,19 +1,7 @@ import unittest -from unittest.mock import patch from tinygrad import Device -from tinygrad.device import Buffer, TinyELF -from tinygrad.helpers import Target -from tinygrad.runtime.ops_cl import CLDevice, CLAllocator, CLCompiler - -@unittest.skipUnless(Device.DEFAULT == "CL", "Runs only on OpenCL") -class TestCLCompileCache(unittest.TestCase): - def test_compile_cached(self): - device = Device[Device.DEFAULT] - src = "__kernel void cached_test(__global int* a) { a[0] = 1; }" - obj = TinyELF(src.encode(), "cached_test", Target("CL"), ()) - device.runtime(obj) - with patch.object(CLCompiler, 'compile', side_effect=RuntimeError("compile should not be called on cache hit")): - device.runtime(obj) +from tinygrad.device import Buffer +from tinygrad.runtime.ops_cl import CLDevice, CLAllocator @unittest.skipUnless(Device.DEFAULT == "CL", "Runs only on OpenCL") class TestCLError(unittest.TestCase): @@ -28,7 +16,7 @@ class TestCLError(unittest.TestCase): def test_invalid_kernel_name(self): device = Device[Device.DEFAULT] with self.assertRaises(RuntimeError) as err: - device.runtime(TinyELF(b"__kernel void test(__global int* a) { a[0] = 1; }", "", Target("CL"), ())) + device.function(b"__kernel void test(__global int* a) { a[0] = 1; }", "") assert str(err.exception) == "OpenCL Error -46: CL_INVALID_KERNEL_NAME" def test_unaligned_copy(self): diff --git a/tinygrad_repo/test/device/cpu/test_cpu.py b/tinygrad_repo/test/device/cpu/test_cpu.py index 043bfcd5bb..2884cda421 100644 --- a/tinygrad_repo/test/device/cpu/test_cpu.py +++ b/tinygrad_repo/test/device/cpu/test_cpu.py @@ -3,7 +3,7 @@ from contextlib import redirect_stdout from tinygrad import Tensor, Device from tinygrad.helpers import Target from tinygrad.renderer.nir import LVPRenderer -from tinygrad.renderer.isa.x86 import X86Renderer +from tinygrad.renderer.x86 import X86Renderer from tinygrad.codegen import to_program @unittest.skipIf(Device.DEFAULT != "CPU", "only run on CPU") diff --git a/tinygrad_repo/test/device/cpu/test_custom_kernel.py b/tinygrad_repo/test/device/cpu/test_custom_kernel.py index addde430b1..e2a9d64722 100644 --- a/tinygrad_repo/test/device/cpu/test_custom_kernel.py +++ b/tinygrad_repo/test/device/cpu/test_custom_kernel.py @@ -1,14 +1,13 @@ import unittest from tinygrad import Tensor, UOp, Context, GlobalCounters from tinygrad.uop.ops import Ops, KernelInfo -from tinygrad.renderer.cstyle import ClangRenderer from test.helpers import assert_kernel_count class TestCustomKernel(unittest.TestCase): @Context(DEV="CPU") def test_simple_from_source(self): a = Tensor.arange(4).clone().realize() - src = ClangRenderer.kernel_typedef + " test_src(int* restrict a) { a[0] = 1; }" + src = "struct args_t { int* a; }; void test_src(const struct args_t* args) { args->a[0] = 1; }" def custom_src_kernel(A:UOp, B:UOp) -> UOp: sink = UOp.sink(A, arg=KernelInfo(name="test_src")) return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(sink.toposort())), UOp(Ops.SOURCE, arg=src),)) @@ -18,7 +17,7 @@ class TestCustomKernel(unittest.TestCase): @Context(DEV="CPU") def test_simple_from_source_alt(self): a = Tensor.arange(4).clone().realize() - src = ClangRenderer.kernel_typedef + " copy(int* restrict out, int* restrict in) { for (int i = 0; i < 4; i++) out[i] = in[i]; }" + src = "struct args_t { int* out; int* in; }; void copy(const struct args_t* args) { for (int i = 0; i < 4; i++) args->out[i] = args->in[i]; }" def custom_src_kernel(out:UOp, inp:UOp) -> UOp: sink = UOp.sink(out, inp, arg=KernelInfo(name="copy")) return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(sink.toposort())), UOp(Ops.SOURCE, arg=src),)) diff --git a/tinygrad_repo/test/device/dsp/test_quantize_onnx.py b/tinygrad_repo/test/device/dsp/test_quantize_onnx.py index 23969674e8..057ecad32e 100644 --- a/tinygrad_repo/test/device/dsp/test_quantize_onnx.py +++ b/tinygrad_repo/test/device/dsp/test_quantize_onnx.py @@ -115,9 +115,11 @@ class TestQuantizeOnnx(unittest.TestCase): unsigned_char128 hi128; }; }; - __attribute__((noinline)) void fxn(unsigned char* restrict __attribute__((align_value(128))) data0, - unsigned char* restrict __attribute__((align_value(128))) data1, - signed char* restrict __attribute__((align_value(128))) data2) { + struct fxn_args { unsigned char* data0; unsigned char* data1; signed char* data2; }; + __attribute__((noinline)) void fxn(const struct fxn_args* args) { + unsigned char* restrict __attribute__((align_value(128))) data0 = args->data0; + unsigned char* restrict __attribute__((align_value(128))) data1 = args->data1; + signed char* restrict __attribute__((align_value(128))) data2 = args->data2; for (int ridx0 = 0; ridx0 < 512; ridx0++) { int alu0 = (ridx0<<9); for (int ridx1 = 0; ridx1 < 4; ridx1++) { diff --git a/tinygrad_repo/test/device/metal/test_metal.py b/tinygrad_repo/test/device/metal/test_metal.py index acf5295707..927879dddf 100644 --- a/tinygrad_repo/test/device/metal/test_metal.py +++ b/tinygrad_repo/test/device/metal/test_metal.py @@ -1,11 +1,11 @@ import unittest from tinygrad.device import CompileError, Device, BufferSpec if Device.DEFAULT=="METAL": - from tinygrad.runtime.ops_metal import MetalDevice, MetalCompiler + from tinygrad.runtime.ops_metal import MetalCompiler @unittest.skipIf(Device.DEFAULT!="METAL", "Metal support required") class TestMetal(unittest.TestCase): def test_alloc_oom(self): - device = MetalDevice("metal") + device = Device["METAL"] with self.assertRaises(MemoryError): device.allocator.alloc(10000000000000000000) @@ -38,7 +38,7 @@ class TestMetal(unittest.TestCase): assert ret is not None def test_failed_newLibraryWithData(self): - device = MetalDevice("metal") + device = Device["METAL"] compiler = MetalCompiler() compiled = compiler.compile(""" #include diff --git a/tinygrad_repo/test/device/qcom/test_qcom.py b/tinygrad_repo/test/device/qcom/test_qcom.py index 7d9fb58dda..5fdaba3af3 100644 --- a/tinygrad_repo/test/device/qcom/test_qcom.py +++ b/tinygrad_repo/test/device/qcom/test_qcom.py @@ -40,7 +40,7 @@ class TestQCOM(unittest.TestCase): def test_dcache_flush(self): from tinygrad.runtime.ops_qcom import dcache_flush buf = (ctypes.c_uint8 * 64)() - dcache_flush().fxn(buf, 0) + dcache_flush()(ctypes.addressof(buf), 0) if __name__ == '__main__': unittest.main() diff --git a/tinygrad_repo/test/device/webgpu/test_renderer_failures.py b/tinygrad_repo/test/device/webgpu/test_renderer_failures.py index 2a7df5c489..142f0c6942 100644 --- a/tinygrad_repo/test/device/webgpu/test_renderer_failures.py +++ b/tinygrad_repo/test/device/webgpu/test_renderer_failures.py @@ -13,6 +13,10 @@ class TestWGSLFailures(unittest.TestCase): ret = _setup_and_test_alu(Ops.MUL, 5.0, UOp.const(float("inf")).cast(dtypes.float32)) self.assertEqual(ret[0], float("inf")) + def test_uint_sub_large_const(self): + x = Tensor([0xffffffff], dtype=dtypes.uint32).contiguous().realize() + self.assertEqual((x - 0x80000001).tolist(), [0x7ffffffe]) + # WGSL has a specific select(alt, val, gate) ternary operator instead of gate?val:alt def test_gated_load(self): a = UOp.param(0, dtypes.int, 4) diff --git a/tinygrad_repo/test/external/external_benchmark_hip_compile.py b/tinygrad_repo/test/external/external_benchmark_hip_compile.py deleted file mode 100644 index d97047b923..0000000000 --- a/tinygrad_repo/test/external/external_benchmark_hip_compile.py +++ /dev/null @@ -1,31 +0,0 @@ -import random, os -from tinygrad.helpers import Timing -from tinygrad.runtime.ops_hip import compile_hip, HIPDevice -from tinygrad.runtime.ops_cl import compile_cl, CLDevice - -# OMP_NUM_THREADS=1 strace -tt -f -e trace=file python3 test/external/external_benchmark_hip_compile.py -# AMD_COMGR_REDIRECT_LOGS=stdout AMD_COMGR_EMIT_VERBOSE_LOGS=1 python3 test/external/external_benchmark_hip_compile.py - -# issue is in https://github.com/ROCm-Developer-Tools/clr/ - -if __name__ == "__main__": - HIPDevice() - CLDevice() - - # warmup - name = "none"+str(random.randint(0, 1000000)) - compile_cl.__wrapped__(f"void {name}() {{}}") - print("compile cl warmed up") - compile_hip.__wrapped__(f"void {name}() {{}}") - print("compile hip warmed up") - - print("**** benchmark ****") - name = "none"+str(random.randint(0, 1000000)) - # this uses AMD_COMGR_ACTION_COMPILE_SOURCE_TO_BC, then it links the lib on the next step - with Timing("compile cl: "): compile_cl.__wrapped__(f"void {name}() {{}}") - # this uses AMD_COMGR_ACTION_COMPILE_SOURCE_WITH_DEVICE_LIBS_TO_BC, much slower - with Timing("compile hip: "): compile_hip.__wrapped__(f"void {name}() {{}}") - os._exit(0) - - - diff --git a/tinygrad_repo/test/external/external_benchmark_op_conv.py b/tinygrad_repo/test/external/external_benchmark_op_conv.py index d77a853c8b..6f5f09dc08 100644 --- a/tinygrad_repo/test/external/external_benchmark_op_conv.py +++ b/tinygrad_repo/test/external/external_benchmark_op_conv.py @@ -95,5 +95,5 @@ gs = sorted(dedup([u for u in ast.toposort() if u.op is Ops.PARAM]), key=lambda bufs = [Buffer(ps.arg.device, g.max_numel() * g.dtype.itemsize).ensure_allocated() for g in gs] gsize, lsize = ps.arg.launch_dims({}) -t = rt(*[b._buf for b in bufs], global_size=gsize, local_size=lsize, vals=ps.arg.vals({}), wait=True) +t = rt(*[b._buf for b in bufs], global_size=gsize, local_size=lsize, wait=True) print(f"{t*1e6:.2f} us") diff --git a/tinygrad_repo/test/external/external_cl_half_max.py b/tinygrad_repo/test/external/external_cl_half_max.py deleted file mode 100644 index 020d806c63..0000000000 --- a/tinygrad_repo/test/external/external_cl_half_max.py +++ /dev/null @@ -1,13 +0,0 @@ -from tinygrad.runtime.ops_cl import CLDevice, CLProgram, compile_cl - -if __name__ == "__main__": - dev = CLDevice() - lib = compile_cl(""" -#pragma OPENCL EXTENSION cl_khr_fp16 : enable -__kernel void test(__global half *out, __global half *a, __global half *b) { - int gid = get_global_id(0); - out[gid] = max(a[gid], b[gid]); -} -""") - prg = CLProgram(dev, "test", lib) - diff --git a/tinygrad_repo/test/external/external_gpu_fail_osx.py b/tinygrad_repo/test/external/external_gpu_fail_osx.py deleted file mode 100644 index 51a458b136..0000000000 --- a/tinygrad_repo/test/external/external_gpu_fail_osx.py +++ /dev/null @@ -1,16 +0,0 @@ -# ugh, OS X OpenCL doesn't support half -from tinygrad.runtime.ops_cl import CLDevice, CLProgram, CLCompiler - -src = """#pragma OPENCL EXTENSION cl_khr_fp16 : enable -__kernel void max_half(__global half* data0, const __global half* data1) { - int gidx0 = get_group_id(0); - data0[gidx0] = max(data1[gidx0], (half)0.0); -}""" - -if __name__ == "__main__": - dev = CLDevice() - print("created device") - lib = CLCompiler(dev, "test").compile(src) - print("created lib", len(lib)) - prg = CLProgram(dev, "max_half", lib) - print("created prg") \ No newline at end of file diff --git a/tinygrad_repo/test/external/external_osx_profiling.py b/tinygrad_repo/test/external/external_osx_profiling.py deleted file mode 100644 index 277ca304ad..0000000000 --- a/tinygrad_repo/test/external/external_osx_profiling.py +++ /dev/null @@ -1,41 +0,0 @@ -from tinygrad.runtime.ops_cl import CLProgram, CL, CLBuffer -from tinygrad import dtypes -import time - -N = 1000000 -a = CLBuffer(N, dtypes.float32) -b = CLBuffer(N, dtypes.float32) -c = CLBuffer(N, dtypes.float32) - -prg = CLProgram("test", """__kernel void test(__global float *a, __global float *b, __global float *c) { - int idx = get_global_id(0); - a[idx] = b[idx] + c[idx]; -}""") -prg.clprgs[0](CL.cl_queue[0], [N,], None, a._buf, b._buf, c._buf) -t1 = time.monotonic_ns() -e1 = prg.clprgs[0](CL.cl_queue[0], [N,], None, a._buf, b._buf, c._buf) -CL.synchronize() -t2 = time.monotonic_ns() -time.sleep(3) -t3 = time.monotonic_ns() -e2 = prg.clprgs[0](CL.cl_queue[0], [N,], None, a._buf, b._buf, c._buf) -CL.synchronize() -t4 = time.monotonic_ns() - -print(e1.profile.queued) -print(e1.profile.submit) -print(e1.profile.start) -print(e1.profile.end) - -print(e1, e2) -print(t2-t1, e1.profile.end - e1.profile.start) -print(t4-t3, e2.profile.end - e2.profile.start) -print(t3-t2, e2.profile.queued-e1.profile.end) -print((t3-t2) / (e2.profile.start-e1.profile.end), "ratio") - -print("ratio since boot", t1/e1.profile.start) - -print(e1.profile.start) -print(e1.profile.end) -print(e2.profile.start) -print(e2.profile.end) diff --git a/tinygrad_repo/test/external/external_test_speed_llama.py b/tinygrad_repo/test/external/external_test_speed_llama.py index 9ed56c7c19..dcb8f2fc25 100644 --- a/tinygrad_repo/test/external/external_test_speed_llama.py +++ b/tinygrad_repo/test/external/external_test_speed_llama.py @@ -10,7 +10,7 @@ from tinygrad.helpers import Profiling class FakeProgram: def __init__(self, name:str, lib:bytes, *args, **kwargs): pass - def __call__(self, *bufs, global_size, local_size, vals=(), wait=False, **kw): pass + def __call__(self, *args, global_size, local_size, wait=False, **kw): pass class FakeAllocator(Allocator[Compiled]): def _alloc(self, sz, options): return BufferStorage(None) diff --git a/tinygrad_repo/test/external/fuzz_symbolic_div.py b/tinygrad_repo/test/external/fuzz_symbolic_div.py index 87e4078d27..b5c6c58df6 100644 --- a/tinygrad_repo/test/external/fuzz_symbolic_div.py +++ b/tinygrad_repo/test/external/fuzz_symbolic_div.py @@ -16,7 +16,7 @@ def get_random_term(ranges, factors): def get_random_expr(ranges, factors): num_terms = random.randint(2,4) x = UOp.usum(*[get_random_term(ranges, factors) for _ in range(num_terms)]) - return x.alu(random.choice([Ops.CDIV, Ops.CMOD]), x.ufix(random.choice(factors)*random.choice([1, 1, 1, -1]))) + return x.alu(random.choice([Ops.TRUNCDIV, Ops.TRUNCMOD]), x.ufix(random.choice(factors)*random.choice([1, 1, 1, -1]))) if __name__ == "__main__": skipped = 0 diff --git a/tinygrad_repo/test/external/process_replay/process_replay.py b/tinygrad_repo/test/external/process_replay/process_replay.py index cb9dc1e8e5..36ba3560c6 100755 --- a/tinygrad_repo/test/external/process_replay/process_replay.py +++ b/tinygrad_repo/test/external/process_replay/process_replay.py @@ -38,6 +38,7 @@ SKIP_PROCESS_REPLAY = (k:="[skip_process_replay]") in os.getenv("COMMIT_MESSAGE" # uncomment this to disable by default #SKIP_PROCESS_REPLAY = not ASSERT_DIFF and not ((k:="[p]") in os.getenv("COMMIT_MESSAGE", "") or k in os.getenv("PR_TITLE", "")) if REF == "master": SKIP_PROCESS_REPLAY = True +if REF == "update_benchmark": SKIP_PROCESS_REPLAY = "[pr]" not in os.getenv("COMMIT_MESSAGE", "").lower() class ProcessReplayWarning(Warning): pass # *** replay the function and convert return values to string @@ -53,7 +54,11 @@ def replay_to_program(p:UOp, ast:UOp, renderer:Renderer) -> tuple[str, str, tupl def to_str(ret:UOp) -> str: src = next(x.arg for x in ret.src if x.op is Ops.SOURCE) # PYTHON renderer pickles UOps, first unpickle and decode here - if device.startswith("PYTHON"): return "\n".join([str(x) for x in pickle.loads(base64.b64decode(src))]) + # a nested PROGRAM prints its LINEAR, its SOURCE/BINARY are pickles of that LINEAR whose bytes depend on object identity + if device.startswith("PYTHON"): + uops = pickle.loads(base64.b64decode(src)) + pickles = {u:u.replace(arg=None) for u in UOp.sink(*uops).toposort(enter_calls=True) if u.op in (Ops.SOURCE, Ops.BINARY)} + return "\n".join([str(x.substitute(pickles, enter_calls=True)) for x in uops]) return src # properly color the name arg ast_repr = codecs.decode(str(input_ast), "unicode_escape") @@ -64,12 +69,12 @@ replayers["do_to_program"] = replay_to_program # *** run replayers on captured rows and print diffs -def diff(offset:int, fxns:dict[str, Callable[..., tuple|None]]) -> None: +def diff(offset:int, fxns:dict[str, Callable[..., tuple|None]], table_name=TABLE_NAME) -> None: if ASSERT_DIFF: warnings.filterwarnings("error", category=ProcessReplayWarning) if early_stop.is_set(): return None conn = db_connection() cur = conn.cursor() - cur.execute(f"SELECT val FROM '{TABLE_NAME}' LIMIT ? OFFSET ?", (PAGE_SIZE, offset)) + cur.execute(f"SELECT val FROM '{table_name}' LIMIT ? OFFSET ?", (PAGE_SIZE, offset)) changed = 0 for row in cur.fetchall(): if changed > MAX_DIFF_PCT: @@ -100,28 +105,32 @@ def diff(offset:int, fxns:dict[str, Callable[..., tuple|None]]) -> None: # *** generic runner to map rows of a table to a function in parallel -def _pmap(fxns:dict[str, Callable]) -> None: +def _pmap(fxns:dict[str, Callable], table_name:str=TABLE_NAME) -> None: conn = db_connection() cur = conn.cursor() - try: row_count = cur.execute(f"select count(*) from '{TABLE_NAME}'").fetchone()[0] + try: row_count = cur.execute(f"select count(*) from '{table_name}'").fetchone()[0] except sqlite3.OperationalError: - raise RuntimeError(f"{TABLE_NAME} isn't accessible in master, did DB_VERSION change?") + raise RuntimeError(f"{table_name} isn't accessible in master, did DB_VERSION change?") finally: cur.close() with ProcessPoolExecutor(multiprocessing.cpu_count(), mp_context=multiprocessing.get_context("spawn")) as pool: with tqdm(total=row_count) as bar: - for _ in pool.map(functools.partial(diff, fxns=fxns), range(0, row_count, s:=min(PAGE_SIZE, row_count))): bar.update(min(s, row_count-bar.n)) + for _ in pool.map(functools.partial(diff, fxns=fxns, table_name=table_name), range(0, row_count, s:=min(PAGE_SIZE, row_count))): + bar.update(min(s, row_count-bar.n)) # *** main loop -if __name__ == "__main__": +def main(fxns:dict[str, Callable], table_name:str=TABLE_NAME) -> None: if SKIP_PROCESS_REPLAY: logging.info("skipping process replay.") exit(0) logging.info(f"running process replay with {ASSERT_DIFF=}") - try: _pmap(replayers) + try: _pmap(fxns, table_name) except Exception as e: logging.info(f"process replay err: {e}") exit(int(ASSERT_DIFF)) + +if __name__ == "__main__": + main(replayers) diff --git a/tinygrad_repo/test/external/process_replay/reset.py b/tinygrad_repo/test/external/process_replay/reset.py index c40b6d0961..f20dfc267d 100755 --- a/tinygrad_repo/test/external/process_replay/reset.py +++ b/tinygrad_repo/test/external/process_replay/reset.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 from tinygrad.helpers import db_connection, VERSION cur = db_connection() -cur.execute(f"drop table if exists process_replay_{VERSION}") +for table in ("process_replay", "emu_process_replay"): + cur.execute(f"drop table if exists {table}_{VERSION}") diff --git a/tinygrad_repo/test/helpers.py b/tinygrad_repo/test/helpers.py index 6fddf79fa2..bae2578103 100644 --- a/tinygrad_repo/test/helpers.py +++ b/tinygrad_repo/test/helpers.py @@ -1,6 +1,6 @@ -import os, sys, time, struct, functools, unittest +import os, sys, gc, time, struct, functools, unittest, contextlib from dataclasses import replace -from typing import Any, Callable +from typing import Any, Callable, Iterator import numpy as np from tinygrad import Tensor, dtypes, Device from tinygrad.device import Buffer @@ -22,7 +22,7 @@ from tinygrad.runtime.ops_python import PythonRenderer def full_rewrite(sink:UOp, ren:Renderer|None=None) -> UOp: if ren is None: ren = Renderer(Target()) if sink.arg is None: sink = sink.replace(arg=KernelInfo()) - return full_rewrite_to_sink(sink, ren, optimize=sink.tag is None) + return full_rewrite_to_sink(sink, ren, optimize=sink.arg is None or not sink.arg.optimized) def get_uops(sink:UOp, ren:Renderer|None=None) -> list[UOp]: """Extract linearized UOps from a sink. Test helper that only does linearization (no render).""" @@ -66,6 +66,13 @@ def check_schedule(t:Tensor|list[Tensor]|UOp, allowed:int, to_prerealize:list[Te raise KernelCountException(allowed, kernel_cnt) return linear, var_vals +@contextlib.contextmanager +def measure_mem() -> Iterator[Callable[[], int]]: + Device.default.synchronize() + gc.collect() + base = GlobalCounters.mem_used + yield lambda: (gc.collect(), GlobalCounters.mem_used - base)[1] + def assert_kernel_count(expected:int): got = GlobalCounters.kernel_count if got != expected: raise KernelCountException(expected, got) @@ -82,7 +89,7 @@ def jit_cache_count(linear:UOp) -> int: return sum(len(call.without_after.arg.aux.kernels) if call_is_hcq(call) else 1 for call in linear.src) def assert_jit_cache_len(fxn, expected_len): - linear = fxn.captured.linear if fxn.captured is not None else None + linear = fxn.captured._linear if fxn.captured is not None else None if linear is None or not linear.src: if expected_len != 0: raise KernelCountException(expected_len, 0) return @@ -149,7 +156,7 @@ def eval_uop(uop:UOp, inputs:list[tuple[DType, list[Any]]]|None=None, vals:tuple prg = to_program(UOp.store(g.index(UOp.const(0)), uop).sink(arg=KernelInfo()), PythonRenderer(Target("PYTHON"))) prog = dev.runtime(prg.to_elf()) out_buf = Buffer("PYTHON", uop.dtype.itemsize, preallocate=True) - prog(out_buf._buf, *[b.buf for b in bufs], vals=vals) + prog(out_buf._buf, *[b.buf for b in bufs], *vals) return out_buf.as_memoryview().cast(uop.dtype.fmt or "").tolist()[0] def to_uops_list(u:list[UOp], ren=None) -> list[UOp]: diff --git a/tinygrad_repo/test/mockgpu/am/amdriver.py b/tinygrad_repo/test/mockgpu/am/amdriver.py index c6a8966d65..37d973cd09 100644 --- a/tinygrad_repo/test/mockgpu/am/amdriver.py +++ b/tinygrad_repo/test/mockgpu/am/amdriver.py @@ -97,6 +97,7 @@ class AMDriver(VirtDriver): def open(self, name, flags, mode, virtfile): return virtfile.fdcls(self._alloc_fd()) + def progress(self): self._emulate_execute() def _emulate_execute(self): if self._executing: return self._executing = True diff --git a/tinygrad_repo/test/mockgpu/am/amgpu.py b/tinygrad_repo/test/mockgpu/am/amgpu.py index 8d44237b65..4f8cf56cc1 100644 --- a/tinygrad_repo/test/mockgpu/am/amgpu.py +++ b/tinygrad_repo/test/mockgpu/am/amgpu.py @@ -1,6 +1,6 @@ # mypy: ignore-errors from __future__ import annotations -import ctypes, struct, functools, os, mmap +import ctypes, struct, functools, os, mmap, tempfile from tinygrad.runtime.autogen.am import am, smu_14_0_2 from tinygrad.runtime.autogen import libc from tinygrad.runtime.support.amd import AMDReg, import_asic_regs @@ -8,6 +8,13 @@ from test.mockgpu.amd.amdgpu import AMDGPU VRAM_SIZE = 512 << 20 +def memfd(name:str) -> int: + if hasattr(os, "memfd_create"): return os.memfd_create(name) + # macos has no memfd_create, use an unlinked temp file + fd, path = tempfile.mkstemp(prefix=name) + os.unlink(path) + return fd + IP_VERSIONS = { am.GC_HWIP: (12, 0, 0), am.SDMA0_HWIP: (7, 0, 0), am.MMHUB_HWIP: (4, 1, 0), am.NBIO_HWIP: (6, 3, 1), am.MP0_HWIP: (14, 0, 2), am.MP1_HWIP: (14, 0, 2), am.HDP_HWIP: (7, 0, 0), am.OSSSYS_HWIP: (7, 0, 0), @@ -263,11 +270,11 @@ class MockMMIOInterface: class MockAMGPU(AMDGPU): def __init__(self, gpuid:int=0): super().__init__(gpuid) - self.vram_fd = libc.memfd_create(b"vram", libc.MFD_CLOEXEC) + self.vram_fd = memfd("vram") os.ftruncate(self.vram_fd, VRAM_SIZE) self.vram_addr = libc.mmap(0, VRAM_SIZE, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED, self.vram_fd, 0) self.vram = (ctypes.c_ubyte * VRAM_SIZE).from_address(self.vram_addr) - self.doorbell_fd = libc.memfd_create(b"doorbell", libc.MFD_CLOEXEC) + self.doorbell_fd = memfd("doorbell") os.ftruncate(self.doorbell_fd, 0x2000) self.arch = "rdna4" self._sysmem_map:dict[int,int] = {} diff --git a/tinygrad_repo/test/mockgpu/amd/amddriver.py b/tinygrad_repo/test/mockgpu/amd/amddriver.py index edd1ef35f8..05dc5f69d7 100644 --- a/tinygrad_repo/test/mockgpu/amd/amddriver.py +++ b/tinygrad_repo/test/mockgpu/amd/amddriver.py @@ -171,6 +171,7 @@ class AMDDriver(VirtDriver): raise RuntimeError(f"unsupported kfd ioctl, {nr} {name}") return 0 + def progress(self): self._emulate_execute() def _emulate_execute(self): if self._executing: return # prevent re-entrancy self._executing = True diff --git a/tinygrad_repo/test/mockgpu/amd/amdgpu.py b/tinygrad_repo/test/mockgpu/amd/amdgpu.py index b7e84bff2c..93ec1273c6 100644 --- a/tinygrad_repo/test/mockgpu/amd/amdgpu.py +++ b/tinygrad_repo/test/mockgpu/amd/amdgpu.py @@ -1,7 +1,6 @@ import ctypes, time from dataclasses import replace from test.mockgpu.gpu import VirtGPU -from test.mockgpu.helpers import PythonRemu from tinygrad.helpers import getbits, to_mv, getenv, DEV from tinygrad.runtime.support import c @@ -41,8 +40,6 @@ WAIT_REG_MEM_FUNCTION_EQ = 3 # == WAIT_REG_MEM_FUNCTION_NEQ = 4 # != WAIT_REG_MEM_FUNCTION_GEQ = 5 # >= -remu = PythonRemu() - def create_sdma_packets(): # TODO: clean up this, if we want to keep it structs = {} @@ -77,6 +74,7 @@ class PM4Executor(AMDQueue): def __init__(self, gpu, base, size, rptr, wptr): self.gpu = gpu self.ib_executor: PM4Executor|None = None + self.kernel_name, self.kernel_text_size = "", 0 super().__init__(base, size, rptr, wptr) def _next_dword(self): @@ -109,6 +107,9 @@ class PM4Executor(AMDQueue): elif op == amd_gpu.PACKET3_DISPATCH_DIRECT: self._exec_dispatch_direct(n) elif op == amd_gpu.PACKET3_INDIRECT_BUFFER: self._exec_indirect_buffer(n) elif op == amd_gpu.PACKET3_EVENT_WRITE: self._exec_event_write(n) + elif op == amd_gpu.PACKET3_NOP: + data = b"".join(self._next_dword().to_bytes(4,"little") for _ in range(n+1)) + self.kernel_name, self.kernel_text_size = data[:-4].rstrip(b"\x00").decode(), int.from_bytes(data[-4:], "little") else: raise RuntimeError(f"PM4: Unknown opcode: {op}") return (self.rptr[0] - prev_rptr) + executed_in_ib @@ -200,8 +201,8 @@ class PM4Executor(AMDQueue): except KeyError: user_data.append(0) prg_sz = 0 - for st,sz in self.gpu.mapped_ranges: - if st <= prg_addr < st+sz: prg_sz = sz - (prg_addr - st) + for st,sz in sorted(self.gpu.mapped_ranges): + if st <= prg_addr+prg_sz < st+sz: prg_sz = st+sz-prg_addr # Get scratch size from COMPUTE_TMPRING_SIZE register # WAVESIZE = ceildiv(lanes * size_per_thread, mem_alignment_size) @@ -212,14 +213,10 @@ class PM4Executor(AMDQueue): wavesize = (tmpring_size >> 12) & 0x3FFF # WAVESIZE field is bits 12:25 scratch_size = wavesize * (16 if self.gpu.arch == "cdna" else 4) # per-thread scratch size in bytes - assert prg_sz > 0, "Invalid prg ptr (not found in mapped ranges)" - # Pass valid memory ranges, rsrc2, scratch_size, arch, and user data registers to the emulator - remu.valid_mem_ranges = self.gpu.mapped_ranges - remu.rsrc2 = rsrc2 - remu.scratch_size = scratch_size - remu.arch = self.gpu.arch - remu.user_data = user_data - err = remu.run_asm(prg_addr, prg_sz, gl[0], gl[1], gl[2], lc[0], lc[1], lc[2], args_addr) + assert 0 < self.kernel_text_size <= prg_sz, "Invalid prg ptr or text size" + from test.mockgpu.amd.emu import run_asm + err = run_asm(prg_addr, self.kernel_text_size, gl[0], gl[1], gl[2], lc[0], lc[1], lc[2], args_addr, + rsrc2, scratch_size, self.gpu.arch, user_data, self.kernel_name) if err != 0: raise RuntimeError("remu does not support the new instruction introduced in this kernel") def _exec_indirect_buffer(self, n): @@ -238,9 +235,6 @@ class PM4Executor(AMDQueue): event_dw = self._next_dword() match (event_dw & 0xFF): # event type case SQTT_EVENTS.THREAD_TRACE_FINISH: - # Get the most recent trace from the emulator (if available) - from test.mockgpu.amd.emu import sqtt_traces - blob = sqtt_traces.pop(0) if sqtt_traces else b'' old_idx = self.gpu.regs.grbm_index for se in range(self.gpu.regs.n_se): self.gpu.regs.grbm_index = 0b011 << 29 | se << 16 # select se, broadcast sa and instance @@ -249,13 +243,7 @@ class PM4Executor(AMDQueue): buf_addr = ((self.gpu.regs[regSQ_THREAD_TRACE_BUF0_SIZE]&0xf)<<32|self.gpu.regs[regSQ_THREAD_TRACE_BUF0_BASE])<<12 else: buf_addr = ((self.gpu.regs[regSQ_THREAD_TRACE_BUF0_BASE_HI])<<32|self.gpu.regs[regSQ_THREAD_TRACE_BUF0_BASE_LO])<<12 - # Use real trace blob for SE 0 (which has itrace enabled), empty blob for other SEs - se_blob = blob if se == 0 else b'' - - # Write blob to trace buffer - if se_blob: ctypes.memmove(buf_addr, se_blob, len(se_blob)) - # RDNA3 has absolute address for wptr, RDNA4 has relative - wptr_val = (((buf_addr if MOCKGPU_ARCH == "rdna3" else 0) + len(se_blob)) // 32) & 0x1FFFFFFF + wptr_val = ((buf_addr if MOCKGPU_ARCH == "rdna3" else 0) // 32) & 0x1FFFFFFF self.gpu.regs[regSQ_THREAD_TRACE_WPTR] = wptr_val self.gpu.regs.grbm_index = old_idx case _: pass # NOTE: for now most events aren't emulated diff --git a/tinygrad_repo/test/mockgpu/amd/call.py b/tinygrad_repo/test/mockgpu/amd/call.py deleted file mode 100644 index b31b746348..0000000000 --- a/tinygrad_repo/test/mockgpu/amd/call.py +++ /dev/null @@ -1,136 +0,0 @@ -import ctypes, itertools -from tinygrad.viz.serve import amd_decode, get_cfg, COND_TAKEN, COND_NOT_TAKEN -from tinygrad.uop.ops import UOp, sint, Ops, KernelInfo, PatternMatcher, UPat, graph_rewrite, rewrite_group, uopfunc -from tinygrad.codegen import to_program -from tinygrad.device import Device -from tinygrad.dtype import Invalid, AddrSpace, dtypes -from tinygrad.helpers import Context, getenv, TracingKey, dedup, unwrap -from tinygrad.runtime.autogen import hsa -from tinygrad.renderer.amd.dsl import EXEC_LO, ttmp -from test.mockgpu.amd.emu import _Ctx, _get_handler, _wave_size, _canonical_info, PC_LO_IDX, PC_HI_IDX, SGPR_COUNT, SCRATCH_STRIDE_IDX, F32_INLINE - -asm_call_counter = itertools.count(1) - -# this is meant to replace the old emulator -@uopfunc -def init_wave(wg:UOp, wave:UOp, sgpr:UOp, vgpr:UOp, lds:UOp, args_ptr:UOp, gx:int, gy:int, lx:int, ly:int, total_threads:int, wave_size:int, - lds_size:int, scratch_size:int, rsrc2:int, arch:str="rdna3", user_data_count:int=0, accvgpr:UOp|None=None): - # define ranges inside a wave - li = UOp.range((wave.eq(0)).where(max(lds_size//4, 1), 0), 2, dtype=dtypes.int) - si = UOp.range(SGPR_COUNT, 3, dtype=dtypes.int) - vi = UOp.range(256*wave_size, 4, dtype=dtypes.int) - # zero ALLOCs - zero_lds = lds.index(li).store(0).end(li) - zero_sgpr = sgpr.after(zero_lds).index(si).store(0).end(si) - zero_vgpr = vgpr.after(zero_sgpr).index(vi).store(0) - zero_agpr = unwrap(accvgpr).after(zero_sgpr).index(vi).store(0) if wave_size == 64 else UOp(Ops.NOOP) - clear_vgpr = UOp.group(zero_vgpr, zero_agpr).end(vi) - # set RANGE registers - gidx, gidy, gidz = wg%gx, (wg//gx)%gy, wg//(gx*gy) - n_lanes = (total_threads-wave*wave_size).minimum(wave_size) - initial:list[tuple[int, sint]] = [*((128+i, i) for i in range(65)), *((193+i, (-i-1)&0xFFFFFFFF) for i in range(16)), *F32_INLINE.items()] - if user_data_count: initial += [(i, UOp.variable(f"user_data_{i}", 0, dtypes.uint32.max, dtypes.uint32)) for i in range(user_data_count)] - else: initial += [(0, args_ptr.cast(dtypes.uint32)), (1, (args_ptr>>32).cast(dtypes.uint32))] - if arch == "rdna4": initial += [(ttmp[7].offset, (gidy&0xFFFF)|((gidz&0xFFFF)<<16)), (ttmp[9].offset, gidx)] - else: - sgpr_id = (rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_USER_SGPR_COUNT) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_USER_SGPR_COUNT_SHIFT - for enabled, gid in [(hsa.AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_X, gidx), - (hsa.AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_Y, gidy), - (hsa.AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_Z, gidz)]: - if rsrc2 & enabled: - initial.append((sgpr_id, gid)) - sgpr_id += 1 - initial += [(EXEC_LO.offset, ((UOp.const(1, dtypes.uint64)< UPat: - reg, null = UPat.const(idx).cast(), UPat.const(124).cast() - return UPat.any(reg, reg.ne(null).where(reg, UPat.const(Invalid))) - -def move_const_idxs(call:UOp) -> UOp|None: - idxs = dedup(u.src[1] for u in call.body.toposort() if u.op is Ops.INDEX and u.src[0].op is Ops.PARAM and u.src[0].arg.name == "vmem" - and u.src[1].op is Ops.CONST) - if not idxs: return None - rep = {idx:UOp.param(len(call.src)-1+i, idx.commit_dtype(), name=f"inst_{i}", addrspace=AddrSpace.ALU) for i,idx in enumerate(idxs)} - return call.replace(src=(call.body.substitute(rep, walk=True), *call.src[1:], *idxs)) - -pm_asm_call = PatternMatcher([ - # remove PC from CALL body - (UPat((Ops.LOAD, Ops.STORE), src=(UPat(Ops.PARAM, name="buf").index(UPat.any(pc_index(PC_LO_IDX), pc_index(PC_HI_IDX))),), allow_any_len=True), - lambda buf: UOp(Ops.NOOP) if buf.arg.name == "sgpr" else None), - # move CONST outside CALL body - (UPat(Ops.CALL, src=(UPat(Ops.SINK),), allow_any_len=True, name="call"), move_const_idxs), -]) - -@rewrite_group(name=lambda *args,ret,**_: TracingKey(f"Lift {(k:=ret.src[0].arg).name}", (("lift", k.function_name),))) -def lift(lib:int, lib_sz:int, gx:int, gy:int, gz:int, lx:int, ly:int, lz:int, rsrc2:int, scratch_size:int, arch:str="rdna3", - user_data_count:int=0, backend:str|None=None) -> UOp: - backend = getenv("ASM_CALL_BACKEND", "CPU") if backend is None else backend - # decode - lib_bytes = ctypes.string_at(lib, lib_sz) - insts = amd_decode(lib_bytes, arch) - cfg = get_cfg(insts)["data"] - # construct CALL graph - wave_size, total_threads = _wave_size(arch), lx*ly*lz - n_waves = (total_threads+wave_size-1)//wave_size - wg = UOp.range(gx*gy*gz, 0) - wave = UOp.range(n_waves, 1) - # alloc register and LDS buffers - sgpr = UOp.alloc((SGPR_COUNT*n_waves,), dtypes.uint32, 0, AddrSpace.REG) - vgpr = UOp.alloc((256*wave_size*n_waves,), dtypes.uint32, 1, AddrSpace.REG) - lds_size = ((rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE_SHIFT)*512 - lds = UOp.alloc((max(lds_size//4, 1),), dtypes.uint32, 3, AddrSpace.REG) - scratch = UOp.alloc((max(scratch_size*wave_size*n_waves, 1),), dtypes.uint8, 4, AddrSpace.REG) - accvgpr = UOp.alloc((256*wave_size*n_waves,), dtypes.uint32, 5, AddrSpace.REG) if wave_size == 64 else vgpr - args_ptr = UOp.variable("args_ptr", 0, dtypes.uint64.max, dtypes.uint64) - lib_addr = UOp.variable("lib", 0, dtypes.uint64.max, dtypes.uint64) - inst_addr = UOp.param(-1, dtypes.uint64, name="inst", addrspace=AddrSpace.ALU) - init = init_wave(wg, wave, sgpr.index(wave*SGPR_COUNT), vgpr.index(wave*256*wave_size), lds, args_ptr, gx, gy, lx, ly, total_threads, wave_size, - lds_size, scratch_size, rsrc2, arch, user_data_count, *([accvgpr.index(wave*256*wave_size)] if wave_size == 64 else [])) - user_vars = [u for u in init.body.toposort() if u.is_variable] - init = init.replace(src=(init.body.substitute({u:UOp.param(len(init.src)-1+i, u.dtype, name=u.arg.name, addrspace=AddrSpace.ALU) - for i,u in enumerate(user_vars)}), *init.src[1:], *user_vars)) - ctx = _Ctx(4, wave_size) - end = init.end(wave) - mem = {ctx.sgpr:(sgpr,SGPR_COUNT),ctx.vgpr:(vgpr,256*wave_size),ctx.vmem:(ctx.vmem,0),ctx.lds:(lds,0),ctx.scratch:(scratch,scratch_size*wave_size)} - if wave_size == 64: mem[ctx.accvgpr] = (accvgpr, 256*wave_size) - for block_idx, (block_pc, block) in enumerate(cfg["blocks"].items()): - wave = UOp.range(n_waves, lib_sz+block_idx) - afters = {b:(buf.shrink(((wave*stride, (wave+1)*stride),)) if stride else buf).after(end, wave) for b,(buf,stride) in mem.items()} - loop_path = cfg["paths"][block_pc].get(block_pc) - loop = UOp.loop(block_pc) if loop_path is not None else None - if loop is not None: afters = {b:arg.after(loop) for b,arg in afters.items()} - branch_cond:UOp|None = None - for off in block: - inst = insts[off] - inst_st = str(inst) - if inst_st.startswith("s_code_end"): continue - if inst_st.startswith(("s_getpc", "s_setpc")): raise AssertionError("getpc and setpc are not allowed in ASM_CALL") - ctx = _Ctx(inst.size(), _wave_size(arch), inst_addr=inst_addr) - sink = _get_handler(inst)(inst, ctx) - *_, canonical_name = _canonical_info(inst, ctx, lib_bytes[off:]) - bufs = sorted((u for u in sink.toposort() if u.op is Ops.PARAM), key=lambda u: u.arg.slot) - args = [lib_addr+off if b is inst_addr else afters.get(b, b.after(loop) if loop is not None else b) for b in bufs] - if ctx.branch_cond is not None and loop_path is not None: - branch_cond = ctx.branch_cond.substitute(dict(zip(bufs, args)), walk=True) - continue - body = sink.substitute({b:b.param_like(i, name=b.arg.name) for i,b in enumerate(bufs)}) - call = body.call(*args, name=canonical_name) - afters.update((b, arg.after(call)) for b, arg in zip(bufs, args) if b is not inst_addr) - if loop is not None: - assert branch_cond is not None and loop_path in (COND_TAKEN, COND_NOT_TAKEN) - if loop_path is COND_NOT_TAKEN: branch_cond = branch_cond.logical_not() - backedge = UOp.sink(*afters.values()).backedge(loop, branch_cond) - afters = {b:arg.after(backedge) for b,arg in afters.items()} - end = UOp.group(*afters.values()).end(wave) - sink = UOp.sink(end.end(wg), arg=KernelInfo(name=f"asm_call n{next(asm_call_counter)}", opts_to_apply=())) - sink = graph_rewrite(sink, pm_asm_call, name="pm_asm_call", bottom_up=True, enter_calls=True) - with Context(NOOPT=1, CHECK_OOB=0, TUPLE_ORDER=0, EMULATED_DTYPES="", CAPTURE_PROCESS_REPLAY=0): - return to_program(sink, Device[backend].renderer) diff --git a/tinygrad_repo/test/mockgpu/amd/emu.py b/tinygrad_repo/test/mockgpu/amd/emu.py index 08821d2cac..2010e01b0e 100644 --- a/tinygrad_repo/test/mockgpu/amd/emu.py +++ b/tinygrad_repo/test/mockgpu/amd/emu.py @@ -5,8 +5,7 @@ # arg=2: vmem - base address 0, INDEX offsets directly to host memory # arg=3: lds - local data share # arg=4: scratch - per-lane scratch memory -from __future__ import annotations -import ctypes, functools, itertools, re, platform, subprocess, tempfile +import ctypes, functools, itertools, re, struct, platform, subprocess, tempfile, traceback from typing import Callable # Set/restore DAZ+FTZ (denormals-are-zero + flush-to-zero) to match RDNA3 default float mode @@ -51,15 +50,16 @@ class _MXCSRContext: if lib is None or not hasattr(self, '_saved'): return lib.set_fpcr(self._saved) -from tinygrad.uop.ops import UOp, Ops, KernelInfo +from tinygrad.uop.ops import UOp, sint, Ops, KernelInfo, uopfunc, AxisType, rewrite_group from tinygrad.dtype import dtypes, AddrSpace from tinygrad.device import Buffer, BufferSpec, Device +from tinygrad.engine.realize import run_linear +from tinygrad.codegen import to_program, to_program_config +from tinygrad.renderer import Renderer +from tinygrad.helpers import Context, DEV, to_mv, ContextVar from tinygrad.runtime.autogen import hsa -from tinygrad.helpers import Context, DEBUG, PROFILE, colored, getenv, ContextVar -from tinygrad.engine.realize import get_runtime -from tinygrad.codegen import to_program +from tinygrad.viz.serve import amd_decode, get_cfg, COND_TAKEN, COND_NOT_TAKEN, UNCOND, get_elf_section -from tinygrad.renderer.amd import decode_inst from tinygrad.runtime.autogen.amd.rdna3.str_pcode import PCODE as PCODE_RDNA3 from tinygrad.runtime.autogen.amd.rdna4.str_pcode import PCODE as PCODE_RDNA4 from tinygrad.runtime.autogen.amd.cdna.str_pcode import PCODE as PCODE_CDNA @@ -71,12 +71,6 @@ from tinygrad.runtime.autogen.amd.common import Fmt, OpType from test.amd.helpers import decode_dpp16 from test.mockgpu.amd.pcode import parse_pcode, _FUNCS, _set_bits, _to_bool, _to_u32, _val_to_bits, _ftz_f32, _bitreverse, _countbits -MASK32 = 0xFFFFFFFF - -# SQTT encoder lives in sqtt_enc.py; traces are consumed by amdgpu.py -from test.mockgpu.amd.sqtt_enc import make_encoder as _make_sqtt_encoder -sqtt_traces: list[bytes] = [] - def _c(val, dtype=dtypes.uint32): return UOp.const(val, dtype) def _u64(lo: UOp, hi: UOp) -> UOp: @@ -120,11 +114,9 @@ def _iattr(inst, name: str, default: int = 0) -> int: v = getattr(inst, name, default) return default if v is None else v # Special registers stored after inline constants (256-259) -PC_LO_IDX, PC_HI_IDX, SCRATCH_STRIDE_IDX = 256, 257, 259 +SCRATCH_STRIDE_IDX = 259 # SGPR buffer: 0-127 = SGPRs, 128-255 = inline constants, 256-259 = special registers SGPR_COUNT = 260 -# Sentinel PC value for s_endpgm -ENDPGM_PC = 0xFFFFFFFFFFFFFFFF def _op_name(inst) -> str: if hasattr(inst, 'opx'): return f"{inst.opx.name}_{inst.opy.name}" # VOPD has opx/opy not op @@ -145,7 +137,6 @@ def _val_to_u32(val: UOp) -> UOp: if val.dtype == dtypes.uint32: return val if val.dtype == dtypes.float32: return val.bitcast(dtypes.uint32) if val.dtype == dtypes.half: return val.bitcast(dtypes.uint16).cast(dtypes.uint32) - if val.dtype in (dtypes.uint16, dtypes.int16): return val.cast(dtypes.uint32) return val.cast(dtypes.uint32) _pcode_fixes = { @@ -238,29 +229,8 @@ def _write_val(bits: int, val: UOp, wfn, reg_or_addr, *args, is_mem: bool = Fals """Write value, splitting 64-bit if needed. bits=64 for 64-bit writes, otherwise 32-bit.""" return _write_64bit(val, wfn, reg_or_addr, is_mem, *args) if bits == 64 else [wfn(reg_or_addr, _to_u32(val), *args)] -def _mem_store(mem: UOp, addr: UOp, val: UOp, active: UOp, addr_bits: int = 32, data_bits: int = 32) -> list[UOp]: - """Conditional memory store with sub-word support. Returns list of store UOps.""" - adt = dtypes.uint64 if addr_bits == 64 else dtypes.uint32 - word_addr = addr >> UOp.const(2, adt) - idx = mem.index(word_addr.valid(active)) - if data_bits == 32: return _mem_store_bytes(mem.bitcast(dtypes.uint8), addr, val, active) - # Sub-word store: read-modify-write with mask - byte_pos = addr.cast(dtypes.uint32) & _c(3) - byte_shift = byte_pos * _c(8) - val_u32, size_mask = val.cast(dtypes.uint32), _c(0xFF if data_bits == 8 else 0xFFFF) - mask = size_mask << byte_shift - new_word = (idx & (mask ^ _c(0xFFFFFFFF))) | ((val_u32 & size_mask) << byte_shift) - if data_bits == 8: return [idx.store(active.where(new_word, idx))] - # 16-bit cross-word case: byte_pos == 3 means value spans two words - is_cross = byte_pos.eq(_c(3)) - cross_word0 = (idx & _c(0x00FFFFFF)) | ((val_u32 & _c(0xFF)) << _c(24)) - store0 = idx.store(active.where(is_cross.where(cross_word0, new_word), idx)) - next_idx = mem.index((word_addr + UOp.const(1, adt)).valid(active & is_cross)) - cross_word1 = (next_idx & _c(0xFFFFFF00)) | ((val_u32 >> _c(8)) & _c(0xFF)) - return [store0, next_idx.store((active & is_cross).where(cross_word1, next_idx))] - def _mem_store_bytes(mem: UOp, addr: UOp, val: UOp, active: UOp, data_bits: int = 32) -> list[UOp]: - """Store to byte-addressable memory (scratch). addr is byte offset, mem is uint8 buffer.""" + """Store to byte-addressable memory. addr is byte offset, mem is uint8 buffer.""" stores = [] val_u32 = val.cast(dtypes.uint32) if val.dtype != dtypes.uint32 else val for i in range(data_bits // 8): @@ -298,8 +268,7 @@ def _int_clamp(op_name: str, srcs: dict) -> UOp | None: # D0 range: 16 for the *_U16/*_I16 result-narrow ops, else 32 (mad*32* D0 is u32/i32; mul operands have op-fmt width) narrow_dt = dtypes.uint16 if is_16bit and '32' not in op_name else (dtypes.int32 if is_signed else dtypes.uint32) wide_dt = dtypes.int64 - narrow_max, narrow_min = ((0xFFFF, 0) if narrow_dt == dtypes.uint16 else - ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0))) + narrow_max, narrow_min = ((0xFFFF, 0) if narrow_dt == dtypes.uint16 else ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0))) def to_mulin(x: UOp) -> UOp: # mul-source: extract the op-fmt-width suboperand with sext for signed mask = (1 << op_bits) - 1 if op_bits == 32: return x.bitcast(narrow_dt) if x.dtype.itemsize == 4 else x.cast(narrow_dt) @@ -322,48 +291,35 @@ def _int_clamp(op_name: str, srcs: dict) -> UOp | None: class _Ctx: """Context for instruction compilation - holds buffers and helpers.""" - __slots__ = ('inst_size', 'dyn_fields', '_axis_id', 'wave_size', 'vgpr', 'accvgpr', 'inst_addr', 'branch_cond') + __slots__ = ('dyn_fields', '_axis_id', 'wave_size', 'vgpr', 'accvgpr', 'branch_cond') sgpr = UOp.param(0, dtypes.uint32, SGPR_COUNT, name="sgpr") vmem = UOp.param(2, dtypes.uint32, 1 << 46, name="vmem") lds = UOp.param(3, dtypes.uint32, 16384, name="lds") scratch = UOp.param(4, dtypes.uint8, 1 << 30, name="scratch") - # Cache PARAM UOps by wave_size so all _Ctx instances with same wave_size share identical UOp references - _vgpr_cache: dict[int, UOp] = {} - _accvgpr_cache: dict[int, UOp] = {} - def __init__(self, inst_size: int, wave_size: int = 32, inst_addr: UOp | None = None): - self.inst_size, self._axis_id, self.wave_size, self.inst_addr = inst_size, 0, wave_size, inst_addr + def __init__(self, wave_size: int = 32): + self._axis_id, self.wave_size = 0, wave_size self.dyn_fields: list[tuple[int, int]] = [] # (lo, hi) of fields read dynamically - if wave_size not in _Ctx._vgpr_cache: _Ctx._vgpr_cache[wave_size] = UOp.param(1, dtypes.uint32, 256 * wave_size, name="vgpr") - self.vgpr = _Ctx._vgpr_cache[wave_size] - if wave_size == 64: - if wave_size not in _Ctx._accvgpr_cache: _Ctx._accvgpr_cache[wave_size] = UOp.param(5, dtypes.uint32, 256 * wave_size, name="accvgpr") - self.accvgpr = _Ctx._accvgpr_cache[wave_size] - else: - self.accvgpr = self.vgpr + self.vgpr = UOp.param(1, dtypes.uint32, 256 * wave_size, name="vgpr") + self.accvgpr = UOp.param(5, dtypes.uint32, 256 * wave_size, name="accvgpr") if wave_size == 64 else self.vgpr self.branch_cond: UOp | None = None def range(self, n: int | None = None) -> UOp: """Create a lane range UOp with unique axis ID.""" if n is None: n = self.wave_size self._axis_id += 1 - return UOp.range(n, self._axis_id, dtype=dtypes.int) + return UOp.range(n, self._axis_id, AxisType.LOOP, dtype=dtypes.int) def unroll_lanes(self, get_lane_bit, exec_mask: UOp, apply_exec: bool = True) -> UOp: """Combine lane bits into a mask using RANGE+REDUCE (32-bit for RDNA, 64-bit for CDNA).""" lane = self.range() - if self.wave_size <= 32: - bit = get_lane_bit(lane).cast(dtypes.uint32) << lane.cast(dtypes.uint32) - result = bit.reduce(lane, arg=Ops.ADD) - else: - bit = get_lane_bit(lane).cast(dtypes.uint64) << lane.cast(dtypes.uint64) - result = bit.reduce(lane, arg=Ops.ADD) + dtype = dtypes.uint32 if self.wave_size <= 32 else dtypes.uint64 + bit = get_lane_bit(lane).cast(dtype) << lane.cast(dtype) + result = bit.reduce(lane, arg=Ops.ADD) return result & exec_mask if apply_exec else result - def inst_word(self, dword_idx: int) -> UOp: - """Read instruction dword from vmem at PC + dword_idx*4.""" - addr = self.inst_addr if self.inst_addr is not None else self.rpc() - return self.vmem.index((addr >> 2) + dword_idx).load() + @staticmethod + def inst_word(dword_idx: int) -> UOp: return UOp.param(-2-dword_idx, dtypes.uint32, name=f"inst_w{dword_idx}", addrspace=AddrSpace.ALU) def inst_field(self, field) -> UOp: """Extract field bits from instruction encoding. Tracks field for canonical key computation.""" @@ -396,26 +352,21 @@ class _Ctx: sign_bit = 1 << (width - 1) return (val.cast(dtypes.int) ^ _c(sign_bit, dtypes.int)) - _c(sign_bit, dtypes.int) - def canonical_mask(self, inst_bytes: bytes) -> tuple[int, int, int]: - """Compute canonical (base, mask, size) for cache lookup. + def canonical_mask(self, inst_bytes: bytes|memoryview) -> tuple[int, int]: + """Compute canonical (base, mask) for cache lookup. base = instruction bits with dynamic fields zeroed - mask = bitmask with 1s for static bits, 0s for dynamic bits - size = instruction size in bytes""" - size = self.inst_size - base = int.from_bytes(inst_bytes[:size], 'little') - mask = (1 << (size * 8)) - 1 # all 1s initially + mask = bitmask with 1s for static bits, 0s for dynamic bits""" + base = int.from_bytes(inst_bytes, 'little') + mask = (1 << (len(inst_bytes) * 8)) - 1 # all 1s initially for lo, hi in self.dyn_fields: field_mask = ((1 << (hi - lo + 1)) - 1) << lo base &= ~field_mask # zero dynamic bits in base mask &= ~field_mask # zero dynamic bits in mask - return base, mask, size + return base, mask def rexec(self) -> UOp: """Read full EXEC mask (32-bit for RDNA, 64-bit for CDNA).""" - lo = self.rsgpr_dyn(_c(EXEC_LO.offset)) - if self.wave_size <= 32: return lo - hi = self.rsgpr_dyn(_c(EXEC_LO.offset + 1)) - return _u64(lo, hi) + return self.rmask(_c(EXEC_LO.offset)) # Dynamic register access (takes UOp index instead of int) def rsgpr_dyn(self, reg: UOp, valid: UOp | None = None) -> UOp: @@ -438,14 +389,14 @@ class _Ctx: def wmask_lane_bit(self, reg: UOp, lane: UOp, val: UOp, exec_mask: UOp) -> list[UOp]: """Set/clear bit `lane` of the mask at `reg` from val for exec-active lanes, preserving memory for inactive lanes""" - active, bit = _lane_active(exec_mask, lane), _to_u32(val) + active, bit, mask32 = _lane_active(exec_mask, lane), _to_u32(val), _c(0xFFFFFFFF) if self.wave_size <= 32: old = self.rsgpr_dyn(reg) mask = _c(1) << lane.cast(dtypes.uint32) - return [self.wsgpr_dyn(reg, active.where((old & (mask ^ _c(MASK32))) | (bit << lane.cast(dtypes.uint32)), old))] + return [self.wsgpr_dyn(reg, active.where((old & (mask ^ mask32)) | (bit << lane.cast(dtypes.uint32)), old))] off = (lane & _c(31, dtypes.int)).cast(dtypes.uint32) mask = _c(1) << off - def half(old: UOp, sel: UOp) -> UOp: return sel.where(active.where((old & (mask ^ _c(MASK32))) | (bit << off), old), old) + def half(old: UOp, sel: UOp) -> UOp: return sel.where(active.where((old & (mask ^ mask32)) | (bit << off), old), old) return [self.wsgpr_dyn(reg, half(self.rsgpr_dyn(reg), lane < _c(32, dtypes.int))), self.wsgpr_dyn(reg + _c(1), half(self.rsgpr_dyn(reg + _c(1)), _c(32, dtypes.int) <= lane))] @@ -470,11 +421,10 @@ class _Ctx: idx = reg.cast(dtypes.int) * _c(self.wave_size, dtypes.int) + lane.cast(dtypes.int) return self.accvgpr.index(idx.valid(valid)).load() if valid is not None else self.accvgpr.index(idx).load() - def waccvgpr_dyn(self, reg: UOp, lane: UOp, val: UOp, exec_mask: UOp, after: UOp | None = None) -> UOp: + def waccvgpr_dyn(self, reg: UOp, lane: UOp, val: UOp, exec_mask: UOp) -> UOp: """Write ACCVGPR with dynamic register index (CDNA only).""" - buf = self.accvgpr.after(after) if after is not None else self.accvgpr offset = reg.cast(dtypes.int) * _c(self.wave_size, dtypes.int) + lane.cast(dtypes.int) - return buf.index(offset.valid(_lane_active(exec_mask, lane))).store(val.cast(dtypes.uint32)) + return self.accvgpr.index(offset.valid(_lane_active(exec_mask, lane))).store(val.cast(dtypes.uint32)) def rsrc_dyn(self, off: UOp, lane: UOp | None, bits: int = 32, literal: UOp | None = None, is_f64: bool = False, do_cast: bool = True) -> UOp: """Read source operand with dynamic offset. Handles SGPR/inline constants (<256), VGPR (>=256). @@ -512,17 +462,6 @@ class _Ctx: return is_vgpr.where(vgpr_val, scalar_val) if lane is not None else scalar_val - def rpc(self) -> UOp: - """Read PC as 64-bit byte address.""" - # Index at PC_LO, then cast to uint64 ptr and load - return _u64(self.rsgpr_dyn(_c(PC_LO_IDX)), self.rsgpr_dyn(_c(PC_HI_IDX))) - - def inc_pc(self) -> list[UOp]: - """Increment PC by instruction size in bytes. Returns [store].""" - new_pc = self.rpc() + UOp.const(self.inst_size, dtypes.uint64) - lo, hi = _split64(new_pc) - return [self.wsgpr_dyn(_c(PC_LO_IDX), lo), self.wsgpr_dyn(_c(PC_HI_IDX), hi)] - def scalar_stores(self, assigns: list[tuple[str, UOp]], sdst_reg: UOp, sdst_size: int = 1) -> list[UOp]: """Generate stores for scalar assigns with dynamic destination register (D0, SCC, EXEC, VCC).""" stores: list[UOp] = [] @@ -539,20 +478,16 @@ class _Ctx: stores.extend([self.wsgpr_dyn(_c(EXEC_LO.offset), lo), self.wsgpr_dyn(_c(EXEC_LO.offset + 1), hi)]) else: stores.append(self.wsgpr_dyn(_c(EXEC_LO.offset), _to_u32(val))) elif dest.startswith('VCC'): stores.extend(self.wmask(_c(VCC_LO.offset), val)) - elif dest.startswith('PC'): # S_SETPC/S_SWAPPC jump: write PC directly (caller skips inc_pc) - lo, hi = _split64(val.cast(dtypes.uint64)) - stores.extend([self.wsgpr_dyn(_c(PC_LO_IDX), lo), self.wsgpr_dyn(_c(PC_HI_IDX), hi)]) + elif dest.startswith('PC'): raise NotImplementedError("AMD lifter does not support direct PC writes") return stores def compile_sop_pcode(self, op, srcs: dict[str, UOp | int], sdst_reg: UOp, sdst_size: int) -> UOp: """Compile a scalar instruction with dynamic destination register.""" pcode = get_pcode(op) - srcs.update(self.base_srcs(self.rexec()), VCC=self.rmask(_c(VCC_LO.offset)), PC=self.rpc().cast(dtypes.int64)) + srcs.update(self.base_srcs(self.rexec()), VCC=self.rmask(_c(VCC_LO.offset))) if 'D0' not in srcs: srcs['D0'] = self.rsgpr_dyn(sdst_reg) # D0 is current dest value for read-modify-write ops _, assigns = parse_pcode(pcode, srcs) - # PC-writing ops (S_SETPC/S_SWAPPC) jump instead of advancing to the next instruction - inc = [] if any(dest.startswith('PC') for dest, _ in assigns) else self.inc_pc() - return UOp.sink(*self.scalar_stores(assigns, sdst_reg, sdst_size), *inc) + return UOp.sink(*self.scalar_stores(assigns, sdst_reg, sdst_size)) def compile_lane_pcode(self, op, inst) -> UOp: """Compile cross-lane ops (READLANE/WRITELANE/PERMLANE) using pcode parser.""" @@ -577,7 +512,7 @@ class _Ctx: for dest, val in assigns: if dest.startswith('D0'): stores.append(self.wsgpr_dyn(vdst_off, val.cast(dtypes.uint32))) elif dest.startswith('VGPR['): stores.append(self.vgpr.index(val[0]).store(val[1].cast(dtypes.uint32))) - return UOp.sink(*stores, *self.inc_pc()) + return UOp.sink(*stores) def base_srcs(self, exec_mask: UOp, lane: UOp | None = None) -> dict[str, UOp | int]: """Pcode environment entries shared by all instructions: EXEC/SCC, rounding mode (emulator always rounds-nearest-even), @@ -592,7 +527,7 @@ class _Ctx: def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: UOp, exec_mask: UOp, opsel_dst_hi: bool | UOp = False, sdst_reg: int | None = None, clmp: int = 0, src0_off: UOp | None = None) -> UOp: - """Compile VOP instruction. Returns sink with stores and inc_pc.""" + """Compile VOP instruction to a sink of stores.""" pcode = get_pcode(op) vcc_reg = sdst_reg if sdst_reg is not None else VCC_LO.offset if 'VCC' not in srcs: srcs['VCC'] = self.rmask(_c(vcc_reg)) @@ -661,7 +596,7 @@ class _Ctx: lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) if lane_stores: stores.append(UOp.sink(*lane_stores).end(lane)) stores.extend(scalar_stores) - return UOp.sink(*stores, *self.inc_pc()) + return UOp.sink(*stores) # ═══════════════════════════════════════════════════════════════════════════════ # INSTRUCTION HANDLERS @@ -670,30 +605,22 @@ class _Ctx: def _compile_sopp(inst: ir3.SOPP | ir4.SOPP, ctx: _Ctx) -> UOp: simm16 = ctx.inst_field_signed(type(inst).simm16).cast(dtypes.int16) if inst.op in (ir3.SOPPOp.S_ENDPGM, ir4.SOPPOp.S_ENDPGM, irc.SOPPOp.S_ENDPGM): - return UOp.sink(ctx.wsgpr_dyn(_c(PC_LO_IDX), UOp.const(0xFFFFFFFF, dtypes.uint32)), - ctx.wsgpr_dyn(_c(PC_HI_IDX), UOp.const(0xFFFFFFFF, dtypes.uint32))) - # S_BARRIER: advance PC past the barrier instruction. The execution loop detects barriers before executing and handles synchronization. - if inst.op in _BARRIER_OPS: return UOp.sink(*ctx.inc_pc()) + return UOp.sink() + # barriers are represented by block boundaries in the lifter. + if inst.op in _BARRIER_OPS: return UOp.sink() # S_NOP and S_WAITCNT are no-ops in emulator (no pipeline/cache to wait on) - if inst.op in (ir3.SOPPOp.S_NOP, ir4.SOPPOp.S_NOP, irc.SOPPOp.S_NOP, irc.SOPPOp.S_WAITCNT): return UOp.sink(*ctx.inc_pc()) + if inst.op in (ir3.SOPPOp.S_NOP, ir4.SOPPOp.S_NOP, irc.SOPPOp.S_NOP, irc.SOPPOp.S_WAITCNT): return UOp.sink() # NOTE: we ignore SOPPs without PCODE if inst.op in _get_pcode_dict(inst.op): pcode = get_pcode(inst.op) - pc_bytes = ctx.rpc() # PC is already 64-bit byte address vcc, exec_val = ctx.rmask(_c(VCC_LO.offset)), ctx.rexec() - srcs: dict[str, UOp|int] = {'PC': pc_bytes.cast(dtypes.int64), 'SIMM16': simm16, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'VCC': vcc, - 'VCCZ': vcc.eq(UOp.const(0, vcc.dtype)).cast(dtypes.uint32), - 'EXECZ': exec_val.eq(UOp.const(0, exec_val.dtype)).cast(dtypes.uint32)} + srcs = {'PC': _c(0, dtypes.uint32), 'SIMM16': simm16, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'VCC': vcc, + 'VCCZ': vcc.eq(UOp.const(0, vcc.dtype)).cast(dtypes.uint32), 'EXECZ': exec_val.eq(UOp.const(0, exec_val.dtype)).cast(dtypes.uint32)} for dest, val in parse_pcode(pcode, srcs)[1]: - if dest.startswith('PC'): - if val.op is Ops.WHERE: ctx.branch_cond = val.src[0] - lo, hi = _split64(val.cast(dtypes.uint64)) - return UOp.sink(ctx.wsgpr_dyn(_c(PC_LO_IDX), lo), ctx.wsgpr_dyn(_c(PC_HI_IDX), hi)) - return UOp.sink(*ctx.inc_pc()) + if dest.startswith('PC') and val.op is Ops.WHERE: ctx.branch_cond = val.src[0] + return UOp.sink() def _compile_smem(inst: ir3.SMEM | ir4.SMEM, ctx: _Ctx) -> UOp: - # Cache invalidation instructions are no-ops in the emulator (we don't model caches) - if '_INV' in inst.op.name: return UOp.sink(*ctx.inc_pc()) # Dynamic sbase field (bits 5:0) - SGPR pair, field value * 2 = register offset sbase = ctx.inst_field(type(inst).sbase) * _c(2) # Dynamic sdata field (bits 12:6) - destination SGPR @@ -705,22 +632,14 @@ def _compile_smem(inst: ir3.SMEM | ir4.SMEM, ctx: _Ctx) -> UOp: soffset_val = _c(0).cast(dtypes.uint64) if not (isinstance(inst, irc.SMEM) and not inst.soffset_en): soffset_val = ctx.rsgpr_dyn(ctx.inst_field(type(inst).soffset)).cast(dtypes.uint64) - addr = _u64(ctx.rsgpr_dyn(sbase), ctx.rsgpr_dyn(sbase + _c(1))) + offset.cast(dtypes.uint64) + soffset_val - # S_LOAD_(DTYPE) series: B32/DWORD=1, B64/DWORDX2=2, U8=0.25, I8=-0.25, etc. - op_name = _op_name(inst) - assert (op_name).startswith('S_LOAD_'), f"unexpected SMEM op: {op_name}" - part = op_name.rsplit('_', 1)[1] # B32, DWORD, DWORDX2, U8, I8, etc. - nval = int(part.removeprefix('DWORD').removeprefix('X') or '1') if 'DWORD' in part else int(part[1:]) / 32 * (-1 if part[0] == 'I' else 1) - ndwords = max(1, int(abs(nval))) - dword_base = addr >> UOp.const(2, dtypes.uint64) - vals = [ctx.vmem.index(dword_base + UOp.const(i, dtypes.uint64)) for i in range(ndwords)] - if abs(nval) < 1: - nbits = int(abs(nval) * 32) - byte_off = (addr & UOp.const(3, dtypes.uint64)).cast(dtypes.uint32) * UOp.const(8, dtypes.uint32) - extracted = (vals[0] >> byte_off) & UOp.const((1 << nbits) - 1, dtypes.uint32) - vals[0] = extracted.cast({8: dtypes.int8, 16: dtypes.int16}[nbits]).cast(dtypes.int32).bitcast(dtypes.uint32) if nval < 0 else extracted - stores = [ctx.wsgpr_dyn(sdata_reg + _c(i), vals[i]) for i in range(ndwords)] - return UOp.sink(*stores, *ctx.inc_pc()) + base = _u64(ctx.rsgpr_dyn(sbase), ctx.rsgpr_dyn(sbase + _c(1))) + byte_offset = offset.cast(dtypes.uint64) + soffset_val + srcs = {'ADDR': base + byte_offset, 'sgpr_base': base, 'offset': byte_offset, + 'SBASE': base, 'SOFFSET': soffset_val, 'OFFSET': offset, '_vmem': ctx.vmem} + funcs = {'CalcScalarGlobalAddr': lambda _base, soffset, offset: base + soffset.cast(dtypes.uint64) + offset.cast(dtypes.uint64)} + _, assigns = parse_pcode(get_pcode(inst.op), srcs, funcs) + stores = [ctx.wsgpr_dyn(sdata_reg + _c(i), val) for i, val in _collect_data_slices(assigns, 'SDATA').items()] + return UOp.sink(*stores) def _compile_sop(inst: ir3.SOP1|ir3.SOP2|ir3.SOPC|ir3.SOPK|ir4.SOP1|ir4.SOP2|ir4.SOPC|ir4.SOPK|irc.SOP1|irc.SOP2|irc.SOPC|irc.SOPK, ctx: _Ctx) -> UOp: bits = inst.canonical_op_bits @@ -740,7 +659,7 @@ def _compile_sop(inst: ir3.SOP1|ir3.SOP2|ir3.SOPC|ir3.SOPK|ir4.SOP1|ir4.SOP2|ir4 srcs: dict[str, UOp|int] = {'S0': s0, 'S1': simm16_sext, 'SIMM16': simm16_sext, 'D0': ctx.rsgpr_dyn(sdst_off)} dst_off, dst_size = sdst_off, 1 # S_GETREG_B32: extract bits from HW register. Handle as special case since HW_REGISTERS is not a normal variable. - # HW register values are stored at SGPR[SGPR_COUNT-16 + hwRegId] by _init_wave. + # HW register values are stored at SGPR[SGPR_COUNT-16 + hwRegId] by init_wave. if 'GETREG' in op_name: hw_reg_id = simm16.cast(dtypes.uint32) & _c(0x3F) offset = (simm16.cast(dtypes.uint32) >> _c(6)) & _c(0x1F) @@ -748,10 +667,10 @@ def _compile_sop(inst: ir3.SOP1|ir3.SOP2|ir3.SOPC|ir3.SOPK|ir4.SOP1|ir4.SOP2|ir4 hw_val = ctx.rsgpr_dyn(_c(SGPR_COUNT - 16) + hw_reg_id) mask = (_c(1) << size) - _c(1) result = (hw_val >> offset) & mask - return UOp.sink(ctx.wsgpr_dyn(sdst_off, result), *ctx.inc_pc()) + return UOp.sink(ctx.wsgpr_dyn(sdst_off, result)) elif isinstance(inst, (ir3.SOP1, ir4.SOP1, irc.SOP1)): # S_BARRIER_SIGNAL: no-op in emulator, barrier sync handled by execution loop - if isinstance(inst, ir4.SOP1) and inst.op in _BARRIER_SOP1_OPS: return UOp.sink(*ctx.inc_pc()) + if isinstance(inst, ir4.SOP1) and inst.op in _BARRIER_SOP1_OPS: return UOp.sink() sdst_off = ctx.inst_field(type(inst).sdst) ssrc0_off = ctx.inst_field(type(inst).ssrc0) srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal)} @@ -760,15 +679,13 @@ def _compile_sop(inst: ir3.SOP1|ir3.SOP2|ir3.SOPC|ir3.SOPK|ir4.SOP1|ir4.SOP2|ir4 sdst_off = ctx.inst_field(type(inst).sdst) ssrc0_off = ctx.inst_field(type(inst).ssrc0) ssrc1_off = ctx.inst_field(type(inst).ssrc1) - srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal), - 'S1': ctx.rsrc_dyn(ssrc1_off, None, bits['s1'], literal)} + srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal), 'S1': ctx.rsrc_dyn(ssrc1_off, None, bits['s1'], literal)} if literal is not None: srcs['SIMM32'] = literal dst_off, dst_size = sdst_off, bits['d'] // 32 elif isinstance(inst, (ir3.SOPC, ir4.SOPC, irc.SOPC)): ssrc0_off = ctx.inst_field(type(inst).ssrc0) ssrc1_off = ctx.inst_field(type(inst).ssrc1) - srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal), - 'S1': ctx.rsrc_dyn(ssrc1_off, None, bits['s1'], literal)} + srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal), 'S1': ctx.rsrc_dyn(ssrc1_off, None, bits['s1'], literal)} dst_off, dst_size = _c(0), 0 # SOPC writes to SCC, not sdst else: raise RuntimeError(f"unknown SOP type: {type(inst).__name__}") @@ -817,8 +734,7 @@ def _dpp16_ctrl(lane: UOp, dpp: int, row_mask: int, bank_mask: int, wave_size: i row_base, lane_in_row = lane_i & _c(~15, dtypes.int), lane_i & _c(15, dtypes.int) row = lane_i // _c(16, dtypes.int) bank = lane_in_row >> _c(2, dtypes.int) - enabled = (((_c(row_mask) >> row.cast(dtypes.uint32)) & _c(1)).ne(_c(0)) & - (((_c(bank_mask) >> bank.cast(dtypes.uint32)) & _c(1)).ne(_c(0)))) + enabled = (((_c(row_mask) >> row.cast(dtypes.uint32)) & _c(1)).ne(_c(0)) & (((_c(bank_mask) >> bank.cast(dtypes.uint32)) & _c(1)).ne(_c(0)))) op, arg = decode_dpp16(dpp) src_lane, valid = lane_i, UOp.const(True) @@ -842,8 +758,7 @@ def _dpp16_ctrl(lane: UOp, dpp: int, row_mask: int, bank_mask: int, wave_size: i def _load_dpp16_src0(ctx: _Ctx, inst, lane: UOp, fallback: UOp) -> UOp: """Load a DPP16-swizzled src0 value from vsrc0.""" - src_lane, enabled, valid = _dpp16_ctrl(lane, _iattr(inst, 'dpp'), _iattr(inst, 'row_mask', 0xf), - _iattr(inst, 'bank_mask', 0xf), ctx.wave_size) + src_lane, enabled, valid = _dpp16_ctrl(lane, _iattr(inst, 'dpp'), _iattr(inst, 'row_mask', 0xf), _iattr(inst, 'bank_mask', 0xf), ctx.wave_size) safe_src_lane = (enabled & valid).where(src_lane, _c(0, dtypes.int)) swizzled = ctx.rvgpr_dyn(ctx.inst_field(type(inst).vsrc0), safe_src_lane) invalid = UOp.const(0, fallback.dtype) if _iattr(inst, 'bc') else fallback @@ -882,7 +797,7 @@ def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc return _c(0) new_result = ctx.unroll_lanes(get_cmp_bit, exec_mask, apply_exec=False) & exec_mask stores = ctx.wmask(sdst_off, new_result) - return UOp.sink(*stores, *ctx.inc_pc()) + return UOp.sink(*stores) # Non-VOPC path: VOP1_SDWA, VOP2_SDWA, VOP2_SDWA_SDST — uses lane loop lane = ctx.range() @@ -920,11 +835,11 @@ def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc init_stores = [ctx.wsgpr_dyn(sdst_off, _c(0)), ctx.wsgpr_dyn(sdst_off + _c(1), _c(0))] stores.extend(ctx.wmask_lane_bit(sdst_off, lane, vcc_val, exec_mask)) if stores: - return UOp.sink(*init_stores, UOp.sink(*stores).end(lane), *ctx.inc_pc()) - return UOp.sink(*init_stores, *ctx.inc_pc()) + return UOp.sink(*init_stores, UOp.sink(*stores).end(lane)) + return UOp.sink(*init_stores) if stores: - return UOp.sink(UOp.sink(*stores).end(lane), *ctx.inc_pc()) - return UOp.sink(*ctx.inc_pc()) + return UOp.sink(UOp.sink(*stores).end(lane)) + return UOp.sink() def _load_vsrc0(ctx: _Ctx, inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | ir3.VOP2_DPP16 | ir4.VOP1 | ir4.VOP1_SDST | ir4.VOP1_DPP16 | ir4.VOP2 | ir4.VOP2_DPP16 | @@ -938,7 +853,7 @@ def _load_vsrc0(ctx: _Ctx, inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3 return s0, None src0_off = ctx.inst_field(type(inst).src0) s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal, is_f64) - if bits['s0'] == 16: # src0 >= 384 means hi half of v[src0-384]. Guard index against OOB access. + if bits['s0'] == 16 and not isinstance(inst, (irc.VOP1, irc.VOP2)): # src0 >= 384 means hi half of v[src0-384]. Guard index against OOB access. src0_hi = src0_off >= _c(384) s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_hi.where(src0_off - _c(384), _c(0)), lane)), s0) return s0, src0_off @@ -954,14 +869,14 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | vdst_reg = ctx.inst_field(type(inst).vdst) # VGPRField: raw ACCVGPR index (0-255) acc_src0_off = ctx.inst_field(type(inst).src0) # SrcField: raw 256 + ACCVGPR index val = ctx.raccvgpr_dyn(acc_src0_off - _c(256), lane) - return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane), *ctx.inc_pc()) + return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane)) lane, exec_mask, bits = ctx.range(), ctx.rexec(), inst.canonical_op_bits literal = ctx.optional_field(inst, 'literal') is_f64 = 'F64' in op_name and 'B64' not in op_name is_float = any(x in op_name for x in ('F16', 'F32', 'F64')) is_dpp16 = hasattr(type(inst), 'dpp') and hasattr(type(inst), 'vsrc0') vdst_reg = ctx.inst_field(type(inst).vdst) - write_hi_half = bits['d'] == 16 and (vdst_reg >= _c(128)) + write_hi_half = bits['d'] == 16 and not isinstance(inst, (irc.VOP1, irc.VOP2)) and (vdst_reg >= _c(128)) if isinstance(write_hi_half, UOp): vdst_reg = write_hi_half.where(vdst_reg - _c(128), vdst_reg) elif write_hi_half: vdst_reg -= 128 if isinstance(inst, (ir3.VOP1, ir4.VOP1, irc.VOP1)): @@ -970,7 +885,7 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | srcs: dict[str, UOp | int] = {'S0': s0, 'D0': d0, 'OPSEL': _c(0)} else: vsrc1_reg = ctx.inst_field(type(inst).vsrc1) - vsrc1_hi = bits['s0'] == 16 and (vsrc1_reg >= _c(128)) + vsrc1_hi = bits['s0'] == 16 and not isinstance(inst, irc.VOP2) and (vsrc1_reg >= _c(128)) if bits['s1'] == 64: s1 = _u64(ctx.rvgpr_dyn(vsrc1_reg, lane), ctx.rvgpr_dyn(vsrc1_reg + _c(1), lane)) d0 = _u64(ctx.rvgpr_dyn(vdst_reg, lane), ctx.rvgpr_dyn(vdst_reg + _c(1), lane)) @@ -998,7 +913,7 @@ def _compile_vopc(inst: ir3.VOPC|ir3.VOPC_DPP16|ir3.VOP3|ir4.VOPC|ir4.VOPC_DPP16 src0_off = ctx.inst_field(type(inst).src0) vsrc1_off = ctx.inst_field(type(inst).vsrc1) # type: ignore[union-attr] # For 16-bit ops, vsrc1 >= 128 means hi-half of v[vsrc1-128] - if bits['s0'] == 16: + if bits['s0'] == 16 and not isinstance(inst, irc.VOPC): vsrc1_hi = vsrc1_off >= _c(128) src1_off = _c(256) + vsrc1_hi.where(vsrc1_off - _c(128), vsrc1_off) else: @@ -1041,7 +956,7 @@ def _compile_vopc(inst: ir3.VOPC|ir3.VOPC_DPP16|ir3.VOP3|ir4.VOPC|ir4.VOPC_DPP16 # CMPX writes EXEC only (hardware verified: e64 CMPX does not write SDST); non-CMPX writes SDST/VCC if is_cmpx: stores = ctx.wmask(_c(EXEC_LO.offset), new_result) else: stores = ctx.wmask(dst_off, new_result) if not is_vopc else ctx.wmask(_c(VCC_LO.offset), new_result) - return UOp.sink(*stores, *ctx.inc_pc()) + return UOp.sink(*stores) def _compile_bitop3(inst, ctx: _Ctx, exec_mask: UOp, bits: dict, op_name: str) -> UOp: @@ -1062,7 +977,7 @@ def _compile_bitop3(inst, ctx: _Ctx, exec_mask: UOp, bits: dict, op_name: str) - for i in range(8): if not (ttbl & (1 << i)): continue result = result | ((s0 if i & 4 else bnot(s0)) & (s1 if i & 2 else bnot(s1)) & (s2 if i & 1 else bnot(s2))) - return UOp.sink(ctx.wvgpr_dyn(vdst_reg, lane, result.cast(dtypes.uint32), exec_mask).end(lane), *ctx.inc_pc()) + return UOp.sink(ctx.wvgpr_dyn(vdst_reg, lane, result.cast(dtypes.uint32), exec_mask).end(lane)) def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: exec_mask = ctx.rexec() @@ -1094,9 +1009,8 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: if 'V_S_' in op_name: src0 = _apply_src_mods(ctx.rsrc_dyn(ctx.inst_field(type(inst).src0), _c(0, dtypes.int), bits['s0'], literal), 0, abs_bits, neg_bits, bits['s0']) srcs = {**ctx.base_srcs(exec_mask, _c(0, dtypes.int)), 'S0': src0} - _, assigns = parse_pcode(get_pcode(inst.op), srcs) - stores = [ctx.wsgpr_dyn(vdst_reg, _val_to_u32(val)) for dest, val in assigns if dest.startswith('D0')] - return UOp.sink(*stores, *ctx.inc_pc()) + pcode_vars, _ = parse_pcode(get_pcode(inst.op), srcs) + return UOp.sink(ctx.wsgpr_dyn(vdst_reg, _val_to_u32(pcode_vars['D0']))) # Regular VOP3 - read operands dynamically lane = ctx.range() @@ -1186,7 +1100,7 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD | irc.VOP3SD, ctx: _Ctx) -> UO vgpr_stores.append(ctx.wvgpr_dyn(vdst_reg, lane3, d0_u32, exec_mask)) # Write carry output (wmask handles lo/hi split for wave64) vcc_writes = ctx.wmask(sdst_off, final_vcc) - return UOp.sink(*vcc_writes, UOp.group(*vgpr_stores).end(lane3), *ctx.inc_pc()) + return UOp.sink(*vcc_writes, UOp.group(*vgpr_stores).end(lane3)) else: return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, sdst_reg=inst.sdst.offset) @@ -1319,11 +1233,10 @@ def _compile_mfma(inst: irc.VOP3P|irc.VOP3PX2, ctx: _Ctx) -> UOp: out_bits = acc.cast(dtypes.uint32) if is_int_out else acc.bitcast(dtypes.uint32) compute_stores.append(acc_writer(vdst_reg + _c(out_reg), compute_lane, out_bits, exec_mask)) compute_phase = UOp.group(*compute_stores).end(compute_lane) - return UOp.sink(read_phase, compute_phase, *ctx.inc_pc()) + return UOp.sink(read_phase, compute_phase) def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P, ctx: _Ctx) -> UOp: - """RDNA3/4 WMMA: D = A@B + C on 16x16 tiles. A/B are unpacked to flat f32/i32 arrays, then all 256 outputs are - computed directly with scalar ops (no lane loop - the wave32 lane structure is baked into the index maps).""" + """RDNA3/4 WMMA: snapshot A/B/C before writing D = A@B + C, allowing source and destination registers to overlap.""" op_name, exec_mask = _op_name(inst), ctx.rexec() vdst_reg = ctx.inst_field(type(inst).vdst) src0_r, src1_r = ctx.inst_field(type(inst).src0) - _c(256), ctx.inst_field(type(inst).src1) - _c(256) @@ -1338,7 +1251,7 @@ def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P, ctx: _Ctx) -> UOp: # read a source element from VGPRs: (src, lane, vgpr, element-in-vgpr) -> f32/i32 def gval(src, lane, vgpr, ridx, *, fmt): v = ctx.rvgpr_dyn(src + _c(vgpr), UOp.const(lane, dtypes.int)) - pkd = v >> UOp.const(ridx * sz, dtypes.uint32) if ridx > 0 else v + pkd = v >> UOp.const(ridx * sz, dtypes.uint32) pkd = pkd & UOp.const((1 << sz) - 1, dtypes.uint32) if not fmt.startswith('IU'): return _FUNCS[f'{fmt.lower()}_to_f32'](pkd) return (pkd << _c(24, dtypes.uint)).bitcast(dtypes.int32) >> _c(24, dtypes.int32) # sign extend @@ -1346,33 +1259,37 @@ def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P, ctx: _Ctx) -> UOp: # RDNA3 f16/bf16: 16 lanes x 8 VGPRs x 2 halves, k maps linearly # RDNA3 iu8: 16 lanes x 4 VGPRs x 4 quarters, k maps linearly # RDNA4: 32 lanes x 4 VGPRs x 2 halves, k bits are scrambled (k[2] goes to lane bit 4) - def read_mat(src, fmt): + def read_mat(src, fmt, row, k): n = 32 // sz # values per vgpr def ab_map(i, k): # (row, k) -> (lane, vgpr, element-in-vgpr) elem, lane = ((k & 3) | ((k >> 1) & 4), i + ((k >> 2) & 1) * 16) if is_rdna4 else (k, i) return lane, elem // n, elem % n - return [gval(src, *ab_map(row, k), fmt=fmt) for row in range(16) for k in range(16)] + return gval(src, *ab_map(row, k), fmt=fmt) - mat_a, mat_b = read_mat(src0_r, a_fmt), read_mat(src1_r, b_fmt) def d_map(m, n): # output (row, col) -> (lane, vgpr) lane_bit, vgpr = (m >> 3, m & 7) if is_rdna4 else (m & 1, m >> 1) return n + lane_bit * 16, vgpr # Accumulator C. RDNA4 f16/bf16 packs two f32 accumulator VGPRs into one f16 VGPR; RDNA3 uses the lo half of each. # src2 may be a VGPR or an inline/scalar constant (128 = int 0, the usual ", 0" C form); the runner must handle both dynamically - out_dt = dtypes.float32 if output_type == "F32" else dtypes.int32 + out_dt = dtypes.int32 if output_type == "I32" else dtypes.float32 cbits = ctx.rsrc_dyn(src2_r, None, 32) cval_const = _FUNCS[f'{output_type.lower()}_to_f32'](cbits & UOp.const(0xFFFF, dtypes.uint32)) \ if output_type in ("F16", "BF16") else cbits.bitcast(out_dt) - if output_type in ("F16", "BF16"): - mat_c = [is_c_vgpr.where(gval(src2_r, *((lane, vgpr // 2, vgpr % 2) if is_rdna4 else (lane, vgpr, 0)), fmt=output_type), cval_const) - for m in range(16) for n in range(16) for lane, vgpr in [d_map(m, n)]] - else: - mat_c = [is_c_vgpr.where(ctx.rvgpr_dyn(src2_r + _c(vgpr), UOp.const(lane, dtypes.int)).bitcast(out_dt), cval_const) - for m in range(16) for n in range(16) for lane, vgpr in [d_map(m, n)]] - mat_d = [sum(mat_a[r*16+k] * mat_b[c*16+k] for k in range(16)) + mat_c[r*16+c] for r in range(16) for c in range(16)] + elem = ctx.range(256) + row, col = elem // 16, elem % 16 + lane, vgpr = d_map(row, col) + cval = gval(src2_r, *((lane, vgpr // 2, vgpr % 2) if is_rdna4 else (lane, vgpr, 0)), fmt=output_type) \ + if output_type in ("F16", "BF16") else ctx.rvgpr_dyn(src2_r + _c(vgpr), lane).bitcast(out_dt) + tmp = UOp.alloc((768,), dtypes.uint32, 0, AddrSpace.REG) + values = [read_mat(src0_r, a_fmt, row, col), read_mat(src1_r, b_fmt, row, col), is_c_vgpr.where(cval, cval_const)] + read_phase = UOp.group(*(tmp.index(elem + i*256).store(v.bitcast(dtypes.uint32)) for i,v in enumerate(values))).end(elem) + data = tmp.after(read_phase) + n = ctx.range(16) + mat_d = [sum(data.index(r*16+k).bitcast(out_dt) * data.index(256+n*16+k).bitcast(out_dt) for k in range(16)) + + data.index(512+r*16+n).bitcast(out_dt) for r in range(16)] - def w_store(m: int, n: int, val: UOp, vgpr_off: int) -> UOp: # store one output element to its (lane, vgpr) slot + def w_store(m: int, n: UOp, val: UOp, vgpr_off: int) -> UOp: # store one output element to its (lane, vgpr) slot lane_i, _ = d_map(m, n) return ctx.wvgpr_dyn(vdst_reg + _c(vgpr_off), UOp.const(lane_i, dtypes.int), val, exec_mask) if output_type in ("F16", "BF16"): @@ -1380,13 +1297,13 @@ def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P, ctx: _Ctx) -> UOp: return ((v.bitcast(dtypes.uint32) >> UOp.const(16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32)) if output_type == "BF16" \ else v.cast(dtypes.half).bitcast(dtypes.uint16).cast(dtypes.uint32) if is_rdna4: # pack 2 outputs per VGPR (adjacent m values share a VGPR) - stores = [w_store(m, n, to_bits(mat_d[m*16+n]) | (to_bits(mat_d[(m+1)*16+n]) << UOp.const(16, dtypes.uint32)), d_map(m, n)[1] // 2) - for n in range(16) for m in range(0, 16, 2)] + stores = [w_store(m, n, to_bits(mat_d[m]) | (to_bits(mat_d[m+1]) << UOp.const(16, dtypes.uint32)), d_map(m, n)[1] // 2) + for m in range(0, 16, 2)] else: # one output per VGPR (lo half) - stores = [w_store(m, n, to_bits(mat_d[m*16+n]), d_map(m, n)[1]) for m in range(16) for n in range(16)] + stores = [w_store(m, n, to_bits(mat_d[m]), d_map(m, n)[1]) for m in range(16)] else: # f32/i32 - stores = [w_store(m, n, mat_d[m*16+n].bitcast(dtypes.uint32), d_map(m, n)[1]) for m in range(16) for n in range(16)] - return UOp.sink(*stores, *ctx.inc_pc()) + stores = [w_store(m, n, mat_d[m].bitcast(dtypes.uint32), d_map(m, n)[1]) for m in range(16)] + return UOp.sink(UOp.group(*stores).end(n)) def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P | irc.VOP3PX2, ctx: _Ctx) -> UOp: op_name = _op_name(inst) @@ -1409,15 +1326,15 @@ def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P | irc.VOP3PX2, ctx: _ if src0_is_acc and not vdst_is_acc: # v_accvgpr_read: VGPR[vdst] = ACCVGPR[src0] val = ctx.raccvgpr_dyn(src0_off - _c(256), lane) - return UOp.sink(ctx.wvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane), *ctx.inc_pc()) + return UOp.sink(ctx.wvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane)) elif vdst_is_acc and not src0_is_acc: # v_accvgpr_write: ACCVGPR[vdst] = src0 (src0 can be VGPR or SGPR/const) src0 = ctx.rsrc_dyn(src0_off, lane, 32) - return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, src0, exec_mask).end(lane), *ctx.inc_pc()) + return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, src0, exec_mask).end(lane)) else: # v_accvgpr_mov: ACCVGPR[vdst] = ACCVGPR[src0] val = ctx.raccvgpr_dyn(src0_off - _c(256), lane) - return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane), *ctx.inc_pc()) + return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane)) lane = ctx.range() exec_mask = ctx.rexec() @@ -1447,7 +1364,7 @@ def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P | irc.VOP3PX2, ctx: _ hi_val = _pk_sel(src1, ctx.inst_field(type(inst).src1), opsel & 2) lo_out, hi_out = _split64(_u64(lo_val, hi_val)) stores = [ctx.wvgpr_dyn(vdst_reg, lane, lo_out, exec_mask), ctx.wvgpr_dyn(vdst_reg + _c(1), lane, hi_out, exec_mask)] - return UOp.sink(UOp.group(*stores).end(lane), *ctx.inc_pc()) + return UOp.sink(UOp.group(*stores).end(lane)) srcs: dict[str, UOp | int] = {} if is_pk_f32: @@ -1529,7 +1446,7 @@ def _compile_vopd(inst: ir3.VOPD | ir4.VOPD, ctx: _Ctx) -> UOp: srcs.update(ctx.base_srcs(exec_mask, lane), VCC=ctx.rmask(_c(VCC_LO.offset))) for dest, val in parse_pcode(pcode, srcs)[1]: if dest.startswith('D0'): all_stores.append(ctx.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask, after=srcy1)) - return UOp.sink(UOp.group(*all_stores).end(lane), *ctx.inc_pc()) + return UOp.sink(UOp.group(*all_stores).end(lane)) def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLAT|ir4.VGLOBAL|ir4.VSCRATCH |irc.DS|irc.FLAT|irc.GLOBAL|irc.SCRATCH, ctx: _Ctx) -> UOp: @@ -1578,7 +1495,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA values = [result[f'thread_out@{i}'] for i in range(ctx.wave_size)] # Snapshot every source before writing: destination and source registers may be identical. reads = UOp(Ops.STACK, src=tuple(values)) - return UOp.sink(*(ctx.wvgpr_dyn(vdst_reg, _c(i), val, exec_mask, after=reads) for i, val in enumerate(values)), *ctx.inc_pc()) + return UOp.sink(*(ctx.wvgpr_dyn(vdst_reg, _c(i), val, exec_mask, after=reads) for i, val in enumerate(values))) # DS_PERMUTE/DS_BPERMUTE: cross-lane VGPR access via pcode if is_lds and 'PERMUTE' in op_name: @@ -1586,7 +1503,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA 'EXEC': exec_mask.cast(dtypes.uint64), '_vgpr': ctx.vgpr, '_wave_size': ctx.wave_size} _, assigns = parse_pcode(pcode, srcs) stores = [ctx.vgpr.index(val[0]).store(val[1].cast(dtypes.uint32)) for dest, val in assigns if dest.startswith('VGPR[')] - return UOp.sink(*stores, *ctx.inc_pc()) + return UOp.sink(*stores) def make_addr(lane: UOp) -> UOp: if is_lds: @@ -1645,7 +1562,8 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA if 'D16' in op_name: data['RETURN_DATA'] = _rvdata(vdst_reg, lane) # RDNA3 uses ADDR/OFFSET, RDNA4 uses vgpr_a/offset (lowercase) + CalcDsAddr function return {'ADDR': addr, 'ADDR_BASE': addr, 'OFFSET': offset, 'OFFSET0': offset0, 'OFFSET1': offset1, '_lds': mem, 'laneId': lane, - 'vgpr_a': ctx.rvgpr_dyn(addr_reg, lane), 'offset': offset, 'offset0': offset0, 'offset1': offset1, **data} + 'vgpr_a': ctx.rvgpr_dyn(addr_reg, lane), 'offset': offset, 'offset0': offset0, 'offset1': offset1, + '_active': _lane_active(exec_mask, lane), **data} active = _lane_active(exec_mask, lane) # saddr < 124 means valid SGPR pair, otherwise use 0 (NULL means no saddr contribution) use_saddr = (saddr_reg < _c(124)) if saddr_reg is not None else UOp.const(False) @@ -1660,8 +1578,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA if is_atomic: atomic_data = _u64(ctx.rvgpr_dyn(vdata_reg, lane), ctx.rvgpr_dyn(vdata_reg + _c(1), lane)) \ if data_bits_mem == 64 else ctx.rvgpr_dyn(vdata_reg, lane) - return {'ADDR': addr, 'DATA': atomic_data, '_vmem': mem, '_active': active, - 'laneId': lane, 'v_addr': vaddr_base, 's_saddr': saddr_base} + return {'ADDR': addr, 'DATA': atomic_data, '_vmem': mem, '_active': active, 'laneId': lane, 'v_addr': vaddr_base, 's_saddr': saddr_base} vdata = _rvdata(vdata_reg, lane).cast(dtypes.uint64) if 'STORE' in op_name \ else _rvdata(vdst_reg, lane) if 'D16' in op_name else UOp.const(0, dtypes.uint32) if 'STORE' in op_name and data_bits_mem >= 64: @@ -1680,8 +1597,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA if is_lds or is_atomic: if data_bits < 32 and is_lds: return [wmem(val[0], val[1], active, data_bits)] return _write_val(data_bits, val[1], wmem, val[0], active, is_mem=True) - if is_scratch: return _mem_store_bytes(mem, val[0], val[1], active, data_bits) - return _mem_store(mem, val[0], val[1], active, 64, data_bits) + return _mem_store_bytes(mem if is_scratch else mem.bitcast(dtypes.uint8), val[0], val[1], active, data_bits) if dest.startswith('RETURN_DATA') and writes_return_data: write_gpr = ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn _wdata = lambda r, v, l, e: write_gpr(r, l, v, e) # noqa: E731 (arg order: reg, val, lane, exec) @@ -1718,7 +1634,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA for i, dest in load_assigns: load_stores.extend(make_stores(dest, lane_assigns[i][1], lane, active, True)) if load_stores: ended.append(UOp.group(*load_stores).end(lane)) - return UOp.sink(*ended, *ctx.inc_pc()) + return UOp.sink(*ended) # Standard path: single lane range writes_return_data = '_RTN' in op_name or (is_lds and (op_name.startswith('DS_LOAD') or op_name.startswith('DS_READ'))) or bool(is_atomic and glc) @@ -1734,7 +1650,7 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA for dword_idx, val in sorted(_collect_data_slices(assigns, 'VDATA', pcode_vars, op_name).items()): stores.append(_wdst(vdst_reg + _c(dword_idx), lane, val, exec_mask)) - return UOp.sink(UOp.group(*stores).end(lane), *ctx.inc_pc()) + return UOp.sink(UOp.group(*stores).end(lane)) def _compile_mubuf(inst: irc.MUBUF, ctx: _Ctx) -> UOp: """CDNA MUBUF: linear buffer address = base + soffset + (stride * index) + vgpr_offset + inst_offset""" @@ -1790,7 +1706,7 @@ def _compile_mubuf(inst: irc.MUBUF, ctx: _Ctx) -> UOp: word_addr = (addr + UOp.const(i * 4, dtypes.uint64)) >> UOp.const(2, dtypes.uint64) val = in_bounds.where(mem.index(word_addr.cast(dtypes.int64).valid(in_bounds)).load(), _c(0)) stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)(vdata + _c(i), lane, val, exec_mask)) - return UOp.sink(UOp.group(*stores).end(lane), *ctx.inc_pc()) + return UOp.sink(UOp.group(*stores).end(lane)) # Dispatch table: instruction type -> handler function. Classes are looked up by name across all three archs. def _inst_kinds(*names: str) -> tuple[type, ...]: @@ -1825,226 +1741,292 @@ def _get_handler(inst: Inst) -> Callable[..., UOp]: # PROGRAM DECODE AND COMPILATION # ═══════════════════════════════════════════════════════════════════════════════ -_canonical_runner_cache: list[tuple[type, int, int, int, tuple[UOp, object]]] = [] # [(inst_type, base, mask, size, (prg, runtime)), ...] - -def _canonical_info(inst:Inst, ctx:_Ctx, inst_bytes:bytes) -> tuple[int, int, int, str]: - base, mask, size = ctx.canonical_mask(inst_bytes) - return base, mask, size, f"{_op_name(inst).lower()}_{base.to_bytes(size, 'little').hex()}" - -@functools.cache -def _get_runner(inst_bytes: bytes, arch: str = "rdna3"): - """Build and compile instruction to (prg, runtime). Cached by instruction bytes, with canonical dedup.""" - inst = decode_inst(inst_bytes, arch) - inst_size = inst.size() - inst_int = int.from_bytes(inst_bytes[:inst_size], 'little') - - # Check if instruction matches any cached canonical pattern (must also match instruction type to avoid variant conflicts) - for inst_type, base, mask, size, entry in _canonical_runner_cache: - if type(inst) is inst_type and inst_size == size and (inst_int & mask) == base: return entry - - ctx = _Ctx(inst_size, _wave_size(arch)) - sink = _get_handler(inst)(inst, ctx) - base, mask, size, canonical_name = _canonical_info(inst, ctx, inst_bytes) - sink = sink.replace(arg=KernelInfo(name=canonical_name)).rtag(1) - - # NOTE: renderer output is not reproducible because of _MXCSRContext. - with Context(NOOPT=1, CHECK_OOB=0, TUPLE_ORDER=0, EMULATED_DTYPES="", CAPTURE_PROCESS_REPLAY=0): - prg = to_program(sink, Device['CPU'].renderer) - runtime = get_runtime('CPU', prg) - _canonical_runner_cache.append((type(inst), base, mask, size, (prg, runtime))) - return prg, runtime - _BARRIER_OPS = {ir3.SOPPOp.S_BARRIER, irc.SOPPOp.S_BARRIER} if hasattr(ir4.SOPPOp, 'S_BARRIER_WAIT'): _BARRIER_OPS.add(ir4.SOPPOp.S_BARRIER_WAIT) _BARRIER_SOP1_OPS: set = set() if hasattr(ir4.SOP1Op, 'S_BARRIER_SIGNAL'): _BARRIER_SOP1_OPS.add(ir4.SOP1Op.S_BARRIER_SIGNAL) -_BRANCH_OPS: set[int] = {op.value for op in (ir3.SOPPOp.S_BRANCH, ir3.SOPPOp.S_CBRANCH_SCC0, ir3.SOPPOp.S_CBRANCH_SCC1, - ir3.SOPPOp.S_CBRANCH_VCCZ, ir3.SOPPOp.S_CBRANCH_VCCNZ, ir3.SOPPOp.S_CBRANCH_EXECZ, ir3.SOPPOp.S_CBRANCH_EXECNZ)} - -def _decode_at(pc: int, arch: str): - """Decode and compile instruction at absolute address pc. Returns (runner, decoded_inst).""" - inst_bytes = bytes((ctypes.c_char * 16).from_address(pc).raw) - inst = decode_inst(inst_bytes, arch) - try: return _get_runner(bytes(inst_bytes[:inst.size() + 4]), arch), inst - except Exception as e: - try: inst_str = repr(inst) - except Exception: inst_str = f"<{type(inst).__name__}>" - raise RuntimeError(f"[emu] Failed to compile {inst_str}: {type(e).__name__}: {e}") from e - -# ═══════════════════════════════════════════════════════════════════════════════ -# WAVE STATE -# ═══════════════════════════════════════════════════════════════════════════════ # Inline float constants (as bit patterns) for GPU instructions F32_INLINE = {240: 0x3f000000, 241: 0xbf000000, 242: 0x3f800000, 243: 0xbf800000, # 0.5, -0.5, 1.0, -1.0 244: 0x40000000, 245: 0xc0000000, 246: 0x40800000, 247: 0xc0800000, 248: 0x3e22f983} # 2.0, -2.0, 4.0, -4.0, 1/(2*pi) -class WaveState: - __slots__ = ('vgpr_buf', 'sgpr_buf', 'accvgpr_buf', '_vgpr_mv', '_sgpr_mv', 'n_lanes', 'wave_size') +instruction_cache:dict[tuple, list[tuple[int, int, tuple[UOp|None, UOp|None]]]] = {} - def __init__(self, n_lanes: int, wave_size: int = 32): - self.n_lanes, self.wave_size = n_lanes, wave_size - vgpr_size = 256 * wave_size - self.vgpr_buf = Buffer('CPU', vgpr_size * 4).ensure_allocated() - self.sgpr_buf = Buffer('CPU', SGPR_COUNT * 4).ensure_allocated() - # CDNA (wave64) has separate ACCVGPR file; RDNA shares with VGPR - if wave_size == 64: - self.accvgpr_buf = Buffer('CPU', vgpr_size * 4).ensure_allocated() - ctypes.memset(self.accvgpr_buf._buf, 0, vgpr_size * 4) - else: - self.accvgpr_buf = self.vgpr_buf - self._vgpr_mv = self.vgpr_buf.host.view(fmt='I').mv - self._sgpr_mv = self.sgpr_buf.host.view(fmt='I').mv - # Zero memory using ctypes memset (much faster than Python loops) - ctypes.memset(self.vgpr_buf._buf, 0, vgpr_size * 4) - ctypes.memset(self.sgpr_buf._buf, 0, SGPR_COUNT * 4) - # Pre-populate inline constants at indices 128-255 - for i in range(65): self._write_sgpr(128 + i, i) # 128-192: integers 0-64 - for i in range(16): self._write_sgpr(193 + i, (-(i + 1)) & MASK32) # 193-208: -1 to -16 - for off, val in F32_INLINE.items(): self._write_sgpr(off, val) # 240-248: float constants - # EXEC mask: for 64-lane waves, set both EXEC_LO and EXEC_HI - if wave_size == 64: - self._write_sgpr(EXEC_LO.offset, (1 << min(n_lanes, 32)) - 1) - self._write_sgpr(EXEC_LO.offset + 1, (1 << max(n_lanes - 32, 0)) - 1 if n_lanes > 32 else 0) - else: - self._write_sgpr(EXEC_LO.offset, (1 << n_lanes) - 1) - self._write_sgpr(PC_LO_IDX, 0) - self._write_sgpr(PC_HI_IDX, 0) - - def _write_sgpr(self, idx: int, val: int): self._sgpr_mv[idx] = val & MASK32 - def _read_sgpr(self, idx: int) -> int: return self._sgpr_mv[idx] - def _write_vgpr(self, reg: int, lane: int, val: int): self._vgpr_mv[reg * self.wave_size + lane] = val & MASK32 - def _read_vgpr(self, reg: int, lane: int) -> int: return self._vgpr_mv[reg * self.wave_size + lane] - - @property - def pc(self) -> int: return self._read_sgpr(PC_LO_IDX) | (self._read_sgpr(PC_HI_IDX) << 32) - @pc.setter - def pc(self, val: int): - self._write_sgpr(PC_LO_IDX, val & MASK32) - self._write_sgpr(PC_HI_IDX, (val >> 32) & MASK32) - -# ═══════════════════════════════════════════════════════════════════════════════ -# EXECUTION -# ═══════════════════════════════════════════════════════════════════════════════ - -def _init_wave(lib: int, wave_start: int, total_threads: int, lx: int, ly: int, lz: int, args_ptr: int, rsrc2: int, - scratch_size: int, arch: str, gidx: int, gidy: int, gidz: int, user_data: list[int]|None, - wave_size: int = 32) -> WaveState: - """Initialize a single wavefront and return WaveState.""" - n_lanes = min(wave_size, total_threads - wave_start) - st = WaveState(n_lanes, wave_size) - st.pc = lib - if user_data: - for i, val in enumerate(user_data): st._write_sgpr(i, val) +# this is meant to replace the old emulator +@uopfunc +def init_wave(wg:UOp, wave:UOp, sgpr:UOp, vgpr:UOp, args_ptr:UOp, gx:int, gy:int, lx:int, ly:int, total_threads:int, wave_size:int, + scratch_size:int, rsrc2:int, arch:str="rdna3", user_data_count:int=0): + ci = UOp.range(81, 2, AxisType.LOOP, dtype=dtypes.int) + init_ints = sgpr.index(ci+128).store((ci<65).where(ci, 64-ci).cast(dtypes.uint32)).end(ci) + fi = UOp.range(len(F32_INLINE), 3, AxisType.LOOP, dtype=dtypes.int) + floats_buf = UOp(Ops.BINARY, arg=struct.pack(f"<{len(F32_INLINE)}I", *F32_INLINE.values())).bitcast(dtypes.uint32) + init_floats = sgpr.index(fi+240).store(floats_buf[fi]).end(fi) + # set RANGE registers + gidx, gidy, gidz = wg%gx, (wg//gx)%gy, wg//(gx*gy) + n_lanes = (total_threads-wave*wave_size).minimum(wave_size) + initial:list[tuple[int, sint]] = [(124, 0)] if wave_size == 32 else [] + if user_data_count: initial += [(i, UOp.variable(f"user_data_{i}", 0, dtypes.uint32.max, dtypes.uint32)) for i in range(user_data_count)] + else: initial += [(0, args_ptr.cast(dtypes.uint32)), (1, (args_ptr>>32).cast(dtypes.uint32))] + if arch == "rdna4": initial += [(ttmp[7].offset, (gidy&0xFFFF)|((gidz&0xFFFF)<<16)), (ttmp[9].offset, gidx)] else: - st._write_sgpr(0, args_ptr & MASK32) - st._write_sgpr(1, (args_ptr >> 32) & MASK32) - if arch == "rdna4": - # workgroup IDs only exist in ttmp registers, not normal SGPRs - st._write_sgpr(ttmp[7].offset, (gidy & 0xFFFF) | ((gidz & 0xFFFF) << 16)) - st._write_sgpr(ttmp[9].offset, gidx) - else: - sgpr_idx = (rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_USER_SGPR_COUNT) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_USER_SGPR_COUNT_SHIFT + sgpr_id = (rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_USER_SGPR_COUNT) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_USER_SGPR_COUNT_SHIFT for enabled, gid in [(hsa.AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_X, gidx), (hsa.AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_Y, gidy), (hsa.AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_Z, gidz)]: if rsrc2 & enabled: - st._write_sgpr(sgpr_idx, gid) - sgpr_idx += 1 - for lane in range(n_lanes): - tid = wave_start + lane - st._write_vgpr(0, lane, ((tid // (lx * ly)) << 20) | (((tid // lx) % ly) << 10) | (tid % lx)) - st._write_sgpr(SCRATCH_STRIDE_IDX, scratch_size) - # Store HW register values at SGPR[SGPR_COUNT-16 .. SGPR_COUNT-1] for s_getreg_b32 emulation. - # HW_ID (hwRegId=4): WAVE_ID[3:0], SIMD_ID[5:4], PIPE_ID[7:6], CU_ID[11:8], ... - wave_idx = wave_start // wave_size # wave index within this workgroup (0, 1, 2, 3 for 256 threads / 64 wave_size) - hw_id = (wave_idx & 0xF) | ((wave_idx & 0x3) << 4) # WAVE_ID = wave_idx, SIMD_ID = wave_idx % 4 - st._write_sgpr(SGPR_COUNT - 16 + 4, hw_id) # HW_REGISTERS[4] = HW_ID - return st + initial.append((sgpr_id, gid)) + sgpr_id += 1 + initial += [(EXEC_LO.offset, ((UOp.const(1, dtypes.uint64)< UOp: + bufs = sorted((u for u in sink.toposort() if u.op is Ops.PARAM), key=lambda u: u.arg.slot) + body = sink.substitute({b:b.param_like(i, name=b.arg.name) for i,b in enumerate(bufs)}, walk=True) + return body.call(*bufs, name=name) + +def instruction_call(inst:Inst, arch:str, renderer:Renderer) -> tuple[UOp|None, UOp|None]: + entries = instruction_cache.setdefault((arch, type(inst), inst.size(), type(renderer), renderer.target, *(x.value for x in to_program_config)), []) + inst_bytes = inst.to_bytes() + bits = int.from_bytes(inst_bytes, "little") + for base, mask, ret in entries: + if bits & mask == base: break + else: + ctx = _Ctx(_wave_size(arch)) + sink = _get_handler(inst)(inst, ctx) + base, mask = ctx.canonical_mask(inst_bytes) + name = f"{_op_name(inst).lower()}_{base.to_bytes(len(inst_bytes), 'little').hex()}" + ret = lower_body(sink, name) if sink.src else None, ctx.branch_cond + entries.append((base, mask, ret)) + call, cond = ret + words = {_Ctx.inst_word(i):_c((bits >> (32*i)) & 0xffffffff) for i in range(inst.size()//4)} + if call is not None: call = call.replace(src=(call.body, *(words.get(b, b) for b in call.src[1:]))) + return call, cond.substitute(words) if cond is not None else None + +def instruction_block(instructions:list[UOp]) -> tuple[UOp, list[UOp]]: + bufs = sorted({b for call in instructions for b in call.src[1:] if b.op is Ops.PARAM}, key=lambda b:b.arg.slot) + params = {b:b.param_like(i, name=b.arg.name) for i,b in enumerate(bufs)} + linear = list(UOp.sink(*params.values()).toposort())[:-1] + seen = set(linear) + for call in instructions: + args = [params.get(b, b) for b in call.src[1:]] + inputs = UOp.sink(*args) + nodes = inputs.toposort(lambda u:u not in seen) + nodes.pop(inputs) + seen.update(nodes) + linear.extend(nodes) + linear.append(call.replace(src=(call.body, *args))) + return UOp(Ops.PROGRAM, src=(sink:=UOp.sink(*linear, arg=KernelInfo("asm_block", optimized=True)), UOp(Ops.LINEAR, src=(*linear, sink)))), bufs + +def thread_branches(insts, blocks:dict[int, list[int]], paths:dict[int, dict[int, int]]): + # thread jump-only blocks. + targets:dict[int, int] = {} + def target(pc:int) -> int: + seen:set[int] = set() + while pc not in targets and pc not in seen and len(blocks[pc]) == 1 and getattr(insts[blocks[pc][0]], "op_name", "") == "S_BRANCH": + seen.add(pc) + pc = next(iter(paths[pc])) + pc = targets.get(pc, pc) + targets.update((src, pc) for src in seen) + return pc + for pc, dsts in list(paths.items()): paths[pc] = {target(dst):kind for dst,kind in dsts.items()} + +def merge_branches(insts, blocks:dict[int, list[int]], paths:dict[int, dict[int, int]]): + groups:dict[tuple, list[int]] = {} + opposite = {"S_CBRANCH_SCC1":"S_CBRANCH_SCC0", "S_CBRANCH_VCCNZ":"S_CBRANCH_VCCZ", "S_CBRANCH_EXECNZ":"S_CBRANCH_EXECZ"} + for pc, pcs in blocks.items(): + if len(paths[pc]) != 2: continue + inst = insts[pcs[-1]] + # complementary tests with reversed destinations are the same branch. + branch_dests = tuple(sorted((dst, kind ^ (inst.op_name in opposite)) for dst,kind in paths[pc].items())) + key = (type(inst), opposite.get(inst.op_name, inst.op_name), + tuple((n, getattr(inst, n)) for n,_ in inst._fields if n not in {"op", "simm16"}), branch_dests) + groups.setdefault(key, []).append(pc) + for group in groups.values(): + if len(group) < 2: continue + header = blocks[group[0]][-1] + dests = paths[group[0]] + for pc in group: + blocks[pc] = blocks[pc][:-1] + paths[pc] = {header:UNCOND} + blocks[header], paths[header] = [header], dests + +# a block has an empty body; a loop contains its header block and the rest of its iteration. +CFGRegion = tuple[int, list["CFGRegion"]] +class IrreducibleCFG(RuntimeError): pass + +def cfg_regions(paths:dict[int, dict[int, int]], entry:int) -> list[CFGRegion]: + # number the reachable blocks in DFS postorder, without recursion on long instruction streams. + order:dict[int, int] = {} + seen, pending = set(), [(entry, False)] + while pending: + pc, visited = pending.pop() + if visited: order[pc] = len(order) + elif pc not in seen: + seen.add(pc) + pending.append((pc, True)) + pending.extend((dst, False) for dst in sorted(paths[pc], reverse=True)) + preds:dict[int, set[int]] = {pc:set() for pc in order} + for pc in order: + for dst in paths[pc]: preds[dst].add(pc) + def build(remaining:set[int]) -> list[CFGRegion]: + regions: list[CFGRegion] = [] + # reverse postorder and predecessor walks partition this region into SCCs in execution order. + for pc in sorted(remaining, key=order.__getitem__, reverse=True): + if pc not in remaining: continue + component, todo = set(), [pc] + while todo: + if (dst:=todo.pop()) not in remaining: continue + remaining.remove(dst) + component.add(dst) + todo.extend(preds[dst]) + body:list[CFGRegion] = [] + if len(component) > 1 or pc in paths[pc]: + if any(preds[dst] - component for dst in component if dst != pc): + raise IrreducibleCFG("irreducible control flow is not supported by the AMD lifter") + # removing the single entry exposes nested loops; the original DFS order still applies. + body = [(pc, [])] + build(component - {pc}) + regions.append((pc, body)) + return regions + return build(set(order)) + +@rewrite_group(replay="emu_process_replay") +def lift(code:bytes, arch:str, renderer:Renderer, dispatch:tuple, name:str="") -> UOp: + insts = amd_decode(code, arch) + for inst in insts.values(): + inst_st = getattr(inst, "op_name", "").lower() + if inst_st.startswith(("s_getpc", "s_setpc")): raise NotImplementedError(f"AMD lifter does not support {inst_st}") + cfg = get_cfg(insts) + thread_branches(insts, cfg["blocks"], cfg["paths"]) + try: regions = cfg_regions(cfg["paths"], 0) + except IrreducibleCFG: + # sharing branch tails can make rotated loops reducible, but adds dispatch blocks otherwise. + merge_branches(insts, cfg["blocks"], cfg["paths"]) + regions = cfg_regions(cfg["paths"], 0) + # propagate possible wave PCs across acyclic regions; loops retain runtime routing. + unconditional:set[int] = set() + possible:set[int]|None = {0} + for pc, body in regions: + if body: possible = None + elif possible is not None: + if possible == {pc}: unconditional.add(pc) + possible = (possible-{pc}) | (set(cfg["paths"][pc]) or {-1}) + routed = len(unconditional) != len(regions) + axes = itertools.count(sum(inst.size() for inst in insts.values())) + gx, gy, gz, lx, ly, lz, rsrc2, scratch_size, user_data_count = dispatch + # construct CALL graph + wave_size, total_threads = _wave_size(arch), lx*ly*lz + n_waves = (total_threads+wave_size-1)//wave_size + wg = UOp.range(gx*gy*gz, 0, AxisType.LOOP, dtype=dtypes.uint64) + wave = UOp.range(n_waves, 1, AxisType.LOOP, dtype=dtypes.int) + # alloc register and LDS buffers + sgpr = UOp.alloc((n_waves*SGPR_COUNT,), dtypes.uint32, 0, AddrSpace.REG) + vgpr = UOp.alloc((n_waves*256*wave_size,), dtypes.uint32, 1, AddrSpace.REG) + lds_size = ((rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE_SHIFT)*512 + lds = UOp.alloc((max(lds_size//4, 1),), dtypes.uint32, 3, AddrSpace.REG) + scratch = UOp.alloc((max(scratch_size*wave_size*n_waves, 1),), dtypes.uint8, 4, AddrSpace.REG) + accvgpr = UOp.alloc((256*wave_size*n_waves,), dtypes.uint32, 5, AddrSpace.REG) if wave_size == 64 else vgpr + args_ptr = UOp.variable("args_ptr", 0, dtypes.uint64.max, dtypes.uint64) + ctx = _Ctx(wave_size) + buffers = {ctx.sgpr:(sgpr, SGPR_COUNT), ctx.vgpr:(vgpr, 256*wave_size), ctx.vmem:(ctx.vmem, 0), + ctx.lds:(lds, 0), ctx.scratch:(scratch, scratch_size*wave_size)} + if wave_size == 64: buffers[ctx.accvgpr] = (accvgpr, 256*wave_size) + routes = UOp.alloc((n_waves,), dtypes.int, 6, AddrSpace.REG) + active = UOp.alloc((1,), dtypes.bool, 7, AddrSpace.REG) + route = UOp.param(6, dtypes.int, (1,), name="route") + taken, fallthrough = [UOp.param(i, dtypes.int, name=name, addrspace=AddrSpace.ALU) for i,name in enumerate(("taken", "fallthrough"), 7)] + buffers[route] = (routes, 1) + init = init_wave(wg, wave, sgpr.shrink(((wave*SGPR_COUNT, (wave+1)*SGPR_COUNT),)), vgpr.shrink(((wave*256*wave_size, (wave+1)*256*wave_size),)), + args_ptr, gx, gy, lx, ly, total_threads, wave_size, scratch_size, rsrc2, arch, user_data_count) + user_vars = [u for u in init.body.toposort() if u.is_variable] + init = init.replace(src=(init.body.substitute({u:UOp.param(len(init.src)-1+i, u.dtype, name=u.arg.name, addrspace=AddrSpace.ALU) + for i,u in enumerate(user_vars)}, walk=True), *init.src[1:], *user_vars)) + initialized = (routes.after(init).index(wave).store(0) if routed else UOp.group(init, wave)).end(wave) + def emit_block(block_pc:int, end:UOp, scopes:tuple[UOp, ...]) -> UOp: + cond = UOp.const(True) + instructions = [] + for off in cfg["blocks"][block_pc]: + inst = insts[off] + inst_st = getattr(inst, "op_name", "").lower() + if inst_st.startswith("s_code_end"): continue + call, branch_cond = instruction_call(inst, arch, renderer) + if branch_cond is not None: cond = branch_cond + if call is not None: instructions.append(call) + values = {} + if routed: + dsts = {kind:dst for dst,kind in cfg["paths"][block_pc].items()} + target = cond.where(taken, fallthrough) if len(dsts) == 2 else taken + values = {taken:UOp.const(dsts[COND_TAKEN], dtypes.int), fallthrough:UOp.const(dsts[COND_NOT_TAKEN], dtypes.int)} if len(dsts) == 2 \ + else {taken:UOp.const(next(iter(dsts.values()), -1), dtypes.int)} + instructions.append(lower_body(route[0].store(target).sink(), "asm_branch")) + if not instructions: return end + body, bufs = instruction_block(instructions) + args = [values[b] if b.addrspace is AddrSpace.ALU else buffers[b][0] for b in bufs] + args += [UOp.const(block_pc, dtypes.int).after(end, *scopes)] + params = [UOp.param(i, a.dtype, a.shape, addrspace=AddrSpace.ALU if i >= len(bufs) or bufs[i].addrspace is AddrSpace.ALU + else AddrSpace.GLOBAL) for i,a in enumerate(args)] + # lower the wave loop with the block, so repeated blocks share its indexing and route check too. + w = UOp.range(n_waves, 0, AxisType.LOOP, dtype=dtypes.int) + gates = () if block_pc in unconditional else (UOp.range(params[bufs.index(route)][w].load().eq(params[-1]).cast(dtypes.int), + 1, AxisType.LOOP, dtype=dtypes.int),) + call_args = [p if b.addrspace is AddrSpace.ALU else p.after(*gates, w).shrink(((w*s, (w+1)*s),)) + if (s:=buffers[b][1]) else p.after(*gates, w) for b,p in zip(bufs, params)] + block = UOp.group(body.call(*call_args, name="asm_block"), *gates, w).end(*gates).end(w).sink() + return block.call(*args, name="asm_waves") + def emit_dispatch(regions:list[CFGRegion], scopes:tuple[UOp, ...], end:UOp) -> UOp: + for pc, body in regions: + if body: + loop = UOp.loop(next(axes)).replace(src=(UOp(Ops.NOOP).after(end),)) + end = emit_dispatch(body, scopes+(loop,), loop) + # waves that exited keep their route while the remaining waves repeat the loop. + check = loop_active(routes.after(end, *scopes, loop), active, UOp.const(pc, dtypes.int), UOp.const(n_waves, dtypes.int)) + end = end.backedge(loop, active.after(check, *scopes, loop)[0].load()) + else: + # complete the block for every participating wave before entering its successors, including barrier successors. + end = emit_block(pc, end, scopes) + return end + + return emit_dispatch(regions, (wg,), initialized).end(wg).sink(ctx.vmem, arg=KernelInfo(name=name, optimized=True)) + +dispatch_cache:dict[tuple, UOp] = {} + +RELIFT = ContextVar("RELIFT", 0) # lift and recompile to assembly N times +REPLAY_EMU = ContextVar("REPLAY_EMU", 0) def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int, rsrc2: int = 0x19c, - scratch_size: int = 0, arch: str = "rdna3", user_data: list[int]|None = None) -> int: + scratch_size: int = 0, arch: str = "rdna3", user_data: list[int]|None = None, name: str = "asm") -> int: """Execute AMD assembly program. scratch_size is private_segment_fixed_size from kernel descriptor (per-lane).""" - if ASM_CALL: - from test.mockgpu.amd.call import lift - prg = lift(lib, lib_sz, gx, gy, gz, lx, ly, lz, rsrc2, scratch_size, arch, len(user_data or []), ASM_CALL_BACKEND) - vals = {"lib":lib, "args_ptr":args_ptr, **{f"user_data_{i}":v for i,v in enumerate(user_data or [])}} - with _MXCSRContext(): - get_runtime(ASM_CALL_BACKEND, prg)(*[0]*len(prg.arg.globals), vals=tuple(vals[v.arg.name] for v in prg.arg.vars)) - return 0 - - program: dict[int, tuple[Callable, list[int], bool, Inst]] = {} # pc -> (fxn, globals, is_barrier, inst) - lds_size = ((rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE_SHIFT) * 512 - total_threads = lx * ly * lz - wave_size = _wave_size(arch) - - # Use Buffer objects with external_ptr=0 for vmem - vmem_buf = Buffer('CPU', 1 << 42, options=BufferSpec(external_ptr=0)).ensure_allocated() - lds_buf = Buffer('CPU', max(lds_size // 4, 1) * 4).ensure_allocated() - # Scratch is per-lane private memory: each wave needs its own region so data spilled before s_barrier survives other waves' execution. - n_waves = -(-total_threads // wave_size) - scratch_buf = Buffer('CPU', scratch_size * wave_size * n_waves).ensure_allocated() if scratch_size else None - - # Initialize SQTT encoder — emits packets inline as instructions execute (only when profiling) - if PROFILE: - sqtt_emit, sqtt_finish, sqtt_finalize = _make_sqtt_encoder() - - def _ensure_compiled(pc: int) -> tuple[Callable, list[int], bool, Inst]: - if pc not in program: - prev_len = len(_canonical_runner_cache) - (prg, runtime), inst = _decode_at(pc, arch) - is_barrier = (isinstance(inst, (ir3.SOPP, ir4.SOPP, irc.SOPP)) and inst.op in _BARRIER_OPS) or \ - (isinstance(inst, (ir4.SOP1,)) and inst.op in _BARRIER_SOP1_OPS) - program[pc] = (runtime.fxn, prg.arg.globals, is_barrier, inst) - if DEBUG >= 3: - msg = f"[emu] PC={pc - lib}: {inst!r}" - print(colored(msg, 'green') if len(_canonical_runner_cache) > prev_len else msg) - return program[pc] - - def _run_workgroup(gidx: int, gidy: int, gidz: int, tracing: bool): - """Initialize all wavefronts for one workgroup and execute them with barrier synchronization. - Each wave runs until it hits s_barrier or s_endpgm. When all waves have stopped, release barrier waves.""" - waves: list[tuple[WaveState, list]] = [] - for wave_start in range(0, total_threads, wave_size): - st = _init_wave(lib, wave_start, total_threads, lx, ly, lz, args_ptr, rsrc2, scratch_size, arch, gidx, gidy, gidz, user_data, wave_size) - scratch_base = scratch_buf._buf + (wave_start // wave_size) * scratch_size * wave_size if scratch_buf else 0 - waves.append((st, [ctypes.c_uint64(st.sgpr_buf._buf), ctypes.c_uint64(st.vgpr_buf._buf), - ctypes.c_uint64(vmem_buf._buf), ctypes.c_uint64(lds_buf._buf), - ctypes.c_uint64(scratch_base if scratch_buf else 0), ctypes.c_uint64(st.accvgpr_buf._buf)])) - done = [False] * len(waves) - for _ in range(10_000_000): - if all(done): return - for wi, (st, c_bufs) in enumerate(waves): - if done[wi]: continue - # Run this wave until barrier or endpgm - for _ in range(1_000_000): - pc = st.pc - if pc == ENDPGM_PC: - done[wi] = True - if tracing: sqtt_finish(wi) - break - fxn, globals_list, is_barrier, inst = _ensure_compiled(pc) - if DEBUG >= 5: print(f" exec gid=({gidx},{gidy},{gidz}) w={wi} PC={pc - lib}: {inst!r}", flush=True) - fxn(*[c_bufs[g] for g in globals_list]) - if tracing: - inst_op = inst.op.value if hasattr(inst, 'op') else 0 - sqtt_emit(wi, inst, (st.pc != ENDPGM_PC and st.pc != pc + inst.size()) if inst_op in _BRANCH_OPS else None) - if is_barrier: break # s_barrier hit: PC already advanced past it, pause this wave - else: raise RuntimeError("exceeded 1M instructions in single wave, likely infinite loop") - # All waves have either hit barrier or endpgm — release barrier waves for next round - raise RuntimeError("exceeded 10M total scheduling rounds") - - # Set DAZ+FTZ during emulator execution, restore afterward to avoid breaking hypothesis tests - # Only trace the first workgroup (like real HW traces one CU/SIMD), subsequent workgroups run but don't add to trace - tracing = bool(PROFILE) + with Context(DEV=DEV.target('CPU', renderer='LLVM'), NOOPT=1, CHECK_OOB=0, TUPLE_ORDER=0, EMULATED_DTYPES="", + CAPTURE_PROCESS_REPLAY=REPLAY_EMU.value, DEFAULT_FLOAT="float32", DEFAULT_INT="int32", TRANSCENDENTAL=1): + code = bytes(to_mv(lib, lib_sz)) + dispatch = (gx, gy, gz, lx, ly, lz, rsrc2, scratch_size, len(user_data or [])) + renderer = Device['CPU'].renderer + if (key:=(code, arch, dispatch, name, type(renderer), renderer.target, *(x.value for x in to_program_config))) not in dispatch_cache: + dispatch_cache[key] = to_program(lift(code, arch, renderer, dispatch, f"emu {name}"), renderer) + relift_lib = code + for relift_count in range(1, RELIFT.value+1): + try: + ret = to_program(lift(relift_lib, arch, Device["AMD"].renderer, dispatch, f"relift_{relift_count} {name}"), Device["AMD"].renderer) + except Exception: + print("relift err", traceback.format_exc()) + break + relift_lib = get_elf_section(ret.src[3].arg, ".text").content + prg = dispatch_cache[key] + vals = {"args_ptr":args_ptr, **{f"user_data_{i}":v for i,v in enumerate(user_data or [])}} with _MXCSRContext(): - for gidz, gidy, gidx in itertools.product(range(gz), range(gy), range(gx)): - _run_workgroup(gidx, gidy, gidz, tracing) - tracing = False # only trace the first workgroup - if lds_size > 0: ctypes.memset(lds_buf._buf, 0, max(lds_size, 4)) # reset LDS for next workgroup - - if PROFILE: sqtt_traces.append(sqtt_finalize()) + vmem_buf = UOp.from_buffer(Buffer('CPU', 1 << 48, options=BufferSpec(external_ptr=0), opaque=0), dtypes.uint32) + args = [vmem_buf] * (max(p.arg.slot for p in prg.kernel_params) + 1) + for p in prg.kernel_params: + if p.addrspace is AddrSpace.ALU: args[p.arg.slot] = p.bind(vals[p.arg.name]) + run_linear(UOp(Ops.LINEAR, src=(prg.call(*args),)), update_stats=False) return 0 diff --git a/tinygrad_repo/test/mockgpu/amd/pcode.py b/tinygrad_repo/test/mockgpu/amd/pcode.py index 2c652d50b1..9d989f77d7 100644 --- a/tinygrad_repo/test/mockgpu/amd/pcode.py +++ b/tinygrad_repo/test/mockgpu/amd/pcode.py @@ -568,8 +568,9 @@ class Parser: if op == '*': return left * right return (left // right) if dtypes.is_int(left.dtype) else (left / right) case '**': return UOp(Ops.EXP2, src=(right.cast(left.dtype),)) if _single_value(left) == 2.0 else left + case '%': return left % right - _PREC = [('||',), ('&&',), ('|',), ('^',), ('&',), ('==', '!=', '<>'), ('>=', '<=', '>', '<'), ('>>', '<<'), ('+', '-'), ('*', '/'), ('**',)] + _PREC = [('||',), ('&&',), ('|',), ('^',), ('&',), ('==', '!=', '<>'), ('>=', '<=', '>', '<'), ('>>', '<<'), ('+', '-'), ('*', '/', '%'), ('**',)] def binop(self, prec: int) -> UOp: if prec >= len(self._PREC): return self.unary() @@ -1449,7 +1450,7 @@ def parse_pcode(pcode: str, srcs: dict[str, UOp | int] | None = None, funcs: dic _, final, _ = parse_block(lines, 0, env, {**_FUNCS, **funcs} if funcs else None, assigns=assigns) sliced = set(d.split('[')[0] for d, _ in assigns if '[' in d) for var, val in final.items(): - if var in ['D0', 'S0', 'SCC', 'VCC', 'EXEC', 'PC', 'RETURN_DATA', 'VDATA'] and isinstance(val, UOp): + if var in ['D0', 'S0', 'SCC', 'VCC', 'EXEC', 'PC', 'RETURN_DATA', 'VDATA', 'SDATA'] and isinstance(val, UOp): if var in sliced and not any(re.match(rf'{var}\.\w+\s*=', l) for l in lines): continue for l in lines: if (m := re.match(rf'{var}\.(\w+(?:\[\w+\])?)', l)): diff --git a/tinygrad_repo/test/mockgpu/amd/replay.py b/tinygrad_repo/test/mockgpu/amd/replay.py new file mode 100644 index 0000000000..f269ba8e41 --- /dev/null +++ b/tinygrad_repo/test/mockgpu/amd/replay.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# usage: replay.py [clang,llvm,amd] [asm] +import functools, re, sys +from dataclasses import replace +from test.mockgpu.amd.emu import lift +from tinygrad.codegen import to_program +from tinygrad.device import Device +from tinygrad.helpers import VERSION, Target +from tinygrad.renderer import Renderer +from tinygrad.renderer.cstyle import ClangRenderer, HIPRenderer +from tinygrad.renderer.llvmir import CPULLVMRenderer +from tinygrad.uop.ops import UOp, Ops +from tinygrad.viz.serve import amd_decode, get_arch, get_cfg, get_elf_section, get_stdout, parse_branch + +def get_output(p:UOp, renderer:Renderer, asm:bool=False) -> str: + nodes = p.toposort(enter_calls=True) + parts = [x.arg for x in nodes if x.op is Ops.SOURCE] + for prg in (x for x in nodes if asm and x.op is Ops.PROGRAM): + binary = next(x.arg for x in prg.src if x.op is Ops.BINARY) + if renderer.target.device != "AMD": + disasm = get_stdout(lambda: renderer.compiler.disassemble(binary)) + parts.append(re.sub(r"^.*:\s+file format", ": file format", disasm, flags=re.MULTILINE)) + continue + text = get_elf_section(binary, ".text") + insts = amd_decode(text.content, get_arch(renderer.target.arch), text.header.sh_addr) + cfg = get_cfg(insts) + labels = {pc:f"%bb.{i}" for i,pc in enumerate(cfg["blocks"])} + blocks = [] + for start, pcs in cfg["blocks"].items(): + lines = [f"{labels[start][1:]}:"] + if successors:=cfg["paths"][start]: lines.append(" successors: " + ", ".join(labels[pc] for pc in successors)) + for pc in pcs: + op = getattr(inst:=insts[pc], "op_name", "").lower() + if (offset:=parse_branch(inst, op)) is not None: lines.append(f" {op} {labels[pc+inst.size()+offset]}") + else: lines.append(f" {inst}") + blocks.append("\n".join(lines)) + parts.append("\n\n".join(blocks)) + return "\n".join(parts) + +def replay_asm(ast:UOp, code:bytes, arch:str, renderer:Renderer, dispatch:tuple, name:str, + backends:tuple[str, ...], asm:bool=False) -> tuple[str, str, tuple]: + compare = lift(code, arch, renderer, dispatch, name) + good_outs, compare_outs = [], [] + for backend in backends: + target = Target("AMD", "HIP", {"rdna3":"gfx1100", "rdna4":"gfx1200", "cdna":"gfx950"}[arch]) if backend == "amd" else \ + replace(Device["CPU"].renderer.target, renderer=backend.upper()) + ren = {"clang":ClangRenderer, "llvm":CPULLVMRenderer, "amd":HIPRenderer}[backend](target) + good_outs.append(get_output(to_program(ast, ren), ren, asm)) + compare_outs.append(get_output(to_program(compare, ren), ren, asm)) + return "\n".join(good_outs), "\n".join(compare_outs), (name, backends, dispatch) + +if __name__ == "__main__": + from test.external.process_replay.process_replay import main + backends = tuple((sys.argv[1] if len(sys.argv) > 1 else "clang").split(",")) + main({"lift":functools.partial(replay_asm, backends=backends, asm=bool(sys.argv[2:]))}, f"emu_process_replay_{VERSION}") diff --git a/tinygrad_repo/test/mockgpu/amd/sqtt_enc.py b/tinygrad_repo/test/mockgpu/amd/sqtt_enc.py deleted file mode 100644 index 924d7ac292..0000000000 --- a/tinygrad_repo/test/mockgpu/amd/sqtt_enc.py +++ /dev/null @@ -1,100 +0,0 @@ -# SQTT trace encoder for the emulator (the decoder lives in tinygrad/renderer/amd/sqtt.py). -# run_asm emits packets inline as instructions execute; finished traces end up in emu.sqtt_traces. -from __future__ import annotations -from tinygrad.renderer.amd.dsl import Inst -from tinygrad.renderer.amd.sqtt import (_build_decode_tables, PACKET_TYPES_RDNA3, PacketType, InstOp, - LAYOUT_HEADER, WAVESTART, WAVEEND, INST, IMMEDIATE, VALUINST) - -_NIB_COUNTS = {cls: nc for _, (cls, nc, *_) in _build_decode_tables(PACKET_TYPES_RDNA3)[0].items()} - -def _emit_nibbles(nibbles: list[int], pkt_cls: type[PacketType], **kwargs): - raw = pkt_cls.encoding.default - for k, v in kwargs.items(): raw = pkt_cls.__dict__[k].set(raw, v) - nibbles.extend((raw >> (i * 4)) & 0xF for i in range(_NIB_COUNTS[pkt_cls])) - -def make_encoder(): - """Build an SQTT trace encoder for the emulator. Returns (emit, finish, finalize).""" - from tinygrad.runtime.autogen.amd.rdna3.enum import SOPPOp as SOPPOp3 - from tinygrad.runtime.autogen.amd.rdna4.enum import SOPPOp as SOPPOp4 - from tinygrad.runtime.autogen.amd.rdna3 import ins as ir3 - from tinygrad.runtime.autogen.amd.rdna4 import ins as ir4 - from tinygrad.runtime.autogen.amd.cdna import ins as irc - import re - - def _kinds(*names: str) -> tuple[type[Inst], ...]: - return tuple(getattr(m, n) for m in (ir3, ir4, irc) for n in names if hasattr(m, n)) - _SOPP, _SMEM, _DS = _kinds('SOPP'), _kinds('SMEM'), _kinds('DS') - _GLOBAL, _FLAT, _SCRATCH = _kinds('GLOBAL', 'VGLOBAL'), _kinds('FLAT', 'VFLAT'), _kinds('SCRATCH', 'VSCRATCH') - _VALU = _kinds('VOP1', 'VOP2', 'VOP3', 'VOP3P', 'VOP3PX2', 'VOPC', 'VOPD', 'VOP3SD', 'VOP3_SDST', 'VOP1_SDST') - - # SOPP classification sets - _SOPP_SKIP = {SOPPOp3.S_ENDPGM.value, SOPPOp3.S_ENDPGM_SAVED.value, SOPPOp3.S_ENDPGM_ORDERED_PS_DONE.value, SOPPOp3.S_DELAY_ALU.value} - _SOPP_IMMEDIATE = {SOPPOp3.S_NOP.value, SOPPOp3.S_CLAUSE.value, SOPPOp3.S_WAITCNT.value, SOPPOp3.S_WAITCNT_DEPCTR.value, - SOPPOp3.S_WAIT_IDLE.value, SOPPOp3.S_WAIT_EVENT.value, SOPPOp3.S_SLEEP.value, SOPPOp3.S_SET_INST_PREFETCH_DISTANCE.value} - for _op in (SOPPOp4.S_WAIT_ALU, SOPPOp4.S_WAIT_LOADCNT, SOPPOp4.S_WAIT_STORECNT, SOPPOp4.S_WAIT_SAMPLECNT, - SOPPOp4.S_WAIT_BVHCNT, SOPPOp4.S_WAIT_EXPCNT, SOPPOp4.S_WAIT_DSCNT, SOPPOp4.S_WAIT_KMCNT, - SOPPOp4.S_WAIT_LOADCNT_DSCNT, SOPPOp4.S_WAIT_STORECNT_DSCNT): - _SOPP_IMMEDIATE.add(_op.value) - _SOPP_BARRIER = {SOPPOp3.S_BARRIER.value} - if hasattr(SOPPOp4, 'S_BARRIER_WAIT'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_WAIT.value) - if hasattr(SOPPOp4, 'S_BARRIER_LEAVE'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_LEAVE.value) - _SOPP_BRANCH = {SOPPOp3.S_BRANCH.value, SOPPOp3.S_CBRANCH_SCC0.value, SOPPOp3.S_CBRANCH_SCC1.value, - SOPPOp3.S_CBRANCH_VCCZ.value, SOPPOp3.S_CBRANCH_VCCNZ.value, - SOPPOp3.S_CBRANCH_EXECZ.value, SOPPOp3.S_CBRANCH_EXECNZ.value} - - # VALU sub-classification patterns - _VALUT_4_RE = re.compile(r'V_(EXP|LOG|RCP|RSQ|SQRT|SIN|COS|CEIL|FLOOR|TRUNC|RNDNE|FRACT|FREXP)_') - _VALUB_2_RE = re.compile(r'V_(LSHLREV|LSHRREV|ASHRREV)_(B|I)64') - _VALUB_4_RE = re.compile(r'V_MAD_(U|I)64') - _VALUB_16_RE = re.compile(r'V_\w+_F64') - - def _valu_op(op_name: str) -> InstOp|None: - if 'CMPX' in op_name: return InstOp.VALU1_WR_EXEC - if _VALUB_2_RE.search(op_name): return InstOp.VALUB_2 - if _VALUB_4_RE.search(op_name): return InstOp.VALUB_4 - if _VALUB_16_RE.search(op_name): return InstOp.VALUB_16 - if _VALUT_4_RE.search(op_name): return InstOp.VALUT_4 - return None - - def _mem_op(t: type[Inst], op_name: str) -> InstOp: - is_store = "STORE" in op_name - if issubclass(t, _DS): return InstOp.LDS_WR_2 if is_store else InstOp.LDS_RD - if issubclass(t, _GLOBAL): return InstOp.SGMEM_WR_2 if is_store else InstOp.SGMEM_RD_1 - if issubclass(t, _FLAT) or issubclass(t, _SCRATCH): return InstOp.FLAT_WR_3 if is_store else InstOp.FLAT_RD_2 - return InstOp.SALU - - nibbles: list[int] = [] - started: set[int] = set() - _emit_nibbles(nibbles, LAYOUT_HEADER, layout=3, sel_a=6) - - def emit(wave_id: int, inst: Inst, branch_taken: bool|None): - """Emit an SQTT packet for one executed instruction.""" - w = wave_id & 0x1F - if wave_id not in started: - _emit_nibbles(nibbles, WAVESTART, delta=1, simd=0, wgp=0, wave=w, id7=wave_id) - started.add(wave_id) - inst_type, inst_op, op_name = type(inst), inst.op.value if hasattr(inst, 'op') else 0, inst.op.name if hasattr(inst, 'op') else "" - if issubclass(inst_type, _SOPP): - if inst_op in _SOPP_SKIP: return - if inst_op in _SOPP_IMMEDIATE: _emit_nibbles(nibbles, IMMEDIATE, delta=1, wave=w) - elif inst_op in _SOPP_BARRIER: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.BARRIER) - elif inst_op in _SOPP_BRANCH: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.JUMP if branch_taken else InstOp.JUMP_NO) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SALU) - elif issubclass(inst_type, _VALU): - if (op := _valu_op(op_name)) is None: _emit_nibbles(nibbles, VALUINST, delta=1, wave=w) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=op) - elif issubclass(inst_type, _SMEM): _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SMEM_RD) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=_mem_op(inst_type, op_name)) - - def finish(wave_id: int): - """Emit WAVEEND for a completed wave.""" - if wave_id in started: _emit_nibbles(nibbles, WAVEEND, delta=1, simd=0, wgp=0, wave=wave_id & 0x1F) - - def finalize() -> bytes: - """Pad and return the encoded SQTT blob.""" - while len(nibbles) % 2 != 0: nibbles.append(0) - nibbles.extend([0] * 32) - while len(nibbles) % 64 != 0: nibbles.append(0) - return bytes(nibbles[i] | ((nibbles[i + 1] if i + 1 < len(nibbles) else 0) << 4) for i in range(0, len(nibbles), 2)) - - return emit, finish, finalize diff --git a/tinygrad_repo/test/mockgpu/cuda/cuda.py b/tinygrad_repo/test/mockgpu/cuda/cuda.py index 9b5519163c..1b212a78ce 100644 --- a/tinygrad_repo/test/mockgpu/cuda/cuda.py +++ b/tinygrad_repo/test/mockgpu/cuda/cuda.py @@ -62,6 +62,7 @@ def cuMemFree_v2(dptr): return free(dptr) def cuMemFreeHost(p): return free(p) def cuMemHostRegister_v2(p, size, flags): return cuda.CUDA_SUCCESS def cuMemHostUnregister(p): return cuda.CUDA_SUCCESS +def cuMemHostGetDevicePointer_v2(pdptr, p, flags): return out(pdptr, p) def cuModuleLoadData(module, image): modules.append(ctypes.create_string_buffer(ctypes.string_at(image))) diff --git a/tinygrad_repo/test/mockgpu/driver.py b/tinygrad_repo/test/mockgpu/driver.py index 7be165cb06..60ae0ab386 100644 --- a/tinygrad_repo/test/mockgpu/driver.py +++ b/tinygrad_repo/test/mockgpu/driver.py @@ -37,3 +37,4 @@ class VirtDriver: self.tracked_addresses = [] def track_address(self, staddr, enaddr, rcb, wcb): self.tracked_addresses.append((staddr, enaddr, rcb, wcb)) def open(self, name, flags, mode, fdcls): raise NotImplementedError() + def progress(self): pass # runs the queues diff --git a/tinygrad_repo/test/mockgpu/helpers.py b/tinygrad_repo/test/mockgpu/helpers.py index 1d3a8648d9..1b9f579814 100644 --- a/tinygrad_repo/test/mockgpu/helpers.py +++ b/tinygrad_repo/test/mockgpu/helpers.py @@ -6,15 +6,3 @@ gpuocelot_lib = c.DLL("ocelot", "gpuocelot") ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int) def ptx_run(source:bytes, n_args:int, args:c.POINTER[ctypes.c_void_p], blck_x:int, blck_y:int, blck_z:int, grid_x:int, grid_y:int, grid_z:int, shared_mem_size:int): pass - -class PythonRemu: - """Python RDNA3/RDNA4 emulator wrapper used by mockgpu.""" - valid_mem_ranges: set[tuple[int, int]] = set() - rsrc2: int = 0x19c # Default: USER_SGPR_COUNT=14, enable X and Y workgroup IDs - scratch_size: int = 0 # private_segment_fixed_size from kernel descriptor - arch: str = "rdna3" # Architecture: rdna3 or rdna4 - user_data: list[int] = [] # All COMPUTE_USER_DATA registers (loaded into s[0:N]) - - def run_asm(self, lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int) -> int: - from test.mockgpu.amd.emu import run_asm - return run_asm(lib, lib_sz, gx, gy, gz, lx, ly, lz, args_ptr, self.rsrc2, self.scratch_size, self.arch, self.user_data) diff --git a/tinygrad_repo/test/mockgpu/hip/hip.py b/tinygrad_repo/test/mockgpu/hip/hip.py new file mode 100644 index 0000000000..135847d7af --- /dev/null +++ b/tinygrad_repo/test/mockgpu/hip/hip.py @@ -0,0 +1,38 @@ +# the hip api on top of mock cuda, kernels run on the amd emulator +import ctypes +from types import SimpleNamespace +from tinygrad.runtime.autogen import hip, libc +from tinygrad.runtime.ops_amd import _amd_program_image +from tinygrad.runtime.ops_hip import HIPIFY +from tinygrad.runtime.support.elf import elf_loader +from test.mockgpu.cuda import cuda as mc +from test.mockgpu.amd.emu import run_asm + +programs:dict[int, tuple] = {} # a function handle is the address of its module's loaded image + +def run_kernel(func:int, args:int, size:int, gx:int, gy:int, gz:int, lx:int, ly:int, lz:int, smem:int) -> bool: + image, data, text_end = programs[func] + entry, entry_size = ctypes.addressof(image) + data.entry_point_offset, text_end - data.entry_point_offset + return run_asm(entry, entry_size, gx, gy, gz, lx, ly, lz, args, data.rsrc2, data.private_segment_size) == 0 +mc.run_kernel = run_kernel + +def hipModuleLoadData(module, image): + elf = libc.Elf64_Ehdr.from_address(image) # the section headers end the elf + gfx1100 = SimpleNamespace(iface=SimpleNamespace(props={"lds_size_in_kb": 64}), target=(11, 0, 0)) + lib = ctypes.string_at(image, elf.e_shoff + elf.e_shnum * elf.e_shentsize) + data, img = _amd_program_image(gfx1100, lib) + text = next(sh.header for sh in elf_loader(lib)[1] if sh.name == ".text") + buf = ctypes.create_string_buffer(img, len(img)) + programs[ctypes.addressof(buf)] = (buf, data, text.sh_addr + text.sh_size) + return mc.out(module, ctypes.addressof(buf)) + +def hipGetDevicePropertiesR0600(prop, device_id): + prop[0].gcnArchName = b"gfx1100" + return hip.hipSuccess + +fns = {fn.__name__: getattr(mc, cu) for cu, fn in HIPIFY.items() if fn.__name__.startswith("hip")} +fns |= dict(hipHostMalloc=mc.cuMemHostAlloc, hipModuleLoadData=hipModuleLoadData, hipGetDevicePropertiesR0600=hipGetDevicePropertiesR0600, + hipMemcpyAsync=lambda dst, src, size, kind, stream: mc.cuMemcpyAsync(dst, src, size, stream), + hipStreamWaitValue64=lambda stream, ptr, value, flags, mask: mc.cuStreamWaitValue64_v2(stream, ptr, value, flags)) +for name, fn in fns.items(): setattr(hip.dll, name, ctypes.CFUNCTYPE(getattr(hip, name).restype, *getattr(hip, name).argtypes)(fn)) +hip.dll._loaded_[hip.dll.nm] = hip.dll diff --git a/tinygrad_repo/test/mockgpu/mockgpu.py b/tinygrad_repo/test/mockgpu/mockgpu.py index c41cdaf224..47321e539a 100644 --- a/tinygrad_repo/test/mockgpu/mockgpu.py +++ b/tinygrad_repo/test/mockgpu/mockgpu.py @@ -1,8 +1,10 @@ import ctypes, time, os, builtins, fcntl, typing -from tinygrad.helpers import DEV, dedup, to_tuple -from tinygrad.device import Compiled +from tinygrad.helpers import DEV, dedup, to_tuple, round_up +from tinygrad.device import Compiled, Device +from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher from tinygrad.runtime.support.system import FileIOInterface, System +from tinygrad.runtime.support.hcq2 import HWQueue from tinygrad.runtime.autogen import libc from test.mockgpu.nv.nvdriver import NVDriver from test.mockgpu.amd.amddriver import AMDDriver @@ -21,10 +23,34 @@ setattr(libc.dll, "mockgpu_doorbell", mockgpu_doorbell) mocked = {t.device for t in DEV.value if t.interface.startswith("MOCK")} def hook_doorbells(s:UOp) -> UOp|None: - hooks = tuple(UOp.custom_function("mockgpu_doorbell").call(b.after(c).getaddr("CPU")) for c in s.toposort() if c.op is Ops.CALL - for b in c.src[1:] if b.op is Ops.ALLOC and "doorbell" in str(b.tag) and to_tuple(b.device)[0].split(":")[0] in mocked) + hooks = tuple(UOp.custom_function("mockgpu_doorbell").call(b.after(c).getaddr(Device[d].host)) for c in s.toposort() if c.op is Ops.CALL + for b in c.src[1:] if b.op is Ops.ALLOC and "doorbell" in str(b.arg.name) and (d:=to_tuple(b.device)[0]).split(":")[0] in mocked) return None if all(h in s.src for h in hooks) else s.replace(src=tuple(dedup(s.src + hooks))) -Compiled.pm_lower = PatternMatcher([(UPat(Ops.SINK, name="s"), hook_doorbells)]) +Compiled.pm_lower += PatternMatcher([(UPat(Ops.SINK, name="s"), hook_doorbells)]) + +@ctypes.CFUNCTYPE(None, ctypes.c_uint64) +def mockgpu_progress(_:int): # compiled spin loops wait on the gpus, which run only when called + for d in drivers: d.progress() +setattr(libc.dll, "mockgpu_progress", mockgpu_progress) + +def kick_gpus(body:UOp, loop:UOp, cond:UOp) -> UOp|None: # each iteration of a spin loop runs the gpus + kick = UOp.custom_function("mockgpu_progress").call(UOp.const(0, dtypes.uint64).after(body)) + return None if kick in cond.src else body.backedge(loop, cond.after(kick)) +Compiled.pm_lower += PatternMatcher([(UPat(Ops.BACKEDGE, src=(UPat(name="body"), UPat(name="loop"), UPat(name="cond"))), kick_gpus)]) + +# this links AMD emulator's kernel back to the original kernel +def emit_kernel_info(ctx:HWQueue, prg:UOp) -> None: + from tinygrad.runtime.ops_amd import AMDComputeQueue, _amd_program_image + if isinstance(ctx, AMDComputeQueue): + from tinygrad.viz.serve import get_elf_section + name = prg.src[0].arg.name.encode() + b"\x00" + text = get_elf_section(prg.src[3].arg, ".text").header + text_size = text.sh_addr + text.sh_size - _amd_program_image(ctx.dev, prg.src[3].arg)[0].entry_point_offset + ctx.pkt3(ctx.pm4.PACKET3_NOP, *memoryview(name.ljust(round_up(len(name), 4), b"\x00")).cast("I"), text_size) + +HWQueue.q_rewrite = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), allow_any_len=True), emit_kernel_info), +]) + HWQueue.q_rewrite original_memoryview = builtins.memoryview class TrackedMemoryView: @@ -44,6 +70,7 @@ class TrackedMemoryView: self.mv = self.mv.cast('B').cast(new_type, **kwargs) return self + def __buffer__(self, flags): return self.mv.__buffer__(flags) @property def nbytes(self): return self.mv.nbytes def __len__(self): return len(self.mv) diff --git a/tinygrad_repo/test/mockgpu/nv/nvdriver.py b/tinygrad_repo/test/mockgpu/nv/nvdriver.py index fe566d59b0..c22695f5d9 100644 --- a/tinygrad_repo/test/mockgpu/nv/nvdriver.py +++ b/tinygrad_repo/test/mockgpu/nv/nvdriver.py @@ -291,6 +291,7 @@ class NVDriver(VirtDriver): self.track_address(st.params.pMemory, st.params.pMemory + st.params.limit + 1, lambda mv,off: None, lambda mv, off: self._gpu_mmio_write(mv, off, None)) return 0 + def progress(self): self._gpu_mmio_write(None, 0, None) def _gpu_mmio_write(self, mv, off, gpu): if self._executing: return # prevent re-entrancy self._executing = True diff --git a/tinygrad_repo/test/mockgpu/usb.py b/tinygrad_repo/test/mockgpu/usb.py index 38bd4d56e2..57c8d0c9f3 100644 --- a/tinygrad_repo/test/mockgpu/usb.py +++ b/tinygrad_repo/test/mockgpu/usb.py @@ -1,8 +1,7 @@ from __future__ import annotations -import ctypes, mmap, struct, sys +import ctypes, mmap, struct from typing import Any, Callable -from tinygrad.runtime.autogen import libusb -if sys.platform != "win32": from tinygrad.runtime.autogen import libc +from tinygrad.runtime.autogen import libusb, libc class MockUSB: def __init__(self, mem): diff --git a/tinygrad_repo/test/null/test_autogen.py b/tinygrad_repo/test/null/test_autogen.py index 4c0e04d0a2..b39a8c0c98 100644 --- a/tinygrad_repo/test/null/test_autogen.py +++ b/tinygrad_repo/test/null/test_autogen.py @@ -1,10 +1,9 @@ import ctypes, struct, subprocess, tempfile, unittest -from tinygrad.helpers import OSX, WIN +from tinygrad.helpers import OSX from tinygrad.runtime.support.c import DLL, record, Field from tinygrad.runtime.support import c from tinygrad.runtime.support.autogen import gen -@unittest.skipIf(WIN, "doesn't compile on windows") class TestC(unittest.TestCase): def compile(self, src): with tempfile.NamedTemporaryFile(suffix=".so") as f: @@ -322,7 +321,6 @@ class TestC(unittest.TestCase): self.assertEqual(result, 43) # 42 + 1 @unittest.skipIf(OSX and ('MTLCompiler' in DLL._loaded_ or 'llvm' in DLL._loaded_), "libclang can't be loaded after MTLCompiler or llvm on OSX") -@unittest.skipIf(WIN, "doesn't compile on windows") class TestAutogen(unittest.TestCase): def run_gen(self, contents): with tempfile.NamedTemporaryFile(mode='w', suffix='.h') as f: diff --git a/tinygrad_repo/test/null/test_call.py b/tinygrad_repo/test/null/test_call.py index b26e342230..1a406e501d 100644 --- a/tinygrad_repo/test/null/test_call.py +++ b/tinygrad_repo/test/null/test_call.py @@ -122,7 +122,7 @@ class TestCallCodegen(unittest.TestCase): v = UOp.variable("external", 1, 8, dtype=dtypes.int) prg = to_program(p.index(0).store(v).sink(arg=KernelInfo("scalar")), ClangRenderer(Target("CPU", arch="x86_64,x86-64"))) - self.assertEqual(prg.arg.vars[0].arg.slot, 1) + self.assertEqual(prg.arg.params[1].arg.slot, 1) linear = UOp(Ops.LINEAR, src=(prg.call(p, v.bind(2)),)) # Slot 1 in the enclosing call must not rewrite the already-compiled program's ABI slot 1. resolved = resolve_linear_call(linear.call(out, UOp.variable("other", 1, 8, dtype=dtypes.int).bind(3))) diff --git a/tinygrad_repo/test/null/test_const_folding.py b/tinygrad_repo/test/null/test_const_folding.py index 92810707ed..c19a95f133 100644 --- a/tinygrad_repo/test/null/test_const_folding.py +++ b/tinygrad_repo/test/null/test_const_folding.py @@ -19,7 +19,7 @@ class TestWeakConstFolding(unittest.TestCase): self.assertEqual((out.op, out.dtype, out.val), (Ops.CONST, dtypes.weakint, 2**41)) def test_float_unaries(self): - for op in (Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL): + for op in (Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIP): out = UOp.const(4.0).alu(op).simplify() self.assertEqual((out.op, out.dtype), (Ops.CONST, dtypes.weakfloat)) @@ -32,7 +32,7 @@ class TestWeakConstFolding(unittest.TestCase): self.assertTrue(nan.ne(nan).simplify().val) def test_invalid_poison(self): - self.assertTrue(UOp.invalid().alu(Ops.CDIV, UOp.const(0)).simplify().is_invalid) + self.assertTrue(UOp.invalid().alu(Ops.TRUNCDIV, UOp.const(0)).simplify().is_invalid) class TestBitcastConstFolding(unittest.TestCase): def test_out_of_range_source_value(self): diff --git a/tinygrad_repo/test/null/test_device.py b/tinygrad_repo/test/null/test_device.py index 699be40ed1..04ef7e313f 100644 --- a/tinygrad_repo/test/null/test_device.py +++ b/tinygrad_repo/test/null/test_device.py @@ -3,7 +3,7 @@ import unittest, os, subprocess from unittest.mock import patch from tinygrad import Tensor, dtypes from tinygrad.device import Device, Compiler, enumerate_devices_str -from tinygrad.helpers import diskcache_get, diskcache_put, getenv, Context, Target, WIN, OSX, DEV +from tinygrad.helpers import diskcache_get, diskcache_put, getenv, Context, Target, OSX, DEV from tinygrad.runtime.support.c import DLL class TestDevice(unittest.TestCase): @@ -71,7 +71,6 @@ class TestDevice(unittest.TestCase): result = subprocess.run(['python3', '-c', 'import runpy, tinygrad; runpy.run_module("tinygrad.device")'], capture_output=True) self.assertEqual(result.returncode, 0, result.stderr.decode()) - @unittest.skipIf(WIN, "skipping windows test") # TODO: subprocess causes memory violation? def test_env_overwrite_default_compiler(self): if Device.DEFAULT == "CPU": from tinygrad.runtime.support.compiler_cpu import ClangCompiler @@ -103,7 +102,6 @@ class TestDevice(unittest.TestCase): shell=True, check=True, env={**os.environ, "DEV": "AMD:HIP"}) else: self.skipTest("only run on CPU/AMD") - @unittest.skipIf(WIN, "skipping windows test") def test_env_online(self): from tinygrad.runtime.support.compiler_cpu import ClangCompiler from tinygrad.runtime.support.compiler_llvm import CPULLVMCompiler diff --git a/tinygrad_repo/test/null/test_dtype_weak.py b/tinygrad_repo/test/null/test_dtype_weak.py index 7c36d3c653..7b86410a75 100644 --- a/tinygrad_repo/test/null/test_dtype_weak.py +++ b/tinygrad_repo/test/null/test_dtype_weak.py @@ -40,7 +40,7 @@ class TestWeakPromotion(unittest.TestCase): self.assertIs(denom.dtype, dtypes.weakfloat) # the setup: the denominator expression itself is weak x, y = Tensor([2048.0], dtype=dtypes.float32)._broadcasted(denom) self.assertIs(y.dtype, dtypes.float32) - recips = [u for u in (x / y)._uop.toposort() if u.op is Ops.RECIPROCAL] + recips = [u for u in (x / y)._uop.toposort() if u.op is Ops.RECIP] self.assertEqual([(u.dtype, u.src[0].dtype) for u in recips], [(dtypes.float32, dtypes.float32)]) with Context(DEFAULT_FLOAT=dtypes.float16): committed = graph_rewrite((UOp.const(1).cast(dtypes.int32) + UOp.const(1.0)).cast(dtypes.float32), pm_commit_weak) diff --git a/tinygrad_repo/test/null/test_encodings.py b/tinygrad_repo/test/null/test_encodings.py index e98362ef38..0917e606d1 100644 --- a/tinygrad_repo/test/null/test_encodings.py +++ b/tinygrad_repo/test/null/test_encodings.py @@ -3,14 +3,13 @@ from dataclasses import replace from tinygrad.helpers import Target from tinygrad.uop.ops import UOp, Ops from tinygrad.dtype import dtypes, DType -from tinygrad.renderer.isa import Register -from tinygrad.renderer.isa.x86 import X86Ops, X86Renderer, RBP, RDI, RSP, RSI, RAX, RDX, XMM, GPR, imm, def_reg +from tinygrad.renderer.x86 import Register, X86Ops, X86Renderer, RBP, RDI, RSP, RSI, RAX, RDX, XMM, GPR, imm -def ins(op, dt, src, tag=None): return UOp(Ops.INS, arg=(op, dt), src=src, tag=tag) -# the operand width is carried by the register in the tag, not by the dtype +# the operand width is carried by the register, not by the dtype def reg(r:Register, size:int) -> Register: return replace(r, size=size) -# the element size of a memory operand comes from the dtype of its base -def ptr(r:Register, dt:DType) -> UOp: return UOp(Ops.INS, arg=(X86Ops.DEFINE, dt), tag=(r,)) +def ins(op, dt, src, r:Register|None=None): return UOp(Ops.INS, arg=(op, dt) + ((r,) if r is not None else ()), src=src) +def reg_placeholder(r:Register): return UOp(Ops.INS, arg=(X86Ops.REG, dtypes.uint64, r)) +def alloc(dt:DType, r:Register): return reg_placeholder(r) class TestEncodingsX86(unittest.TestCase): # NOTE: x86 supports a single displacement as memory address and index without base memory address @@ -19,133 +18,136 @@ class TestEncodingsX86(unittest.TestCase): # displacement of 0 isn't emitted def test_base_address(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RDI, dtypes.int32), UOp(Ops.NOOP), imm(dtypes.int8, 0)), reg(RDI, 4)) + load = ins(X86Ops.MOV, dtypes.int32, (alloc(dtypes.int32, RDI), UOp(Ops.NOOP), imm(dtypes.int8, 0), imm(dtypes.uint8, 4)), reg(RDI, 4)) # mov edi, dword ptr [rdi] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("8B 3F")) # rsp/r12 require a sib byte when used as base memory address def test_rsp_base_address(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RSP, dtypes.int32), UOp(Ops.NOOP), imm(dtypes.int8, 0)), reg(RSP, 4)) + load = ins(X86Ops.MOV, dtypes.int32, (alloc(dtypes.int32, RSP), UOp(Ops.NOOP), imm(dtypes.int8, 0), imm(dtypes.uint8, 4)), reg(RSP, 4)) # mov esp, dword ptr [rsp] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("8B 24 24")) # rbp/r13 require a displacement when used as base memory address def test_rbp_base_address(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RBP, dtypes.int32), UOp(Ops.NOOP), imm(dtypes.int8, 0)), reg(RBP, 4)) + load = ins(X86Ops.MOV, dtypes.int32, (alloc(dtypes.int32, RBP), UOp(Ops.NOOP), imm(dtypes.int8, 0), imm(dtypes.uint8, 4)), reg(RBP, 4)) # mov ebp, dword ptr [rbp + 0] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("8B 6D 00")) # test [base + index*scale] def test_base_index_address(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RAX, dtypes.int32), def_reg(RDX), imm(dtypes.int8, 0)), reg(RAX, 4)) + load = ins(X86Ops.MOV, dtypes.int32, (alloc(dtypes.int32, RAX), reg_placeholder(RDX), imm(dtypes.int8, 0), imm(dtypes.uint8, 4)), reg(RAX, 4)) # mov eax, dword ptr [rax + rdx*4] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("8B 04 90")) # rsp as index means no index def test_rsp_index_address(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RAX, dtypes.int32), def_reg(RSP), imm(dtypes.int8, 0)), reg(RAX, 4)) + load = ins(X86Ops.MOV, dtypes.int32, (alloc(dtypes.int32, RAX), reg_placeholder(RSP), imm(dtypes.int8, 0), imm(dtypes.uint8, 4)), reg(RAX, 4)) # mov eax, dword ptr [rax] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("8B 00")) # however r12 is a valid index def test_r12_index_address(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RAX, dtypes.int32), def_reg(GPR[12]), imm(dtypes.int8, 0)), reg(RAX, 4)) + address = (alloc(dtypes.int32, RAX), reg_placeholder(GPR[12]), imm(dtypes.int8, 0), imm(dtypes.uint8, 4)) + load = ins(X86Ops.MOV, dtypes.int32, address, reg(RAX, 4)) # mov eax, dword ptr [rax + r12*4] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("42 8B 04 A0")) # test [base + index*scale + 8bit disp] def test_complex_address_8bit_disp(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RDI, dtypes.int32), def_reg(RSI), imm(dtypes.int8, 10)), reg(RDI, 4)) + address = (alloc(dtypes.int32, RDI), reg_placeholder(RSI), imm(dtypes.int8, 10), imm(dtypes.uint8, 4)) + load = ins(X86Ops.MOV, dtypes.int32, address, reg(RDI, 4)) # mov edi, dword ptr [rdi + rsi*4 + 0xa] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("8B 7C B7 0A")) # test [base + index*scale + 32bit disp] def test_complex_address_32bit_disp(self): - load = ins(X86Ops.MOV, dtypes.int32, (ptr(RDI, dtypes.int32), def_reg(RSI), imm(dtypes.int32, 10000)), reg(RDI, 4)) + address = (alloc(dtypes.int32, RDI), reg_placeholder(RSI), imm(dtypes.int32, 10000), imm(dtypes.uint8, 4)) + load = ins(X86Ops.MOV, dtypes.int32, address, reg(RDI, 4)) # mov edi, dword ptr [rdi + rsi*4 + 0x2710] self.assertEqual(bytes.fromhex(self.encode(load)), bytes.fromhex("8B BC B7 10 27 00 00")) # 8bit variants of legacy instructions subtract 1 from opcode def test_8bit_legacy_encoding(self): - cast = ins(X86Ops.MOVSX, dtypes.int32, (def_reg(reg(RDX, 1)),), reg(RAX, 4)) + cast = ins(X86Ops.MOVSX, dtypes.int32, (reg_placeholder(reg(RDX,1)),), reg(RAX, 4)) # movsx eax, dl self.assertEqual(bytes.fromhex(self.encode(cast)), bytes.fromhex("0F BE C2")) # accessing lower 8 bits of rsp, rbp, rsi, rdi requires rex prefix def test_lower_8bits_reg(self): - cast = ins(X86Ops.MOVSX, dtypes.int32, (def_reg(reg(RDI, 1)),), reg(RAX, 4)) + cast = ins(X86Ops.MOVSX, dtypes.int32, (reg_placeholder(reg(RDI,1)),), reg(RAX, 4)) # movsx eax, dil self.assertEqual(bytes.fromhex(self.encode(cast)), bytes.fromhex("40 0F BE C7")) # test 16 bit variant of legacy instruction def test_16bit_legacy_encoding(self): - cast = ins(X86Ops.MOVSX, dtypes.int16, (def_reg(reg(RDX, 1)),), reg(RAX, 2)) + cast = ins(X86Ops.MOVSX, dtypes.int16, (reg_placeholder(reg(RDX,1)),), reg(RAX, 2)) # movsx ax, dl self.assertEqual(bytes.fromhex(self.encode(cast)), bytes.fromhex("66 0F BE C2")) # test 64 bit variant of legacy instruction def test_64bit_legacy_encoding(self): - cast = ins(X86Ops.MOVSX, dtypes.int64, (def_reg(reg(RDX, 1)),), reg(RAX, 8)) + cast = ins(X86Ops.MOVSX, dtypes.int64, (reg_placeholder(reg(RDX, 1)),), reg(RAX, 8)) # movsx rax, dl self.assertEqual(bytes.fromhex(self.encode(cast)), bytes.fromhex("48 0F BE C2")) - # the width comes from the register in the tag, the dtype doesn't take part in the encoding + # the width comes from the register, the dtype doesn't take part in the encoding def test_width_from_register(self): for size, expected in ((1, "8A C2"), (2, "66 8B C2"), (4, "8B C2"), (8, "48 8B C2")): - mov = ins(X86Ops.MOV, dtypes.float32, (def_reg(reg(RDX, size)),), reg(RAX, size)) + mov = ins(X86Ops.MOV, dtypes.float32, (reg_placeholder(reg(RDX, size)),), reg(RAX, size)) self.assertEqual(bytes.fromhex(self.encode(mov)), bytes.fromhex(expected)) # test compact vex encoding def test_compact_vex_encoding(self): - xmm0, xmm1 = def_reg(XMM[0]), def_reg(XMM[1]) + xmm0, xmm1 = reg_placeholder(XMM[0]), reg_placeholder(XMM[1]) add = ins(X86Ops.VADDSS, dtypes.float32, (xmm0, xmm1), XMM[0]) # vaddss xmm0, xmm0, xmm1 self.assertEqual(bytes.fromhex(self.encode(add)), bytes.fromhex("C5 FA 58 C1")) # test long vex encoding def test_long_vex_encoding(self): - xmm0, xmm8 = def_reg(XMM[0]), def_reg(XMM[8]) + xmm0, xmm8 = reg_placeholder(XMM[0]), reg_placeholder(XMM[8]) add = ins(X86Ops.VADDSS, dtypes.float32, (xmm0, xmm8), XMM[0]) # vaddss xmm0, xmm0, xmm8 self.assertEqual(bytes.fromhex(self.encode(add)), bytes.fromhex("C4 C1 7A 58 C0")) # test encoding where register is in the immediate field def test_reg_in_imm_field(self): - xmm0, xmm1, xmm2 = def_reg(XMM[0]), def_reg(XMM[1]), def_reg(XMM[2]) + xmm0, xmm1, xmm2 = reg_placeholder(XMM[0]), reg_placeholder(XMM[1]), reg_placeholder(XMM[2]) blend = ins(X86Ops.VBLENDVPS, dtypes.float32, (xmm0, xmm1, xmm2), XMM[0]) # vblendvps xmm0, xmm0, xmm1, xmm2 self.assertEqual(bytes.fromhex(self.encode(blend)), bytes.fromhex("C4 E3 79 4A C1 20")) # when writting to mem the uop takes the store form where dtype is void and there's no definition def test_write_mem(self): - address = (ptr(RDI, dtypes.int32), def_reg(RSI), imm(dtypes.int8, 10)) - xmm0 = def_reg(XMM[0]) + address = (alloc(dtypes.int32, RDI), reg_placeholder(RSI), imm(dtypes.int8, 10), imm(dtypes.uint8, 4)) + xmm0 = reg_placeholder(XMM[0]) extr = ins(X86Ops.VPEXTRD, dtypes.void, address + (xmm0, imm(dtypes.uint8, 0))) # vpextrd dword ptr [rdi + rsi*4 + 0xa], xmm0, 0 self.assertEqual(bytes.fromhex(self.encode(extr)), bytes.fromhex("C4 E3 79 16 44 B7 0A 00")) # test two address instruction with fused load works def test_two_address_load(self): - address = (ptr(RDI, dtypes.int32), def_reg(RSI), imm(dtypes.int8, 10)) + address = (alloc(dtypes.int32, RDI), reg_placeholder(RSI), imm(dtypes.int8, 10), imm(dtypes.uint8, 4)) cmove = ins(X86Ops.CMOVE, dtypes.int32, address, reg(RAX, 4)) # cmove eax, dword ptr [rdi + rsi*4 + 0xa] self.assertEqual(bytes.fromhex(self.encode(cmove)), bytes.fromhex("0F 44 44 B7 0A")) # test instruction where displacement and imm have the same value def test_disp_imm_same_value(self): - address = (ptr(RDI, dtypes.int8), def_reg(RSI), imm(dtypes.int8, 10)) + address = (alloc(dtypes.int8, RDI), reg_placeholder(RSI), imm(dtypes.int8, 10), imm(dtypes.uint8, 1)) mov = ins(X86Ops.MOVi, dtypes.void, address + (imm(dtypes.int8, 10),)) # mov byte ptr [rdi + rsi + 0xa], 0xa self.assertEqual(bytes.fromhex(self.encode(mov)), bytes.fromhex("40 C6 44 37 0A 0A")) - address = (ptr(RDI, dtypes.int32), def_reg(RSI), imm(dtypes.int32, 10)) + address = (alloc(dtypes.int32, RDI), reg_placeholder(RSI), imm(dtypes.int32, 10), imm(dtypes.uint8, 4)) imul = ins(X86Ops.IMULi, dtypes.int32, address + (imm(dtypes.int32, 10),), reg(RDI, 4)) # imul edi, dword ptr [rdi + rsi*4 + 0xa], 0xa self.assertEqual(bytes.fromhex(self.encode(imul)), bytes.fromhex("69 BC B7 0A 00 00 00 0A 00 00 00")) # cmoves have the cmp as the last src even though it is not explicitly used, the cmp doesn't define a reg and is ignored in the encoding def test_cmove_ignore_cmp(self): - cmove = ins(X86Ops.CMOVE, dtypes.int32, (def_reg(reg(RAX, 4)), UOp(Ops.INS, arg=(X86Ops.CMP, dtypes.void))), reg(RDX, 4)) + cmove = ins(X86Ops.CMOVE, dtypes.int32, (reg_placeholder(reg(RAX, 4)), UOp(Ops.INS, arg=(X86Ops.CMP, dtypes.void))), reg(RDX, 4)) # cmove edx, eax self.assertEqual(bytes.fromhex(self.encode(cmove)), bytes.fromhex("0F 44 D0")) diff --git a/tinygrad_repo/test/null/test_gen_float4.py b/tinygrad_repo/test/null/test_gen_float4.py index 3cc3ff0b82..15f4960d5c 100644 --- a/tinygrad_repo/test/null/test_gen_float4.py +++ b/tinygrad_repo/test/null/test_gen_float4.py @@ -117,7 +117,7 @@ class TestFloat4(unittest.TestCase): assert TestFloat4.count_float4(uops) == (1, 1) def test_float4_aligned_variable(self): - x = Variable('x', 0, 4, multiple_of=4).bind(4) + x = Variable('k', 0, 1).bind(1)*4 a = Tensor.empty(4).realize() b = Tensor.empty(12).realize().shrink(((x, x+4),)) c = a + b @@ -131,7 +131,7 @@ class TestFloat4(unittest.TestCase): assert TestFloat4.count_float4(uops) == (2, 1) def test_float4_unaligned_variable(self): - x = Variable('x', 0, 4, multiple_of=2).bind(4) + x = Variable('k', 0, 2).bind(2)*2 a = Tensor.empty(4).realize() b = Tensor.empty(12).realize().shrink(((x, x+4),)) c = a + b diff --git a/tinygrad_repo/test/null/test_graph_rewrite.py b/tinygrad_repo/test/null/test_graph_rewrite.py index 9c6a09578e..9807999538 100644 --- a/tinygrad_repo/test/null/test_graph_rewrite.py +++ b/tinygrad_repo/test/null/test_graph_rewrite.py @@ -116,14 +116,6 @@ class TestSubstitute(unittest.TestCase): ret = substitute(ret, {a.sin():a.sqrt(), n1.sin():n1.sqrt()}) self.assertIs(ret, a.sqrt().sqrt()) - def test_tagged_replace(self): - a = UOp.variable('a', 0, 10) - b = UOp.variable('b', 0, 10) - ret = (a+4).replace(tag=1) - ret = substitute(ret, {a:b}) - # the srcs are rewritten but we keep tag - self.assertIs(ret, (b+4).replace(tag=1)) - matchers = strat.sampled_from([PatternMatcher, TrackedPatternMatcher]) class TestRecurse(unittest.TestCase): diff --git a/tinygrad_repo/test/null/test_hcq2.py b/tinygrad_repo/test/null/test_hcq2.py index f3558a1dc0..3ee9c7c8b1 100644 --- a/tinygrad_repo/test/null/test_hcq2.py +++ b/tinygrad_repo/test/null/test_hcq2.py @@ -1,24 +1,26 @@ import unittest -from typing import cast from types import SimpleNamespace from collections import defaultdict from unittest.mock import patch -from tinygrad import Device, Tensor, TinyJit, dtypes -from tinygrad.device import Buffer, Compiled, ProfileGraphEvent +from tinygrad import Device, Tensor, TinyJit, Variable, dtypes +from tinygrad.device import Compiled, ProfileGraphEvent from tinygrad.helpers import Context, unwrap, to_tuple -from tinygrad.dtype import AddrSpace from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo -from tinygrad.engine.realize import compile_linear, link_linear, get_call_arg_uops, lower_and_compile, run_linear +from tinygrad.engine.realize import compile_linear, get_call_arg_uops, lower_and_compile, run_linear import tinygrad.runtime.support.hcq2 as hcq2 from tinygrad.runtime.support.hcq2 import HCQInfo def lower_hcq(*body:UOp) -> UOp: # body through lower_call, as a linear - return UOp(Ops.LINEAR, src=(unwrap(hcq2.lower_call(UOp.sink(*body, arg=KernelInfo("test"), tag=1).call(aux=HCQInfo(("CPU",))))),)) + return UOp(Ops.LINEAR, src=(unwrap(hcq2.lower_call(UOp.sink(*body, arg=KernelInfo("test", optimized=True)).call(aux=HCQInfo(("CPU",))))),)) def chain(x:Tensor, n:int) -> Tensor: for _ in range(n): x = (x + 1).contiguous() return x +def add_n(n:UOp, C:UOp, B:UOp) -> UOp: + i = UOp.range(C.numel(), 0) + return C[i].store(B[i] + n).end(i).sink(arg=KernelInfo(name="add_n")) + def chain_input(value:int=2, device="NULL") -> Tensor: return Tensor.full((4,), value, dtype=dtypes.int32, device=device).contiguous().realize() def compiled_chain(n:int, jit=False, device="NULL") -> tuple[Tensor, UOp, list[UOp]]: @@ -39,14 +41,21 @@ def scheduled(*ts:Tensor, **kwargs) -> list[UOp]: lin = orig(l, profile) batches.extend(c for c in lin.src if c.op is Ops.CALL and isinstance(c.arg.aux, HCQInfo)) return lin - with patch.object(hcq2, "sched_batches", track): compile_linear(ts[0].schedule_linear(*ts[1:]), **kwargs) + with patch.object(hcq2, "sched_batches", track): compile_linear(ts[0].linear_with_vars(*ts[1:])[0], **kwargs) return batches +def submits(batch:UOp) -> list[UOp]: # the submit calls of a batch sink in order + calls, s = list[UOp](), batch.src[0] + while s.op is Ops.AFTER: calls, s = [s.src[0]] + calls, s.src[-1] + return calls def queues(batch:UOp) -> dict[tuple[str, str], list[UOp]]: - return {(lin.arg[0][0], lin.arg[1]): list(lin.src) for lin in (s.src[1].without_after for s in batch.body.src)} -def calls(batch:UOp) -> list[UOp]: return [c for cmds in queues(batch).values() for c in cmds if c.op is Ops.CALL] + return {(q.device, q.arg.name): list(c.body.src) for c in submits(batch.body) for q in [c.src[1]]} +def ins_name(c:UOp) -> str|None: return c.body.arg.name if c.op is Ops.CALL and c.body.op is Ops.CUSTOM_FUNCTION else None +def calls(batch:UOp) -> list[UOp]: return [c for cmds in queues(batch).values() for c in cmds if c.op is Ops.CALL and ins_name(c) is None] def devices_of(call:UOp) -> set[str]: return {to_tuple(a.device)[0] for a in get_call_arg_uops(call)} +def is_slots(buf:UOp) -> bool: return (b:=hcq2.unwrap_view(buf)[0]).op is Ops.ALLOC and b.arg.name == "slots" + def word(u:UOp) -> tuple[UOp, int]: if u.op is Ops.INDEX: return (v:=hcq2.unwrap_view(u.src[0]))[0], v[1] + u.src[1].val * u.dtype.itemsize return hcq2.unwrap_view(u) @@ -55,8 +64,8 @@ T = 5 # the timeline value the last submitted batch of every device signals def run(batch:UOp, done:dict[str, int]|None=None, prio:list|None=None) -> tuple[list, dict, dict]: qs, cs, mem = queues(batch), calls(batch), defaultdict(int) for d in batch.arg.aux.device: - mem[word(hcq2.timeline((d,)))] = (done or {}).get(d, T) - mem[word(hcq2.timeline((d,)).index(1))] = T + mem[word(hcq2.timeline(d))] = (done or {}).get(d, T) + mem[word(hcq2.timeline(d).index(1))] = T def val(u:UOp) -> int: if u.op is Ops.CONST: return u.val if u.op is Ops.LOAD: return mem[word(u.src[0])] @@ -66,16 +75,16 @@ def run(batch:UOp, done:dict[str, int]|None=None, prio:list|None=None) -> tuple[ for q in (list(qs) if prio is None else prio): if not (cmds:=qs[q]): continue c = cmds[0] - if c.op is Ops.INS and c.arg[0].startswith("wait"): - sig, target = mem[word(c.src[0])], val(c.src[1]) - if sig != target if c.arg[0] == "wait_eq" else sig < target: continue - elif c.op is Ops.INS and c.arg[0] == "store": - mem[w:=word(c.src[0])] = val(c.src[1]) - if w[0].tag == "timeline": log.append(w[0].device) - elif c.op is Ops.CALL: log.append(cs.index(c)) + if (name:=ins_name(c)) is not None and name.startswith("wait"): + sig, target = mem[word(c.src[1])], val(c.src[2]) + if sig != target if name == "wait_eq" else sig < target: continue + elif name == "store": + mem[w:=word(c.src[1])] = val(c.src[2]) + if w[0].arg.name == "timeline": log.append(w[0].device) + elif c in cs: log.append(cs.index(c)) cmds.pop(0) break - else: return log, {q: len(cmds) for q, cmds in qs.items()}, {d: mem[word(hcq2.timeline((d,)))] for d in batch.arg.aux.device} + else: return log, {q: len(cmds) for q, cmds in qs.items()}, {d: mem[word(hcq2.timeline(d))] for d in batch.arg.aux.device} def rotations(batch:UOp) -> list[list]: return [(qs:=list(queues(batch)))[i:] + qs[:i] for i in range(len(queues(batch)))] def orders(batch:UOp) -> set[tuple[int, ...]]: return {tuple(x for x in run(batch, prio=p)[0] if isinstance(x, int)) for p in rotations(batch)} @@ -85,44 +94,44 @@ class TestHCQ2Deps(unittest.TestCase): b = UOp.param(0, dtypes.float32, 64, device=("NULL", "NULL:1")) for view in [b.mselect(0).shrink(((8, 16),)), b.shrink(((8, 16),)).mselect(0), b.shrink(((4, 32),)).mselect(0).shrink(((4, 12),))]: tracker = hcq2.DepsTracker() - tracker.access_resources([view], [0], 0) - self.assertEqual(tracker.access_resources([b.mselect(1)], [], 1), []) - self.assertEqual(tracker.access_resources([b.mselect(0).shrink(((16, 24),))], [], 2), []) - self.assertEqual(tracker.access_resources([b.mselect(0).shrink(((12, 20),))], [], 3), [0]) + tracker.access_resources([(view, True)], 0) + self.assertEqual(tracker.access_resources([(b.mselect(1), False)], 1), []) + self.assertEqual(tracker.access_resources([(b.mselect(0).shrink(((16, 24),)), False)], 2), []) + self.assertEqual(tracker.access_resources([(b.mselect(0).shrink(((12, 20),)), False)], 3), [0]) def test_disjoint_write_preserves_dependencies(self): b = UOp.param(0, dtypes.uint8, 16, device="NULL") - for write in ([], [0]): + for write in (False, True): tracker = hcq2.DepsTracker() - tracker.access_resources([b.shrink(((0, 4),))], write, 0) - self.assertEqual(tracker.access_resources([b.shrink(((4, 8),))], [0], 1), []) - self.assertEqual(tracker.access_resources([b.shrink(((0, 4),))], [0], 2), [0]) + tracker.access_resources([(b.shrink(((0, 4),)), write)], 0) + self.assertEqual(tracker.access_resources([(b.shrink(((4, 8),)), True)], 1), []) + self.assertEqual(tracker.access_resources([(b.shrink(((0, 4),)), True)], 2), [0]) def test_partial_write_preserves_dependencies(self): b = UOp.param(0, dtypes.uint8, 16, device="NULL") - for write in ([], [0]): + for write in (False, True): tracker = hcq2.DepsTracker() - tracker.access_resources([b], write, 0) - self.assertEqual(tracker.access_resources([b.shrink(((4, 12),))], [0], 1), [0]) - self.assertEqual(tracker.access_resources([b.shrink(((0, 4),))], [0], 2), [0]) - self.assertEqual(tracker.access_resources([b.shrink(((12, 16),))], [0], 3), [0]) - self.assertEqual(tracker.access_resources([b.shrink(((4, 12),))], [], 4), [1]) + tracker.access_resources([(b, write)], 0) + self.assertEqual(tracker.access_resources([(b.shrink(((4, 12),)), True)], 1), [0]) + self.assertEqual(tracker.access_resources([(b.shrink(((0, 4),)), True)], 2), [0]) + self.assertEqual(tracker.access_resources([(b.shrink(((12, 16),)), True)], 3), [0]) + self.assertEqual(tracker.access_resources([(b.shrink(((4, 12),)), False)], 4), [1]) def test_write_waits_for_all_readers(self): b = UOp.param(0, dtypes.uint8, 16, device="NULL") tracker = hcq2.DepsTracker() - self.assertEqual(tracker.access_resources([b], [], 0), []) - self.assertEqual(tracker.access_resources([b], [], 1), []) - self.assertEqual(tracker.access_resources([b], [0], 2), [0, 1]) - self.assertEqual(tracker.access_resources([b], [], 3), [2]) + self.assertEqual(tracker.access_resources([(b, False)], 0), []) + self.assertEqual(tracker.access_resources([(b, False)], 1), []) + self.assertEqual(tracker.access_resources([(b, True)], 2), [0, 1]) + self.assertEqual(tracker.access_resources([(b, False)], 3), [2]) def test_aliases_do_not_wait_on_themselves(self): b = UOp.param(0, dtypes.uint8, 16, device="NULL") - for write in ([0], [1], [0, 1]): + for whole, part in ((True, False), (False, True), (True, True)): tracker = hcq2.DepsTracker() - tracker.access_resources([b], [0], 0) - self.assertEqual(tracker.access_resources([b, b.shrink(((4, 12),))], write, 1), [0]) - self.assertEqual(tracker.access_resources([b.shrink(((4, 12),))], [0], 2), [1]) + tracker.access_resources([(b, True)], 0) + self.assertEqual(tracker.access_resources([(b, whole), (b.shrink(((4, 12),)), part)], 1), [0]) + self.assertEqual(tracker.access_resources([(b.shrink(((4, 12),)), True)], 2), [1]) class TestHCQ2Schedule(unittest.TestCase): def setUp(self): @@ -145,11 +154,21 @@ class TestHCQ2Schedule(unittest.TestCase): def scheduled(self, *ts:Tensor) -> list[UOp]: return [self.check(b) for b in scheduled(*ts)] def batch(self, *ts:Tensor) -> UOp: return unwrap(self.scheduled(*ts)[0]) + def kernels(self, batch:UOp) -> list[str]: + return [ins_name(c) or "kernel" for c in queues(batch)[("NULL", "COMPUTE:0")] if ins_name(c) in ("exec_barrier", None)] def test_kernels_run_in_order(self): self.assertEqual(orders(self.batch(chain(self.x, 3))), {(0, 1, 2)}) - def test_a_peer_kernel_runs_after_the_copy_that_feeds_it(self): - self.assertEqual(orders(self.batch((self.x.to("NULL:1") + 1).contiguous())), {(0, 1)}) + def test_exec_barrier(self): + self.assertEqual(self.kernels(self.batch(chain(self.x, 3))), ["kernel", "exec_barrier", "kernel", "exec_barrier", "kernel"]) + a = (self.x + 1).contiguous() + self.assertEqual(self.kernels(self.batch((a + 1).contiguous(), (a + 2).contiguous())), ["kernel", "exec_barrier", "kernel", "kernel"]) + + def test_a_copy_waits_for_the_kernel_when_the_scalar_is_first(self): + n, B = Tensor(Variable("n", 0, 10, dtypes.int).bind(5)), Tensor.ones(4, dtype=dtypes.int).contiguous().realize() + b = self.batch(Tensor.custom_kernel(n, Tensor.empty(4, dtype=dtypes.int), B, fxn=add_n)[1].to("NULL:1")) + kernel = next(i for i, c in enumerate(calls(b)) if c.body.op is Ops.PROGRAM) + self.assertEqual({o[0] for o in orders(b)}, {kernel}) def test_lanes_of_a_sharded_kernel_do_not_wait_for_each_other(self): s = Tensor.ones(8).contiguous().realize().shard(("NULL", "NULL:1"), axis=0).contiguous().realize() @@ -188,33 +207,23 @@ class TestHCQ2Profile(unittest.TestCase): def test_slots_addressed_by_the_device(self): pm = PatternMatcher([(UPat((Ops.LOAD, Ops.STORE), src=(UPat(Ops.INDEX, src=(UPat.var("buf"), UPat())),), allow_any_len=True), - lambda buf: buf.getaddr(Device["NULL"].host) if hcq2.unwrap_view(buf)[0].tag == "slots" else None)]) - with patch.object(Device["NULL"], "pm_lower", pm): self.test_profiling_reports_a_range_per_kernel(n=3) + lambda buf: buf.getaddr(Device["NULL"].host) if is_slots(buf) else None)]) + with patch.object(Compiled, "pm_lower", pm + Compiled.pm_lower): self.test_profiling_reports_a_range_per_kernel(n=3) class TestHCQ2Link(unittest.TestCase): def setUp(self): self.enterContext(Context(DEV="NULL")) - def test_links_serve_any_input(self): - a, inputs = chain_input(), list[UOp]() - linear = compile_linear(chain(a, 2).schedule_linear(), input_uops=inputs, cache=True) - linked = link_linear(linear, input_uops=inputs) - self.assertIs(link_linear(linear, input_uops=[chain_input(3).uop.base, *inputs[1:]]), linked) - bufs = [cast(Buffer, u.buffer) for u in linked.toposort() if u.op is Ops.BUFFER and u.addrspace is AddrSpace.GLOBAL] - self.assertNotIn(a.uop.base.buffer, bufs) - words = [w for b in bufs if b.options.external_ptr and b.nbytes % 8 == 0 for w in b.host.view(fmt='Q')[:]] - self.assertNotIn(cast(Buffer, a.uop.base.buffer)._buf, words) - def test_eager_templates_compile_once(self): self.assertIs(compiled_chain(3)[1], compiled_chain(3)[1]) def test_repeated_word_loops(self): # for (i..10) cmdbuf[off[i]] = var - var, offs = UOp.alloc((1,), dtypes.uint32, device="CPU").rtag("var"), [4 * i * i for i in range(10)] + var, offs = UOp.alloc((1,), dtypes.uint32, device="CPU", name="var"), [4 * i * i for i in range(10)] var = hcq2.patch(var, [], bytes(4)) # initialize at link: the allocator may return a reused buffer hq = SimpleNamespace(blob=bytearray(offs[-1] + 4), patches=[(o, var.index(0).load()) for o in offs], devs=("CPU",), queue="COPY:0", deps=()) lowered = lower_hcq(var.after(hcq2.encode_cmdbuf(hq, device="CPU")).index(0).store(var.index(0).load() + 1)) self.assertEqual(len([u for u in lowered.src[0].without_after.src[0].toposort() if u.op is Ops.RANGE]), 1) - linked = hcq2.hcq_link(lower_and_compile(lowered), allow_cache=False) + linked = hcq2.hcq_link(lower_and_compile(lowered)) args = zip(lowered.src[0].without_after.src[1:], linked.src[0].without_after.src[1:]) - cmdbuf = next(b.buffer for p, b in args if p.without_after.tag == "cmdbuf_copy_0") + cmdbuf = next(b.buffer for p, b in args if p.without_after.arg.name == "cmdbuf_copy_0") for step in range(3): run_linear(linked, jit=True) self.assertEqual([cmdbuf.host.view(fmt="I")[o // 4] for o in offs], [step] * 10) diff --git a/tinygrad_repo/test/null/test_isel.py b/tinygrad_repo/test/null/test_isel.py index 5449969195..32e3654969 100644 --- a/tinygrad_repo/test/null/test_isel.py +++ b/tinygrad_repo/test/null/test_isel.py @@ -1,9 +1,10 @@ import unittest -from tinygrad.helpers import Target +from tinygrad import Tensor +from tinygrad.helpers import Target, Context +from tinygrad.codegen import to_program from tinygrad.uop import Ops from tinygrad.uop.ops import UOp, dtypes, graph_rewrite -from tinygrad.renderer.isa.x86 import X86Renderer, X86Ops -from tinygrad.renderer.isa import IselContext +from tinygrad.renderer.x86 import X86Renderer, X86Ops, IselContext # INDEX on a register value with a constant index extracts a single element (the old GEP) def lane(y:UOp, i:int) -> UOp: return y.index(UOp.cconst(i, dtypes.int)) @@ -24,7 +25,7 @@ class TestIselX86(unittest.TestCase): a = UOp.variable("a", 0, 0, dtypes.int32) b = UOp.variable("b", 0, 0, dtypes.int32) c = (a < b).where(a, b) - d = (a != b).where(a, b) + d = a.alu(Ops.CMPNE, b).where(a, b) f = c + d n = self.isel_rewrite(f) self.assertTrue(n.src[0].arg[0] is X86Ops.CMOVL and n.src[1].arg[0] is X86Ops.CMOVNE) @@ -49,5 +50,9 @@ class TestIselX86(unittest.TestCase): # displacement is the constant in "a" scaled to the buffer element size, dtype is int8 when the value fits otherwise int32 self.assertTrue(n.src[2].dtype is dtypes.int8 and n.src[2].src[0].op is Ops.CONST and n.src[2].src[0].val == 4) + @Context(SPEC=2, NOOPT=1) + def test_register_store_is_effect(self): + to_program(Tensor.empty(16, 16).sum(1).schedule_linear().src[-1].src[0], X86Renderer(Target(device="CPU", arch="x86_64"))) + if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/null/test_linearizer.py b/tinygrad_repo/test/null/test_linearizer.py index 4fd0b9f2ba..114b418f4e 100644 --- a/tinygrad_repo/test/null/test_linearizer.py +++ b/tinygrad_repo/test/null/test_linearizer.py @@ -1,11 +1,11 @@ import unittest from tinygrad.codegen.opt import Opt, OptOps -from tinygrad.uop.ops import Ops, GroupOp, AxisType +from tinygrad.uop.ops import UOp, Ops, GroupOp, AxisType from tinygrad.device import Device from tinygrad.tensor import Tensor from tinygrad.codegen import to_program from tinygrad.dtype import DType, dtypes, AddrSpace -from tinygrad.renderer.isa import ISARenderer +from tinygrad.renderer.x86 import ISARenderer from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.renderer.llvmir import AMDLLVMRenderer from tinygrad.helpers import Target @@ -59,6 +59,20 @@ class TestLinearizer(unittest.TestCase): num_ops = len([uop for uop in uops if uop.op in GroupOp.ALU]) assert num_ops == 0, "more alu uops than needed" + def test_stack_select_folds_into_index(self): + x = Tensor.empty(7, 4).realize() + r = Tensor.stack(x.const_like(0), x, x.const_like(0)).reshape(21, 4).shrink(((4, 19), None)).contiguous() + uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], []), renderer=Device[Device.DEFAULT].renderer).src[1].src) + self.assertEqual([u.op for u in uops if u.op in {Ops.DIV, Ops.MOD, Ops.TRUNCDIV, Ops.TRUNCMOD}], []) + + def test_const_table_even_index(self): + # idx*2 is even, so the gather never selects the odd entry 5 and the store folds to 0 + idx = UOp.variable("idx", 0, 3).bind(0) + out = Tensor.const((0, 0, 0, 5, 0, 0, 0, 0), dtypes.int)[idx*2].to("NULL", force=True) + uops = to_program(out.linear_with_vars()[0].src[-1].src[0], Device["NULL"].renderer).src[1].src + self.assertNotIn(Ops.WHERE, [u.op for u in uops]) + self.assertEqual(next(u for u in uops if u.op is Ops.STORE).src[1].vmax, 0) + def test_sum_acc_dtype(self): for tensor_dtype, acc_dtype in ( (dtypes.bool, dtypes.int), (dtypes.int16, dtypes.int), (dtypes.float16, dtypes.float), (dtypes.bfloat16, dtypes.float)): diff --git a/tinygrad_repo/test/null/test_llm_tokenizer.py b/tinygrad_repo/test/null/test_llm_tokenizer.py index 25215d2115..fa7f42f10d 100644 --- a/tinygrad_repo/test/null/test_llm_tokenizer.py +++ b/tinygrad_repo/test/null/test_llm_tokenizer.py @@ -1,8 +1,7 @@ -import unittest, base64, functools, re, sys, time, unicodedata +import unittest, base64, functools, re, time, unicodedata, json, pathlib, tempfile from tinygrad.llm.cli import SimpleTokenizer, FallbackTemplate from tinygrad.helpers import fetch -@unittest.skipIf(sys.platform == 'win32', "fetch race condition on Windows") class TestLLMTokenizer(unittest.TestCase): @functools.cached_property def llama_tok(self): @@ -105,8 +104,43 @@ class TestLLMTokenizer(unittest.TestCase): self.assertTrue(tok.is_end(0)) # eom ends generation self.assertFalse(tok.is_end(2)) + def test_hf_tokens(self): + with tempfile.TemporaryDirectory() as folder: + path = pathlib.Path(folder) + vocab = {"h": 0, "i": 1, "hi": 2, "Ġ": 3, "!": 4, "[gMASK]": 5, "hello world": 9, "1": 10, "2": 11, "3": 12, "123": 13} + added = [dict(content=t, id=i, special=i != 9) for t, i in (("[gMASK]", 5), ("", 6), ("", 7), ("", 8), ("hello world", 9))] + (path / "tokenizer.json").write_text(json.dumps(dict(model=dict(vocab=vocab), added_tokens=added))) + (path / "config.json").write_text(json.dumps(dict(eos_token_id=[6, 7, 8]))) + for preset in ("glm4", "llama3", "tekken"): + with self.subTest(preset=preset): + tok = SimpleTokenizer.from_hf(str(path), preset=preset) + self.assertEqual(tok.preset, preset) + self._test_coding(tok, "hi hi!", [2, 3, 2, 4]) + self._test_coding(tok, "h!i", [0, 4, 1]) + self._test_coding(tok, "hi [gMASK]hello world!", [2, 3, 5, 9, 4, 6]) + self._test_coding(tok, "123", [10, 11, 12] if preset == "tekken" else [13]) + self.assertEqual((tok.eos_id, tok.eot_id, tok.eom_id), (6, 7, 8)) + self.assertEqual([tok.is_end(i) for i in range(5, 10)], [False, True, True, True, False]) + + def test_hf_metadata(self): + with tempfile.TemporaryDirectory() as folder: + path = pathlib.Path(folder) + (path / "tokenizer.json").write_text(json.dumps(dict(model=dict(vocab={"": 0})))) + (path / "config.json").write_text(json.dumps(dict(eos_token_id=1))) + tok = SimpleTokenizer.from_hf(path, preset="llama3") + self.assertIsNone(tok.bos_id) + self.assertIsNone(tok.chat_template) + self.assertEqual((tok.eos_id, tok.eot_id, tok.eom_id), (1, None, None)) + self.assertTrue(tok.is_end(1)) + (path / "tokenizer_config.json").write_text(json.dumps(dict(bos_token="", chat_template="config template"))) + tok = SimpleTokenizer.from_hf(path, preset="llama3") + self.assertEqual(tok.bos_id, 0) + self.assertEqual(tok.chat_template, "config template") + (path / "chat_template.jinja").write_text("{{ messages[0].content }}") + self.assertEqual(SimpleTokenizer.from_hf(path, preset="llama3").chat_template, "{{ messages[0].content }}") + def test_tekken_gpt4o_split(self): - split = {p: SimpleTokenizer({}, {}, p)._split_to_word.findall for p in ("tekken", "gpt-4o")} + split = {p: SimpleTokenizer({}, {}, p)._split_to_word.findall for p in ("tekken", "gpt-4o", "kimi-k2")} shared = { "HelloWorld": ["Hello", "World"], " ÜNICODE": [" ÜNICODE"], # Ü: non-ascii upper joins the run @@ -117,6 +151,7 @@ class TestLLMTokenizer(unittest.TestCase): for s, want in shared.items(): self.assertEqual(split["tekken"](s), want, f"tekken {s!r}") self.assertEqual(split["gpt-4o"](s), want, f"gpt-4o {s!r}") + self.assertEqual(split["kimi-k2"](s), want, f"kimi-k2 {s!r}") differ = [ ("12345", list("12345"), ["123", "45"]), # digits: tekken single, o200k groups {1,3} ("it's I'M don'T", ["it", "'s", " I", "'M", " don", "'T"], ["it's", " I'M", " don'T"]), # contraction: o200k inline suffix @@ -124,6 +159,8 @@ class TestLLMTokenizer(unittest.TestCase): for s, tk, go in differ: self.assertEqual(split["tekken"](s), tk, f"tekken {s!r}") self.assertEqual(split["gpt-4o"](s), go, f"gpt-4o {s!r}") + self.assertEqual(split["kimi-k2"](s), go, f"kimi-k2 {s!r}") + self.assertEqual(split["kimi-k2"]("a.\n/b"), ["a", ".\n", "/b"]) def test_stream_decoder(self): """stream_decoder buffers incomplete UTF-8: token 25677 has 3/4 of emoji, token 138 completes it.""" diff --git a/tinygrad_repo/test/null/test_multitensor.py b/tinygrad_repo/test/null/test_multitensor.py index ba5b2ee216..26cc3d249e 100644 --- a/tinygrad_repo/test/null/test_multitensor.py +++ b/tinygrad_repo/test/null/test_multitensor.py @@ -1,6 +1,7 @@ import gc, unittest from tinygrad import Tensor, Device, nn, GlobalCounters, TinyJit, dtypes, UOp from tinygrad.uop.ops import Ops +from tinygrad.dtype import AddrSpace from tinygrad.helpers import Context from tinygrad.nn.state import get_parameters, get_state_dict from test.helpers import not_support_multi_device, needs_second_gpu @@ -131,8 +132,6 @@ class TestMultiRamUsage(unittest.TestCase): return x, w1, w2 def run_layers(n_layers): - GlobalCounters.reset() - @TinyJit def f(x, w1, w2): for _ in range(n_layers): @@ -144,8 +143,10 @@ class TestMultiRamUsage(unittest.TestCase): r = f(*a) del a, r - gc.collect() - return GlobalCounters.mem_used + # get the data itself. cmdbuf memory might grow a bit + linked = next(iter(f.captured._linears.values()))[0] + bufs = [u.buffer for u in linked.toposort(enter_calls=True) if u.op is Ops.BUFFER and u.addrspace is AddrSpace.GLOBAL] + return sum(b.nbytes for b in {b for mb in bufs for b in getattr(mb, "bufs", [mb])} if not b.options.cpu_access) mem_2 = run_layers(2) mem_4 = run_layers(4) diff --git a/tinygrad_repo/test/null/test_pattern_matcher.py b/tinygrad_repo/test/null/test_pattern_matcher.py index 20dc78f9f4..f5c8f9aa5e 100644 --- a/tinygrad_repo/test/null/test_pattern_matcher.py +++ b/tinygrad_repo/test/null/test_pattern_matcher.py @@ -5,23 +5,23 @@ from tinygrad.uop.ops import PatternMatcher, UPat class TestPatternMatcher(unittest.TestCase): def test_simple_match(self): - matcher = PatternMatcher([(UPat(Ops.CONST, name="x", dtype=dtypes.weakfloat), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(Ops.CONST, name="x", dtype=dtypes.weakfloat), lambda x: x.sink())]) c1 = UOp.const(1.0) c2 = UOp.const(1) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) self.assertEqual(matcher.rewrite(c2), None) def test_upat_any(self): def test(a, x=None, y=None, z=None): #print(x,y,z) - if y is not None: return (a+y).rtag() + if y is not None: return (a+y).sink() matcher = PatternMatcher([ (UPat.var("a")+UPat.any(UPat.var("x"), UPat.var("y"), UPat.var("z")), test), ]) v1 = UOp.variable("a", 0, 10) v2 = UOp.variable("b", 0, 10) c1 = v1+v2 - self.assertEqual(matcher.rewrite(c1), c1.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) def test_minimum_len(self): matcher = PatternMatcher([ @@ -60,43 +60,43 @@ class TestPatternMatcher(unittest.TestCase): self.assertEqual(len(ctx), 1) def test_uop(self): - matcher = PatternMatcher([(UPat(Ops.CONST, name="x"), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(Ops.CONST, name="x"), lambda x: x.sink())]) c1 = UOp.const(1.0) c2 = UOp(Ops.ADD, src=(c1, c1)) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) self.assertEqual(matcher.rewrite(c2), None) def test_uop_set(self): - matcher = PatternMatcher([(UPat((Ops.CONST, Ops.CAST), name="x"), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat((Ops.CONST, Ops.CAST), name="x"), lambda x: x.sink())]) c1 = UOp.const(False) c2 = UOp(Ops.CAST, arg=dtypes.int, src=(c1,)) c3 = UOp.const(1.0) c4 = UOp(Ops.ADD, src=(c3, c3)) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) - self.assertEqual(matcher.rewrite(c2), c2.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) + self.assertEqual(matcher.rewrite(c2), c2.sink()) self.assertEqual(matcher.rewrite(c4), None) def test_arg(self): matcher = PatternMatcher([ - (UPat(Ops.CONST, arg=0, name="x"), lambda x: x.rtag()), - (UPat(Ops.CONST, arg=False, name="x"), lambda x: x.rtag()), - (UPat(Ops.MAX, name="x"), lambda x: x.rtag()), + (UPat(Ops.CONST, arg=0, name="x"), lambda x: x.sink()), + (UPat(Ops.CONST, arg=False, name="x"), lambda x: x.sink()), + (UPat(Ops.MAX, name="x"), lambda x: x.sink()), ]) c1 = UOp.const(0.0) c2 = UOp.const(False) c3 = UOp(Ops.MAX, src=(c1, c1)) c4 = UOp(Ops.MUL, src=(c1, c1)) c5 = UOp.const(-1) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) - self.assertEqual(matcher.rewrite(c2), c2.rtag()) - self.assertEqual(matcher.rewrite(c3), c3.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) + self.assertEqual(matcher.rewrite(c2), c2.sink()) + self.assertEqual(matcher.rewrite(c3), c3.sink()) self.assertEqual(matcher.rewrite(c4), None) self.assertEqual(matcher.rewrite(c5), None) def test_filter_arg(self): matcher = PatternMatcher([ (UPat(Ops.MUL, src=[UPat(Ops.CONST, name="c"), UPat(Ops.CONST, arg=2)], name="x"), - lambda x,c: x.rtag() if c.val in {1, -1} else None) + lambda x,c: x.sink() if c.val in {1, -1} else None) ]) y1 = UOp.const(1) y2 = UOp.const(2) @@ -106,46 +106,46 @@ class TestPatternMatcher(unittest.TestCase): c3 = UOp(Ops.MUL, src=(y3, y2)) c4 = UOp(Ops.MUL, src=(y2, y1)) c5 = UOp(Ops.MUL, src=(y2, y3)) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) self.assertEqual(matcher.rewrite(c2), None) - self.assertEqual(matcher.rewrite(c3), c3.rtag()) - self.assertEqual(matcher.rewrite(c4), c4.rtag()) - self.assertEqual(matcher.rewrite(c5), c5.rtag()) + self.assertEqual(matcher.rewrite(c3), c3.sink()) + self.assertEqual(matcher.rewrite(c4), c4.sink()) + self.assertEqual(matcher.rewrite(c5), c5.sink()) def test_dup_name(self): - matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=(UPat(Ops.CONST, name="y"), UPat(Ops.CONST, name="y"))), lambda x, y: x.rtag())]) + matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=(UPat(Ops.CONST, name="y"), UPat(Ops.CONST, name="y"))), lambda x, y: x.sink())]) y1 = UOp.const(1.0) y2 = UOp.const(1.0) c1 = UOp(Ops.ADD, src=(y1, y1)) c2 = UOp(Ops.ADD, src=(y1, y2)) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) - self.assertEqual(matcher.rewrite(c2), c1.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) + self.assertEqual(matcher.rewrite(c2), c1.sink()) def test_dtype(self): # a concrete const dtype lives on the pair's CAST - matcher = PatternMatcher([(UPat(Ops.CAST, name="x", dtype=dtypes.float32), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(Ops.CAST, name="x", dtype=dtypes.float32), lambda x: x.sink())]) c1 = UOp.const(1.0).cast(dtypes.float32) c2 = UOp.const(1.0).cast(dtypes.float64) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) self.assertEqual(matcher.rewrite(c2), None) def test_dtype_set(self): - matcher = PatternMatcher([(UPat(Ops.CAST, name="x", dtype={dtypes.float32, dtypes.float64}), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(Ops.CAST, name="x", dtype={dtypes.float32, dtypes.float64}), lambda x: x.sink())]) c1 = UOp.const(1.0).cast(dtypes.float32) c2 = UOp.const(1.0).cast(dtypes.float64) c3 = UOp.const(1.0).cast(dtypes.float16) c4 = UOp.const(1).cast(dtypes.int) - self.assertEqual(matcher.rewrite(c1), c1.rtag()) - self.assertEqual(matcher.rewrite(c2), c2.rtag()) + self.assertEqual(matcher.rewrite(c1), c1.sink()) + self.assertEqual(matcher.rewrite(c2), c2.sink()) self.assertEqual(matcher.rewrite(c3), None) self.assertEqual(matcher.rewrite(c4), None) def test_src_one(self): - matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=(UPat(Ops.CONST), UPat(Ops.CONST))), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=(UPat(Ops.CONST), UPat(Ops.CONST))), lambda x: x.sink())]) c1 = UOp.const(1.0) c2 = UOp.const(2.0) c3 = UOp(Ops.ADD, src=(c1,c2)) - self.assertEqual(matcher.rewrite(c3), c3.rtag()) + self.assertEqual(matcher.rewrite(c3), c3.sink()) self.assertEqual(matcher.rewrite(c2), None) # that CONST/ALU -> ALU/CONST rewrite is now instant """ @@ -158,7 +158,7 @@ class TestPatternMatcher(unittest.TestCase): """ def test_src_permutations(self): - matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=[UPat(Ops.CONST), UPat(GroupOp.ALU)]), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=[UPat(Ops.CONST), UPat(GroupOp.ALU)]), lambda x: x.sink())]) c1 = UOp.const(1.0) c2 = UOp.const(2.0) c3 = UOp(Ops.ADD, src=(c1,c2)) @@ -166,21 +166,21 @@ class TestPatternMatcher(unittest.TestCase): c5 = UOp(Ops.ADD, src=(c2,c3)) c6 = UOp(Ops.ADD, src=(c3,c4)) self.assertEqual(matcher.rewrite(c3), None) - self.assertEqual(matcher.rewrite(c4), c4.rtag()) - self.assertEqual(matcher.rewrite(c5), c5.rtag()) + self.assertEqual(matcher.rewrite(c4), c4.sink()) + self.assertEqual(matcher.rewrite(c5), c5.sink()) self.assertEqual(matcher.rewrite(c6), None) def test_src_repeat(self): - matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=UPat(Ops.CONST)), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(GroupOp.ALU, name="x", src=UPat(Ops.CONST)), lambda x: x.sink())]) c1 = UOp.const(1.0) c2 = UOp.const(2.0) c3 = UOp(Ops.ADD, src=(c1,c2)) c4 = UOp(Ops.ADD, src=(c2,c3)) - self.assertEqual(matcher.rewrite(c3), c3.rtag()) + self.assertEqual(matcher.rewrite(c3), c3.sink()) self.assertEqual(matcher.rewrite(c4), None) def test_allow_len(self): - matcher = PatternMatcher([(UPat(Ops.MULACC, name="x", src=(UPat(Ops.CONST),), allow_any_len=True), lambda x: x.rtag())]) + matcher = PatternMatcher([(UPat(Ops.MULACC, name="x", src=(UPat(Ops.CONST),), allow_any_len=True), lambda x: x.sink())]) c1 = UOp.const(1.0) c2 = UOp.const(2.0) c3 = UOp.const(3.0) @@ -189,7 +189,7 @@ class TestPatternMatcher(unittest.TestCase): c6 = UOp(Ops.MULACC, src=(c1,c2,c3)) self.assertEqual(matcher.rewrite(c4), None) self.assertEqual(matcher.rewrite(c5), None) - self.assertEqual(matcher.rewrite(c6), c6.rtag()) + self.assertEqual(matcher.rewrite(c6), c6.sink()) def test_deep_src_permutations(self): c1 = UOp.const(1.0) diff --git a/tinygrad_repo/test/null/test_randomness.py b/tinygrad_repo/test/null/test_randomness.py index 73a1120d44..77859bfd1d 100644 --- a/tinygrad_repo/test/null/test_randomness.py +++ b/tinygrad_repo/test/null/test_randomness.py @@ -1,22 +1,7 @@ import unittest from tinygrad import dtypes, Tensor -from tinygrad.codegen import to_program -from tinygrad.helpers import Target -from tinygrad.renderer.cstyle import ClangRenderer -from tinygrad.uop.ops import Ops class TestRandomness(unittest.TestCase): - def test_threefry_doesnt_use_long(self): - # Use a renderer without native THREEFRY or 64-bit pointer arithmetic in the UOps. - renderer = ClangRenderer(Target("CPU", arch="x86_64,x86-64")) - linear = Tensor.rand(20).schedule_linear() - for call in linear.src: - ast = call.src[0] - if ast.op is Ops.SINK: - prg = to_program(ast, renderer=renderer) - for u in tuple(prg.src[1].src): - self.assertNotIn(u.dtype, {dtypes.long, dtypes.ulong}, msg=f"long found in {prg.src[0].arg.name}") - def test_rand_like(self): empty = Tensor.empty((80, 44)) rand = Tensor.rand_like(empty) diff --git a/tinygrad_repo/test/null/test_real_world.py b/tinygrad_repo/test/null/test_real_world.py index 74b9469bfa..30e3145f7b 100644 --- a/tinygrad_repo/test/null/test_real_world.py +++ b/tinygrad_repo/test/null/test_real_world.py @@ -30,7 +30,7 @@ def helper_test(nm, gen, model, max_memory_allowed, max_kernels_allowed, all_jit tms.append(time.perf_counter_ns() - st) mem_used = (GlobalCounters.mem_used - global_mem_used) / 1e9 - kernels_used = jit_cache_count(model.captured.linear) if getattr(model, "captured", None) is not None else None + kernels_used = jit_cache_count(model.captured._linear) if getattr(model, "captured", None) is not None else None print(f"{nm}: used {mem_used/1e9:.2f} GB and {kernels_used} kernels in {min(tms)/1e6:.2f} ms") assert mem_used < max_memory_allowed, f"{nm} used more than {max_memory_allowed:.3f} GB - {mem_used:.3} GB used" assert (max_memory_allowed - mem_used) / max_memory_allowed < 0.2, f"{max_memory_allowed:.3f} GB is too far from {mem_used:.3} GB used" diff --git a/tinygrad_repo/test/null/test_renderer_failures.py b/tinygrad_repo/test/null/test_renderer_failures.py index 4a9ee090d4..d5ba9bda07 100644 --- a/tinygrad_repo/test/null/test_renderer_failures.py +++ b/tinygrad_repo/test/null/test_renderer_failures.py @@ -38,7 +38,7 @@ class TestWGSLFailures(unittest.TestCase): b = UOp.param(0, dtypes.char, 4) word = b.index(UOp.const(0).cast(dtypes.int)).cast(dtypes.uint32) store = word.store(word.load() & UOp.const(0xffffff00).cast(dtypes.uint32)) - src = WGSLRenderer(Target("WEBGPU")).render(UOp.sink(store, arg=KernelInfo()).toposort()) + src = WGSLRenderer(Target("WEBGPU")).render(list(UOp.sink(store, arg=KernelInfo()).toposort())) self.assertIn("atomicAnd(&data0_4[0],4294967040u);", src) self.assertNotIn("atomicAdd", src) diff --git a/tinygrad_repo/test/null/test_schedule.py b/tinygrad_repo/test/null/test_schedule.py index 1f81fc0c52..7c0c94c247 100644 --- a/tinygrad_repo/test/null/test_schedule.py +++ b/tinygrad_repo/test/null/test_schedule.py @@ -1,13 +1,13 @@ # schedule tests that pass on NULL backend (no copyout needed) -import unittest, time, gc +import unittest, time from typing import cast from tinygrad import nn, dtypes, Device, Tensor, getenv -from tinygrad.helpers import GlobalCounters, Context, all_same +from tinygrad.helpers import Context, all_same from tinygrad.uop.ops import UOp, Ops, GroupOp, UPat, KernelInfo from tinygrad.dtype import AddrSpace from tinygrad.engine.realize import run_linear, compile_linear from tinygrad.codegen import to_program, full_rewrite_to_sink -from test.helpers import check_schedule, assert_kernel_count, KernelCountException, jit_cache_count +from test.helpers import check_schedule, KernelCountException, jit_cache_count, measure_mem def _realize_weights(m): for p in nn.state.get_parameters(m): p.realize() @@ -209,16 +209,6 @@ class TestSchedule(unittest.TestCase): expr = a/a check_schedule(expr, 1) - def test_div_collapse(self): - a = Tensor.full((4,), 1.0).contiguous().realize() - b = Tensor.full((4,), 2.0).contiguous().realize() - c = Tensor.full((4,), 3.0).contiguous().realize() - GlobalCounters.reset() - expr = (a/b)/c - expr.realize() - assert_kernel_count(1) - self.assertLessEqual(GlobalCounters.global_ops, 4*3) - # NOTE: this is causing "LAZYCACHE=1 incorrectly reuses contiguous const" #4562 # should contiguous dedup? @unittest.skip("we do the exact opposite now") @@ -611,13 +601,10 @@ class TestSchedule(unittest.TestCase): def test_conv2d_half(self): self.test_conv2d(4, dtype=dtypes.half) def test_schedule_mem_used_with_inputs(self): - Tensor.ones(256).contiguous().realize() # hcq2 caches the linked schedule with its buffers - gc.collect() - base = GlobalCounters.mem_used - x = Tensor.ones(256).contiguous().realize() - (x+Tensor.ones(256).contiguous()).schedule_linear() - gc.collect() - self.assertEqual(GlobalCounters.mem_used-base, 1024) + with measure_mem() as used: + x = Tensor.ones(256).contiguous().realize() + (x+Tensor.ones(256).contiguous()).schedule_linear() + self.assertEqual(used(), 1024) def test_image_dot_f16_fusion(self): with Context(FLOAT16=1, OPENPILOT_HACKS=1): @@ -722,7 +709,8 @@ class TestSchedule(unittest.TestCase): for call in linear.src: if call.src[0].op is not Ops.SINK: continue sink = full_rewrite_to_sink(call.src[0], Device[call.device].renderer) - reduce_kernels += any(u.op is Ops.STORE and u.addrspace is AddrSpace.REG and u.ranges for u in sink.toposort()) + # a reduce kernel is identified by a STORE into a REG buffer (STORE has void dtype, check the addrspace of its target) + reduce_kernels += any(u.op is Ops.STORE and u.src[0].addrspace is AddrSpace.REG and u.ranges for u in sink.toposort()) self.assertEqual(reduce_kernels, 1) def test_push_through_reshape(self): @@ -1413,7 +1401,7 @@ class TestSchedule(unittest.TestCase): def test_pow_neg_05_is_rsqrt(self): t = Tensor([1.0, 2.0, 3.0]) ** -0.5 - self.assertEqual(self._alu_from_tensor(t), [Ops.RECIPROCAL, Ops.SQRT]) + self.assertEqual(self._alu_from_tensor(t), [Ops.RECIP, Ops.SQRT]) def test_pow_2_has_1_mul(self): t = Tensor([1.0, 2.0, 3.0]) ** Tensor(2.0) @@ -1698,13 +1686,10 @@ class TestSchedule(unittest.TestCase): check_schedule(out, 2) def test_schedule_mem_used(self): - Tensor.ones(256).contiguous().realize() # hcq2 caches the linked schedule with its buffers - gc.collect() - base = GlobalCounters.mem_used - Tensor.ones(256).contiguous().realize() - Tensor.ones(5, 5).contiguous().schedule_linear() - gc.collect() - self.assertEqual(GlobalCounters.mem_used-base, 0) + with measure_mem() as used: + Tensor.ones(256).contiguous().realize() + Tensor.ones(5, 5).contiguous().schedule_linear() + self.assertEqual(used(), 0) def test_const_schedule(self): constv = Tensor.empty(2, 2).const_like(10).uop diff --git a/tinygrad_repo/test/null/test_simplify_valid_idx.py b/tinygrad_repo/test/null/test_simplify_valid_idx.py index e317e25447..5d5f73951c 100644 --- a/tinygrad_repo/test/null/test_simplify_valid_idx.py +++ b/tinygrad_repo/test/null/test_simplify_valid_idx.py @@ -2,15 +2,16 @@ import unittest, itertools from dataclasses import replace from tinygrad.codegen.late.coalesce import indexing_simplify, pm_simplify_add_image +from tinygrad import Tensor from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, Ops, graph_rewrite from tinygrad.uop.weak import pm_commit_weak -from tinygrad.uop.symbolic import simplify_valid, sym, pm_move_where_on_load +from tinygrad.uop.symbolic import simplify_valid, uop_given_valid, sym, pm_move_where_on_load from tinygrad.helpers import Context from test.helpers import full_rewrite from test.null.test_uop_symbolic import check_uop_against_string -# symbolic-only idx + valid simplification (no late lowering of FLOORDIV/FLOORMOD) +# symbolic-only idx + valid simplification (no late lowering of DIV/MOD) def simplify_valid_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move_where_on_load, name="simplify_valid_idx") # image-aware idx + valid simplification: adds the codegen-layer matcher that drops provably in-bounds gates def simplify_image_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move_where_on_load+indexing_simplify, name="simplify_image_idx") @@ -40,7 +41,7 @@ class TestValidIdxSimplification(unittest.TestCase): load = get_gated_load_uop(gate, idx) self.check(load, "0", - "(((lidx0+(gidx0*4))<19)!=True)") + "((lidx0+(gidx0*4))<19).eq(False)") def test_simplify_within_valid1(self): ridx0 = Range(0, 4) @@ -61,6 +62,13 @@ class TestValidIdxSimplification(unittest.TestCase): valid = (alu0 < 57) & (alu0 >= 1) self.assertIsNone(simplify_valid(valid)) + def test_simplex_const_term_keeps_lanes_aligned(self): + # the 1 in the valid is not a simplex variable, lane +1 must simplify like the other lanes + a, b, c, d = Variable("a", 0, 7), Variable("b", 0, 7), Variable("c", 0, 3), Variable("d", 0, 1) + valid = ((a+b+1)<1).ne(True) & ((c+d)<1).ne(True) + base = ((a*8+b+c*128+d*64+1000)%1024)*4 + self.assertTrue(all(uop_given_valid(valid, base+k).src[0] is uop_given_valid(valid, base+0).src[0] for k in range(1, 4))) + def test_bitwise_and_is_not_a_valid(self): ridx0 = Range(0, 16) self.assertEqual(simplify_valid_idx(UOp.sink((ridx0 & UOp.const(12, dtypes.int)) & ridx0)).src[0].render(), "((i32)(r0)&12&(i32)(r0))") @@ -191,7 +199,7 @@ class TestValidIdxSimplification(unittest.TestCase): load = get_gated_load_uop(valid, idx) self.check(load, "1", - "((((r0+r1)<1)!=True)&(((r2+r3)<1)!=True))") + "(((r0+r1)<1).eq(False)&((r2+r3)<1).eq(False))") def test_valid_with_non_const_rhs(self): ridx0 = Range(0, 1024) @@ -232,11 +240,11 @@ class TestImageSimplification(unittest.TestCase): # should match any one of the AND clause and drop the matched statement from valid valid = (gidx0<1).ne(True) & (gidx1<1).ne(True) load = get_load_image_uop(shape, valid, (gidx0+1, gidx1-1)) - self.check(load, "((gidx0<1)!=True)", "(gidx0+1)", "(gidx1+-1)") + self.check(load, "(gidx0<1).eq(False)", "(gidx0+1)", "(gidx1+-1)") valid = (gidx0<1).ne(True) & (gidx1<1).ne(True) load = get_load_image_uop(shape, valid, (gidx0, gidx1-1)) - self.check(load, "((gidx0<1)!=True)", "gidx0", "(gidx1+-1)") + self.check(load, "(gidx0<1).eq(False)", "gidx0", "(gidx1+-1)") def test_idx_lt_bound(self): # (idx1 < image_bound) ? (..., idx1) : 0 can drop the valid @@ -471,7 +479,7 @@ class TestImageSimplification(unittest.TestCase): self.check(load, None, "(gidx0+lidx0*1024+r0*1024+lidx1*128+-3168)", "0") except AssertionError: # TODO: fold valid - self.check(load, "(((lidx1<1)!=True)&((lidx0+r0)<19))", + self.check(load, "((lidx1<1).eq(False)&((lidx0+r0)<19))", "(gidx0+lidx1*128+(lidx0*1024+r0*1024)+-3168)", "0") def test_simplify10(self): @@ -490,7 +498,7 @@ class TestImageSimplification(unittest.TestCase): self.check(load, None, "(lidx2+gidx0*4+lidx0*1024+r0*1024+lidx1*256+-3264)", "0") except AssertionError: # TODO: fold valid - self.check(load, "(((lidx1<1)!=True)&((lidx0+r0)<11))", + self.check(load, "((lidx1<1).eq(False)&((lidx0+r0)<11))", "(lidx2+gidx0*4+lidx1*256+(lidx0*1024+r0*1024)+-3264)", "0") def test_drop_non_monotonic_window(self): @@ -513,6 +521,10 @@ class TestImageSimplification(unittest.TestCase): off = graph_rewrite(load.sink(), pm_commit_weak+indexing_simplify).src[0].src[0] self.assertEqual(off.src[1].get_valid(), UOp.const(True)) + def test_int64_index_into_image_with_emulated_long(self): + with Context(DEV="NULL:OPENCL:,IMAGE_PITCH_ALIGNMENT=64", IMAGE=1, EMULATED_DTYPES="long"): + Tensor.empty(96, 128)[Tensor([3, 1, 5], dtype=dtypes.int64)].realize() + class TestImageStore(unittest.TestCase): def test_half_store_converts_lane_by_lane(self): # a half4 stored to a half image converts to float per lane: a half4->float4 CAST is not valid OpenCL diff --git a/tinygrad_repo/test/null/test_symbolic_tensor.py b/tinygrad_repo/test/null/test_symbolic_tensor.py index 6d035f0dba..09cd8438ea 100644 --- a/tinygrad_repo/test/null/test_symbolic_tensor.py +++ b/tinygrad_repo/test/null/test_symbolic_tensor.py @@ -67,7 +67,7 @@ class TestSymbolicReshape(unittest.TestCase): def test_reshape_infer_multiple_of(self): for multiple in (1, 2, 3, 4, 8): - var = Variable("n", multiple, 8*multiple, multiple_of=multiple) + var = Variable("n", multiple, 8*multiple) for n in (var, var.bind(2*multiple)): with self.subTest(multiple=multiple, bound=n.is_bound_var): self.assertEqual(Tensor.empty(8*multiple, 4)[:n].reshape(n, -1).shape, (n, 4)) diff --git a/tinygrad_repo/test/null/test_tensor_metadata.py b/tinygrad_repo/test/null/test_tensor_metadata.py deleted file mode 100644 index e35f0d62b1..0000000000 --- a/tinygrad_repo/test/null/test_tensor_metadata.py +++ /dev/null @@ -1,127 +0,0 @@ -import unittest -from tinygrad import Tensor, dtypes -from tinygrad.tensor import _METADATA -from tinygrad.engine.realize import capturing -from tinygrad.helpers import Context - -@unittest.skip("tensor metadata is no longer supported") -class TestTensorMetadata(unittest.TestCase): - def setUp(self) -> None: - _METADATA.set(None) - self._ctx = Context(SCACHE=0) - self._ctx.__enter__() - def tearDown(self) -> None: - self._ctx.__exit__(None, None, None) - - @unittest.skip("why would this be true?") - def test_exclude_noop_metadata(self): - a = Tensor.rand(4, 4)*1 - self.assertEqual(a.uop.metadata[0].name, "__mul__") - k = a.schedule_linear().src[-1] - self.assertEqual([m.name for m in k.arg.metadata], ["rand"]) - - @unittest.skip("metadata not reaching kernel schedule") - def test_exclude_const_metadata(self): - a = Tensor.arange(4) - b = Tensor.full((4,), -1, dtype=dtypes.int).contiguous() - sched = a.schedule_linear(b) - self.assertEqual([m.name for m in sched.src[0].arg.metadata], ["arange"]) - self.assertEqual([m.name for m in sched.src[1].arg.metadata], ["contiguous"]) - - def test_matmul(self): - x = Tensor.rand(3) - W = Tensor.rand(3, 3) - out = x.matmul(W) - self.assertEqual(out.uop.metadata[0].name, "matmul") - si = out.schedule_linear().src[-1] - self.assertEqual(len(si.arg.metadata), 1) - self.assertEqual(si.arg.metadata[0].name, "matmul") - - def test_relu(self): - x = Tensor.rand(3) - out = x.relu() - self.assertEqual(out.uop.metadata[0].name, "relu") - si = out.schedule_linear().src[-1] - self.assertEqual(len(si.arg.metadata), 1) - self.assertEqual(si.arg.metadata[0].name, "relu") - - @unittest.skip("assign metadata no longer captured") - def test_assign(self): - x = Tensor.empty(10, 10).realize() - x.assign(Tensor.ones(10, 10).contiguous()) - si = x.schedule_linear().src[-1] - self.assertEqual(len(si.arg.metadata), 1) - self.assertEqual(si.arg.metadata[0].name, "assign") - - def test_complex(self): - x = Tensor.rand(3) - y = Tensor.rand(3) - out = x.relu() * y.sigmoid() - self.assertEqual(out.uop.metadata[0].name, "__mul__") - self.assertEqual(out.uop.src[0].metadata[0].name, "relu") - self.assertEqual(out.uop.src[1].metadata[0].name, "sigmoid") - si = out.schedule_linear().src[-1] - self.assertEqual(len(si.arg.metadata), 3) - self.assertEqual(set(m.name for m in si.arg.metadata), {"relu", "sigmoid", "__mul__"}) - - @unittest.skip("flaky") - def test_complex_backward(self): - x = Tensor.rand(3).realize() - y = Tensor.rand(3).realize() - out = (x.relu() * y.sigmoid()).sum() - self.assertEqual(out.uop.metadata[0].name, "sum") - out.backward() - self.assertEqual(x.grad.uop.metadata[0].name, "relu") - #self.assertTrue(x.grad.uop.metadata[0].backward) # TODO: backward flag is False - self.assertEqual(y.grad.uop.metadata[0].name, "sigmoid") - #self.assertTrue(y.grad.uop.metadata[0].backward) # TODO: backward flag is False - si = out.schedule_linear(x.grad, y.grad).src[-1] - #self.assertEqual(len(si.arg.metadata), 3, f"failed with {si.arg.metadata}") - # skip numpy, this is schedule cache - self.assertSetEqual(set(m.name for m in si.arg.metadata if m.name != "numpy"), {"sigmoid", "relu"}) - #bw = [m for m in si.metadata if m.backward] - #self.assertEqual(len(bw), 1) - #self.assertEqual(bw[0].name, "sigmoid") - - def test_tracemeta_0(self): - with Context(TRACEMETA=0): - x = Tensor.rand(3) - y = Tensor.rand(3) - out = (x.relu() * y.sigmoid()).sum() - self.assertIsNone(out.uop.metadata) - self.assertIsNone(out.uop.src[0].metadata) - si = out.schedule_linear().src[-1] - self.assertEqual(si.arg.metadata, ()) - - def _has_metadata(self, h, name): - linears = [] - capturing.append(type("", (), {"add_linear": lambda _, linear: linears.append(linear)})()) - try: h.realize() - finally: capturing.clear() - calls = [call for linear in linears for call in linear.src] - return any(m.name == name for call in calls for m in call.arg.metadata) - - def test_metadata_survives_realize_pending_assign(self): - shared = Tensor.rand(4) - c = Tensor.zeros(8).contiguous().realize() - c[:4].assign(shared) - self.assertTrue(self._has_metadata(c[:4].relu(), "relu")) - - @unittest.expectedFailure - def test_metadata_lost_realize_pending_assign(self): - shared = Tensor.rand(4) - c = Tensor.zeros(8).contiguous().realize() - c[:4].assign(shared) - self.assertTrue(self._has_metadata((c[:4] + shared).relu(), "relu")) - -class TestTraceMetaShutdown(unittest.TestCase): - def test_tracemeta_del_no_shutdown_error(self): - import subprocess, os - result = subprocess.run(['python3', '-c', 'from tinygrad import Tensor\n' - 'x=Tensor.eye(3); (x@x).sum().backward()'], - env={**os.environ, "TRACEMETA": "2"}, capture_output=True) - self.assertEqual(result.returncode, 0) - self.assertNotIn(b"Exception", result.stderr) - -if __name__ == '__main__': - unittest.main() diff --git a/tinygrad_repo/test/null/test_uir_render.py b/tinygrad_repo/test/null/test_uir_render.py index cdfd5e6c82..72d8c4bbe4 100644 --- a/tinygrad_repo/test/null/test_uir_render.py +++ b/tinygrad_repo/test/null/test_uir_render.py @@ -2,7 +2,7 @@ import unittest, re, ast as pyast from typing import Any from tinygrad import Tensor from tinygrad.dtype import dtypes, DType, AddrSpace, Invalid -from tinygrad.uop import Ops +from tinygrad.uop import Ops, GroupOp from tinygrad.uop.ops import AxisType, UOp, graph_rewrite, ParamArg, KernelInfo from tinygrad.uop.movement import mop_cleanup from tinygrad.uop.render import render_uir @@ -36,11 +36,10 @@ def _parse_const(tok:str) -> UOp: def _parse_paramarg(rest:str, name:str|None) -> ParamArg: kv = _kv(rest) - def i(k:str) -> int|None: return int(kv[k]) if k in kv else None dev = kv.get("device") return ParamArg(int(kv["slot"]), _DTYPES_BY_NAME[kv["dtype"]], tuple(map(int, kv["bounds"].strip("[]").split(","))) if "bounds" in kv else None, - i("multiple_of"), name, AddrSpace[kv["addrspace"]] if "addrspace" in kv else AddrSpace.GLOBAL, + name, AddrSpace[kv["addrspace"]] if "addrspace" in kv else AddrSpace.GLOBAL, pyast.literal_eval(dev) if dev and dev[0] in "'(" else dev, kv.get("volatile") == "true") def parse_ssa(text:str) -> UOp: @@ -66,7 +65,7 @@ def parse_ssa(text:str) -> UOp: name, argstr = argstr[1:end], argstr[end+1:].strip() srcs = [parse_tok(t) for t in _src_tok.findall(srcstr)] arg: Any = None - if op in {Ops.PARAM, Ops.BUFFER, Ops.ALLOC}: arg = _parse_paramarg(argstr, name) + if op in GroupOp.Defines: arg = _parse_paramarg(argstr, name) elif op is Ops.RANGE: t, *ids = argstr.split() arg = (AxisType[t], *map(int, ids)) @@ -86,7 +85,7 @@ def parse_ssa(text:str) -> UOp: def _strip_buffers(root:UOp) -> UOp: # realized BUFFERs carry runtime state the wire can't; a BUFFER with no binding is exactly an ALLOC - subs = {b: b.replace(op=Ops.ALLOC, arg=ParamArg(b.arg.slot, b.arg.dtype, b.arg.vmin_vmax, b.arg.multiple_of, + subs = {b: b.replace(op=Ops.ALLOC, arg=ParamArg(b.arg.slot, b.arg.dtype, b.arg.vmin_vmax, b.arg.name, b.arg.addrspace, b.arg.device, b.arg.volatile)) for b in root.toposort() if b.op is Ops.BUFFER and isinstance(b.arg, ParamArg)} return root.substitute(subs, walk=True, name="strip buffers for wire format test") if subs else root diff --git a/tinygrad_repo/test/null/test_uop_graph.py b/tinygrad_repo/test/null/test_uop_graph.py index 2541a643bd..800e08922a 100644 --- a/tinygrad_repo/test/null/test_uop_graph.py +++ b/tinygrad_repo/test/null/test_uop_graph.py @@ -1,9 +1,9 @@ import unittest, pytest, weakref from tinygrad import dtypes, Variable from tinygrad.dtype import AddrSpace -from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite, GroupOp, AxisType, broadcast_axes, KernelInfo +from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite, AxisType, broadcast_axes, KernelInfo from tinygrad.uop.symbolic import sym -from tinygrad.codegen.simplify import pm_reduce_unparented +from tinygrad.codegen.simplify import pm_reduce_collapse from test.helpers import full_rewrite, to_uops_list simple_pm = PatternMatcher([ @@ -19,9 +19,6 @@ class TestTuplize(unittest.TestCase): a, b = UOp.const(1), UOp.const(2) pairs = [(a, UOp.const(1)), (a, b), (a, a+b), (a+b, b+a), (UOp.sink(a, b), UOp.sink(a, b))] for x, y in pairs: self.assertEqual(x.tuplize == y.tuplize, x is y) - # tags are not part of the order: tag-only differences order neither way, like value-equal plain tuples - self.assertFalse(a.tuplize < a.rtag("tagged").tuplize) - self.assertFalse(a.rtag("tagged").tuplize < a.tuplize) def test_deep_shared_subgraphs(self): # long equal prefixes with a difference at the bottom: correct order, and the pathological case for value equality @@ -31,7 +28,7 @@ class TestTuplize(unittest.TestCase): self.assertNotEqual(a.tuplize < b.tuplize, b.tuplize < a.tuplize) def test_does_not_retain_uops(self): - a = UOp.const(1).rtag(object()) + a = UOp(Ops.NOOP, arg=object()) ref = weakref.ref(a) _ = a.tuplize del a @@ -300,19 +297,6 @@ class TestUOpGraph(unittest.TestCase): assert u.op is not Ops.WHERE if u.op is Ops.LOAD: self.assertIs(u.src[1], expected) - def test_where_on_gated_load_with_cast(self): - ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.int, 100) - gate_idx = ridx0.valid((ridx0<50)) - ld = d0.index(gate_idx).cast(dtypes.float) - w = (ridx0<50).where(ld, 5.0) - out = UOp.param(1, dtypes.float, 100) - uops = to_uops_list([out.index(ridx0).store(w)]) - expected = full_rewrite(UOp.const(5, dtypes.int).sink()).src[0] - for u in uops: - assert u.op is not Ops.WHERE - if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: self.assertIs(u.src[1], expected) - def test_where_on_casted_gated_load_extra_cond(self): ridx0 = UOp.range(100, 0) d0 = UOp.param(0, dtypes.float, 100) @@ -448,7 +432,6 @@ class TestUOpGraph(unittest.TestCase): class TestReduceCollapse(unittest.TestCase): def test_multi_range_reduce_add(self): """Test that (x + y).reduce(r1, r2) distributes over multiple ranges""" - from tinygrad.codegen.simplify import pm_reduce_collapse # Create two ranges r1 = UOp.range(3, 0) r2 = UOp.range(4, 1) @@ -462,14 +445,12 @@ class TestReduceCollapse(unittest.TestCase): # Should become add of two separate reduces self.assertEqual(result.op, Ops.ADD) - def test_bool_reduce_unparented_keeps_dtype(self): - for op in (Ops.ADD, Ops.MUL): - self.assertIs(graph_rewrite(UOp.const(True).reduce(UOp.range(4, 0), arg=op), pm_reduce_unparented).dtype, dtypes.bool) - - def test_max_reduce_unparented_empty_range(self): - n = Variable("n", 0, 4) - ret = graph_rewrite(UOp.const(1.0).reduce(UOp.range(n, 0), arg=Ops.MAX), pm_reduce_unparented) - self.assertEqual(ret.substitute({n: n.const_like(0)}).simplify().arg, float("-inf")) + def test_lt_lift_skips_unsigned(self): + # x+y x x= 8, 0, 1, "((a<8)!=True)") + self.helper_test_variable(Variable("a", 3, 8) >= 8, 0, 1, "(a<8).eq(False)") def test_ge(self): self.helper_test_variable(Variable("a", 3, 8) >= 77, 0, 0, "False") self.helper_test_variable(Variable("a", 3, 8) >= 9, 0, 0, "False") - self.helper_test_variable(Variable("a", 3, 8) >= 8, 0, 1, "((a<8)!=True)") - self.helper_test_variable(Variable("a", 3, 8) >= 4, 0, 1, "((a<4)!=True)") + self.helper_test_variable(Variable("a", 3, 8) >= 8, 0, 1, "(a<8).eq(False)") + self.helper_test_variable(Variable("a", 3, 8) >= 4, 0, 1, "(a<4).eq(False)") self.helper_test_variable(Variable("a", 3, 8) >= 3, 1, 1, "True") self.helper_test_variable(Variable("a", 3, 8) >= 2, 1, 1, "True") @@ -108,7 +108,7 @@ class TestSymbolic(unittest.TestCase): def test_multiple_of_cancellation(self): for multiple in (1, 2, 3, 4, 8): - var = UOp.variable("n", multiple, 8*multiple, multiple_of=multiple) + var = UOp.variable("n", multiple, 8*multiple) for n in (var, var.bind(2*multiple)): for factor in (-multiple, -1, 2, multiple, multiple+1): with self.subTest(multiple=multiple, bound=n.is_bound_var, factor=factor): @@ -117,14 +117,6 @@ class TestSymbolic(unittest.TestCase): self.assertEqual(((-n*factor)//(-n)).ssimplify(), factor) self.assertIs(UOp.gcd(n*factor, n).simplify(), n) - def test_gcd_multiple_of_without_shared_factors(self): - n = UOp.variable("n", 6, 60, multiple_of=6) - m = UOp.variable("m", 4, 40, multiple_of=4) - self.assertEqual(UOp.gcd(n, m).ssimplify(), 2) - self.assertEqual((n%3).ssimplify(), 0) - self.assertEqual((m%2).ssimplify(), 0) - self.assertIs((n<12).simplify(), n<12) - def test_divides_exact(self): a = Variable("a", 1, 8) b = Variable("b", 1, 8) @@ -373,14 +365,14 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable(Variable("a", 0, 5)*4 < 13, 0, 1, "(a<4)") self.helper_test_variable(Variable("a", 0, 5)*4 < 16, 0, 1, "(a<4)") self.helper_test_variable(Variable("a", -5, 5)*4 < -13, 0, 1, "(a<-3)") - self.helper_test_variable(Variable("a", -5, 5)*-4 < 13, 0, 1, "((a*-1)<4)") + self.helper_test_variable(Variable("a", -5, 5)*-4 < 13, 0, 1, "(-4= 12, 0, 1, "((a<3)!=True)") - self.helper_test_variable(Variable("a", 0, 5)*4 >= 13, 0, 1, "((a<4)!=True)") + self.helper_test_variable(Variable("a", 0, -c1)*c0 < c1, 0, 1, f"({2**53-1}= 12, 0, 1, "(a<3).eq(False)") + self.helper_test_variable(Variable("a", 0, 5)*4 >= 13, 0, 1, "(a<4).eq(False)") def test_div_div(self): self.helper_test_variable((Variable("a", 0, 1800)//10)//9, 0, 20, "(a//90)") @@ -476,13 +468,26 @@ class TestSymbolic(unittest.TestCase): z = UOp.const(0.0) self.assertTrue(math.isnan(graph_rewrite(z/z, symbolic_simple, bottom_up=True).arg)) + def test_two_stage_const_folding(self): + x = UOp.variable("x", 0, 10) + self.assertIs(graph_rewrite((x+1)+2, symbolic_simple), x+3) + + def test_unpack_uint64_high(self): + x, y = UOp.variable("x", 0, 2**32-1, dtypes.uint64), UOp.variable("y", 0, 2**32-1, dtypes.uint32) + self.assertIs(graph_rewrite((((x<<32) | y.cast(dtypes.uint64)) >> 32).cast(dtypes.uint32), symbolic_simple), x.cast(dtypes.uint32)) + def test_masked_shr_fold(self): x = UOp.variable('x', 0, 255, dtype=dtypes.uint32) self.helper_test_variable((x & -4) >> 2, 0, 63, "(x>>2)") - def test_masked_idiv_fold(self): + def test_masked_shr_fold_committed_mask(self): x = UOp.variable('x', 0, 255, dtype=dtypes.uint32) - self.helper_test_variable((x & -4) // 4, 0, 63, "(x//4)") + self.helper_test_variable((x & UOp.const(-4, dtypes.uint32)) >> 2, 0, 63, "(x>>2)") + + def test_signed_const_wraps(self): + c = UOp.const(2**31, dtypes.int32) + self.assertEqual(c.val, -2**31) + self.assertIs((UOp.const(1, dtypes.int32) < c).simplify().arg, False) def test_bool_or_not_tautology(self): a = Variable("a", 0, 10) @@ -772,14 +777,14 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable((30 * b + 1) % 18 + ((30 * b + 1) // 18) * 18, 1, 3001, "((b*30)+1)") def test_div_partial_quotient(self): - # FLOORDIV should extract partial quotients when const_factor > divisor, matching what FLOORMOD already does + # DIV should extract partial quotients when const_factor > divisor, matching what MOD already does # (f*x+c)//d -> (f%d*x+c)//d + (f//d)*x when f >= d b = Variable("b", 0, 100) self.helper_test_variable((31*b+1)//18, 0, 172, "(((b*13)+1)//18+b)") self.helper_test_variable((19*b+3)//7, 0, 271, "(((b*5)+3)//7+(b*2))") def test_gcd_with_remainder(self): - # gcd_with_remainder: factor GCD out of non-constant terms and denominator + # nest_by_factor on the GCD of non-constant terms and denominator a = Variable("a", 0, 2) self.helper_test_variable((a*4)//6, 0, 1, "(a*2//3)") self.helper_test_variable((a*4+1)//6, 0, 1, "(a*2//3)") @@ -790,6 +795,11 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable((a*4+2)%6, 0, 4, "((a*2+1)%3*2)") self.helper_test_variable((a*4+3)%6, 1, 5, "((a*2+1)%3*2+1)") + def test_gcd_with_remainder_negative(self): + a, b = Variable("a", -3, 5), Variable("b", 0, 7) + self.helper_test_variable((a*18+1)//4, -14, 22, "(a*9//2)") + self.helper_test_variable((a*12+b*18+5)//12, -3, 15, "((a*2+b*3)//2)") + def test_div_by_factor_tie_break(self): a = Variable("a", 0, 1) b = Variable("b", 0, 1) @@ -817,7 +827,7 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable((x%4)*-2 + ((x//4)%2)*-8, -14, 0, "(x%8*-2)") def test_mod_nest_by_factor(self): - # (a*f+b) % (f*k) = (a%k)*f + b when 0<=b x0, and <=> c*d x0 idx = Variable("idx", 0, 24) self.helper_test_variable((idx//4<3), 0, 1, "(idx<12)") self.helper_test_variable(((idx-20)//4<-3), 0, 1, "(idx<8)") self.helper_test_variable(((idx-10)//4<0), 0, 1, "(idx<10)") - self.helper_test_variable((idx//-4<-3), 0, 1, "(12 (x // c) % k requires k>0; (x % (k*c)) % c -> x % c is unconditional for c>0 @@ -959,16 +965,16 @@ class TestSymbolic(unittest.TestCase): c = Variable("c", 0, 3) d = Variable("d", -3, 3) self.helper_test_variable((a<2), 0, 1, "(a<2)") - self.helper_test_variable((a<=2), 0, 1, "((21), 0, 1, "(1=1), 0, 1, "((a<1)!=True)") - self.helper_test_variable((a<1).ne(True), 0, 1, "((a<1)!=True)") - self.helper_test_variable((a+b<1).ne(True), 0, 1, "(((a+b)<1)!=True)") - self.helper_test_variable((a*3+b*4<1).ne(True), 0, 1, "(((a+b)<1)!=True)") - self.helper_test_variable((a*(-3)+b*4<1).ne(True), 0, 1, "((((a*-3)+(b*4))<1)!=True)") # negative coeff, should not be simplified - self.helper_test_variable((a*3+d*4<1).ne(True), 0, 1, "((((a*3)+(d*4))<1)!=True)") # var can be negative, should not be simplified - self.helper_test_variable((a+b+c*2<1).ne(True), 0, 1, "((((a+b)+c)<1)!=True)") - self.helper_test_variable((a+b*2+c*4<1).ne(True), 0, 1, "((((a+b)+c)<1)!=True)") + self.helper_test_variable((a>=1), 0, 1, "(a<1).eq(False)") + self.helper_test_variable((a<1).ne(True), 0, 1, "(a<1).eq(False)") + self.helper_test_variable((a+b<1).ne(True), 0, 1, "((a+b)<1).eq(False)") + self.helper_test_variable((a*3+b*4<1).ne(True), 0, 1, "((a+b)<1).eq(False)") + self.helper_test_variable((a*(-3)+b*4<1).ne(True), 0, 1, "(((a*-3)+(b*4))<1).eq(False)") # negative coeff, should not be simplified + self.helper_test_variable((a*3+d*4<1).ne(True), 0, 1, "(((a*3)+(d*4))<1).eq(False)") # var can be negative, should not be simplified + self.helper_test_variable((a+b+c*2<1).ne(True), 0, 1, "(((a+b)+c)<1).eq(False)") + self.helper_test_variable((a+b*2+c*4<1).ne(True), 0, 1, "(((a+b)+c)<1).eq(False)") def test_mul_by_zero_casted_to_emulated_dtype(self): # a float zero cast to bfloat16 is one committed const, so the mul-by-zero fold still sees it @@ -980,13 +986,13 @@ class TestSymbolic(unittest.TestCase): # CAST(bool -> int) != 0 -> cond self.helper_test_variable(cond.cast(dtypes.int).ne(0), 0, 1, "(a<2)") # CAST(bool -> int) != 1 -> !cond - self.helper_test_variable(cond.cast(dtypes.int).ne(1), 0, 1, "((a<2)!=True)") + self.helper_test_variable(cond.cast(dtypes.int).ne(1), 0, 1, "(a<2).eq(False)") # CAST(bool -> int) != c (c not in {0,1}) -> always True (CAST is 0 or 1) self.helper_test_variable(cond.cast(dtypes.int).ne(2), 1, 1, "True") self.helper_test_variable(cond.cast(dtypes.int).ne(-1), 1, 1, "True") # CAST(bool -> index) folds too self.helper_test_variable(cond.cast(dtypes.weakint).ne(0), 0, 1, "(a<2)") - self.helper_test_variable(cond.cast(dtypes.weakint).ne(1), 0, 1, "((a<2)!=True)") + self.helper_test_variable(cond.cast(dtypes.weakint).ne(1), 0, 1, "(a<2).eq(False)") def test_where_removal(self): cond = Variable("a", 0, 3) < 2 @@ -994,13 +1000,15 @@ class TestSymbolic(unittest.TestCase): self.helper_test_variable(cond, 0, 1, "(a<2)") self.helper_test_variable(cond.where(u1, u0), 0, 1, "(a<2)") self.helper_test_variable(cond.where(u1, u0).where(u1, u0), 0, 1, "(a<2)") - self.helper_test_variable(cond.where(u0, u1), 0, 1, "((a<2)!=True)") + self.helper_test_variable(cond.where(u0, u1), 0, 1, "(a<2).eq(False)") self.helper_test_variable(cond.where(u0, u1).where(u0, u1), 0, 1, "(a<2)") def test_equivalent_const_max(self): x = Variable("x", -10, 10) self.helper_test_variable((x < 0).where(0, x), 0, 10, "x.maximum(0)") self.helper_test_variable((0 < x).where(x, 0), 0, 10, "x.maximum(0)") + self.helper_test_variable(((x+3) < 0).where(0, x+3), 0, 13, "(x+3).maximum(0)") + self.helper_test_variable((5 < x).where(5, x), -10, 5, "(5>6)*18725)>>17) instead of (int)((((long)(ridx0)*1198373)>>29)) self.assertNotIn(dtypes.long, [x.dtype for x in uops]) - self.assertNotIn(Ops.CDIV, [x.op for x in uops]) + self.assertNotIn(Ops.TRUNCDIV, [x.op for x in uops]) @unittest.expectedFailure def test_fast_idiv_overflow(self): @@ -408,25 +431,41 @@ class TestFastIdiv(unittest.TestCase): g = UOp.param(0, dtypes.uint32, 8) c = UOp.const(7).cast(dtypes.uint) l = UOp(Ops.LOAD, src=(g.index(c),)) - a = UOp(Ops.CDIV, src=(l, c)) + a = UOp(Ops.TRUNCDIV, src=(l, c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) Device[Device.DEFAULT].renderer.render(uops) ops = [x.op for x in uops] self.assertIn(Ops.SHR, ops) - self.assertNotIn(Ops.CDIV, ops) + self.assertNotIn(Ops.TRUNCDIV, ops) def test_disable_fast_idiv(self): g = UOp.param(0, dtypes.uint32, 4) c = UOp.const(3) l = g.index(c) - a = UOp(Ops.CDIV, src=(l, c)) + a = UOp(Ops.TRUNCDIV, src=(l, c)) with Context(DISABLE_FAST_IDIV=1): uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) ops = [x.op for x in uops] self.assertNotIn(Ops.SHR, ops) - self.assertIn(Ops.CDIV, ops) + self.assertIn(Ops.TRUNCDIV, ops) + +class TestNegLowering(unittest.TestCase): + def test_int_neg(self): + for dt in dtypes.ints: + x = UOp.param(0, dt, 1).index(UOp.const(0)) + self.assertIn(Ops.NEG, [u.op for u in to_uops_list([-x], ren=CStyleLanguage(Target()))], dt) class TestUOpMethod(unittest.TestCase): + def test_cmpeq_canonical(self): + a, b = UOp.variable("a", 0, 10), UOp.variable("b", 0, 10) + eq = a.eq(b) + self.assertIs(eq.op, Ops.CMPEQ) + self.assertIs(eq.dtype, dtypes.bool) + self.assertIs(a.ne(b), eq.eq(False)) + self.assertIs(eq.logical_not(), eq.eq(False)) + self.assertIs(eq.logical_not().logical_not().simplify(), eq) + self.assertIs(eq.eq(True).simplify(), eq) + @unittest.skip("uops lt no longer ordered") def test_compare_alu_same_src_different_arg(self): a = UOp.const(2.0) @@ -457,6 +496,7 @@ class TestUOpMethod(unittest.TestCase): x = Tensor.empty(*shape, dtype=dtypes.int).uop self.assertIs((x < x).simplify(), x.const_like(False, dtypes.bool)) self.assertIs((x != x).simplify(), x.const_like(False, dtypes.bool)) + self.assertIs(x.eq(x).simplify(), x.const_like(True, dtypes.bool)) def test_replace(self): x = UOp.param(0, dtypes.int, 1) @@ -628,5 +668,30 @@ class TestAssembly(unittest.TestCase): self.assertIn(Ops.CMPEQ, ops) self.assertNotIn(Ops.CMPNE, ops) + def test_use_cmpne(self): + x = UOp.param(0, dtypes.uint32, 8).index(UOp.const(0)) + uops = to_uops_list([x.ne(7)], ren=self.renderer) + self.renderer.render(uops) + self.assertEqual(sum(u.op is Ops.CMPNE for u in uops), 1) + self.assertNotIn(Ops.CMPEQ, [u.op for u in uops]) + + def test_int_compare_without_negation(self): + for dt in (dtypes.int32, dtypes.uint32): + x = UOp.param(0, dt, 8).index(UOp.const(0)) + for comp in [(x<7).ne(True), (x.const_like(7) dict: return decode_profile(get_profile(VizData(), lst)) class TestVizProfiler(unittest.TestCase): + def test_tensor_calls(self): + a = Tensor.empty(2, device="NULL") + with save_viz(): + a.relu() + count = len(cpu_events) + with Context(PROFILE=0): a.sigmoid() + self.assertEqual(len(cpu_events), count) + events = [e for e in cpu_events if isinstance(e, ProfileRangeEvent) and e.device == "USER"] + self.assertEqual(events[-1].name.display_name, "relu") + self.assertGreater(len(events), 1) + self.assertTrue(all(events[-1].st <= e.st <= e.en <= events[-1].en for e in events)) + + def test_tensor_call_exception(self): + a = Tensor.empty(2, device="NULL") + with save_viz(): + with self.assertRaises(ValueError): a.reshape(3) + a.relu() + events = [e for e in cpu_events if isinstance(e, ProfileRangeEvent) and e.device == "USER"] + self.assertIn("reshape", [e.name.display_name for e in events]) + self.assertEqual(events[-1].name.display_name, "relu") + def test_transfer_uses_copy_device(self): with save_viz(): a = Tensor.ones(1, device="NULL").contiguous().realize() @@ -624,15 +630,13 @@ class TestVizProfiler(unittest.TestCase): ProfileDeviceEvent(device='NV:1:SDMA:0', tdiff=decimal.Decimal(-50)), ProfileGraphEvent(ents=[ProfileGraphEntry(device='NV', name='E_25_4n2', st_id=0, en_id=1), ProfileGraphEntry(device='NV:1:SDMA:0', name='NV -> NV:1', st_id=2, en_id=3)], - deps=[[], [0]], sigs=[decimal.Decimal(1000), decimal.Decimal(1002), decimal.Decimal(1004), decimal.Decimal(1008)])] j = load_profile(prof) tracks = list(j['layout']) self.assertEqual(tracks[0], 'NV') - self.assertEqual(tracks[1], 'NV Graph') - self.assertEqual(tracks[2], 'NV:1:SDMA:0') + self.assertEqual(tracks[1], 'NV:1:SDMA:0') nv_events = j['layout']['NV']['events'] self.assertEqual(nv_events[0]['name'], 'E_25_4n2') @@ -643,17 +647,12 @@ class TestVizProfiler(unittest.TestCase): self.assertEqual(sdma_events[0]['name'], 'NV -> NV:1') self.assertEqual(sdma_events[0]['st'], 954) - graph_events = j['layout']['NV Graph']['events'] - self.assertEqual(graph_events[0]['st'], nv_events[0]['st']) - self.assertEqual(graph_events[0]['st']+graph_events[0]['dur'], sdma_events[0]['st']+sdma_events[0]['dur']) - def test_graph_copy_bandwidth(self): sz = 256*1024*1024 dur = 10_000 prof = [ProfileDeviceEvent(device='NV', tdiff=decimal.Decimal(-1000)), ProfileDeviceEvent(device='NV:1:SDMA:0', tdiff=decimal.Decimal(-50)), ProfileGraphEvent(ents=[ProfileGraphEntry(device='NV:1:SDMA:0', name=TracingKey("NV -> NV:1", ret=sz), st_id=0, en_id=1)], - deps=[[]], sigs=[decimal.Decimal(1004), decimal.Decimal(1004+dur)])] j = load_profile(prof) @@ -668,12 +667,11 @@ class TestVizProfiler(unittest.TestCase): ProfileRangeEvent(device='NV:1', name='E_3', st=decimal.Decimal(1000), en=decimal.Decimal(1010)), ProfileRangeEvent(device='NV:SDMA:0', name='COPY', st=decimal.Decimal(1000), en=decimal.Decimal(1010)), ProfileGraphEvent(ents=[ProfileGraphEntry(device='NV', name='E_2', st_id=0, en_id=1)], - deps=[[]], sigs=[decimal.Decimal(1000), decimal.Decimal(1010)])] + sigs=[decimal.Decimal(1000), decimal.Decimal(1010)])] j = load_profile(prof) - # graph grouped with its device, memory at the end - self.assertListEqual(list(j['layout']), ['NV', 'NV Graph', 'NV:SDMA:0', 'NV:1']) + # memory at the end + self.assertListEqual(list(j['layout']), ['NV', 'NV:SDMA:0', 'NV:1']) - @unittest.skipIf(sys.platform == 'win32', "TODO: ops_amd import fails on windows") def test_multi_sdma_ordering(self): props = {"gfx_target_version": 0} D, St, En = decimal.Decimal, decimal.Decimal(1000), decimal.Decimal(1010) @@ -694,14 +692,14 @@ class TestVizProfiler(unittest.TestCase): # graph spanning compute + copy on GPU 0 ProfileGraphEvent(ents=[ProfileGraphEntry(device='AMD', name='E_1', st_id=0, en_id=1), ProfileGraphEntry(device='AMD:SDMA:0', name='COPY0', st_id=2, en_id=3)], - deps=[[], [0]], sigs=[St, En, St, En]), + sigs=[St, En, St, En]), # memory alloc on both GPUs ProfilePointEvent(device='AMD', name='alloc', key=0, arg={"nbytes":4096}, ts=St), ProfilePointEvent(device='AMD:1', name='alloc', key=1, arg={"nbytes":2048}, ts=St)] j = load_profile(prof) - # graph grouped with its device, memory at the end + # memory at the end self.assertListEqual(list(j['layout']), - ['AMD', 'AMD Graph', 'AMD:SDMA:0', 'AMD:SDMA:1', + ['AMD', 'AMD:SDMA:0', 'AMD:SDMA:1', 'AMD:1', 'AMD:1:SDMA:0', 'AMD:1:SDMA:1', 'AMD Memory', 'AMD:1 Memory']) @@ -959,7 +957,7 @@ class TestCfg(unittest.TestCase): self.assertEqual(len(references["r0"]), 2) insts = [cfg["pc_tokens"][pc][0]["st"] for pc in references["r0"]] self.assertEqual(insts, ['s_mov_b32', 's_cmp_eq_u64']) - end_block = [" ".join(t["st"] for t in cfg["pc_tokens"][pc]) for pc in list(cfg["blocks"].values())[-1]] + end_block = next(b for b in cfg["blocks"].values() if cfg["pc_tokens"][b[-1]][0]["st"] == "s_endpgm") self.assertEqual(len(end_block), 1) def test_loop(self): @@ -1184,14 +1182,16 @@ class TestCLI(unittest.TestCase): with save_viz() as viz: for _ in range(CNT:=4): # use kernel names unique to this test - Tensor.custom_kernel(Tensor.empty(4, device="NULL"), fxn=lambda _: UOp.sink(arg=KernelInfo("k1_test_viz_dedup")))[0].realize() - Tensor.custom_kernel(Tensor.empty(8, device="NULL"), fxn=lambda _: UOp.sink(arg=KernelInfo("k2_test_viz_dedup")))[0].realize() - with write_files(viz) as files, Context(NO_COLOR=1): - name = run_cli(*files, "-s", "NULL")[0]["name"] - with Context(DEBUG=3): - select = run_cli(*files, "-s", "NULL", name) - self.assertEqual(len([s for s in select if s.get("value")]), 1, "debug output was not deduped") - self.assertEqual(len([s for s in select if s.get("device") == "NULL"]), CNT, f"expected 4 runs for {name}") + Tensor.custom_kernel(Tensor.empty(4, device="NULL"), fxn=lambda _: UOp.sink(arg=KernelInfo(f"{colored('k1', 'cyan')}_viz")))[0].realize() + Tensor.custom_kernel(Tensor.empty(8, device="NULL"), fxn=lambda _: UOp.sink(arg=KernelInfo(f"{colored('k2', 'cyan')}_viz")))[0].realize() + Tensor.custom_kernel(Tensor.empty(16, device="NULL"), fxn=lambda _: UOp.sink(arg=KernelInfo(f"{colored('g1', 'cyan')}_viz")))[0].realize() + with write_files(viz) as files, Context(DEBUG=3): + k1q = run_cli(*files, "-s", "NULL", "-q", "k1") + both_kq = run_cli(*files, "-s", "NULL", "-q", "k1|k2") + self.assertEqual(len([s for s in k1q if s.get("value")]), 1, "debug output was not deduped") + self.assertEqual(len([s for s in k1q if s.get("dur_ms")]), CNT, "expected 4 runs for k1") + self.assertEqual(len([s for s in both_kq if s.get("value")]), 2) + self.assertEqual(len([s for s in both_kq if s.get("dur_ms")]), CNT*2) @needs_tracked_pm def test_call_graph(self): @@ -1244,13 +1244,18 @@ class TestCLI(unittest.TestCase): with save_viz() as viz, Context(SCACHE=0): Tensor.custom_kernel(Tensor.empty(1, device="CPU"), fxn=kernel)[0].realize() with write_files(viz) as files: + programs = [s["name"] for s in run_cli(*files, "-s", "TINY") if s.get("name", "").startswith("do_to_program for ")] rewrites = run_cli(*files, "-s", "TINY", "do_to_program for nested_calls", "--ls", json_fmt=False)[0]["out"].split("\n") with Context(NO_COLOR=1): uops = run_cli(*files, "-s", "TINY", "do_to_program for nested_calls", "View UOp List", json_fmt=False)[0]["out"] + source = run_cli(*files, "-s", "TINY", "do_to_program for nested_calls", "View Source")[1]["value"] codegen_count = [s for s in rewrites if "View Output AST" in s] - self.assertEqual(len(codegen_count), 4) + self.assertCountEqual(programs, [f"do_to_program for {name}" for name in ("inner", "outer", "nested_calls")]) + self.assertEqual(len(codegen_count), 1) self.assertIn(" = linear ", uops) self.assertIn(" = call ", uops) + self.assertEqual(len([s for s in source.splitlines() if "inner(" in s]), 4) # define + declare + 2 calls + self.assertEqual(len([s for s in source.splitlines() if "nested_calls(" in s]), 1) @needs_tracked_pm def test_nested_calls_schedule_ls(self): diff --git a/tinygrad_repo/test/runtime/test_arange.py b/tinygrad_repo/test/runtime/test_arange.py index 2749ce0e93..bd46d8546f 100644 --- a/tinygrad_repo/test/runtime/test_arange.py +++ b/tinygrad_repo/test/runtime/test_arange.py @@ -112,6 +112,15 @@ class TestIndexing(unittest.TestCase): X = dataset[idxs] np.testing.assert_equal(X.numpy(), 0) + def test_index_collapse_no_valid_lane(self): + x = Tensor.arange(4).cast(dtypes.float32) + data = Tensor([1., 2., 3., 4.]).realize() + for val, expected in ((2., 0.), (1., 2.), (9., 4.), (16., 0.)): + with self.subTest(val=val): + self.assertEqual(((x*x) == val).where(data, 0.).sum().item(), expected) + self.assertEqual(((x*x) == 2.).where(data, 0.).tolist(), [0.]*4) + self.assertEqual(((x*x) == 2.).reshape(2, 2).where(data.reshape(2, 2), 0.).sum(axis=1).tolist(), [0.]*2) + def test_index_mnist(self, noopt=1, op_limit=512*784*13, split_reduceop=0): # WEBGPU generates more ops due to bitpacking of < 4-byte dtypes if Device.DEFAULT == "WEBGPU": op_limit *= 15 diff --git a/tinygrad_repo/test/runtime/test_assign.py b/tinygrad_repo/test/runtime/test_assign.py index f640750eac..6fc3c7fb11 100644 --- a/tinygrad_repo/test/runtime/test_assign.py +++ b/tinygrad_repo/test/runtime/test_assign.py @@ -99,9 +99,11 @@ class TestAssign(unittest.TestCase): for T in (1, 2, 10, 100): x = Tensor([0, 0]).realize() buf = x.uop.base.realized + GlobalCounters.reset() for _ in range(T): x[0] += 1 x.realize() + assert_kernel_count(T) assert x.tolist() == [T, 0] assert x.uop.base.realized is buf @@ -193,14 +195,6 @@ class TestAssign(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "cycle"): # TODO: broken now, raises np.testing.assert_allclose(new.numpy(), 4) - def test_assign_diamond_contiguous(self): - a = Tensor.ones(4).contiguous().realize() - times_a = a*3 - a.assign(Tensor.full((4,), 2.)) - new = a.contiguous() + times_a-1 - with self.assertRaisesRegex(RuntimeError, "cycle"): # TODO: broken now, raises - np.testing.assert_allclose(new.numpy(), 4) - def test_assign_diamond_possible(self): a = Tensor.ones(4).contiguous().realize() times_a = a*3 @@ -208,13 +202,6 @@ class TestAssign(unittest.TestCase): new = a + (times_a-1).contiguous() np.testing.assert_allclose(new.numpy(), 4) - def test_assign_diamond_possible_contiguous(self): - a = Tensor.ones(4).contiguous().realize() - times_a = a*3 - a.assign(Tensor.full((4,), 2.).contiguous()) - new = a + (times_a-1).contiguous() - np.testing.assert_allclose(new.numpy(), 4) - def test_assign_diamond_both_contiguous(self): a = Tensor.ones(4).contiguous().realize() times_a = a*3 @@ -431,7 +418,20 @@ class TestAssign(unittest.TestCase): a.realize() self.assertListEqual(a.tolist(), [[2.,2.,2.,2.],[2.,2.,2.,2.],[3.,3.,3.,3.], [3.,3.,3.,3.]]) - # TODO: is there a way to sneak in a permute such that it returns the wrong answer? + def test_flipped_assignment_from_self(self): + a = Tensor([0, 1, 2]).realize() + a[::-1] = a + self.assertEqual(a.tolist(), [2, 1, 0]) + + def test_add_row_from_self(self): + a = Tensor([[0, 1], [2, 3], [4, 5]]).realize() + a += a[0] + self.assertEqual(a.tolist(), [[0, 2], [2, 4], [4, 6]]) + + def test_roll_assignment_from_self(self): + a = Tensor([[0, 1], [2, 3]]).realize() + a.assign(a.roll(1, 0)) + self.assertEqual(a.tolist(), [[2, 3], [0, 1]]) def test_overlapping_shrink_assignment_forward(self): # Forward shift: read index > write index in overlap @@ -453,6 +453,12 @@ class TestAssign(unittest.TestCase): with Context(NOOPT=1): a[shift:N].assign(a[0:N-shift]).realize() np.testing.assert_allclose(a.numpy(), expected) + def test_overlapping_shrink_assignment_pending(self): + a = Tensor.arange(6).clone().realize() + a += 1 + a[1:] = a[:-1] + self.assertEqual(a.tolist(), [1, 1, 2, 3, 4, 5]) + def test_nonoverlapping_shrink_assignment(self): # TODO: non-overlapping shrinks don't actually need contiguous, could be 1 kernel with smarter range analysis a = Tensor.arange(100).float().clone().realize() @@ -637,21 +643,6 @@ class TestAssignOrdering(unittest.TestCase): self.assertEqual(r1.item(), 4) self.assertEqual(r2.item(), 8) - def test_write_read_write_chain(self): - """Write, read, write chain - middle read must complete before second write.""" - buf = Tensor.zeros(4).contiguous().realize() - buf.assign(Tensor.ones(4) * 3) - mid_sum = buf.sum() # lazy read, should be 12 - buf.assign(Tensor.ones(4) * 5) - final_sum = buf.sum() # lazy read, should be 20 - # Realize in "wrong" order - final first - self.assertEqual(final_sum.realize().item(), 20) - try: - self.assertEqual(mid_sum.realize().item(), 12) - except AssertionError: - # TODO: this is wrong - self.assertEqual(mid_sum.realize().item(), 20) - def test_slice_read_then_full_write(self): """Read from slice, then overwrite full buffer - WAR dependency works for full buffer assigns.""" buf = Tensor([1.,2.,3.,4.]).contiguous().realize() @@ -674,26 +665,16 @@ class TestAssignOrdering(unittest.TestCase): buf[4:8].assign(buf[0:4].contiguous()) np.testing.assert_equal(buf.numpy(), [1, 2, 3, 4, 1, 2, 3, 4]) + @unittest.expectedFailure def test_swap_slices(self): """Swap two non-overlapping slices - requires reading both before writing.""" + # TODO: broken now, left reads buf after the first slice write. left should be the original [1, 2, 3, 4] # without .realize() on temps: values not captured before overwriting buf = Tensor([1, 2, 3, 4, 5, 6, 7, 8]).contiguous().realize() left = buf[0:4].clone() # lazy - not captured yet right = buf[4:8].clone() # lazy - not captured yet buf[0:4].assign(right).realize() # this works buf[4:8].assign(left).realize() # left now reads from modified buf! - try: - np.testing.assert_equal(buf.numpy(), [5, 6, 7, 8, 1, 2, 3, 4]) - except AssertionError: - # TODO: broken now - np.testing.assert_equal(buf.numpy(), [5, 6, 7, 8, 5, 6, 7, 8]) - - # with .realize() on temps: values captured before writes - buf = Tensor([1, 2, 3, 4, 5, 6, 7, 8]).contiguous().realize() - left = buf[0:4].clone().realize() - right = buf[4:8].clone().realize() - buf[0:4].assign(right).realize() - buf[4:8].assign(left).realize() np.testing.assert_equal(buf.numpy(), [5, 6, 7, 8, 1, 2, 3, 4]) def test_reduction_after_partial_assign(self): @@ -790,18 +771,26 @@ class TestAssignOrdering(unittest.TestCase): np.testing.assert_allclose(param.item(), p_np, atol=1e-5) def test_war_reader_already_depends_on_write(self): + # TODO: x_expr reads x before the assign but fuses with a read after it. should be [2, 4, 15] once that read is staged x = Tensor([1.0]).contiguous().realize() y = Tensor([2.0]).contiguous().realize() x_expr = x + 10 # 11, x is read here, before the assign x.assign(x * 2) y.assign(y + x) z = y + x_expr + with self.assertRaises(RuntimeError): Tensor.realize(x, y, z) + + @unittest.expectedFailure + def test_war_reader_of_disjoint_slice(self): + # TODO: the stale read of x[2:4] fuses into z's kernel with the new y, like the sibling. staging that read gives [18, 20] + x = Tensor([1.0, 2.0, 3.0, 4.0]).contiguous().realize() + y = Tensor([0.0, 0.0]).contiguous().realize() + stale = x[2:4] + 10 + x[0:2].assign(Tensor([5.0, 6.0])) + y.assign(y + x[0:2]) + z = y + stale Tensor.realize(x, y, z) - try: - np.testing.assert_allclose([x.item(), y.item(), z.item()], [2.0, 4.0, 15.0]) - except AssertionError: - # TODO: broken now, x_expr reads x after the assign - np.testing.assert_allclose([x.item(), y.item(), z.item()], [2.0, 4.0, 16.0]) + np.testing.assert_equal(z.numpy(), [18.0, 20.0]) def test_war_multi_read_then_assign(self): devices = ("CPU:0", "CPU:1") @@ -821,24 +810,6 @@ class TestAssignOrdering(unittest.TestCase): buf[2:3].assign(Tensor.full((1,), 3.0)) self.assertEqual(buf.sum().realize().item(), 6.0) -# TODO: assigns into views of unrealized non-BUFFER bases are silently dropped - def test_read_before_two_assigns(self): - g = Tensor.full((2,), 4.0).realize() - before = g + 1 # 5 - g.assign(0.0) - g.assign(g + 4) - with self.assertRaisesRegex(RuntimeError, "cycle"): # TODO: broken now, raises - np.testing.assert_allclose((before + g).numpy(), 9) - - def test_read_between_two_assigns(self): - a = Tensor.ones(4).realize() - b = Tensor.full((4,), 10.).realize() - a.assign(b + 1) # a == 11 - v1 = a * 3 # reads 11 -> 33 - a.assign(b + 100) # a == 110 - with self.assertRaisesRegex(RuntimeError, "cycle"): # TODO: broken now, ideally v1 is realized between the assigns - np.testing.assert_allclose((a + v1).numpy(), 143) - def test_two_reads_between_three_assigns(self): a = Tensor.zeros(4).realize() first = a + 100 @@ -852,26 +823,21 @@ class TestAssignOrdering(unittest.TestCase): a = Tensor.ones(4).realize() before = a * 3 a[0:2].assign(Tensor.full((2,), 2.)) - out = (a + (before - 1)).numpy() - try: - np.testing.assert_allclose(out, [4, 4, 3, 3]) - except AssertionError: - # TODO: broken now, before reads the two assigned elements after the assign - np.testing.assert_allclose(out, [7, 7, 3, 3]) + with self.assertRaisesRegex(RuntimeError, "cycle"): # TODO: broken now, raises: one kernel reads a from before and after the slice assign + np.testing.assert_allclose((a + (before - 1)).numpy(), [4, 4, 3, 3]) + @unittest.expectedFailure def test_read_before_assign_survives_a_realize(self): + # TODO: broken now, before is computed again from the assigned value. before should be 3 a = Tensor.ones(4).realize() before = a * 3 a.assign(Tensor.full((4,), 5.)) a.realize() - out = before.numpy() - try: - np.testing.assert_allclose(out, 3) - except AssertionError: - # TODO: broken now, before is computed again from the assigned value - np.testing.assert_allclose(out, 15) + np.testing.assert_allclose(before.numpy(), 3) + @unittest.expectedFailure def test_loss_read_after_step_is_the_pre_step_loss(self): + # TODO: broken now, loss is computed again from the updated weight. loss should be the pre-step 6.0 from tinygrad import nn w = Tensor([2.]).contiguous().realize() x = Tensor([3.]).realize() @@ -880,49 +846,37 @@ class TestAssignOrdering(unittest.TestCase): loss = (w*x).sum() # 6.0 loss.backward() opt.step() # w becomes 1.7 - out = loss.item() - try: - self.assertAlmostEqual(out, 6.0, places=5) - except AssertionError: - # TODO: broken now, loss is computed again from the updated weight - self.assertAlmostEqual(out, 5.1, places=5) + self.assertAlmostEqual(loss.item(), 6.0, places=5) + @unittest.expectedFailure def test_rand_realized_out_of_order(self): + # TODO: broken now, r[0] returns the fourth set of numbers. r[0] should be the first set Tensor.manual_seed(1) r = [Tensor.rand(4) for _ in range(4)] r[3].realize() out_of_order = r[0].numpy() Tensor.manual_seed(1) in_order = [Tensor.rand(4).numpy() for _ in range(4)] - try: - np.testing.assert_equal(out_of_order, in_order[0]) - except AssertionError: - # TODO: broken now, r[0] returns the fourth set of numbers - np.testing.assert_equal(out_of_order, in_order[3]) + np.testing.assert_equal(out_of_order, in_order[0]) + @unittest.expectedFailure def test_batchnorm_stats_are_realized(self): + # TODO: broken now, the stat update is never run because nothing reads it. running_mean should be realized from tinygrad import nn bn, x = nn.BatchNorm(4), Tensor.randn(2, 4, 3, 3).realize() with Context(TRAINING=1): bn(x).realize() - try: - self.assertTrue(bn.running_mean.uop.base.is_realized) - except AssertionError: - # TODO: broken now, the stat update is never run because nothing reads it - self.assertFalse(bn.running_mean.uop.base.is_realized) + self.assertTrue(bn.running_mean.uop.base.is_realized) + @unittest.expectedFailure def test_batchnorm_under_jit_counts_every_call(self): + # TODO: broken now, only the calls whose stat update happened to be captured are counted. every call should count from tinygrad import nn bn, x = nn.BatchNorm(4), Tensor.randn(8, 4, 2, 2).realize() @TinyJit def step(t): with Context(TRAINING=1): return bn(t).sum().realize() for _ in range(4): step(x) - out = bn.num_batches_tracked.item() - try: - self.assertEqual(out, 4) - except AssertionError: - # TODO: broken now, only the calls whose stat update happened to be captured are counted - self.assertEqual(out, 2) + self.assertEqual(bn.num_batches_tracked.item(), 4) def test_assign_from_unrealized_tensor_does_not_alias(self): a = Tensor.full((4,), 7.).realize() @@ -962,11 +916,7 @@ class TestAssignToUnrealizedView(unittest.TestCase): c = t.to("CPU:1") # unrealized COPY self.assertIs(c.uop.base.op, Ops.COPY) c[:, 1:2].assign(Tensor.ones(2,1, dtype=dtypes.int).to("CPU:1").contiguous().realize()) - try: - self.assertEqual(c.tolist(), [[0,1],[0,1]]) - except AssertionError: - # TODO: broken now - self.assertEqual(c.tolist(), [[0,0],[0,0]]) + self.assertEqual(c.tolist(), [[0,1],[0,1]]) def test_contiguous(self): t = Tensor([[1,2],[3,4]]).contiguous().realize() @@ -998,22 +948,14 @@ class TestAssignToUnrealizedView(unittest.TestCase): cb = t.contiguous_backward() # unrealized CONTIGUOUS_BACKWARD self.assertIs(cb.uop.base.op, Ops.CONTIGUOUS_BACKWARD) cb[:, 1:2].assign(Tensor.ones(2,1, dtype=dtypes.int).contiguous().realize()) - try: - self.assertEqual(cb.tolist(), [[1,1],[3,1]]) - except AssertionError: - # TODO: broken now - self.assertEqual(cb.tolist(), [[1,2],[3,4]]) + self.assertEqual(cb.tolist(), [[1,1],[3,1]]) def test_detach_copy(self): t = Tensor.zeros(2,2, dtype=dtypes.int).to("CPU:0").contiguous().realize() d = t.to("CPU:1").detach() # DETACH(unrealized COPY) self.assertIs(d.uop.base.op, Ops.COPY) d[:, 1:2].assign(Tensor.ones(2,1, dtype=dtypes.int).to("CPU:1").contiguous().realize()) - try: - self.assertEqual(d.tolist(), [[0,1],[0,1]]) - except AssertionError: - # TODO: broken now - self.assertEqual(d.tolist(), [[0,0],[0,0]]) + self.assertEqual(d.tolist(), [[0,1],[0,1]]) def test_detach_contiguous(self): t = Tensor([[1,2],[3,4]]).contiguous().realize() @@ -1028,43 +970,39 @@ class TestAssignToUnrealizedView(unittest.TestCase): c = a + b # unrealized ADD self.assertIs(c.uop.base.op, Ops.ADD) c[:2].assign(Tensor([99, 99]).realize()) - try: - self.assertEqual(c.tolist(), [99,99,10,12]) - except AssertionError: - # TODO: broken now, silently dropped - self.assertEqual(c.tolist(), [6,8,10,12]) + self.assertEqual(c.tolist(), [99,99,10,12]) def test_reduce(self): a = Tensor([[1,2],[3,4]]).contiguous().realize() r = a.sum(axis=0) # unrealized REDUCE self.assertIs(r.uop.base.op, Ops.REDUCE) r[:1].assign(Tensor([99]).realize()) - try: - self.assertEqual(r.tolist(), [99,6]) - except AssertionError: - # TODO: broken now, silently dropped - self.assertEqual(r.tolist(), [4,6]) + self.assertEqual(r.tolist(), [99,6]) def test_cast(self): a = Tensor([1,2,3,4]).contiguous().realize() c = a.float() # unrealized CAST self.assertIs(c.uop.base.op, Ops.CAST) c[:2].assign(Tensor([99, 99], dtype=dtypes.float).realize()) - try: - self.assertEqual(c.tolist(), [99,99,3,4]) - except AssertionError: - # TODO: broken now, silently dropped - self.assertEqual(c.tolist(), [1,2,3,4]) + self.assertEqual(c.tolist(), [99,99,3,4]) + + def test_write_reading_pending_write(self): + a, b = Tensor([1., 1]).realize()+0, Tensor([2., 2]).realize()+0 + b[:1].assign(b[:1] + a[:1]*0 + 1) + a[:1].assign(a[:1] - b[:1]) + self.assertEqual(a.tolist(), [-2, 1]) + self.assertEqual(b.tolist(), [3, 2]) + + def test_broadcast_const(self): + w = Tensor.empty(4).const_like(1.0) # EXPAND of a const, a view of no tensor + w.assign(Tensor([1., 2, 3, 4])) + self.assertEqual(w.tolist(), [1, 2, 3, 4]) def test_const(self): c = Tensor(5).reshape(1, 1).expand(2, 2) self.assertIs(c.uop.base.op, Ops.CONST) - c[:, 1:2].assign(Tensor.ones(2,1, dtype=dtypes.int).contiguous().realize()) - try: - self.assertEqual(c.tolist(), [[5,1],[5,1]]) - except AssertionError: - # TODO: broken now, silently dropped - self.assertEqual(c.tolist(), [[5,5],[5,5]]) + with self.assertRaises(RuntimeError): + c[:, 1:2].assign(Tensor.ones(2,1, dtype=dtypes.int).contiguous().realize()) def test_detach_assignment_preserves_earlier_update(self): x = Tensor([1., 2.]).detach() @@ -1187,11 +1125,7 @@ class TestMultiAssign(unittest.TestCase): out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0) ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() out[:, 2:3].assign(ones).realize() - try: - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - except AssertionError: - # TODO: broken now, the write is dropped - self.assertListEqual(out.tolist(), [[0,0,0,0], [0,0,0,0], [0,0,0,0], [0,0,0,0]]) + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) def test_multi_assign_var_offset(self): out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() diff --git a/tinygrad_repo/test/runtime/test_attention.py b/tinygrad_repo/test/runtime/test_attention.py index 58b7d59ae4..ee12332321 100644 --- a/tinygrad_repo/test/runtime/test_attention.py +++ b/tinygrad_repo/test/runtime/test_attention.py @@ -2,7 +2,8 @@ import unittest import numpy as np from tinygrad import Tensor, dtypes, nn from tinygrad.llm.model import ( - GatedDeltaNetBlock, SSMConfig, TransformerBlock, TransformerConfig, + GatedDeltaNetBlock, SSMConfig, TransformerBlock, TransformerConfig, MLATransformerBlock, Indexer, IndexerConfig, + HyperConnection, HCConfig, apply_rope as apply_rope_new, precompute_freqs_cis, pairwise_topk, ) from test.helpers import slow @@ -69,6 +70,59 @@ class TestAttention(unittest.TestCase): expected = apply_rope_new(k[..., :rope_dim], block.freqs_cis[:seqlen]).cat(k[..., rope_dim:], dim=-1) np.testing.assert_allclose(block.cache_kv[0, :, :, :seqlen, :].numpy(), expected.numpy(), rtol=1e-5, atol=1e-5) + def test_hyperconnection(self): + hc = HyperConnection(self._make_config(dim=1, hc=HCConfig(count=2, sinkhorn_iters=1, eps=0.0))) + hc.fn.weight = Tensor.zeros(8, 2) + hc.base["weight"] = Tensor([0., 0., 0., 0., float(np.log(3)), 0., 0., 0.]) + # pre averages streams [1, 3] to 2, the sublayer squares it to 4, and post adds 4 to both streams + # mixing rows [3/4, 1/4], [1/2, 1/2] normalize to [3/5, 1/3], [2/5, 2/3] + expected = [[[[4 + (3/5 + 3*2/5)], [4 + (1/3 + 3*2/3)]]]] + for mask in (None, Tensor([0., -np.inf])): + with self.subTest(mask=mask is not None): + actual, returned_mask = hc(Tensor([[[[1.], [3.]]]]), lambda x: (x.square(), mask)) + self.assertIs(returned_mask, mask) + np.testing.assert_allclose(actual.numpy(), expected, rtol=1e-6, atol=1e-7) + + def test_indexer(self): + indexer = Indexer(self._make_config(dim=2, q_lora_rank=2, indexer=IndexerConfig(n_heads=1, head_dim=2, top_k=2, kpool=2, norm_eps=1e-6))) + indexer.attn_q_b.weight, indexer.attn_k.weight = Tensor.eye(2), Tensor.eye(2) + indexer.proj.weight, indexer.compressor_gate.weight = Tensor.ones(1, 2), Tensor.zeros(2, 2) + x, q = Tensor([[[2., 0.], [2., 0.], [0., 2.], [0., 2.]]]), Tensor([[[1., -1.], [1., -1.], [1., -1.], [-1., 1.]]]) + indexer._init_state(x) + # the first three queries prefer the first pool, the final query selects the completed second pool + expected = np.where([[1, 0, 0, 0], [1, 1, 0, 0], [1, 1, 1, 0], [0, 0, 1, 1]], 0., -np.inf) + np.testing.assert_array_equal(indexer(x, q, 0).numpy(), expected[None, None]) + + def test_indexer_no_kpool(self): + # glm-dsa: every key is scored on its own (no k-pools) and the first rope_dim dims are rotated, interleaved + idx = IndexerConfig(n_heads=1, head_dim=2, top_k=2, kpool=1, norm_eps=1e-6, rope_dim=2) + indexer = Indexer(self._make_config(dim=2, q_lora_rank=2, indexer=idx)) + indexer.attn_q_b.weight, indexer.attn_k.weight = Tensor.eye(2), Tensor.eye(2) + indexer.proj.weight = Tensor.ones(1, 2) + x, q = Tensor([[[2., 0.], [2., 0.], [0., 2.], [0., 2.]]]), Tensor([[[1., -1.], [1., -1.], [1., -1.], [-1., 1.]]]) + indexer._init_state(x) + # reference: interleaved rope on q/k, relu scores, top-2 of the causal positions + pos = np.arange(4) + cos, sin = np.cos(pos), np.sin(pos) # rope_dim=2 -> one frequency, theta=10000**0 + def irope(v): + return np.concatenate([v[:, 0::2]*cos[:, None] - v[:, 1::2]*sin[:, None], v[:, 0::2]*sin[:, None] + v[:, 1::2]*cos[:, None]], axis=-1) + scores = np.maximum(irope(q.numpy()[0]) @ irope(x.numpy()[0]).T, 0) / 2**0.5 + expected = np.full((4, 4), -np.inf) + for t in range(4): expected[t, np.argsort(scores[t, :t+1])[-2:]] = 0.0 + np.testing.assert_allclose(indexer(x, q, 0, precompute_freqs_cis(2, 4, 10000.0)).numpy(), expected[None, None]) + + def test_mla_indexer_mask(self): + idx = IndexerConfig(n_heads=1, head_dim=2, top_k=2, kpool=2, norm_eps=1e-6) + block = MLATransformerBlock(self._make_config(dim=2, kv_lora_rank=2, q_lora_rank=2, max_context=5, indexer=idx)) + block.attn_q_a.weight = Tensor([[1., 1.], [-1., -1.]]) + block.indexer.attn_q_b.weight, block.indexer.attn_k.weight = Tensor.eye(2), Tensor.eye(2) + block.indexer.proj.weight, block.indexer.compressor_gate.weight = Tensor.ones(1, 2), Tensor.zeros(2, 2) + x = Tensor([[[2., 0.], [2., 0.], [0., 2.], [0., 2.], [0., 2.]]]) + block._init_state(x) + # every query prefers the first pool, and incomplete pools contribute only their visible tail + expected = np.where([[1, 0, 0, 0, 0], [1, 1, 0, 0, 0], [1, 1, 1, 0, 0], [1, 1, 0, 0, 0], [1, 1, 0, 0, 1]], 0., -np.inf) + np.testing.assert_array_equal(block._indexer_mask(x, 0, None).numpy(), expected[None, None]) + class TestGatedDeltaNetBlock(unittest.TestCase): def _tensor_linspace(self, start:float, stop:float, shape:tuple[int, ...]) -> Tensor: return Tensor.linspace(start, stop, int(np.prod(shape)), dtype=dtypes.float32).reshape(*shape) @@ -101,6 +155,8 @@ class TestGatedDeltaNetBlock(unittest.TestCase): def _cache_views(self, block:GatedDeltaNetBlock) -> tuple[np.ndarray, np.ndarray]: if hasattr(block, 'conv_state'): return block.conv_state.numpy(), block.recurrent_state.numpy() + elif hasattr(block, 'conv_states'): + return np.concatenate([s.numpy() for s in block.conv_states], -1), block.recurrent_state.numpy() else: conv_flat = (block.ssm_conv_kernel - 1) * block.conv_channels cache = block.delta_cache.numpy() @@ -109,7 +165,7 @@ class TestGatedDeltaNetBlock(unittest.TestCase): return conv_state, recurrent_state def _reset_state(self, block:GatedDeltaNetBlock): - Tensor.realize(block.conv_state.assign(block.conv_state.const_like(0)), + Tensor.realize(*[s.assign(s.const_like(0)) for s in (block.conv_states if hasattr(block, 'conv_states') else [block.conv_state])], block.recurrent_state.assign(block.recurrent_state.const_like(0))) def _linear_np(self, x:np.ndarray, weight:np.ndarray) -> np.ndarray: @@ -219,8 +275,9 @@ class TestGatedDeltaNetBlock(unittest.TestCase): # q/k are normalized as x/sqrt(sum(x²)+1e-6) like FLA: tiny vectors aren't scaled up to unit length ssm = SSMConfig(conv_kernel=4, state_size=8, group_count=2, time_step_rank=2, inner_size=16, kda=True) block = GatedDeltaNetBlock(self._make_config(dim=16, n_heads=2, ssm=ssm), ssm) - block.attn_qkv.weight = Tensor([1e-4]*32 + [1.0]*16).reshape(48, 1).expand(48, 16) / 16 # x of ones: q = k = 1e-4, v = 1 - block.ssm_conv1d["weight"], block.ssm_beta.weight = Tensor.ones(48, 4), Tensor.zeros(2, 16) + block.attn_q.weight = block.attn_k.weight = Tensor.full((16, 16), 1e-4/16) # x of ones: q = k = 1e-4, v = 1 + block.attn_v.weight, block.ssm_beta.weight = Tensor.full((16, 16), 1/16), Tensor.zeros(2, 16) + block.ssm_conv1d_q["weight"] = block.ssm_conv1d_k["weight"] = block.ssm_conv1d_v["weight"] = Tensor.ones(16, 4) x = Tensor.ones(1, 1, 16) block._init_state(x) block._attention(x, 0).realize() diff --git a/tinygrad_repo/test/runtime/test_buffer.py b/tinygrad_repo/test/runtime/test_buffer.py index 318441e1c0..7d3b0e667f 100644 --- a/tinygrad_repo/test/runtime/test_buffer.py +++ b/tinygrad_repo/test/runtime/test_buffer.py @@ -1,5 +1,5 @@ import unittest -from tinygrad.device import Buffer +from tinygrad.device import Buffer, Device from tinygrad.helpers import Context class TestBuffer(unittest.TestCase): @@ -38,7 +38,7 @@ class TestBuffer(unittest.TestCase): def test_cache_owned_storage_only(self): for opaque in (None, memoryview(bytearray(8))): with self.subTest(imported=opaque is not None), Context(LRU=1): - b = Buffer("PYTHON", 8, opaque=opaque) + b = Buffer("PYTHON", 8, opaque=opaque, allocator=type(Device["PYTHON"].allocator)(Device["PYTHON"])) buf = b._buf b.deallocate() self.assertEqual(b._buf is buf, opaque is None) diff --git a/tinygrad_repo/test/runtime/test_custom_kernel.py b/tinygrad_repo/test/runtime/test_custom_kernel.py index fe68e65707..868bed705c 100644 --- a/tinygrad_repo/test/runtime/test_custom_kernel.py +++ b/tinygrad_repo/test/runtime/test_custom_kernel.py @@ -1,8 +1,10 @@ -import unittest -from tinygrad import Tensor, UOp, GlobalCounters, Context, Device +import unittest, functools +from tinygrad import Tensor, UOp, GlobalCounters, Context, Device, Variable, TinyJit +from tinygrad.engine.realize import run_linear, get_runtime +from tinygrad.codegen import to_program import numpy as np from tinygrad.dtype import AddrSpace, dtypes, Invalid -from tinygrad.helpers import getenv +from tinygrad.helpers import getenv, DEV from tinygrad.schedule.rangeify import BufferizeOpts from tinygrad.uop.ops import KernelInfo, AxisType, Ops, uopfunc from tinygrad.codegen.opt import Opt, OptOps @@ -35,6 +37,25 @@ def custom_ignore_first_kernel(C:UOp, A:UOp, B:UOp) -> UOp: i = UOp.range(C.numel(), 0) return C[i].store(B[i] + 1).end(i).sink(arg=KernelInfo(name=f"ignore_first_{C.numel()}")) +def custom_add_var_kernel(*srcs:UOp, n_slot:int) -> UOp: + C, *Bs = [s.flatten() for i,s in enumerate(srcs) if i != n_slot] + i = UOp.range(C.numel(), 0) + out = functools.reduce(lambda a,b: a+b, [B[i] for B in Bs]) + srcs[n_slot] + return C[i].store(out).end(i).sink(arg=KernelInfo(name=f"add_var_{n_slot}_{len(Bs)}")) + +def run_kernel_args(out:Tensor, name:str) -> list[str|None]: + linear, var_vals = out.linear_with_vars() + sink = [u for u in linear.toposort(enter_calls=True) if u.op is Ops.SINK and isinstance(u.arg, KernelInfo) and u.arg.name == name][0] + sig = to_program(sink, Device[Device.DEFAULT].renderer).to_elf().signature + run_linear(linear, var_vals) + return [nm for nm, *_ in sig] + +def custom_ignore_first_var_kernel(C:UOp, A:UOp, B:UOp, n:UOp) -> UOp: + C, B = C.flatten(), B.flatten() + k = UOp.variable("k", 0, 100, dtype=dtypes.int) + i = UOp.range(C.numel(), 0) + return C[i].store(B[i] + n + k).end(i).sink(arg=KernelInfo(name=f"ignore_first_var_{C.numel()}")) + def custom_sum(B:UOp, A:UOp) -> UOp: i = UOp.range(A.shape[0], 0, axis_type=AxisType.LOOP) B = B[0].set(0.0) @@ -180,6 +201,61 @@ class TestCustomKernel(unittest.TestCase): out = Tensor.custom_kernel(Tensor.empty(4), a, b, fxn=custom_ignore_first_kernel)[0] self.assertEqual(out.tolist(), [2, 3, 4, 5]) + def test_scalar_arg_any_position(self): + for n_slot in range(3): + with self.subTest(n_slot=n_slot): + srcs = [Tensor.empty(4, dtype=dtypes.int), Tensor([1, 2, 3, 4], dtype=dtypes.int).realize()] + srcs.insert(n_slot, Tensor(Variable("n", 0, 100, dtypes.int).bind(5))) + out = Tensor.custom_kernel(*srcs, fxn=functools.partial(custom_add_var_kernel, n_slot=n_slot))[1 if n_slot == 0 else 0] + self.assertEqual(run_kernel_args(out, f"add_var_{n_slot}_1"), ["n" if i == n_slot else None for i in range(3)]) + self.assertEqual(out.tolist(), [6, 7, 8, 9]) + + @unittest.skipIf(Device.DEFAULT == "PYTHON", "needs a device that isnt the default") + def test_scalar_arg_first_other_device(self): + srcs = [Tensor(Variable("n", 0, 100, dtypes.int).bind(5), device="PYTHON"), Tensor.empty(4, dtype=dtypes.int, device="PYTHON"), + Tensor([1, 2, 3, 4], dtype=dtypes.int, device="PYTHON").realize()] + out = Tensor.custom_kernel(*srcs, fxn=functools.partial(custom_add_var_kernel, n_slot=0))[1] + self.assertEqual(out.tolist(), [6, 7, 8, 9]) + + def test_scalar_arg_with_other_variable(self): + other = (Tensor([1, 2, 3, 4], dtype=dtypes.int) + Tensor(Variable("m", 0, 100, dtypes.int).bind(70))).contiguous() + srcs = [Tensor.empty(4, dtype=dtypes.int), Tensor([1, 2, 3, 4], dtype=dtypes.int).realize(), Tensor(Variable("n", 0, 100, dtypes.int).bind(5))] + out = Tensor.custom_kernel(*srcs, fxn=functools.partial(custom_add_var_kernel, n_slot=2))[0] + Tensor.realize(other, out) + self.assertEqual(out.tolist(), [6, 7, 8, 9]) + + def test_scalar_arg_past_registers(self): + # 9 inputs with the scalar between pointers, x86 passes the ones after the 6th (4th on windows) on the stack. every buffer differs + srcs = [Tensor.empty(4, dtype=dtypes.int)] + [Tensor([k, 2*k, 3*k, 4*k], dtype=dtypes.int).realize() for k in range(1, 8)] + srcs.insert(4, Tensor(Variable("n", 0, 100, dtypes.int).bind(5))) + out = Tensor.custom_kernel(*srcs, fxn=functools.partial(custom_add_var_kernel, n_slot=4))[0] + self.assertEqual(run_kernel_args(out, "add_var_4_7"), [None]*4 + ["n"] + [None]*4) + self.assertEqual(out.tolist(), [33, 61, 89, 117]) + + def test_unused_arg_then_var(self): + srcs = (Tensor.empty(4, dtype=dtypes.int), Tensor.empty(4, dtype=dtypes.int), Tensor([1, 2, 3, 4], dtype=dtypes.int).realize(), + Tensor(Variable("n", 0, 100, dtypes.int).bind(5))) + out = Tensor.custom_kernel(*srcs, fxn=custom_ignore_first_var_kernel)[0] + linear, var_vals = out.linear_with_vars() + run_linear(linear, {**var_vals, "k": 10}) + self.assertEqual(out.tolist(), [16, 17, 18, 19]) + + def test_scalar_arg_jit(self): + # the scalar changes every call, a replay must not keep the value it was captured with + kernel = functools.partial(custom_add_var_kernel, n_slot=0) + jf = TinyJit(lambda B, n: Tensor.custom_kernel(Tensor(n), Tensor.empty(4, dtype=dtypes.int), B, fxn=kernel)[1].realize()) + B, n = Tensor([1, 2, 3, 4], dtype=dtypes.int).realize(), Variable("n", 0, 100, dtypes.int) + for k in range(1, 6): self.assertEqual(jf(B, n.bind(k)).tolist(), [1+k, 2+k, 3+k, 4+k]) + + def test_scalar_arg_jit_fixed_bind(self): + # m is bound inside the jitted function so it's not an input, a replay keeps m=10 while the input n changes + kernel = functools.partial(custom_add_var_kernel, n_slot=0) + def f(B, n): + Bm = Tensor.custom_kernel(Tensor(Variable("m", 0, 100, dtypes.int).bind(10)), Tensor.empty(4, dtype=dtypes.int), B, fxn=kernel)[1] + return Tensor.custom_kernel(Tensor(n), Tensor.empty(4, dtype=dtypes.int), Bm, fxn=kernel)[1].realize() + jf, B, n = TinyJit(f), Tensor([1, 2, 3, 4], dtype=dtypes.int).realize(), Variable("n", 0, 100, dtypes.int) + for k in range(1, 6): self.assertEqual(jf(B, n.bind(k)).tolist(), [11+k, 12+k, 13+k, 14+k]) + def test_sum(self): a = Tensor([1.0, 2, 3, 4, 5]) tst = Tensor.empty(1) @@ -292,6 +368,13 @@ class TestCustomKernel(unittest.TestCase): a = Tensor([1., 2, 3, 4]) self.assertEqual(Tensor.custom_kernel(Tensor.empty(4), a, fxn=kernel)[0].tolist(), [4., 5, 6, 7]) + def test_range_bound_after_outer_range(self): + def kernel(C:UOp) -> UOp: + i = UOp.range(3, 0, AxisType.LOOP) + j = UOp.range(UOp.const(2, dtypes.weakint).after(i), 1, AxisType.LOOP) + return C[0].set(C.after(i, j)[0] + 1, end=(j, i)).sink(arg=KernelInfo()) + self.assertEqual(Tensor.custom_kernel(Tensor.zeros(1, dtype=dtypes.int).contiguous(), fxn=kernel)[0].tolist(), [6]) + def test_gemm_multi(self): devs = ("CPU:0", "CPU:1") N = 16 @@ -367,6 +450,21 @@ class TestCustomKernel(unittest.TestCase): x = Tensor.empty(4).uop self.assertEqual(Tensor(x.after(x.store(5), x.store(Invalid))).tolist(), [5]*4) + def test_store_behind_value_after_kept(self): + def kernel(A:UOp, loop:bool) -> UOp: + x = UOp.const(1, dtypes.weakint).after(A[0].store(7)) + return (A[(r:=UOp.range(x, 0))+1].store(9).end(r) if loop else A[x].store(9)).sink(arg=KernelInfo(opts_to_apply=())) + for loop in (False, True): + out = Tensor.custom_kernel(Tensor.zeros(2, dtype=dtypes.int).contiguous().realize(), fxn=functools.partial(kernel, loop=loop))[0] + self.assertEqual(out.tolist(), [7, 9]) + + def test_deleted_store_keeps_deps(self): + for f in (lambda a, s: s[1].store(s[1].load()), lambda a, s: s[UOp.invalid()].store(5), + lambda a, s: s[1].store(UOp.invalid()), lambda a, s: a[UOp.invalid()].store(s[1].load())): + out = Tensor.custom_kernel(Tensor.zeros(3, dtype=dtypes.int).contiguous(), + fxn=lambda a: f(a, a.after(a[0].store(7))).sink(arg=KernelInfo(opts_to_apply=())))[0] + self.assertEqual(out.tolist(), [7, 0, 0]) + def test_expand_view_invalid_assign_keeps_uncovered_reads(self): x = Tensor([[10., 11., 12., 13.], [20., 21., 22., 23.], [30., 31., 32., 33.], [40., 41., 42., 43.]]).realize() v = x[:1, :].expand(4, 4) @@ -454,18 +552,20 @@ class TestCustomKernel(unittest.TestCase): def test_custom_kernel_sched_copy(self): self.test_custom_kernel_sched(use_custom=True) @unittest.skipIf(Device.DEFAULT == "CPU", "test needs to copy from CPU to another device") - def test_custom_kernel_source_copy(self): + def test_call_to_program(self): + t = Tensor([-1]).realize() + # render a source string from tinygrad.codegen import do_to_program + source:str = do_to_program(custom_add_one_kernel(t.uop.param_like(0), t.uop.param_like(1)), Device[t.device].renderer).src[2].arg + assert isinstance(source, str) + # call to PROGRAM def custom_source(out:UOp, inp:UOp) -> UOp: - prg_uop = do_to_program(custom_add_one_kernel(out, inp), Device[out.device].renderer) - # construct a plain Ops.PROGRAM sink = UOp.sink(out.base, inp.base, arg=KernelInfo("add_one_1")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(sink.toposort())),)+prg_uop.src[2:]) - out = Tensor([-1]).realize() + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(sink.toposort())), UOp(Ops.SOURCE, arg=source))) cpu_src = Tensor([2], device="CPU").realize() - out = Tensor.custom_kernel(out, cpu_src.to(out.device), fxn=custom_source)[0] - cp = out.to("CPU").realize() - self.assertEqual(out.tolist(), [3]) + t = Tensor.custom_kernel(t, cpu_src.to(t.device), fxn=custom_source)[0] + cp = t.to("CPU").realize() + self.assertEqual(t.tolist(), [3]) self.assertEqual(cp.tolist(), [3]) def test_sliced_buffer_function(self): @@ -481,19 +581,20 @@ class TestCustomKernel(unittest.TestCase): assert_kernel_count(2 if x[0].uop.contiguous_view() is None else 1) self.assertEqual(y.tolist(), [1, 2, 3, 4]) - @unittest.skip("this shouldn't be expected to work") def test_inplace_transpose(self): def custom_assign_row_max_kernel(A:UOp) -> UOp: row = UOp.range(A.shape[0], 0) col = UOp.range(A.shape[1], 1) - return A[row, col].store(A[row].max(axis=0)).end(col).end(row).sink(arg=KernelInfo(name=f"assign_row_max_{A.numel()}")) - a = Tensor.arange(4).clone().realize() - a = Tensor.custom_kernel(a.reshape(2, 2).T, fxn=custom_assign_row_max_kernel)[0] - self.assertEqual(a.flatten().tolist(), [2, 2, 3, 3]) + return A[row, col].store(A[row].max(axis=0)).end(col).end(row).sink(arg=KernelInfo(name=f"assign_row_max_{A.numel()}", opts_to_apply=())) + a = Tensor.arange(4).reshape(2, 2).clone().realize() + a = Tensor.custom_kernel(a.T, fxn=custom_assign_row_max_kernel)[0] + with self.assertRaises(AssertionError): + self.assertEqual(a.flatten().tolist(), [2, 2, 3, 3]) self.assertEqual(a.shape, (2, 2)) @unittest.skipUnless((isinstance(Device[Device.DEFAULT].renderer, CStyleLanguage) or Device.DEFAULT == "PYTHON") and Device.DEFAULT != "WEBGPU", "binary not supported on this backend") + @unittest.skipIf(Device.DEFAULT == "AMD" and DEV.interface.startswith("MOCK"), "AMD emulator doesn't support direct pc read and write") def test_binary(self): payload = bytes(range(256)) def kernel(out:UOp): @@ -502,8 +603,13 @@ class TestCustomKernel(unittest.TestCase): return out[i].store(data[i]).end(i).sink(arg=KernelInfo(name="binary", opts_to_apply=())) self.assertEqual(Tensor.empty(len(payload), dtype=dtypes.uint8).custom_kernel(fxn=kernel)[0].tolist(), list(payload)) -@unittest.skipUnless(Device.DEFAULT == "CPU" and isinstance(Device[Device.DEFAULT].renderer, (CStyleLanguage, LLVMRenderer)), - "calls in kernels render on CPU") + def test_one_lane_stack_runtime_index(self): + def kernel(A:UOp): return A[0].store(UOp.stack(UOp.const(42, dtypes.int)).index(A[1].load())).sink(arg=KernelInfo(opts_to_apply=())) + self.assertEqual(Tensor.zeros(2, dtype=dtypes.int).contiguous().custom_kernel(fxn=kernel)[0].tolist(), [42, 0]) + +@unittest.skipUnless((Device.DEFAULT == "CPU" and isinstance(Device[Device.DEFAULT].renderer, (CStyleLanguage, LLVMRenderer))) + or Device.DEFAULT == "AMD" and isinstance(Device[Device.DEFAULT].renderer, CStyleLanguage), + "calls in kernel not implemented in this renderer") class TestCallInKernel(unittest.TestCase): def test_simple(self): N = getenv("N", 4) @@ -569,6 +675,18 @@ class TestCallInKernel(unittest.TestCase): out = Tensor.custom_kernel(Tensor.zeros(4, dtype=dtypes.int).clone().realize(), a, fxn=kernel)[0] self.assertEqual(out.tolist(), [3, 6, 9, 12]) + def test_call_scalar_arg(self): + @uopfunc + def add(out:UOp, src:UOp, val:UOp) -> UOp: + return out[0].store(src[0]+val).sink() + + def kernel(out:UOp, src:UOp) -> UOp: + i = UOp.range(4, 0) + return add(out[i], src[i], i).end(i).sink(arg=KernelInfo(name="call_scalar_arg", opts_to_apply=())) + + x = Tensor([1, 2, 3, 4], dtype=dtypes.int).realize() + self.assertEqual(Tensor.empty_like(x).custom_kernel(x, fxn=kernel)[0].tolist(), [1, 3, 5, 7]) + def test_call_with_gated_store(self): @uopfunc def gated_store(out:UOp, idx:UOp): @@ -590,11 +708,11 @@ class TestCallInKernel(unittest.TestCase): def test_call_body_range_is_not_ours(self): @uopfunc def mul(out:UOp, A:UOp): - k = UOp.range(4, 0) + k = UOp.range(4, 0, AxisType.LOOP) return out[k].store(A[k]*3).end(k).sink() def kernel(C:UOp, A:UOp): - m, i, call = UOp.range(3, 1), UOp.range(4, 0), mul(C, A) + m, i, call = UOp.range(3, 1, AxisType.LOOP), UOp.range(4, 0, AxisType.LOOP), mul(C, A) self.assertIn(i, call.body.toposort()) end_m = C.after(call)[m].store(A[m]).end(m) return C.after(end_m)[i].store(A[i]+1).end(i).sink(arg=KernelInfo(name="call_body_range", opts_to_apply=())) @@ -626,18 +744,91 @@ class TestCallInKernel(unittest.TestCase): def test_uopfunc(self): @uopfunc def axpy(out:UOp, x:UOp, a:int) -> UOp: # a is a trace-time constant: two bodies, two C functions - i = UOp.range(x.shape[0], 0) + i = UOp.range(x.shape[0], 0, AxisType.LOOP) return out[i].store(out[i] + x[i] * a).end(i).sink() + @uopfunc + def triple(out:UOp, x:UOp): return axpy(out, x, 3).sink() + def kernel(C:UOp, A:UOp) -> UOp: - C = C.after(axpy(C.after(axpy(C, A, 2)), A, 3)) - i = UOp.range(A.shape[0], 1) # a loop after the calls + C = C.after(triple(C.after(axpy(C, A, 2)), A)) + i = UOp.range(A.shape[0], 1, AxisType.LOOP) # a loop after the calls return C[i].store(C[i] + 1).end(i).sink(arg=KernelInfo(name="uopfunc", opts_to_apply=())) a = Tensor([1, 2, 3, 4], dtype=dtypes.int).realize() out = Tensor.custom_kernel(Tensor.zeros(4, dtype=dtypes.int).contiguous().realize(), a, fxn=kernel)[0] self.assertEqual(out.tolist(), [6, 11, 16, 21]) + def test_copy_offset_slices(self): + @uopfunc + def copy_block(dst:UOp, src:UOp) -> UOp: + i = UOp.range(4, 0, AxisType.UPCAST) + return dst[i].store(src[i]).end(i).sink() + + def kernel(dst:UOp, src:UOp) -> UOp: + return copy_block(dst[1:5], src[1:5]).sink(arg=KernelInfo(opts_to_apply=())) + + src = Tensor([0., 1., 2., 3., 4.]).realize() + dst = Tensor([-1.] * 6).realize() + out = dst.custom_kernel(src, fxn=kernel)[0] + self.assertEqual(out.tolist(), [-1., 1., 2., 3., 4., -1.]) + + @unittest.skipIf(Device.DEFAULT == "CPU", "test needs to copy from CPU to another device") + def test_call_to_copy(self): + @uopfunc + def copy(out:UOp, A:UOp): + i = UOp.range(out.shape[0], 0, AxisType.LOOP) + return out[i].store(A[i]).end(i).sink() + + def forward(out1:UOp, out2:UOp, A:UOp): + call = copy(out1, A) + i = UOp.range(out2.shape[0], 0, AxisType.LOOP) + inline = out2[i].store(A[i]).end(i).sink() + return UOp.group(call, inline).sink(arg=KernelInfo(name="forward", opts_to_apply=())) + + x = Tensor([1, 2, 3, 4], device="CPU").realize().to(Device.DEFAULT) + y, z = x.empty_like(), x.empty_like() + y, z, _ = Tensor.custom_kernel(y, z, x, fxn=forward) + y, z = y.to("CPU"), z.to("CPU") + Tensor.realize(y, z) + + self.assertEqual(y.tolist(), z.tolist()) + self.assertEqual(x.tolist(), z.tolist()) + + @unittest.skipIf(Device[Device.DEFAULT].renderer.inline_prefix is not None, "backend inlines calls instead of linking") + def test_linker(self): + @uopfunc + def plus2(out:UOp, A:UOp): return out[0].store(A[0]+2).sink() + + @uopfunc + def copy(out:UOp, A:UOp): return out[0].store(A[0]).sink() + + def kernel(A:UOp, extra=False): + A = A.flatten() + i = UOp.range(A.shape[0], 0) + tmp = UOp.alloc_like(A, addrspace=AddrSpace.REG) + if extra: tmp = tmp.after(copy(tmp[i], A[i])) + return A[i].store(tmp[i].after(plus2(tmp[i], A[i]))).end(i).sink(arg=KernelInfo(name="test_kernel", opts_to_apply=())) + + x = Tensor.custom_kernel(Tensor.arange(4).clone().realize(), fxn=kernel)[0] + run_linear(lin:=x.schedule_linear()) + + prg = to_program(lin.src[0].body, Device[Device.DEFAULT].renderer) + assert "+2" not in prg.src[2].arg, f"found callee body in caller kernel {prg.src[2].arg}" + call = next(u for u in prg.src[1].src if u.op is Ops.CALL) + first_binary = call.body.src[3] + call_runtime = get_runtime(Device.DEFAULT, call.body) + self.assertListEqual(x.tolist(), Tensor.arange(4).add(2).tolist()) + + # launch a different caller with the same callee + x = Tensor.custom_kernel(Tensor.arange(5).clone().realize(), fxn=lambda A: kernel(A, extra=True))[0] + run_linear(lin:=x.schedule_linear()) + prg2 = to_program(lin.src[0].body, Device[Device.DEFAULT].renderer).src[1] + self.assertIs(call.body, next(u.body for u in prg2.src if u.op is Ops.CALL and u.arg.name == "plus2")) + self.assertIs(first_binary, next(u.body.src[3] for u in prg2.src if u.op is Ops.CALL and u.arg.name == "plus2")) + self.assertIs(call_runtime, get_runtime(Device.DEFAULT, call.body)) + self.assertListEqual(x.tolist(), Tensor.arange(5).add(2).tolist()) + class TestCustomKernelInput(unittest.TestCase): def _test_mop(self, mop_fxn, max_kernels): # default: input is BUFFER diff --git a/tinygrad_repo/test/runtime/test_dtype.py b/tinygrad_repo/test/runtime/test_dtype.py index 8574bfbb31..73b7ede869 100644 --- a/tinygrad_repo/test/runtime/test_dtype.py +++ b/tinygrad_repo/test/runtime/test_dtype.py @@ -78,7 +78,8 @@ class TestDType(unittest.TestCase): def test_const_kernel(self): if not get_available_cast_dtypes(self.DTYPE): raise unittest.SkipTest("dtype does not run here") - _assert_eq(Tensor.ones((4,4), dtype=self.DTYPE).clone(), self.DTYPE, np.ones((4,4))) + for value in ([1, 1e-5] if dtypes.is_float(self.DTYPE) else [1]): + _assert_eq(Tensor.full((4,4), value, dtype=self.DTYPE).clone(), self.DTYPE, np.full((4,4), truncate[self.DTYPE](value))) def test_same_size_ops(self): for dtype in get_available_cast_dtypes(self.DTYPE): @@ -137,6 +138,15 @@ class TestFp8sConversions(unittest.TestCase): # e4m3 and the fnuz fp8s have no inf, so their extremes (the MAX/MIN reduce identities) are finite values that round trip for dt in dtypes.fp8s: self.assertEqual(Tensor([dt.min, dt.max], dtype=dt).float().tolist(), [dt.min, dt.max]) + @Context(EMULATED_DTYPES="fp8e4m3,fp8e5m2,fp8e4m3fnuz,fp8e5m2fnuz") + def test_cast_rounds(self): + for dt in dtypes.fp8s: self.assertEqual(Tensor([1.0625]).cast(dt).float().tolist(), [1.0]) + + @Context(EMULATED_DTYPES="fp8e4m3,fp8e5m2,fp8e4m3fnuz,fp8e5m2fnuz") + def test_cast_nonfinite(self): + x = [math.inf, -math.inf, math.nan] + for dt in dtypes.fp8s: self.assertEqual(Tensor(x).cast(dt).bitcast(dtypes.uint8).tolist(), [float_to_fp8(v, dt) for v in x]) + @given(strat.floats(width=32, allow_subnormal=True, allow_nan=False, allow_infinity=False, min_value=-FP8E4M3_MAX, max_value=FP8E4M3_MAX)) def test_float_to_fp8e4m3(self, x): np.testing.assert_equal(float_to_fp8(x, dtypes.fp8e4m3), torch.tensor(x, dtype=torch.float8_e4m3fn).view(torch.uint8).item()) @@ -219,6 +229,10 @@ class TestBFloat16DTypeCast(unittest.TestCase): original_to_float32 = original_tensor.cast(dtypes.float32) np.testing.assert_allclose(back_to_float32.numpy(), original_to_float32.numpy(), rtol=1e-2, atol=1e-3) + @Context(EMULATED_DTYPES="bf16") + def test_cast_denormal_flushes(self): + self.assertEqual(Tensor([0x7f8000], dtype=dtypes.uint32).bitcast(dtypes.float32).cast(dtypes.bfloat16).bitcast(dtypes.uint16).tolist(), [0]) + def test_f16_to_bf16_edge_cases(self): edge_cases = Tensor([0.0, -0.0, float('inf'), float('-inf'), float('nan')], dtype=dtypes.float16) converted = edge_cases.cast(dtypes.bfloat16).cast(dtypes.float32) @@ -250,6 +264,12 @@ class TestEmulatedHalf(TestHalfDType): @classmethod def tearDownClass(cls): cls.stack.close() + def test_cast_overflow(self): self.assertEqual(Tensor([65519., 65520.]).cast(dtypes.half).float().tolist(), [65504., math.inf]) + def test_cast_underflow(self): + self.assertEqual(Tensor([2**-14 - 2**-25, 2**-14 - 2**-25 - 2**-38]).cast(dtypes.half).float().tolist(), [2**-14, 0.]) + def test_cast_nan_payload(self): + self.assertEqual(Tensor([0x7f800001], dtype=dtypes.uint32).bitcast(dtypes.float32).cast(dtypes.half).bitcast(dtypes.uint16).tolist(), [0x7e00]) + class TestFloatDType(TestDType): DTYPE = dtypes.float @@ -387,6 +407,11 @@ class TestUint64DType(TestDType): @unittest.skipIf(dtypes.double not in supported_dtypes, "needs float64") def test_uint64_cast_double(self): assert Tensor([2**32 + 1], dtype=dtypes.uint64).cast(dtypes.double).numpy() == 2**32 + 1 + def test_uint64_cast_float_rounds_once(self): + self.assertEqual(Tensor([2**25+3], dtype=dtypes.uint64).cast(dtypes.float32).tolist(), [2**25+4]) + @unittest.skipIf(dtypes.double not in supported_dtypes, "needs float64") + def test_uint64_cast_double_rounds_once(self): + self.assertEqual(Tensor([2**63+2**10+1], dtype=dtypes.uint64).cast(dtypes.double).tolist(), [2**63+2**11]) @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "PTX does indexing math with longs") class TestEmulatedUInt64DType(TestUint64DType): diff --git a/tinygrad_repo/test/runtime/test_dtype_alu.py b/tinygrad_repo/test/runtime/test_dtype_alu.py index 21221e45a1..1af2409b77 100644 --- a/tinygrad_repo/test/runtime/test_dtype_alu.py +++ b/tinygrad_repo/test/runtime/test_dtype_alu.py @@ -7,7 +7,7 @@ from tinygrad.runtime.ops_python import from_storage_scalar from tinygrad.renderer.ptx import PTXRenderer from tinygrad.renderer.nir import NIRRenderer from tinygrad.renderer.llvmir import CPULLVMRenderer -from tinygrad.renderer.isa.x86 import X86Renderer +from tinygrad.renderer.x86 import X86Renderer from tinygrad.uop import Ops import numpy as np import pytest @@ -274,6 +274,99 @@ class TestDTypeALU(unittest.TestCase): @given(ht.uint64, ht.uint64, strat.sampled_from(integer_binary_operations)) def test_uint64(self, a, b, op): universal_test(a, b, dtypes.uint64, op) + def test_unsigned_add_lt(self): + for dt in (dtypes.uint32, dtypes.uint64): + x = Tensor.arange(10, dtype=dt) + Tensor([dt.max-4], dtype=dt) + self.assertEqual((x < 3).tolist(), [i in (5, 6, 7) for i in range(10)]) + + def test_unsigned_narrow_wrap(self): + for dt in (dtypes.uint8, dtypes.uint16): + x = Tensor([dt.max, 3], dtype=dt) + self.assertEqual(((x+1)>>4).tolist(), [0, 0]) + self.assertEqual(((x-4)>>4).tolist(), [dt.max>>4, dt.max>>4]) + self.assertEqual(((x*2)>>4).tolist(), [dt.max>>4, 0]) + self.assertEqual(((-x)>>4).tolist(), [0, dt.max>>4]) + self.assertEqual(((x<<4)>>4).tolist(), [dt.max>>4, 3]) + self.assertEqual(((x*x)>>4).tolist(), [0, 0]) + self.assertEqual(((x+1)<1).tolist(), [True, False]) + + def test_pow_one_keeps_dtype(self): + self.assertEqual(((1**Tensor([2], dtype=dtypes.uint8))+255).cast(dtypes.uint32).tolist(), [0]) + self.assertEqual((1**Tensor([2], dtype=dtypes.int8)).bitcast(dtypes.uint8).tolist(), [1]) + + def test_div_add_div_no_overflow(self): + for dt in (dtypes.uint32, dtypes.uint64): + x = Tensor([dt.max-1], dtype=dt) + self.assertEqual(((x//2+1)//2).tolist(), [(dt.max+1)//4]) + x = Tensor([2**31-2], dtype=dtypes.int32) + self.assertEqual(((x//2+2**29)//2).tolist(), [805306367]) + + def test_div_div_no_overflow(self): + self.assertEqual(((Tensor([2**31-1, -5], dtype=dtypes.int32)//65536)//65536).tolist(), [0, -1]) + + def test_add_add_no_overflow(self): + self.assertEqual((((Tensor([-2**31, 1-2**31], dtype=dtypes.int32)+2**30)+2**30)//2).tolist(), [0, 0]) + + def test_neg_zero(self): + self.assertEqual((1/-Tensor([0., -0.])).tolist(), [-math.inf, math.inf]) + + def test_neg_add_half_zero(self): + self.assertEqual((1/-(Tensor([0.])+Tensor([-0.])).half()).tolist(), [-math.inf]) + + def test_neg_add_no_overflow(self): + self.assertEqual((-(Tensor([1, 2], dtype=dtypes.int32)+(-2**31))==2**31-1).tolist(), [True, False]) + + def test_neg_add_signed_zero(self): + self.assertEqual((1/-(Tensor([0.])+Tensor([-0.]))).tolist(), [-math.inf]) + + def test_add_mul_no_overflow(self): + self.assertEqual((((Tensor([-715827887, 0], dtype=dtypes.int32)+5)*3)==-2**31+2).tolist(), [True, False]) + + def test_add_lt_no_overflow(self): + self.assertEqual(((Tensor([0, -715827882], dtype=dtypes.int32)*3+15) < -2**31+8).tolist(), [False, False]) + + def test_mul_mul_no_overflow(self): + self.assertEqual(((Tensor([-1, 0], dtype=dtypes.int32)*65536)*32768//2).tolist(), [-2**30, 0]) + + def test_combine_terms_no_overflow(self): + t, u = Tensor([-1, 0], dtype=dtypes.int32), Tensor([0, 0], dtype=dtypes.int32) + self.assertEqual(((t*2**30+t*2**30)//2).tolist(), [-2**30, 0]) + self.assertEqual(((t+t*(2**31-1))//2).tolist(), [-2**30, 0]) + self.assertEqual((((u+t*(2**31-1))+t)//2).tolist(), [-2**30, 0]) + + def test_fused_div_matches_unfused(self): + x, y, z = Tensor([3e38, 1e30, 1.7]), Tensor([1e20, 1e30, 1.3]), Tensor([1e20, 1e10, 3.1]) + for f, g in ((lambda: x*y, lambda p: p/y), (lambda: x/y, lambda p: p/z), (lambda: z*z, lambda p: 1/p), (lambda: z*z*z, lambda p: 1/p)): + np.testing.assert_equal(g(f()).numpy(), g(f().realize()).numpy()) + + @unittest.skipIf(Device.DEFAULT == "CL", "rusticl fdiv scaling reassociates (x*c)*0.25 past inf") + def test_fused_recip_const_matches_unfused(self): + x = Tensor([3e38, 1e30, 1.7]) + np.testing.assert_equal((1/(x*3)).numpy(), (1/(x*3).realize()).numpy()) + + def test_unsigned_mul_max(self): + for dt in (dtypes.uint32, dtypes.uint64): + x = Tensor([dt.max//2, dt.max//2+1], dtype=dt) + self.assertEqual((x*2).max().item(), dt.max-1) + + def test_unsigned_wrap_bounds(self): + for dt in (dtypes.uint32, dtypes.uint64): + h = 1 << (dt.bitsize-1) + u, x = Tensor([h], dtype=dt).maximum(h), Tensor([1], dtype=dt) + self.assertEqual([(t < 1).item() for t in (u+h, u*2, u<<1, -x, x^-1)], [True, True, True, False, False]) + + def test_unsigned_const_canonical(self): + for dt in (dtypes.uint8, dtypes.uint32, dtypes.uint64): + c, z = Tensor(-1, dtype=dt), Tensor([1], dtype=dt)*0-1 + self.assertEqual((c>>1).item(), dt.max>>1) + self.assertEqual((z>>1).item(), dt.max>>1) + self.assertEqual((z//2).item(), dt.max//2) + self.assertEqual((z<3).item(), False) + self.assertEqual(z.maximum(5).item(), dt.max) + self.assertEqual(Tensor(-1, dtype=dtypes.uint32).cast(dtypes.int64).item(), 2**32-1) + self.assertEqual((Tensor(255, dtype=dtypes.uint8)+1).cast(dtypes.uint16).item(), 0) + self.assertEqual((Tensor([5], dtype=dtypes.uint32)//Tensor(-1, dtype=dtypes.uint32)).item(), 0) + @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "PTX does indexing math with longs") @given(ht.uint64, ht.uint64, strat.sampled_from(integer_binary_operations)) @Context(EMULATED_DTYPES="long") diff --git a/tinygrad_repo/test/runtime/test_dtype_weak.py b/tinygrad_repo/test/runtime/test_dtype_weak.py index 9f0b932baf..f7de544e3f 100644 --- a/tinygrad_repo/test/runtime/test_dtype_weak.py +++ b/tinygrad_repo/test/runtime/test_dtype_weak.py @@ -18,11 +18,6 @@ class TestWeakPromotion(unittest.TestCase): self.assertEqual(c.item(), value) self.assertEqual(t.contiguous().dtype, weak) - def test_assign_into_weak_commits(self): - t = Tensor.const(0.5) - t.assign(Tensor(1.0, dtype=dtypes.default_float)) - self.assertEqual((t.dtype, t.item()), (dtypes.default_float, 1.0)) - def test_copysign_meets_operands(self): r = Tensor([2], dtype=dtypes.uint8).copysign(Tensor([1], dtype=dtypes.uint32)) self.assertEqual((r.dtype, r.tolist()), (dtypes.uint32, [2])) diff --git a/tinygrad_repo/test/runtime/test_edgecases.py b/tinygrad_repo/test/runtime/test_edgecases.py index 317de30370..6d712ab4f3 100644 --- a/tinygrad_repo/test/runtime/test_edgecases.py +++ b/tinygrad_repo/test/runtime/test_edgecases.py @@ -129,15 +129,10 @@ class TestAssignIssues(unittest.TestCase): t.shrink(((1, 3), (1, 3))).assign(Tensor.ones(2, 2)) np.testing.assert_allclose(t.numpy(), torch_tensor.numpy()) - @unittest.expectedFailure def test_assign_broadcast(self): - # broadcasting during assign should behave like PyTorch - # NOTE: we don't want implicit dtype casting (int64 -> float32 loses precision), so this fails - torch_tensor = torch.zeros(3, 5) - torch_tensor[:] = torch.arange(5) - t = Tensor.zeros(3, 5) - t.assign(Tensor.arange(5)) - np.testing.assert_allclose(t.numpy(), torch_tensor.numpy()) + # assign broadcasts like PyTorch but does not cast implicitly (int -> float can lose precision), so a dtype mismatch raises + with self.assertRaises(RuntimeError): + Tensor.zeros(3, 5).assign(Tensor.arange(5)) class TestUOpValidationIssue(unittest.TestCase): # these fail with UOp verification error. diff --git a/tinygrad_repo/test/runtime/test_function.py b/tinygrad_repo/test/runtime/test_function.py index 2533caee50..4bd99be325 100644 --- a/tinygrad_repo/test/runtime/test_function.py +++ b/tinygrad_repo/test/runtime/test_function.py @@ -350,6 +350,13 @@ class TestFunction(unittest.TestCase): np.testing.assert_equal(a.numpy(), [1,2,3]) np.testing.assert_equal(b.numpy(), [10,20,30]) + def test_offset_slice(self): + @function(precompile=True) + def square(x:Tensor) -> Tensor: return x * x + + x = Tensor([0., 1., 2., 3., 4.]).realize() + self.assertEqual(square(x[1:]).tolist(), [1., 4., 9., 16.]) + class TestFunctionMulti(unittest.TestCase): devices_2 = ("CPU:0", "CPU:1") diff --git a/tinygrad_repo/test/runtime/test_gradient.py b/tinygrad_repo/test/runtime/test_gradient.py index 8f100d7867..53dc3c32f8 100644 --- a/tinygrad_repo/test/runtime/test_gradient.py +++ b/tinygrad_repo/test/runtime/test_gradient.py @@ -118,7 +118,7 @@ class TestTensorGradient(unittest.TestCase): self.assertAlmostEqual(dp.item(), 7*math.cos(0.5), places=5) def test_max_backward_many_ties(self): - t = Tensor.ones(70000, dtype=dtypes.half).contiguous() + t = Tensor.ones(70000, dtype=dtypes.bfloat16).contiguous() np.testing.assert_allclose(t.max().gradient(t)[0].sum().numpy(), 1.0, atol=1e-3) class TestMultiOutputGradient(unittest.TestCase): diff --git a/tinygrad_repo/test/runtime/test_hcq2.py b/tinygrad_repo/test/runtime/test_hcq2.py index fa3d580a90..ce54b14635 100644 --- a/tinygrad_repo/test/runtime/test_hcq2.py +++ b/tinygrad_repo/test/runtime/test_hcq2.py @@ -1,6 +1,7 @@ import unittest, gc, struct, ctypes, threading, numpy as np +from unittest.mock import patch from tinygrad import Device, Tensor, TinyJit, Variable, dtypes, GlobalCounters -from tinygrad.device import Buffer, BufferSpec +from tinygrad.device import Buffer from tinygrad.dtype import AddrSpace from tinygrad.helpers import Context from tinygrad.uop.ops import Ops, UOp, uopfunc @@ -11,9 +12,9 @@ from tinygrad.runtime.autogen import libc from tinygrad.runtime.support.c import init_c_struct_t import tinygrad.runtime.support.hcq2 as hcq2 from tinygrad.runtime.support.hcq2 import HCQ_DEVS, all_devices_in, hcq_compile_cache -from test.null.test_hcq2 import chain, chain_input, compiled_chain, lower_hcq +from test.null.test_hcq2 import compiled_chain, lower_hcq -def cpu_buf(size:int=1, dtype=dtypes.uint8, tag=None, **kwargs) -> UOp: return UOp.alloc((size,), dtype, device="CPU", **kwargs).rtag(tag) +def cpu_buf(size:int=1, dtype=dtypes.uint8, **kwargs) -> UOp: return UOp.alloc((size,), dtype, device="CPU", **kwargs) @unittest.skipUnless(all_devices_in(Device.DEFAULT, HCQ_DEVS) and not Device.DEFAULT.startswith("NULL"), "hcq2 device required") class TestHCQ2Schedule(unittest.TestCase): @@ -21,10 +22,10 @@ class TestHCQ2Schedule(unittest.TestCase): for jit in (False, True): with self.subTest(jit=jit): out, compiled, inputs = compiled_chain(2, jit=jit, device=Device.DEFAULT) - linked = link_linear(compiled, input_uops=inputs, allow_cache=not jit) + linked = link_linear(compiled, input_uops=inputs) before = tuple(inputs) for linear in (compiled, linked): self.assertIs(compile_linear(linear, input_uops=inputs, cache=not jit), linear) - self.assertIs(link_linear(linked, input_uops=inputs, allow_cache=not jit), linked) + self.assertIs(link_linear(linked, input_uops=inputs), linked) self.assertEqual(tuple(inputs), before) run_linear(linked, input_uops=inputs, jit=True, wait=True) self.assertEqual(out.tolist(), [4] * 4) @@ -55,6 +56,18 @@ class TestHCQ2Schedule(unittest.TestCase): run_linear(UOp(Ops.LINEAR, src=(copyout, copyin, copyout)), wait=True) self.assertEqual(bytes(host.host[:]), bytes(new.host[:])) + @patch.object(type(Device["NULL"]), "host", "CPU") + def test_staging(self): + table = [("NULL", "NULL:1"), ("NULL", "CPU"), ("CPU", "NULL"), ("CL", "CPU"), ("CL", "METAL"), ("CL", "CPU", "NULL"),] + for path in table: + with self.subTest(path): + try: Device[path[0]], Device[path[-1]] # TODO: do not open devices? + except Exception: self.skipTest(f"{path} needs both devices") + copy = UOp.param(0, dtypes.uint8, 16, device=path[-1]).store_call(UOp.param(1, dtypes.uint8, 16, device=path[0])) + staged = hcq2.stage_copy(copy, *copy.src[1:3]) + copies = (copy,) if staged is None else staged.src + self.assertEqual((copies[0].src[2].device, *(c.src[1].device for c in copies)), path) + @unittest.skipIf(Device.DEFAULT == "METAL", "unified memory: METAL copies on the host and maps nothing") def test_map_cpu_buffer_preserves_contents(self): src = Buffer("CPU", 16, preallocate=True) @@ -72,24 +85,13 @@ class TestHCQ2Schedule(unittest.TestCase): def f(a): return (a * 2 + 1).contiguous().realize() for _ in range(3): out = f(x) self.assertEqual(out.to("CPU").tolist(), [2.0 * i + 1] * 1024) - step(1) # warms the programs, templates and rings + for i in (1,2): step(i) # warms the programs and templates gc.collect() used = GlobalCounters.mem_used - for i in range(2, 5): step(i) + for i in range(3, 6): step(i) gc.collect() self.assertEqual(GlobalCounters.mem_used, used) - def test_jit_has_no_rt_buffers(self): - # a one shot link borrows ring slots, a jit's link owns its buffers: nothing it keeps may come from the ring - dev = Device[Device.DEFAULT] - specs = (BufferSpec(cpu_access=True), BufferSpec(host=True, uncached=True, cpu_access=True)) # device data, signals - ranges = [((b:=dev.rt_buffer(spec))._buf, b._buf + b.nbytes) for spec in specs] - x, f = chain_input(device=dev.device), TinyJit(lambda a: chain(a, 2).realize()) - for _ in range(2): f(x) - for u in f.captured.linear.toposort(enter_calls=True): - if u.op is Ops.BUFFER and u.addrspace is AddrSpace.GLOBAL and (buf:=u.buffer).device == dev.device: - self.assertFalse(any(buf._buf < end and start < buf._buf + buf.nbytes for start, end in ranges)) - # the fence and the ffi run on the CPU runtime, which is always available @unittest.skipUnless(isinstance(Device["CPU"].renderer, (CStyleLanguage, LLVMRenderer)), "CALL is rendered in C style and LLVM only") class TestHCQ2Fence(unittest.TestCase): @@ -99,15 +101,15 @@ class TestHCQ2Fence(unittest.TestCase): self.addCleanup(lambda: self.tl.__setitem__(0, self.tl[1])) def test_a_schedule_waits_for_its_previous_run(self): - slots = UOp.alloc((4,), dtypes.uint64, device="CPU").rtag("slots") - program = lower_and_compile(lower_hcq(UOp.custom_function("hcq_fence").call(slots[0:2], slots[2:4]))) - linked = hcq2.hcq_link(program, allow_cache=False) - (i,) = [i for i, p in enumerate(program.src[0].without_after.src[1:]) if p.without_after.tag == "slots"] + slots = UOp.alloc((4,), dtypes.uint64, device="CPU", name="slots") + program = lower_and_compile(lower_hcq(fence:=hcq2.make_fence({"CPU": slots}, {"CPU": 1}), *hcq2.announce({"CPU": slots}, {"CPU": 1}, fence))) + linked = hcq2.hcq_link(program) + (i,) = [i for i, p in enumerate(program.src[0].without_after.src[1:]) if p.without_after.arg.name == "slots"] slots_mv = linked.src[0].without_after.src[1 + i].buffer.host.view(fmt='Q') - slots_mv[2], base = 7, self.tl[1] + slots_mv[0], base = 7, self.tl[1] run_linear(linked, jit=True) - self.assertEqual((self.tl[1], slots_mv[0], slots_mv[2]), (base + 1, base + 1, 0), "the run is announced, recorded, the signal re-armed") + self.assertEqual((self.tl[1], slots_mv[2], slots_mv[0]), (base + 1, base + 1, 0), "the run is announced, recorded, the signal re-armed") t = threading.Thread(target=run_linear, args=(linked,), kwargs={"jit": True}, daemon=True) t.start() @@ -122,14 +124,14 @@ class TestHCQ2Fence(unittest.TestCase): class TestHCQ2FFI(unittest.TestCase): @staticmethod def _run(body:UOp) -> list[UOp]: - linear = hcq2.hcq_link(lower_and_compile(lower_hcq(body)), allow_cache=False) - assert hcq2.hcq_link(linear, allow_cache=False) is linear, "a linked linear links to itself, with the refs its call keeps" + linear = hcq2.hcq_link(lower_and_compile(lower_hcq(body))) + assert hcq2.hcq_link(linear) is linear, "a linked linear links to itself, with the refs its call keeps" run_linear(linear, jit=True) return [u for u in linear.src[0].without_after.src[1:] if u.op is Ops.BUFFER] def test_ffi_ccall(self): with Context(HCQ_RUNTIME_DEV="CPU"): - out = cpu_buf(dtype=dtypes.int32, slot=1, tag="ffi_result") + out = cpu_buf(dtype=dtypes.int32, slot=1, name="ffi_result") bufs = self._run(out.index(0).store(hcq2.ccall(libc.dll.ffs, 0x10))) self.assertEqual(next(b for b in bufs if b.dtype is dtypes.int).buffer.host.view(fmt='i')[0], 5) @@ -141,7 +143,7 @@ class TestHCQ2FFI(unittest.TestCase): with Context(HCQ_RUNTIME_DEV=host): src = hcq2.cstruct(init_c_struct_t(4, (("value", ctypes.c_uint32, 0),)), value=42) - out = cpu_buf(2, dtypes.uint32, tag="ffi_result") + out = cpu_buf(2, dtypes.uint32, name="ffi_result") bufs = self._run(copy_pair(out, src.bitcast(dtypes.uint32))) self.assertEqual(list(next(b for b in bufs if b.dtype is dtypes.uint32).buffer.host.view(fmt='I')), [42, 42]) def test_nested_ffi_call_python(self): self.test_nested_ffi_call("PYTHON") @@ -160,7 +162,7 @@ class TestHCQ2FFI(unittest.TestCase): with Context(HCQ_RUNTIME_DEV="CPU"): inner = hcq2.cstruct(init_c_struct_t(8, (("pad", ctypes.c_uint32, 0), ("value", ctypes.c_uint32, 4))), value=42) outer = hcq2.cstruct(init_c_struct_t(8, (("ptr", ctypes.c_uint64, 0),)), ptr=inner[4:8].getaddr("CPU")) - out = cpu_buf(dtype=dtypes.uint32, tag="result") + out = cpu_buf(dtype=dtypes.uint32, name="result") copied = hcq2.ccall(libc.memcpy, out.index(0), outer.bitcast(dtypes.uint64).index(0).load(), 4) bufs = self._run(out.after(copied).index(0).load()) self.assertEqual(next(b for b in bufs if b.dtype is dtypes.uint32).buffer.host.view(fmt='I')[0], 42) @@ -178,7 +180,7 @@ class TestHostCalls(unittest.TestCase): return UOp.from_buffer(Buffer("CPU", n * dtype.itemsize, initial_value=bytes(n * dtype.itemsize)), dtype) @staticmethod def _run(fxn, *bufs:UOp, **var_vals:int) -> list: # first buffer is the output - run_linear(hcq2.hcq_link(lower_and_compile(lower_hcq(fxn(*bufs))), allow_cache=False), var_vals, jit=True) + run_linear(hcq2.hcq_link(lower_and_compile(lower_hcq(fxn(*bufs)))), var_vals, jit=True) return bufs[0].buffer.host.view(fmt=bufs[0].dtype.fmt)[:] def test_no_addrs_no_placeholders(self): @@ -198,7 +200,8 @@ class TestHostCalls(unittest.TestCase): self.assertEqual(self._run(addr, self._buf(1), b), [b.buffer.get_buf("CPU") + 4]) def test_sink_calls_function(self): # a bare sink with args calls a function that writes one arg and takes the address of the other - def sink(o:UOp, b:UOp): return addr_of(UOp.param(0, dtypes.uint64, 1), UOp.param(1, dtypes.uint8, 16)).sink().call(o, b, name="sink") + def sink(o:UOp, b:UOp): + return addr_of(UOp.param(0, dtypes.uint64, 1, name="o"), UOp.param(1, dtypes.uint8, 16, name="b")).sink().call(o, b, name="sink") outs, b = [self._buf(1) for _ in range(2)], self._buf(16, dtypes.uint8) for out in outs: self.assertEqual(self._run(sink, out, b), [b.buffer.get_buf("CPU")]) # the second run reuses the program @@ -219,7 +222,7 @@ class TestHostCalls(unittest.TestCase): def test_nested_placeholders(self): # storage a function keeps for itself @uopfunc def keep(o:UOp): - tmps = [cpu_buf(dtype=dtypes.uint64, tag="tmp") for _ in range(2)] + tmps = [cpu_buf(dtype=dtypes.uint64, name="tmp") for _ in range(2)] return UOp.sink(*[o.index(i).store(t.after(t.index(0).store(7 + i)).index(0).load()) for i, t in enumerate(tmps)]) @uopfunc def top(o:UOp): return keep(o).sink() @@ -228,7 +231,7 @@ class TestHostCalls(unittest.TestCase): def test_addr_of_placeholder(self): # the address is the one of the buffer the placeholder links to @uopfunc def keep(o:UOp): - tmp = cpu_buf(dtype=dtypes.uint64, tag="tmp") + tmp = cpu_buf(dtype=dtypes.uint64, name="tmp") return UOp.sink(tmp.index(0).store(7), o.index(0).store(tmp.getaddr("CPU"))) self.assertEqual(ctypes.c_uint64.from_address(self._run(keep, self._buf(1))[0]).value, 7) @@ -272,7 +275,7 @@ class TestHostCalls(unittest.TestCase): def test_one_function_for_any_placeholder(self): # a function names its params: what it is called on does not make another function @uopfunc def put(out:UOp, v:UOp): return out.index(0).store(v).sink() - a, b = [cpu_buf(dtype=dtypes.uint64, tag=t) for t in ("cb", "enc")] + a, b = [cpu_buf(dtype=dtypes.uint64, name=t) for t in ("cb", "enc")] lowered = lower_hcq(put(a, UOp.const(1, dtypes.uint64)), put(b, UOp.const(2, dtypes.uint64))) self.assertEqual(len({c.body for c in lowered.toposort(enter_calls=True) if c.op is Ops.CALL and c.arg.name == "put"}), 1) diff --git a/tinygrad_repo/test/runtime/test_invalid_tensor.py b/tinygrad_repo/test/runtime/test_invalid_tensor.py index 972c35d777..5efb83b580 100644 --- a/tinygrad_repo/test/runtime/test_invalid_tensor.py +++ b/tinygrad_repo/test/runtime/test_invalid_tensor.py @@ -97,6 +97,10 @@ class TestInvalidTensor(unittest.TestCase): out = mask.where(Tensor([1.0, 2.0, 3.0, 4.0]), Invalid).sum() self._invalid_test_helper(out, [10.0]) + def test_where_reduce_identity(self): + out = (Tensor.arange(4) < 2).where(Tensor([-1.0, -2.0, -3.0, -4.0]), Invalid) + self.assertEqual([out.sum().item(), out.prod().item(), out.max().item()], [-3.0, 2.0, -1.0]) + def test_invalid_unary(self): mask = Tensor.arange(4) < 2 out = mask.where(Tensor([1.0, 2.0, 3.0, 4.0]), Tensor.full((4,), Invalid, dtype=dtypes.float, buffer=False).sqrt()) diff --git a/tinygrad_repo/test/runtime/test_jit.py b/tinygrad_repo/test/runtime/test_jit.py index 4ad75fee8a..a0a43d1e8a 100644 --- a/tinygrad_repo/test/runtime/test_jit.py +++ b/tinygrad_repo/test/runtime/test_jit.py @@ -5,7 +5,7 @@ from test.helpers import assert_jit_cache_len, not_support_multi_device, needs_s from tinygrad import Tensor, TinyJit, Device, dtypes, Context, UOp from tinygrad.helpers import DEV, GlobalCounters from tinygrad.engine.jit import JitError -from tinygrad.renderer.isa.x86 import X86Renderer +from tinygrad.renderer.x86 import X86Renderer def _simple_test(add, extract=lambda x: x, N=10): for _ in range(5): @@ -615,10 +615,8 @@ class TestJitFree(unittest.TestCase): return x for _ in range(5): fxn(Tensor([1])) self.assertEqual(x.item(), 6) - pre_free = GlobalCounters.mem_used fxn.captured.free_intermediates() - savings_after_free = pre_free - GlobalCounters.mem_used - self.assertEqual(savings_after_free, 0) + self.assertTrue(x.uop.base.buffer.is_allocated()) fxn(Tensor([2])) self.assertEqual(x.item(), 8) diff --git a/tinygrad_repo/test/runtime/test_kernel_opts.py b/tinygrad_repo/test/runtime/test_kernel_opts.py index 93d8de5991..22f2bea744 100644 --- a/tinygrad_repo/test/runtime/test_kernel_opts.py +++ b/tinygrad_repo/test/runtime/test_kernel_opts.py @@ -3,6 +3,7 @@ from tinygrad import Device, Tensor, dtypes from tinygrad.codegen.opt import Opt, OptOps, KernelOptError from tinygrad.uop.ops import AxisType from tinygrad.codegen.opt.postrange import Scheduler +from tinygrad.helpers import DEV # TODO: write a clean version of this from test.runtime.test_linearizer import helper_linearizer_opt @@ -68,6 +69,7 @@ class TestKernelOpts(unittest.TestCase): [Opt(OptOps.SPLIT, 5, (0, AxisType.LOCAL, True)), Opt(OptOps.SPLIT, 1, (0, AxisType.LOCAL))], ]) + @unittest.skipIf(Device.DEFAULT == "AMD" and DEV.interface.startswith("MOCK"), "AMD emulator doesn't support direct pc read and write") def test_unrolled_padded_cumsum(self): Tensor.manual_seed(7) a = Tensor.rand(13, 17) @@ -203,6 +205,11 @@ class TestKernelOpts(unittest.TestCase): with self.assertRaises(KernelOptError): helper_linearizer_opt(a@b, [[Opt(OptOps.SPLIT, 2, (0, AxisType.UPCAST)), Opt(OptOps.PADTO, 2, 8)]]) + def test_padto_cmpeq_sum(self): + a, b = Tensor(list(range(17))), Tensor([i%3 for i in range(17)]) + # The extra lanes compare 0 == 0, but must not contribute to the reduction. + helper_linearizer_opt((a == b).sum(), [[Opt(OptOps.PADTO, 0, 32)]]) + def test_padto_sum_ok(self): N = 18 # NOTE: this setup prevents 17 * 17 contiguous merged into one dimension diff --git a/tinygrad_repo/test/runtime/test_linearizer.py b/tinygrad_repo/test/runtime/test_linearizer.py index 9d31ea19c8..8b7b24fe07 100644 --- a/tinygrad_repo/test/runtime/test_linearizer.py +++ b/tinygrad_repo/test/runtime/test_linearizer.py @@ -11,7 +11,7 @@ from tinygrad.helpers import Context, dedup, TC_SELECT, TC_OPT, DEV from tinygrad.dtype import dtypes, AddrSpace from tinygrad.renderer.ptx import PTXRenderer from tinygrad.renderer.cstyle import CUDARenderer -from tinygrad.renderer.isa import ISARenderer +from tinygrad.renderer.x86 import ISARenderer from test.helpers import replace_opts, check_schedule from test.runtime.test_softmax_fusion import single_kernel_softmax MOCKGPU = DEV.interface.startswith("MOCK") diff --git a/tinygrad_repo/test/runtime/test_llama_kernels.py b/tinygrad_repo/test/runtime/test_llama_kernels.py index ec611df3b2..8127741dd5 100644 --- a/tinygrad_repo/test/runtime/test_llama_kernels.py +++ b/tinygrad_repo/test/runtime/test_llama_kernels.py @@ -35,6 +35,11 @@ class TestFusedCE(unittest.TestCase): def test_fused_ce_1_2_16(self): run_fused_ce(1, 2, 16, label_smoothing=0.2) def test_fused_ce_2_16_128(self): run_fused_ce(2, 16, 128) def test_fused_ce_4_128_1024(self): run_fused_ce(4, 128, 1024, label_smoothing=0.2) + def test_fused_ce_grad_with_logits_reader(self): + # squaring makes the backward's upstream grad read the forward's loss too. the backward must not pin logits behind the forward + logits = Tensor.randn(1, 2, 16).cast(dtypes.bfloat16).contiguous().realize() + targets = Tensor.randint(1, 2, high=16, dtype=dtypes.int32).realize() + (fused_ce_loss(logits, targets).square() * logits.float().sum()).gradient(logits)[0].realize() # note: this is the shape used in llama 8b #def test_fused_ce_smoothing_16_1024_128256(self): run_fused_ce(16, 1024, 128256, label_smoothing=0.2) diff --git a/tinygrad_repo/test/runtime/test_llm_mla.py b/tinygrad_repo/test/runtime/test_llm_mla.py index f7f3bacc4b..afbc3c35f5 100644 --- a/tinygrad_repo/test/runtime/test_llm_mla.py +++ b/tinygrad_repo/test/runtime/test_llm_mla.py @@ -1,7 +1,9 @@ import unittest import numpy as np -from tinygrad import Tensor +from tinygrad import Tensor, dtypes, Device from tinygrad.llm.model import TransformerConfig, apply_rope, MLATransformerBlock, precompute_freqs_cis +from tinygrad.llm.hf import e4m3fn_to_fnuz +from test.runtime.test_llm_quantized import fp8_reference class TestMLA(unittest.TestCase): def _make_config(self, **kwargs): @@ -12,12 +14,29 @@ class TestMLA(unittest.TestCase): } | kwargs) def test_mla_attention_matches_naive(self): - config = self._make_config(max_context=16) + for interleave in (False, True): + with self.subTest(interleave=interleave): self._test_mla_attention(interleave) + + def _test_mla_attention(self, interleave, fnuz=None): + config = self._make_config(max_context=16, rope_interleave=interleave, rope_dim=4) block = MLATransformerBlock(config) c = config B, T = 1, 4 q_nope_head_dim = c.head_dim - c.rope_dim + block.attn_k_b['weight'] = Tensor.randn(c.n_heads, c.kv_lora_rank, q_nope_head_dim) * .1 + block.attn_v_b['weight'] = Tensor.randn(c.n_heads, c.v_head_dim, c.kv_lora_rank) * .1 + fp8_weights = [] + if fnuz is not None: + block.kv_b_block_size = (4, 8) + for projection, scale_shape in ((block.attn_k_b, (4, 2, 3)), (block.attn_v_b, (4, 2, 2))): + bits = Tensor.randint(projection['weight'].shape, high=73, dtype=dtypes.uint8) + bits |= Tensor.randint(bits.shape, high=2, dtype=dtypes.uint8) << 7 + w, s = bits.bitcast(dtypes.fp8e4m3), Tensor.rand(scale_shape, dtype=dtypes.float32) * .49 + .01 + fp8_weights.append((bits.numpy(), s.numpy())) + if fnuz: w, s = e4m3fn_to_fnuz(w, s) + projection['weight'].replace(w) + projection['weight_scale_inv'] = s x = Tensor.randn(B, T, c.dim) x_norm = block.attn_norm(x) @@ -26,39 +45,53 @@ class TestMLA(unittest.TestCase): q = block.attn_q(x_norm).reshape(B, T, c.n_heads, c.head_dim).transpose(1, 2) q_nope, q_rope = q[..., :q_nope_head_dim], q[..., q_nope_head_dim:] freqs = precompute_freqs_cis(c.rope_dim, 16, c.rope_theta) - q_rope = apply_rope(q_rope, freqs[0:T]) + q_rope = apply_rope(q_rope, freqs[0:T], interleave) kv_a = block.attn_kv_a_mqa(x_norm) c_kv = block.attn_kv_a_norm(kv_a[..., :c.kv_lora_rank]) k_rope = kv_a[..., c.kv_lora_rank:].reshape(B, T, 1, c.rope_dim).transpose(1, 2) - k_rope = apply_rope(k_rope, freqs[0:T]) + k_rope = apply_rope(k_rope, freqs[0:T], interleave) - # --- Naive (non-absorbed): expand K and V, do standard attention --- - k_nope_naive = c_kv.unsqueeze(1) @ block.attn_k_b["weight"] # (B, H, T, nope) - k_naive = k_nope_naive.cat(k_rope.expand(-1, c.n_heads, -1, -1), dim=-1) # (B, H, T, nope+rope) - v_naive = c_kv.unsqueeze(1) @ block.attn_v_b["weight"].transpose(-1, -2) # (B, H, T, v_dim) - - q_naive = q_nope.cat(q_rope, dim=-1) scale = 1.0 / c.head_dim ** 0.5 - scores_naive = (q_naive @ k_naive.transpose(-1, -2)) * scale - # causal mask mask = Tensor.full((1, 1, T, T), float("-inf")).triu(1) - attn_naive = (scores_naive + mask).softmax(-1) @ v_naive # (B, H, T, v_dim) - out_naive = block.attn_output(attn_naive.transpose(1, 2).reshape(B, T, -1)) + if fnuz is None: + # --- Naive (non-absorbed): expand K and V, do standard attention --- + k_nope_naive = c_kv.unsqueeze(1) @ block.attn_k_b["weight"] # (B, H, T, nope) + k_naive = k_nope_naive.cat(k_rope.expand(-1, c.n_heads, -1, -1), dim=-1) # (B, H, T, nope+rope) + v_naive = c_kv.unsqueeze(1) @ block.attn_v_b["weight"].transpose(-1, -2) # (B, H, T, v_dim) + + q_naive = q_nope.cat(q_rope, dim=-1) + scores_naive = (q_naive @ k_naive.transpose(-1, -2)) * scale + attn_naive = (scores_naive + mask).softmax(-1) @ v_naive # (B, H, T, v_dim) + out_naive = block.attn_output(attn_naive.transpose(1, 2).reshape(B, T, -1)) # --- Absorbed: q_nope @ wk_b^T, then dot with compressed kv --- - q_nope_abs = q_nope @ block.attn_k_b["weight"].transpose(-1, -2) # (B, H, T, lora) + q_nope_abs = (q_nope @ block.attn_k_b["weight"].transpose(-1, -2) if fnuz is None else + Tensor(fp8_reference(q_nope.numpy(), *fp8_weights[0], (8, 4), fnuz))) # (B, H, T, lora) q_abs = q_nope_abs.cat(q_rope, dim=-1) # (B, H, T, lora+rope) k_abs = c_kv.reshape(B, 1, T, c.kv_lora_rank).cat(k_rope.reshape(B, 1, T, c.rope_dim), dim=-1) scores_abs = (q_abs @ k_abs.transpose(-1, -2)) * scale attn_abs = (scores_abs + mask).softmax(-1) # attn @ v_compressed @ wv_b v_compressed = c_kv.reshape(B, 1, T, c.kv_lora_rank) - attn_abs_out = (attn_abs @ v_compressed) @ block.attn_v_b["weight"].transpose(-1, -2) + attn_compressed = attn_abs @ v_compressed + attn_abs_out = (attn_compressed @ block.attn_v_b["weight"].transpose(-1, -2) if fnuz is None else + Tensor(fp8_reference(attn_compressed.numpy(), *fp8_weights[1], (4, 8), fnuz))) out_abs = block.attn_output(attn_abs_out.transpose(1, 2).reshape(B, T, -1)) # Compare - naive_np = out_naive.realize().numpy() abs_np = out_abs.realize().numpy() - np.testing.assert_allclose(naive_np, abs_np, atol=1e-4, rtol=1e-4, - err_msg="Absorbed MLA should match naive MLA") + if fnuz is None: + np.testing.assert_allclose(out_naive.numpy(), abs_np, atol=1e-4, rtol=1e-4, + err_msg="Absorbed MLA should match naive MLA") + block._init_state(x_norm) + prefill = block._attention(x_norm[:, :3], 0).realize() + decode = block._attention(x_norm[:, 3:], 3).realize() + np.testing.assert_allclose(prefill.cat(decode, dim=1).numpy(), abs_np, atol=2e-5, rtol=2e-5) + + def test_fp8_projections(self): + formats = [fnuz for fnuz, dt in ((False, dtypes.fp8e4m3), (True, dtypes.fp8e4m3fnuz)) + if dt in Device[Device.DEFAULT].renderer.supported_dtypes()] + if not formats: self.skipTest('requires native FP8') + for fnuz in formats: + with self.subTest(fnuz=fnuz): self._test_mla_attention(True, fnuz) diff --git a/tinygrad_repo/test/runtime/test_llm_quantized.py b/tinygrad_repo/test/runtime/test_llm_quantized.py index 8ab1345399..7a34eba157 100644 --- a/tinygrad_repo/test/runtime/test_llm_quantized.py +++ b/tinygrad_repo/test/runtime/test_llm_quantized.py @@ -1,8 +1,12 @@ import gc, itertools, unittest, weakref import numpy as np -from tinygrad import Tensor, UOp, dtypes, function, Device -from tinygrad.llm.kernels.amd import Linear, amd_custom_kernels_supported, QUANT_SIZES, HALFWORD_QUANTS, iq4_half_lut, _iq_grid +from tinygrad import Tensor, UOp, dtypes, function, Device, TinyJit +from tinygrad.helpers import Context +from tinygrad.llm.kernels.amd import (Linear, fp8_matmul, amd_custom_kernels_supported, QUANT_SIZES, HALFWORD_QUANTS, SMALL_BLOCK_QUANTS, + iq4_half_lut, _iq_grid) from tinygrad.llm.gguf import ggml_data_to_tensor +from tinygrad.llm.model import ExpertWeights +from tinygrad.llm.hf import e4m3fn_to_fnuz from test.helpers import not_support_multi_device class QuantLinearMixin: @@ -15,7 +19,8 @@ class QuantLinearMixin: elif ggml_type == 10: packed[:, 80:82] = np.array([0.001], dtype=np.float16).view(np.uint8) packed[:, 82:84] = np.array([0.0002], dtype=np.float16).view(np.uint8) - elif ggml_type == 20: packed.reshape(-1, 18)[:, :2] = np.array([0.001], dtype=np.float16).view(np.uint8) + elif ggml_type in SMALL_BLOCK_QUANTS: + packed.reshape(-1, SMALL_BLOCK_QUANTS[ggml_type]*2)[:, :2] = np.array([0.001], dtype=np.float16).view(np.uint8) else: packed[:, :2] = np.array([0.001], dtype=np.float16).view(np.uint8) if ggml_type in (12, 13): packed[:, 2:4] = np.array([0.0002], dtype=np.float16).view(np.uint8) raw = Tensor(np.pad(packed.flatten(), (4, 0))).contiguous().realize()[4:] @@ -106,6 +111,17 @@ class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): raw.assign(raw.full_like(1)).realize() np.testing.assert_array_equal(linear.weight.bitcast(dtypes.uint8).numpy(), np.ones(type_size, dtype=np.uint8)) + def test_small_block_quant_weights(self): + for ggml_type, halfwords in SMALL_BLOCK_QUANTS.items(): + with self.subTest(ggml_type=ggml_type): + packed = np.arange(25*halfwords*2, dtype=np.uint8) + raw = Tensor(packed).realize()[0:] + if raw.uop.contiguous_view() is None: self.skipTest("requires buffer views") + linear = Linear(800, 1, bias=False) + linear.set_quantized(ggml_data_to_tensor(raw, 800, ggml_type).reshape(1, 800)) + self.assertEqual(linear.ggml_type, ggml_type) + np.testing.assert_array_equal(linear.weight.bitcast(dtypes.uint8).numpy(), packed) + @unittest.skipIf(Device.DEFAULT == "WEBGPU", "slow on WEBGPU") def test_quant_linear_fallback(self): if amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("run with DISABLE_AMD_KERNELS=1") @@ -114,4 +130,113 @@ class TestQ8Quantize(QuantLinearMixin, unittest.TestCase): with self.subTest(ggml_type=typ): self._test_quant_linear(typ, QUANT_SIZES[typ], in_features=256, out_features=16, token_counts=(1, 3), bias=True, custom=False) +def fp8_decode(bits, fnuz=False): + exponent, mantissa = ((bits >> 3) & 15).astype(np.int32), (bits & 7).astype(np.float32) + value = np.ldexp(np.where(exponent == 0, mantissa, 8 + mantissa), np.maximum(exponent, 1) - (11 if fnuz else 10)) + return np.where(bits & 128, -value, value) + +def fp8_reference(x, bits, scale, block_size, fnuz=False): + bn, bk = block_size + if fnuz: bits, scale = np.where(bits == 128, 0, bits), scale * 2 + w = fp8_decode(bits, fnuz) + k, n = w.shape[-1], w.shape[-2] + x = np.pad(x, [(0, 0)] * (x.ndim-1) + [(0, (-k) % bk)]).reshape(*x.shape[:-1], -1, bk) + limit = 224.0 if fnuz else 448.0 + xs = np.maximum(np.abs(x).max(-1, keepdims=True), 1e-10) / limit + q = np.clip(x / xs, -limit, limit) + step = np.exp2(np.maximum(np.frexp(q)[1] - 4, -10 if fnuz else -9)).astype(np.float32) + x = (np.rint(q / step) * step * xs).reshape(*x.shape[:-2], -1)[..., :k] + w = w * scale.repeat(bn, -2).repeat(bk, -1)[..., :n, :k] + return x @ w.swapaxes(-1, -2) + +class TestBlockFP8(unittest.TestCase): + def setUp(self): + supported = Device[Device.DEFAULT].renderer.supported_dtypes() + self.formats = [fnuz for fnuz, dt in ((False, dtypes.fp8e4m3), (True, dtypes.fp8e4m3fnuz)) if dt in supported] + + def test_fnuz_conversion(self): + bits = np.array([i for i in range(256) if i not in (127, 255)], np.uint8).reshape(2, 127) + scale = np.array([[0.125], [0.5]], np.float32) + w, s = e4m3fn_to_fnuz(Tensor(bits).bitcast(dtypes.fp8e4m3), Tensor(scale)) + converted = w.bitcast(dtypes.uint8).numpy() + np.testing.assert_array_equal(converted, np.where(bits == 128, 0, bits)) + expected = fp8_decode(bits) * scale + np.testing.assert_array_equal(fp8_decode(converted, fnuz=True) * s.numpy(), expected) + if True in self.formats: np.testing.assert_array_equal((w.float() * s).numpy(), expected) + + def test_dense(self): + if not self.formats: self.skipTest('requires native FP8') + rng = np.random.default_rng(1) + for fnuz, tc, (n, k, tokens) in itertools.product(self.formats, (0, 1), ((132, 260, 3), (256, 256, 16), (16, 256, 1))): + with Context(TC=tc), self.subTest(fnuz=fnuz, tc=tc, n=n, k=k): + x = rng.normal(size=(2, tokens, k)).astype(np.float32) + x[0, 0] = 0 + bits = rng.integers(0, 73, size=(n, k), dtype=np.uint8) # finite magnitudes from 0 to 4 + bits |= rng.integers(0, 2, size=bits.shape, dtype=np.uint8) << 7 + scales = rng.uniform(.01, .5, ((n+127)//128, (k+127)//128)).astype(np.float32) + w, s = Tensor(bits).bitcast(dtypes.fp8e4m3), Tensor(scales) + if fnuz: w, s = e4m3fn_to_fnuz(w, s) + layer = Linear(k, n, bias=False) + layer.weight, layer.weight_scale_inv = w, s + expected = fp8_reference(x, bits, scales, (128, 128), fnuz) + np.testing.assert_allclose(layer(Tensor(x)).numpy(), expected, atol=2e-5, rtol=2e-5) + + def test_fp32_fallback(self): + for tc, fnuz in itertools.product((0, 1), (False, True)): + with Context(TC=tc), self.subTest(tc=tc, fnuz=fnuz): + x = Tensor([[[1.125, 0., 0., 0.]], [[-1.125, 0., 0., 0.]], [[0., 0., 0., 0.]]]) + w = Tensor([[57, 0, 0, 0], [60, 0, 0, 0]], dtype=dtypes.uint8).bitcast(dtypes.fp8e4m3) + scale = Tensor.ones(1, 1) + if fnuz: w, scale = e4m3fn_to_fnuz(w, scale) + np.testing.assert_allclose(fp8_matmul(x, w, scale, (2, 4)).numpy(), [[[1.265625, 1.6875]], [[-1.265625, -1.6875]], [[0., 0.]]], rtol=1e-6) + + def _routed_experts(self, fnuz): + rng = np.random.default_rng(2) + x = rng.normal(size=(2, 3, 1, 24)).astype(np.float32) + bits = rng.integers(0, 73, size=(4, 16, 24), dtype=np.uint8) # finite magnitudes from 0 to 4 + bits |= rng.integers(0, 2, size=bits.shape, dtype=np.uint8) << 7 + scales = rng.uniform(.01, .5, (4, 4, 3)).astype(np.float32) + selected = np.array([[[0, 2], [2, 0], [3, 1]], [[1, 1], [0, 3], [2, 1]]], np.int32) + layer = ExpertWeights(4, 24, 16) + w, scale = Tensor(bits).bitcast(dtypes.fp8e4m3), Tensor(scales) + if fnuz: w, scale = e4m3fn_to_fnuz(w, scale) + layer.weight, layer.weight_scale_inv, layer.weight_block_size = w, scale, (4, 8) + return layer, x, bits, scales, selected + + def test_routed_experts(self): + if not self.formats: self.skipTest('requires native FP8') + for fnuz in self.formats: + with self.subTest(fnuz=fnuz): + layer, x, bits, scales, selected = self._routed_experts(fnuz) + expected = fp8_reference(x[..., None, :], bits[selected], scales[selected], (4, 8), fnuz).squeeze(-2) + np.testing.assert_allclose(layer(Tensor(selected), Tensor(x)).numpy(), expected, atol=2e-6, rtol=2e-5) + + def test_routed_experts_jit(self): + if not self.formats: self.skipTest('requires native FP8') + for fnuz in self.formats: + with self.subTest(fnuz=fnuz): + layer, x, bits, scales, selected = self._routed_experts(fnuz) + run = TinyJit(lambda sel, h: layer(sel, h).realize()) + for ids in (selected, selected, selected[..., ::-1].copy()): + got = run(Tensor(ids).realize(), Tensor(x).realize()).numpy() + expected = fp8_reference(x[..., None, :], bits[ids], scales[ids], (4, 8), fnuz).squeeze(-2) + np.testing.assert_allclose(got, expected, atol=2e-6, rtol=2e-5) + + @unittest.skipIf(not_support_multi_device(), "no multi") + def test_tensor_parallel(self): + if not self.formats: self.skipTest('requires native FP8') + rng, devices = np.random.default_rng(3), (Device.DEFAULT, f"{Device.DEFAULT}:1") + x = rng.normal(size=(1, 3, 256)).astype(np.float32) + bits = rng.integers(0, 73, size=(256, 256), dtype=np.uint8) # finite magnitudes from 0 to 4 + bits |= rng.integers(0, 2, size=bits.shape, dtype=np.uint8) << 7 + scales = rng.uniform(.01, .5, (2, 2)).astype(np.float32) + for fnuz, axis in itertools.product(self.formats, (0, 1)): + with self.subTest(fnuz=fnuz, axis=axis): + w, scale = Tensor(bits).bitcast(dtypes.fp8e4m3), Tensor(scales) + if fnuz: w, scale = e4m3fn_to_fnuz(w, scale) + expected = fp8_reference(x, bits, scales, (128, 128), fnuz) + w = w.bitcast(dtypes.uint8).shard(devices, axis).bitcast(w.dtype) + got = fp8_matmul(Tensor(x).shard(devices, 2 if axis == 1 else None), w, scale.shard(devices, axis), (128, 128)) + np.testing.assert_allclose(got.to(Device.DEFAULT).numpy(), expected, atol=1e-5, rtol=2e-5) + if __name__ == "__main__": unittest.main() diff --git a/tinygrad_repo/test/runtime/test_llm_tp.py b/tinygrad_repo/test/runtime/test_llm_tp.py index 63179a1040..610d33b18c 100644 --- a/tinygrad_repo/test/runtime/test_llm_tp.py +++ b/tinygrad_repo/test/runtime/test_llm_tp.py @@ -5,7 +5,7 @@ from tinygrad import Device, Tensor, Variable, nn from tinygrad.helpers import DEV from tinygrad.llm.model import Transformer, TransformerConfig, SSMConfig from tinygrad.llm.gguf import gguf_load, gguf_parse, gguf_shard, ggml_data_to_tensor -from tinygrad.llm.kernels.amd import Linear, amd_custom_kernels_supported +from tinygrad.llm.kernels.amd import Linear, ExpertWeights, amd_custom_kernels_supported from test.helpers import not_support_multi_device DEVICES = (Device.DEFAULT, f"{Device.DEFAULT}:1") @@ -98,6 +98,24 @@ class TestTensorParallel(unittest.TestCase): got = layer(Tensor(x).shard(DEVICES, 2 if axis == 1 else None)).float().to(Device.DEFAULT).numpy() np.testing.assert_allclose(got, ref, rtol=2e-2, atol=2e-2*np.abs(ref).max()) + def test_quantized_experts(self): + # the moe decode kernel on the packed shards: gate/up split the output features, down splits the input features + if not amd_custom_kernels_supported(DEVICES[0]): self.skipTest("needs the AMD custom kernels") + E, O, I = shape = (4, 8, 512) + typ = Q.Q4_K + data = random_data(shape, typ).reshape(-1) + w = ggml_data_to_tensor(Tensor(data, device='CPU'), E*O*I, typ).reshape(shape).float().numpy() + sel = np.array([[[3, 1]]], dtype=np.int32) + for axis in (1, 2): + with self.subTest(axis=axis): + layer = ExpertWeights(E, I, O) + layer.weight = gguf_shard({'w': (Tensor(data, device='CPU'), shape, typ)}, DEVICES, {'w': axis})['w'].half() + x = np.random.default_rng(1).normal(size=(1, 1, 1 if axis == 1 else 2, I)).astype(np.float16) + ref = np.stack([w[e] @ x[0, 0, j % x.shape[2]].astype(np.float32) for j, e in enumerate(sel[0, 0])]) + got = layer(Tensor(sel).shard(DEVICES), Tensor(x).shard(DEVICES, 3 if axis == 2 else None)).float().to(Device.DEFAULT).numpy() + self.assertEqual(layer.ggml_type, typ) + np.testing.assert_allclose(got[0, 0], ref, rtol=2e-2, atol=2e-2*np.abs(ref).max()) + @unittest.skipIf(DEV.interface.startswith("MOCK"), "too heavy for mock GPUs") def test_model(self): rng = np.random.default_rng(42) diff --git a/tinygrad_repo/test/runtime/test_multitensor.py b/tinygrad_repo/test/runtime/test_multitensor.py index a75305e802..3d1927041e 100644 --- a/tinygrad_repo/test/runtime/test_multitensor.py +++ b/tinygrad_repo/test/runtime/test_multitensor.py @@ -91,6 +91,15 @@ class TestMultiTensor(unittest.TestCase): self.assertEqual(scalar.shape, ()) self.assertEqual(scalar.uop.base.buffer.nbytes, scalar.dtype.itemsize) + def test_full_axis(self): + x = Tensor.full((4, 6), 7, device=devices_2, axis=-1).realize() + self.assertEqual((x.shape, x.uop.axis), ((4, 6), 1)) + self.assertEqual(x.uop.base.buffer.nbytes, 12 * x.dtype.itemsize) + np.testing.assert_equal(x.numpy(), np.full((4, 6), 7)) + z = Tensor.zeros(4, 6, device=devices_2, axis=0).realize() + self.assertEqual((z.uop.axis, z.uop.base.buffer.nbytes), (0, 12 * z.dtype.itemsize)) + np.testing.assert_equal(z.numpy(), np.zeros((4, 6))) + @unittest.expectedFailure # TODO: fix def test_shard_empty(self): GlobalCounters.reset() @@ -1152,8 +1161,8 @@ class TestShrinkMultiTensorShardedAxis(unittest.TestCase): to_add.append((Tensor.ones(2, 8) * i).shard(devices)) added:list[Tensor] = [] - for bound, a in zip(x.uop.bounds, to_add): - added.append(x[bound[0]:bound[1]] + a) + for i, a in enumerate(to_add): + added.append(x[2*i:2*i+2] + a) output = added[0].cat(*added[1:]) expected = np.arange(64).reshape((8,8)) + np.array([[0,0,1,1,2,2,3,3] for _ in range(8)]).T @@ -1266,8 +1275,8 @@ class TestBatchNorm(unittest.TestCase): bns.append(bn) bn_ts = [] - for bound, bn in zip(x.uop.bounds, bns): - bni = bn(x[bound[0]:bound[1]]) + for i, bn in enumerate(bns): + bni = bn(x[2*i:2*i+2]) bn_ts.append(bni) bn_ts[0].cat(*bn_ts[1:]).numpy() diff --git a/tinygrad_repo/test/runtime/test_ops.py b/tinygrad_repo/test/runtime/test_ops.py index a203cc4722..220c42b618 100644 --- a/tinygrad_repo/test/runtime/test_ops.py +++ b/tinygrad_repo/test/runtime/test_ops.py @@ -1,4 +1,4 @@ -import time, math, unittest, functools, platform, warnings, sys +import time, math, unittest, functools, warnings, sys import numpy as np import torch from tinygrad.helpers import getenv, DEBUG, DEV, IMAGE, Context, to_tuple @@ -305,7 +305,6 @@ class TestOps(TensorTestCase): lambda x: torch.where(x > 0.5, 4, 2).type(torch.int32).permute((1, 0)), lambda x: (x > 0.5).where(4, 2).clone().permute((1, 0)), forward_only=True) - @unittest.skipIf(Device.DEFAULT == "WEBGPU", "software vulkan evaluates a NaN != x as false") def test_where_nan_cond(self): # a NaN compares false against everything except !=. for fxn in (lambda x: x<1, lambda x: x>1, lambda x: x!=1, lambda x: x==1): @@ -330,6 +329,22 @@ class TestOps(TensorTestCase): helper_test_op(None, fxn, fxn, forward_only=True, vals=[[s0], [s1]]) def test_cmp_eq(self): self._test_cmp(lambda x,y: x==y, reverse=False) + def test_cmp_ne(self): self._test_cmp(lambda x,y: x!=y) + + def test_cmp_eq_ne_specials(self): + vals = [[math.nan, math.inf, -math.inf, 0.0, -0.0, 1.0], [math.nan, math.inf, math.inf, -0.0, 0.0, 2.0]] + for fxn in (lambda x,y: x==y, lambda x,y: x!=y): + helper_test_op(None, fxn, forward_only=True, vals=vals) + helper_test_op(None, lambda x: fxn(x, x), forward_only=True, vals=vals[:1]) + + def test_cmp_eq_padding(self): + # Padding a comparison fills False; comparing two zero-padded inputs fills True. + for fxn in (lambda x,y: x==y, lambda x,y: x!=y): + helper_test_op(None, lambda x,y: torch.nn.functional.pad(fxn(x,y), (1,2)).float().sum(), + lambda x,y: fxn(x,y).pad((1,2)).float().sum(), vals=[[1,2,3], [1,0,3]], forward_only=True) + helper_test_op(None, lambda x,y: fxn(torch.nn.functional.pad(x, (1,2)), torch.nn.functional.pad(y, (1,2))).float().sum(), + lambda x,y: fxn(x.pad((1,2)), y.pad((1,2))).float().sum(), vals=[[1,2,3], [1,0,3]], forward_only=True) + def test_cmp_gt(self): self._test_cmp(lambda x,y: x>y) def test_cmp_ge(self): self._test_cmp(lambda x,y: x>=y) def test_cmp_lt(self): self._test_cmp(lambda x,y: x 0 else float('-inf' if a==0 else 'nan')) @unittest.skipIf(Device.DEFAULT == "CPU", 'not supported as uop') def test_sin(self): self._test_uop_fxn(Ops.SIN, lambda a: math.sin(a)) - def test_recip(self): self._test_uop_fxn(Ops.RECIPROCAL, lambda a: 1/a if a != 0 else float('inf')) + def test_recip(self): self._test_uop_fxn(Ops.RECIP, lambda a: 1/a if a != 0 else float('inf')) def test_sqrt(self): self._test_uop_fxn(Ops.SQRT, lambda a: math.sqrt(a) if a >= 0 else float('nan')) + def test_sqrt_edgecases(self): + for a, expected in [(-math.inf, math.nan), (-1., math.nan), (0., 0.), (4., 2.), (math.inf, math.inf), (math.nan, math.nan)]: + with self.subTest(a=a): + np.testing.assert_equal(_test_single_value([a], Ops.SQRT, (dtypes.float32,)), expected) def test_add(self): self._test_bop_fxn(Ops.ADD, lambda a,b: a+b) def test_mul(self): self._test_bop_fxn(Ops.MUL, lambda a,b: a*b) def test_max(self): self._test_bop_fxn(Ops.MAX, lambda a,b: max(a,b)) def test_cmplt(self): self._test_bop_fxn(Ops.CMPLT, lambda a,b: a UOp: # a void RANGE is a bound-less loop header: a jump target with no induction variable. @@ -54,6 +53,21 @@ def pressure_loop_kernel(C:UOp, n=13) -> UOp: return C[0].store(i[0].load()).sink(arg=KernelInfo(name="pressure_loop", opts_to_apply=())) +def weighted_loop_kernel(C:UOp, n:int, N:int) -> UOp: + # every load has its own weight, a load read from another load's register changes the sum + vs = [C[j+1].load() for j in range(n)] + l = UOp.loop(0) + + i = UOp.placeholder((1,), dtypes.int, 0, addrspace=AddrSpace.REG) + i = i.after(i[0].store(0)) + + cur = i.after(l)[0].load() + inc = cur + sum((v ^ cur) * (j+1) for j,v in enumerate(vs)) + st = i[0].store(inc) + i = i.after(st.backedge(l, inc < N)) + + return C[0].store(i[0].load()).sink(arg=KernelInfo(name="weighted_loop", opts_to_apply=())) + def wait_ext_kernel() -> UOp: sig = UOp.param(0, dtypes.int, 1, volatile=True) l = UOp.loop(0) @@ -112,19 +126,21 @@ class TestWaitLoop(unittest.TestCase): c.realize() self.assertEqual(c.item(), 25) - # TODO: x86's lower_loop builds an Ops.IF node after regalloc, which fails spec_full - @(unittest.expectedFailure if isinstance(Device[Device.DEFAULT].renderer, X86Renderer) else lambda f: f) def test_wait_loop_spec(self): c = Tensor.custom_kernel(Tensor.empty(1, dtype=dtypes.int), fxn=functools.partial(wait_loop_kernel, N=7))[0] with Context(SPEC=2): c.realize() self.assertEqual(c.item(), 7) - @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, X86Renderer), "TODO: do-while loop under register pressure segfaults on x86") def test_loop_carried_registers(self): # more loads live across the backedge than any register file (x86 15 gprs, arm64 31, sass 255, rdna3 256 vgprs) c = Tensor.custom_kernel(Tensor.ones(301, dtype=dtypes.int), fxn=functools.partial(pressure_loop_kernel, n=300))[0] self.assertEqual(c[0].item(), 2) + def test_loop_carried_weighted_registers(self): + # two iterations, the first sum is 2870 + c = Tensor.custom_kernel(Tensor(list(range(21)), dtype=dtypes.int).contiguous(), fxn=functools.partial(weighted_loop_kernel, n=20, N=2871))[0] + self.assertEqual(c[0].item(), 2870 + sum((j ^ 2870) * j for j in range(1, 21))) + def test_register_pressure_loop(self): c = Tensor.zeros(16, dtype=dtypes.int).contiguous() c = Tensor.custom_kernel(c, fxn=pressure_loop_kernel)[0] diff --git a/tinygrad_repo/tinygrad/codegen/__init__.py b/tinygrad_repo/tinygrad/codegen/__init__.py index 4c70087788..e78c95196d 100644 --- a/tinygrad_repo/tinygrad/codegen/__init__.py +++ b/tinygrad_repo/tinygrad/codegen/__init__.py @@ -1,20 +1,20 @@ -from dataclasses import replace +from dataclasses import replace, dataclass, field import itertools, functools from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, USE_TC from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT, TC_MIN_GLOBALS, TracingKey, Context, panic -from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType +from tinygrad.uop.ops import PatternMatcher, graph_rewrite, MapCtx, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType from tinygrad.uop.weak import pm_lower_weak, pm_commit_weak, pm_cast_const from tinygrad.uop.render import render_uir from tinygrad.uop.spec import type_verify, spec_tensor, spec_program from tinygrad.renderer import Renderer, Estimates -from tinygrad.renderer.isa import ISARenderer, IselContext +from tinygrad.renderer.x86 import ISARenderer, IselContext from tinygrad.dtype import dtypes, AddrSpace # import all pattern matchers here from tinygrad.codegen.gpudims import pm_group_gpudims, pm_range_to_special -from tinygrad.uop.symbolic import sym, symbolic_simple, symbolic, pm_move_where_on_load, pm_clean_up_group_sink, pm_remove_invalid, invalid_gate +from tinygrad.uop.symbolic import sym, symbolic_simple, symbolic, pm_move_where_on_load, pm_clean_up_group_sink, pm_remove_invalid from tinygrad.uop.movement import mop_cleanup -from tinygrad.codegen.decomp.dtype import pm_dtype_decomps +from tinygrad.codegen.decomp.dtype import DTypeDecompCtx, pm_dtype_decomps from tinygrad.codegen.decomp.op import get_late_rewrite_patterns, get_simplifying_rewrite_patterns from tinygrad.codegen.decomp.transcendental import get_transcendental_patterns from tinygrad.codegen.late.coalesce import indexing_simplify @@ -26,12 +26,12 @@ from tinygrad.schedule.prepare import pm_mops from tinygrad.codegen.late.linearizer import CFGContext, pm_split_ends, pm_add_control_flow, linearize from tinygrad.codegen.late.regalloc import LinearScanRegallocContext, pm_regalloc_rewrite from tinygrad.codegen.late.coalesce import memory_coalescing, pm_simplify_add_image -from tinygrad.helpers import all_same, all_int, argsort, partition, to_function_name +from tinygrad.helpers import all_same, argsort, partition, to_function_name from tinygrad.uop.ops import _broadcast_shape, identity_element from tinygrad.schedule.rangeify import BufferizeOpts def do_number_param(ctx:tuple[int, dict[str, int]], x:UOp): # after the params, one slot per name - if x.is_variable: return x.replace(arg=replace(x.arg, slot=ctx[0] + ctx[1].setdefault(x.arg.name, len(ctx[1])))) + if x.is_variable and x.arg.slot == -1: return x.replace(arg=replace(x.arg, slot=ctx[0] + ctx[1].setdefault(x.arg.name, len(ctx[1])))) pm_number_params = PatternMatcher([ (UPat(Ops.PARAM, name="x"), do_number_param), @@ -101,10 +101,10 @@ pm_wmma_add = PatternMatcher([ (UPat(Ops.WMMA, name="wmma") + UPat.var("add"), lambda add, wmma: UOp(wmma.op, src=(wmma.src[0], wmma.src[1], wmma.src[2]+add), arg=wmma.arg)), # push permute/reshape to the other side of the add - (UPat(Ops.PERMUTE, src=(UPat(Ops.WMMA, name="wmma"),), name="permute") + UPat.var("add"), - lambda wmma,permute,add: (wmma + add.permute(argsort(permute.arg))).permute(permute.arg)), - (UPat(Ops.PERMUTE, src=(UPat(Ops.RESHAPE, src=(UPat(Ops.WMMA, name="wmma"), UPat()), name="reshape"),), name="permute") + UPat.var("add"), - lambda wmma,reshape,permute,add: (wmma + add.permute(argsort(permute.arg)).reshape(wmma.shape)).reshape(reshape.shape).permute(permute.arg)), + (UPat(Ops.PERMUTE, src=(UPat(Ops.WMMA, name="wmma"), UPat()), name="permute") + UPat.var("add"), + lambda wmma,permute,add: (wmma + add.permute(argsort(permute.marg))).permute(permute.marg)), + (UPat(Ops.PERMUTE, src=(UPat(Ops.RESHAPE, src=(UPat(Ops.WMMA, name="wmma"), UPat()), name="reshape"), UPat()), name="permute") + UPat.var("add"), + lambda wmma,reshape,permute,add: (wmma + add.permute(argsort(permute.marg)).reshape(wmma.shape)).reshape(reshape.shape).permute(permute.marg)), ]) pm_expand_broadcast = pm_wmma_add+PatternMatcher([ @@ -139,20 +139,10 @@ devectorizer2 = pm_mops+PatternMatcher([ (UPat(Ops.INDEX, src=(UPat.var('x'),)), lambda x: x), # unpack WMMA (UPat(Ops.WMMA, name="u"), do_stack_wmma), - # stacked INDEX is many INDEX - (UPat(Ops.INDEX, src=(UPat(GroupOp.Defines, name="b"), UPat(Ops.STACK, name="s")), name="x"), - lambda b,s,x: UOp.stack(*[x.replace(src=(b,u)) for u in s.src])), - # INDEX into RESHAPE moves the RESHAPE - (UPat(Ops.INDEX, src=(UPat(GroupOp.Defines, name="b"), UPat(Ops.RESHAPE, name="s"))), - lambda b,s: b.index(s.src[0]).reshape(s.shape)), - # RESHAPE a void is removed (hack for AFTER) - (UPat(Ops.RESHAPE, dtype=dtypes.void, name="x"), lambda x: x.src[0]), # reshape of a single element shaped value to scalar is an index (UPat(Ops.RESHAPE, name="x"), lambda x: x.src[0].index(0) if x.marg == () and x.src[0].shape == (1,) else None), - # EXPAND on scalar -> nested STACKs with the same shape - (UPat(Ops.EXPAND, src=(UPat.var("x"), UPat()), name="out"), - lambda x,out: functools.reduce(lambda x,s: UOp.stack(*([x]*s)), reversed(out.shape), x) - if x.shape == () and all_int(out.shape) and 0 not in out.shape else None), + # a scalar index into a one-lane STACK is its lane, 0 is the only in-bounds index + (UPat(Ops.INDEX, src=(UPat(Ops.STACK, src=(UPat.var("x"),)), UPat.var("i"))), lambda x,i: x if i.shape == () else None), ]) def fix_group_for_reduce(x:UOp): @@ -170,11 +160,11 @@ def fix_group_for_reduce(x:UOp): return buf.reduce(*reduce_loop, arg=(x.arg[0], 0)) def merge_reduce_ends(sink:UOp): - # merge ENDs that share the same range and nesting context (only those created by reduce_to_acc) + # merge ENDs that share the same range and nesting context # ENDs at different nesting depths get cloned RANGEs so each RANGE maps to one END range_to_ends: dict[tuple[UOp, ...], list[UOp]] = {} for u in sink.backward_slice: - if u.op is Ops.END and u.tag == "mergeable": range_to_ends.setdefault(u.src[1:], []).append(u) + if u.op is Ops.END: range_to_ends.setdefault(u.src[1:], []).append(u) subs: dict[UOp, UOp] = {} next_axis = max((u.axis_id[0] for u in sink.backward_slice if u.op is Ops.RANGE), default=-1) + 1 for r, ends in range_to_ends.items(): @@ -189,39 +179,31 @@ def merge_reduce_ends(sink:UOp): for e in group: subs[e] = merged return sink.substitute(subs) if subs else None -def reduce_ranges_to_acc(ctx:itertools.count, r:UOp): +def lower_reduce(ctx:itertools.count, r:UOp): + # horizontal: int, how many leading axes of inp are reduced in place, the rest of the reduce is over the ranges in src[1:] + inp, (op, horizontal) = r.src[0], r.arg + vals = [inp.index(*idx) for idx in itertools.product(*[range(inp.max_shape[a]) for a in range(horizontal)])] if horizontal else [inp] + ret = functools.reduce(lambda x,y: x.alu(op, y), vals) + if len(r.src) == 1: return ret acc = UOp.alloc_like(r, next(ctx), AddrSpace.REG) - input_ranges = tuple(x for x in r.src[0].ranges if x not in r.src[1:]) - acc_init = acc.after(*input_ranges).store(UOp.const(identity_element(r.arg[0], r.dtype))) + input_ranges = tuple(x for x in inp.ranges if x not in r.src[1:]) + acc_init = acc.after(*input_ranges).store(UOp.const(identity_element(op, r.dtype))) acc_initted = acc.after(acc_init, *r.src[1:]) - inp = r.src[0].reduce(arg=r.arg) if r.arg[1] else r.src[0] - acc_out = acc_initted.store(acc_initted.alu(r.arg[0], inp)).end(*r.src[1:]).rtag("mergeable") + acc_out = acc_initted.store(acc_initted.alu(op, ret)).end(*r.src[1:]) return acc.after(acc_out) -def expand_horizontal_reduce(r:UOp): - inp = r.src[0] - vals = [inp.index(*idx) for idx in itertools.product(*[range(inp.max_shape[a]) for a in range(r.arg[1])])] - return functools.reduce(lambda x,y: x.alu(r.arg[0], y), vals) - -# an Invalid in a REDUCE source is that reduce's identity -pm_reduce_identity = PatternMatcher([ - (invalid_gate.reduce(allow_any_len=True, name="red"), lambda red,cond,x,i: - red.replace(src=(cond.where(x, x.const_like(identity_element(red.arg[0], red.dtype))),)+red.src[1:])), -]) - pm_reduce_local = pm_wmma_add+PatternMatcher([ # fix group for reduce (UPat(Ops.REDUCE, name="x"), fix_group_for_reduce), # remove reduces - (UPat(Ops.REDUCE, src=(UPat(), UPat()), allow_any_len=True, name="r"), reduce_ranges_to_acc), - (UPat(Ops.REDUCE, src=(UPat(),), name="r"), expand_horizontal_reduce), + (UPat(Ops.REDUCE, name="r"), lower_reduce), (UPat(Ops.SINK, name="sink"), merge_reduce_ends), ])+pm_clean_up_group_sink def is_shape_changing_bitcast(u:UOp): return u.op is Ops.BITCAST and u.shape != u.src[0].shape def maybe_load(u:UOp): return u.load() if u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL, AddrSpace.REG) else u pm_add_loads = PatternMatcher([ - (UPat(GroupOp.Elementwise|{Ops.REDUCE,Ops.WMMA,Ops.STACK}, name="x"), + (UPat(GroupOp.Elementwise|{Ops.WMMA,Ops.STACK}, name="x"), lambda x: None if is_shape_changing_bitcast(x) else x.replace(src=tuple(map(maybe_load, x.src)))), (UPat(Ops.STORE, name="x"), lambda x: x.replace(src=(x.src[0], maybe_load(x.src[1]))+x.src[2:])), ]) @@ -238,11 +220,11 @@ pm_add_local_buffers = PatternMatcher([ # float ALUs need a float operand # make that cast explicit before the decomps, which expand SIN/LOG2/EXP2 into float polynomials and assert a float operand pm_cast_float_alu = PatternMatcher([ - (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL), src=(UPat(name="x"),), name="u"), + (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIP), src=(UPat(name="x"),), name="u"), lambda u,x: u.replace(src=(x.cast(u.dtype),)) if x.dtype != u.dtype else None), ]) -def _is_local_store(x:UOp): return x.op is Ops.STORE and x.addrspace is AddrSpace.LOCAL +def _is_local_store(x:UOp): return x.op is Ops.STORE and x.src[0].addrspace is AddrSpace.LOCAL def add_raw_barrier(after:UOp): # loads from a LOCAL buffer that depend (via AFTER) on stores to LOCAL memory need a workgroup barrier @@ -284,20 +266,20 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_load_collapse, name="load collapse") # split ranges - sink = graph_rewrite(sink, pm_split_ranges+pm_flatten_range, ctx={}, name="split ranges") + sink = graph_rewrite(sink, pm_split_ranges+pm_flatten_range, init_ctx=MapCtx, name="split ranges") # symbolic (NOTE: this is a requirement for pm_simplify_ranges to be correct) sink = graph_rewrite(sink, sym+pm_flatten_range, name="initial symbolic") # optimize (schedule) the AST - sink = graph_rewrite(sink, pm_flatten_range+pm_simplify_ranges, ctx={}, name="simplify ranges") + sink = graph_rewrite(sink, pm_flatten_range+pm_simplify_ranges, init_ctx=MapCtx, name="simplify ranges") # do postrange optimization, BEAM or hand_coded_optimizations sink = apply_opts(sink, ren, beam=ast.arg.beam) # ** expander (expand_rewrite) ** # reduce_unparented: a REDUCE whose src folded to a CONST (e.g. x*0) has no parented ranges, collapse it before the expander - sink = graph_rewrite(sink, sym+pm_move_where_on_load+pm_flatten_range+pm_reduce_unparented+pm_reduce_identity, name="postopt symbolic") + sink = graph_rewrite(sink, sym+pm_move_where_on_load+pm_flatten_range+pm_reduce_unparented, name="postopt symbolic") # expand sink = graph_rewrite(sink, expander, ctx=build_range_map(sink), name="expander") @@ -325,7 +307,9 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # do memory coalescing (late) sink = memory_coalescing(sink, ren) - sink = graph_rewrite(sink, symbolic_simple+pm_simplify_add_image, name="add images", ctx=({}, ren), bottom_up=True) + # an image is read_only or write_only, so a PARAM that is both loaded and stored stays a buffer + loaded, stored = [{u.src[0].buf_uop for u in sink.toposort() if u.op is op} for op in (Ops.LOAD, Ops.STORE)] + sink = graph_rewrite(sink, pm_simplify_add_image, name="add images", ctx=({}, ren, loaded & stored), bottom_up=True) # extra symbolic before decomp. crashes without this? # NOTE: also run indexing_simplify here, while the index is still weakint and (x+y)*c -> x*c+y*c applies @@ -355,7 +339,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: get_transcendental_patterns(supported_ops, TRANSCENDENTAL>=2) sink = graph_rewrite(sink, pm_decomp, ctx=ren, name="late decompositions") # Transcendental decompositions can introduce unsupported dtypes, e.g. uint64 in Payne-Hanek reduction. - sink = graph_rewrite(sink, pm_dtype_decomps+pm_commit_weak, ctx=(set(), ren), name="decomp dtypes") + sink = graph_rewrite(sink, pm_dtype_decomps+pm_commit_weak, ctx=DTypeDecompCtx(ren), name="decomp dtypes") sink = graph_rewrite(sink, pm_move_gates_from_index, name="move gates from index") # final rules for the renderer (without sym) @@ -364,7 +348,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_final_rewrite+pm_remove_invalid, ctx=ren, name="final rewrite") # commit every const still bare so no renderer reads one - sink = graph_rewrite(sink, pm_cast_const, name="cast consts") + sink = graph_rewrite(sink, pm_cast_const, name="cast consts", walk=True) # add implicit barriers (stores/loads through LOCAL memory ordered by AFTER or across loop iterations need workgroup barriers) sink = graph_rewrite(sink, pm_implicit_barriers, name="add implicit barriers") @@ -376,7 +360,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_add_control_flow, ctx=CFGContext(sink), name="add control flow", bottom_up=True) # put the variables in slots - num_params = max([x.arg.slot + 1 for x in sink.toposort() if x.op is Ops.PARAM and not x.is_variable], default=0) + num_params = max([x.arg.slot + 1 for x in sink.toposort() if x.op is Ops.PARAM and x.arg.slot != -1], default=0) sink = graph_rewrite(sink, pm_number_params, ctx=(num_params, {}), name="number variables", walk=True) if VIZ: graph_rewrite(sink, PatternMatcher([]), name="View Output AST") @@ -414,14 +398,24 @@ def line_rewrite(lst:list[UOp], pm:PatternMatcher, ctx=None) -> list[UOp]: newlst.extend(ret[1]) return newlst -def lower_call(ctx:Renderer, call:UOp) -> UOp: - body = graph_rewrite(call.body, pm_lower_calls, ctx=ctx, name="lower calls", walk=True) - body = full_rewrite_to_sink(body, ctx, optimize=False) - body = UOp(Ops.LINEAR, src=tuple(line_rewrite(linearize(body), pm_linearize_cleanups+pm_alloc_to_buf)), arg=to_function_name(call.arg.name)) - return call.replace(src=(body,)+call.src[1:]) +@dataclass +class LowerCallsCtx: + renderer: Renderer + names: dict[str, dict[UOp, int]] = field(default_factory=dict) + +def lower_call(ctx:LowerCallsCtx, call:UOp, b:UOp) -> UOp|None: + if b.op is Ops.PROGRAM and b.src[-1].op is Ops.BINARY: return None + bodies = ctx.names.setdefault(name:=to_function_name(call.arg.name), {}) + ki = KernelInfo(name + (f"_{i}" if (i:=bodies.setdefault(b, len(bodies))) else ""), optimized=True) + if ctx.renderer.inline_prefix: + b = graph_rewrite(b, pm_lower_calls, ctx=ctx, name="lower calls", walk=True) + b = UOp(Ops.LINEAR, src=tuple(line_rewrite(list(b.src[1].src) if b.op is Ops.PROGRAM \ + else linearize(full_rewrite_to_sink(b, ctx.renderer, optimize=False)), pm_linearize_cleanups+pm_alloc_to_buf)), arg=ki.name) + else: b = to_program(b.substitute({b.src[0]:b.src[0].replace(arg=ki)}) if b.op is Ops.PROGRAM else b.replace(arg=ki), ctx.renderer) + return call.replace(src=(b,)+call.src[1:]) pm_lower_calls = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.SINK),), allow_any_len=True, name="call"), lower_call), + (UPat(Ops.CALL, src=(UPat((Ops.SINK, Ops.PROGRAM), name="b"),), allow_any_len=True, name="call"), lower_call), ]) def do_linearize(ctx:Renderer, prg:UOp, sink:UOp) -> UOp: @@ -436,6 +430,7 @@ def do_linearize(ctx:Renderer, prg:UOp, sink:UOp) -> UOp: lst = sorted(lst, key=lambda u: u.op is not Ops.INS or bool(u.src)) regalloc_ctx = LinearScanRegallocContext(lin_ctx, lst, ctx) lst = line_rewrite(lst, pm_regalloc_rewrite, regalloc_ctx) + lst = lin_ctx.insert_linearized(lst) lst = line_rewrite(lst, ctx.post_regalloc_matcher, lin_ctx) if DEBUG >= 4: print(ctx.asm_str(lst, sink.arg.function_name)) return prg.replace(src=prg.src + (UOp(Ops.LINEAR, src=tuple(lst)),)) @@ -444,7 +439,8 @@ def do_estimates(prg:UOp, sink:UOp, lin:UOp) -> UOp|None: if sink.arg.estimates is not None: return None return prg.replace(src=(sink.replace(arg=replace(sink.arg, estimates=Estimates.from_uops(lin.src, ignore_indexing=True))),)+prg.src[1:]) -def do_assemble(ctx:Renderer, prg:UOp, lin:UOp) -> UOp: +def do_assemble(ctx:Renderer, prg:UOp, lin:UOp) -> UOp|None: + if getattr(type(ctx), "asm") is Renderer.asm: return None src = "\n".join(str(u.arg[0]) for u in lin.src) if DEBUG >= 4: print(src) binary = ctx.asm(prg, lin) @@ -468,7 +464,7 @@ pm_to_program = PatternMatcher([ (UPat(Ops.PROGRAM, src=(UPat(), UPat(Ops.LINEAR), UPat(Ops.SOURCE, name="source")), name="prg"), do_compile), ]) -@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(k.function_name, ret.key),ret=renderer), replay=True) +@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(ret.key,k.function_name),ret=renderer),replay="process_replay") @Context(ALLOW_DEVICE_USAGE=0) def do_to_program(ast:UOp, renderer:Renderer) -> UOp: """ @@ -481,12 +477,12 @@ def do_to_program(ast:UOp, renderer:Renderer) -> UOp: Returns: The Ops.PROGRAM with SINK/LINEAR/SOURCE/BINARY. """ + if VIZ: graph_rewrite(ast, PatternMatcher([]), name="View Base AST") + ast = graph_rewrite(ast, pm_lower_calls, ctx=LowerCallsCtx(renderer), name="lower calls", walk=True) if ast.op is Ops.PROGRAM: prg = ast elif ast.op is Ops.SINK: assert isinstance(ast.arg, KernelInfo), "requires KernelInfo on arg to to_program" - if VIZ: graph_rewrite(ast, PatternMatcher([]), name="View Base AST") - ast = graph_rewrite(ast, pm_lower_calls, ctx=renderer, name="lower calls", walk=True) - full_sink = full_rewrite_to_sink(ast, renderer, optimize=ast.tag is None) + full_sink = full_rewrite_to_sink(ast, renderer, optimize=not ast.arg.optimized) prog_info = ProgramInfo.from_sink(full_sink, renderer.target) # instruction selection if isinstance(renderer, ISARenderer): diff --git a/tinygrad_repo/tinygrad/codegen/decomp/dtype.py b/tinygrad_repo/tinygrad/codegen/decomp/dtype.py index 548a9697e0..d9fee876d4 100644 --- a/tinygrad_repo/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad_repo/tinygrad/codegen/decomp/dtype.py @@ -1,4 +1,4 @@ -from dataclasses import replace +from dataclasses import dataclass, field, replace from tinygrad.dtype import dtypes, DType, AddrSpace, truncate from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES from tinygrad.uop import GroupOp @@ -21,7 +21,6 @@ def reindex(idx:UOp, off:int, mul=2) -> UOp: def l2i(op: Ops, dt: DType, *uops:UOp): zero = UOp.const(0, dt) if len(uops) == 2: a0, a1 = uops - elif len(uops) == 3: a0, a1, b0 = uops # a shift's count is a single word elif len(uops) == 4: a0, a1, b0, b1 = uops match op: case Ops.NEG: return l2i(Ops.SUB, dt, zero, zero, *uops) @@ -53,7 +52,7 @@ def l2i(op: Ops, dt: DType, *uops:UOp): (a00, a01), (b00, b01) = unpack32(a0), unpack32(b0) mid = l2i(Ops.ADD, dt, shl(a00*b01, 16).bitcast(dt), shr(a00*b01, 16).bitcast(dt), shl(a01*b00, 16).bitcast(dt), shr(a01*b00, 16).bitcast(dt)) return l2i(Ops.ADD, dt, *mid, (a00*b00).bitcast(dt), (a01*b01).bitcast(dt) + a0*b1 + a1*b0) - case Ops.CDIV | Ops.CMOD: + case Ops.TRUNCDIV | Ops.TRUNCMOD: # TAOCP Algorithm 4.3.1D could be faster here, but must be parameterized over the width of b if dt == dtypes.int: ua0, ua1, ub0, ub1 = a0.bitcast(dtypes.uint), a1.bitcast(dtypes.uint), b0.bitcast(dtypes.uint), b1.bitcast(dtypes.uint) @@ -70,8 +69,8 @@ def l2i(op: Ops, dt: DType, *uops:UOp): if dt == dtypes.int: (nq0, nq1), (nr0, nr1) = l2i(Ops.BITCAST, dt, *l2i(Ops.NEG, dtypes.uint, *q)), l2i(Ops.BITCAST, dt, *l2i(Ops.NEG, dtypes.uint, *r)) (q0, q1), (r0, r1) = l2i(Ops.BITCAST, dt, *q), l2i(Ops.BITCAST, dt, *r) - return (a_neg.where(nr0, r0), a_neg.where(nr1, r1)) if op == Ops.CMOD else ((a_neg^b_neg).where(nq0, q0), (a_neg^b_neg).where(nq1, q1)) - return r if op == Ops.CMOD else q + return (a_neg.where(nr0, r0), a_neg.where(nr1, r1)) if op == Ops.TRUNCMOD else ((a_neg^b_neg).where(nq0, q0), (a_neg^b_neg).where(nq1, q1)) + return r if op == Ops.TRUNCMOD else q case Ops.CMPLT: return (a1 < b1) | ((a1.eq(b1)) & (a0.bitcast(dtypes.uint) < b0.bitcast(dtypes.uint))) case Ops.CMPEQ: return a0.eq(b0) & a1.eq(b1) case Ops.CMPNE: return a0.ne(b0) | a1.ne(b1) @@ -86,13 +85,10 @@ def l2i_define(x:UOp) -> UOp: size = shape_to_shape_arg((x.max_numel()*2,)) if x.shape else x.src[0] return x.replace(arg=replace(x.arg, dtype=l2i_dt[x.dtype]), src=(size, *x.src[1:])) -def split_l2i(ctx:dict, op: Ops, dt: DType, *uops:UOp): - # l2i does arithmetic on its inputs; rules enter here to split them to 32-bit words first, l2i recurses on itself. - # both word halves of a node ask for the same split, so ctx memos it for the pass - if (key:=(op, dt, uops)) not in ctx: ctx[key] = l2i(op, dt, *graph_rewrite(UOp.sink(*uops), pm_long_decomp, ctx=ctx, bottom_up=True).src) - return ctx[key] - -def l2i_rewrite_idx(ctx:dict, idx:UOp) -> UOp: return graph_rewrite(idx, pm_long_decomp, ctx=ctx, bottom_up=True) +# the pair of a long is the long bitcast to two words, its words are the pair's lanes +def l2i_pair(x:UPat) -> UPat: return UPat(Ops.BITCAST, src=(UPat(Ops.STACK, src=(x,)),), name='p') +def l2i_words(*uops:UOp) -> list[UOp]: + return flatten(((p:=UOp(Ops.STACK, src=(u,)).bitcast(l2i_dt[u.dtype])).index(0), p.index(1)) if u.dtype in l2i_dt else (u,) for u in uops) # ***** floats ***** f2f_dt = { f:getattr(dtypes, f"uint{f.bitsize}") for f in dtypes.floats } @@ -114,28 +110,27 @@ def f2f(v, fr:DType, to:DType, sat=True): is_nan = (nosign.eq(shl(1, fm + fe) - 1) if fr == dtypes.fp8e4m3 else exp.eq(shl(1, fe) - 1)) return (sign | exp.eq(0).where(0, is_nan.where(nan, norm))).bitcast(to) elif fe >= te and fm > tm: - v = f2f_clamp(v.bitcast(fr), to, sat).bitcast(f2f_dt[fr]) sign, nosign = shr(v, fs - ts) & shl(1, ts - 1), v & (shl(1, fs - 1) - 1) - norm = (rne(nosign, fm - tm) - shl(fb - tb, tm)).cast(f2f_dt[to]) - underflow = (shr(v, fm) & (shl(1, fe) - 1)) < (1 + fb - tb) - nan_mantissa = (shl(1, tm) - 1) if to == dtypes.fp8e4m3 else (shr(nosign, fm - tm) & (shl(1, tm) - 1)) - nan = (sign | nan_mantissa | shl(shl(1, te) - 1, tm)).cast(f2f_dt[to]) - is_nan = (shr(v, fm) & (shl(1, fe) - 1)).eq(shl(1, fe) - 1) - if to in dtypes.fp8_fnuz: return is_nan.where(shl(1, ts - 1), underflow.where(0, sign.cast(f2f_dt[to]) | norm)) - return is_nan.where(nan, sign.cast(f2f_dt[to]) | underflow.where(0, norm)) + # denormals are zero, but within half a denormal ulp of the smallest normal rounds up to it + min_norm = shl(1 + fb - tb, fm) + underflow = nosign < max(min_norm - shl(1, fm - tm), shl(1, fm)) + norm = rne(nosign.maximum(min_norm), fm - tm) - shl(fb - tb, tm) + # top is one past the largest finite, inf or the nan of formats without inf + top = 0x7f if to == dtypes.fp8e4m3 else 0x80 if to in dtypes.fp8_fnuz else shl(shl(1, te) - 1, tm) + norm = norm.minimum(top - (sat and to in dtypes.fp8s)) + inf, nan = shl(shl(1, fe) - 1, fm), top | (shl(1, tm) - 1 if to in dtypes.fp8s else shl(1, tm - 1)) + # fnuz has one nan and no -0 + if to in dtypes.fp8_fnuz: return (nosign >= inf).where(top, underflow.where(0, sign | norm)).cast(f2f_dt[to]) + return (sign | (nosign > inf).where(nan, nosign.eq(inf).where(top, underflow.where(0, norm)))).cast(f2f_dt[to]) else: raise NotImplementedError(f"unsupported decomp {fr} -> {to}") -def f2f_clamp(val:UOp, dt:DType, sat=True) -> UOp: - e, m = dtypes.finfo(dt) - if dt in dtypes.fp8_fnuz: max_exp, max_man = (1 << e) - 1, (1 << m) - 1 - else: max_exp, max_man = ((1 << e) - 1, (1 << m) - 2) if dt == dtypes.fp8e4m3 else ((1 << e) - 2, (1 << m) - 1) - mx = val.const_like(2.0**(max_exp - exponent_bias(dt)) * (1.0 + max_man / (1 << m))) - sat = mx if dt in dtypes.fp8s and sat else val.const_like(float('inf')) - # FIXME: CMPLT of nan is undefined - return val.ne(val).where(val, (val < -mx).where(-sat, (mx < val).where(sat, val))) +@dataclass(frozen=True) +class FloatDecompCtx: + fr: DType + to: DType def f2f_load(x: UOp, fr:DType, to:DType) -> UOp: - storage_idx = graph_rewrite(x.src[0], pm_float_decomp, ctx=(fr, to), bottom_up=True) + storage_idx = graph_rewrite(x.src[0], pm_float_decomp, ctx=FloatDecompCtx(fr, to), bottom_up=True) if (n:=x.max_numel()) == 1: return f2f(storage_idx.load(*x.src[1:]), fr, to) return UOp(Ops.STACK, src=tuple(f2f(reindex(storage_idx, i, 1).load(*x.src[1:]), fr, to) for i in range(n))) @@ -143,83 +138,77 @@ def f2f_store(st, idx, val, fr:DType, to:DType): if (n:=val.max_numel()) == 1: return st.replace(src=(idx, f2f(val.bitcast(f2f_dt[to]), to, fr))) return UOp.group(*(st.replace(src=(reindex(idx, i, 1), f2f(val.index(i).bitcast(f2f_dt[to]), to, fr))) for i in range(n))) -def f2f_rewrite(ctx:tuple, x:UOp) -> UOp: return graph_rewrite(x, pm_float_decomp, ctx=ctx, bottom_up=True) +def f2f_rewrite(ctx:FloatDecompCtx, x:UOp) -> UOp: return graph_rewrite(x, pm_float_decomp, ctx=ctx, bottom_up=True) -# tag is the 32-bit word this node becomes - (0 for the low word, 1 for the high, the dtype the consumer wants) pm_long_decomp: PatternMatcher = PatternMatcher([ # the decomp's own bottom-up rewrite can mint bare consts mid-flight: word splitting commits them at the long sibling's dtype (UPat(GroupOp.All, name='x'), lambda x: commit_weak_consts(x, next((s.dtype for s in x.src if s.dtype in l2i_dt), None))), (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), name="x"), l2i_define), - (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: - reindex(x, x.tag[0]).replace(tag=None) if x.tag is not None else None), - (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val')), name='st'), lambda st,idx,val: - st.replace(src=(idx.rtag((0, dt:=l2i_dt[idx.dtype])), val.rtag((0, dt)))).group( - st.replace(src=(idx.rtag((1, dt)), val.rtag((1, dt))))) if val.tag is None else None), - (UPat(GroupOp.Comparison, src=[UPat.var('a', tuple(l2i_dt.keys())), UPat()], name="x"), lambda ctx,a,x: - split_l2i(ctx, x.op, dt:=l2i_dt[a.dtype], *flatten((s.rtag((0, dt)), s.rtag((1, dt))) for s in x.src))), - (UPat(Ops.CAST, tuple(l2i_dt.keys()), src=(UPat.var('a', tuple(l2i_dt.keys())),), name="x"), lambda ctx,a,x: - split_l2i(ctx, Ops.BITCAST, l2i_dt[x.dtype], a.rtag((0, dt:=l2i_dt[a.dtype])), a.rtag((1, dt)))[x.tag[0]]), - # a const splits by value; the general CAST arm below would drop its high word - (UPat(Ops.CAST, src=(UPat(Ops.CONST, name='c'),), tag={(w, dt) for w in (0, 1) for dt in l2i_dt.values()}, name='x'), - lambda x,c: UOp.const(truncate[x.tag[1]](c.val >> (32*x.tag[0])), x.tag[1])), - (UPat(Ops.CAST, tuple(l2i_dt.keys()), src=(UPat.var('a'),), name="x"), lambda ctx,a,x: - split_l2i(ctx, x.op, x.dtype, a)[x.tag[0]] if x.tag is not None else None), - (UPat(Ops.CAST, src=(UPat.var('a', tuple(l2i_dt.keys())),), name="x"), lambda ctx,a,x: - split_l2i(ctx, x.op, x.dtype, a.rtag((0, dt:=l2i_dt[a.dtype])), a.rtag((1, dt))) if x.dtype not in l2i_dt and a.tag is None else None), - (UPat((Ops.SHL, Ops.SHR), tuple(l2i_dt.keys()), src=(UPat.var('a'), UPat.var('b')), name="x"), lambda ctx,a,b,x: - split_l2i(ctx, x.op, dt:=l2i_dt[x.dtype], a.rtag((0, dt)), a.rtag((1, dt)), b.rtag((0, dt)))[x.tag[0]] if x.tag is not None else None), - (UPat(Ops.WHERE, tuple(l2i_dt.keys()), src=(UPat.var('c'), UPat.var('a'), UPat.var('b')), name="x"), lambda ctx,a,b,c,x: - split_l2i(ctx, x.op, dt:=l2i_dt[x.dtype], c, a.rtag((0, dt)), a.rtag((1, dt)), b.rtag((0, dt)), b.rtag((1, dt)))[x.tag[0]] - if x.tag is not None else None), - (UPat((*(GroupOp.ALU - GroupOp.Comparison - {Ops.SHL, Ops.SHR, Ops.WHERE}), Ops.BITCAST), tuple(l2i_dt.keys()), name="x"), lambda ctx,x: - split_l2i(ctx, x.op, l2i_dt[x.dtype], *flatten((a.rtag((0, l2i_dt[x.dtype])), a.rtag((1, l2i_dt[x.dtype]))) for a in x.src))[x.tag[0]] - if x.tag is not None else None), - (UPat(Ops.LOAD, tuple(l2i_dt.keys()), src=(UPat.var('idx'),), name='x'), lambda ctx,x,idx: - reindex(l2i_rewrite_idx(ctx, idx), x.tag[0]).replace(tag=None).load() if x.tag is not None else None) + # long stores, comparisons and casts out of long consume the words of their long srcs, a stored word is no longer long + (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val', tuple(l2i_dt.keys()))), name='st'), lambda st,idx,val: + UOp.group(*(st.replace(src=(reindex(idx, w), v)) for w,v in enumerate(l2i_words(val))))), + (UPat(GroupOp.Comparison, src=[UPat.var('a', tuple(l2i_dt.keys())), UPat()], name="x"), lambda a,x: l2i(x.op, l2i_dt[a.dtype], *l2i_words(*x.src))), + (UPat(Ops.CAST, src=(UPat.var('a', tuple(l2i_dt.keys())),), name="x"), lambda a,x: + l2i(x.op, x.dtype, *l2i_words(a)) if x.dtype not in l2i_dt else None), + # a pair becomes the STACK of the long's two words + (l2i_pair(UPat(Ops.LOAD, src=(UPat.var('idx'),))), lambda p,idx: UOp(Ops.STACK, src=tuple(reindex(idx, w).load() for w in (0, 1)))), + # a const splits by value, the general CAST below would drop its high word + (l2i_pair(UPat(Ops.CAST, src=(UPat(Ops.CONST, name='c'),))), lambda p,c: + UOp(Ops.STACK, src=tuple(UOp.const(truncate[p.dtype](c.val >> (32*w)), p.dtype) for w in (0, 1)))), + (l2i_pair(UPat(Ops.CAST, src=(UPat.var('a'),), name='x')), lambda p,a,x: + UOp(Ops.STACK, src=l2i(x.op, x.dtype, a)) if a.dtype not in l2i_dt else None), + # a cast between long dtypes reinterprets the words, a bare const src commits at the long dtype + (l2i_pair(UPat(GroupOp.Elementwise, name='x')), lambda p,x: + UOp(Ops.STACK, src=l2i(Ops.BITCAST if x.op is Ops.CAST else x.op, p.dtype, *l2i_words(*commit_weak_consts(x, x.dtype).src)))), + (UPat(Ops.INDEX, src=(UPat(Ops.STACK, name='s'), UPat.cvar('w'))), lambda s,w: s.src[w.arg]), ]) -# float decomposition patterns - ctx is (fr, to) tuple +# float decomposition patterns pm_float_decomp: PatternMatcher = PatternMatcher([ (UPat(GroupOp.Defines, name="x"), lambda ctx,x: - UOp(x.op, src=x.src, arg=replace(x.arg, dtype=f2f_dt[ctx[0]]), tag=ctx[0]) if x.dtype == ctx[0] else None), + x.replace(arg=replace(x.arg, dtype=f2f_dt[ctx.fr])) if x.dtype == ctx.fr else None), # INDEX into a LOAD/STACK selects a lane of an already converted value, the load rules below own those (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(GroupOp.All-{Ops.LOAD, Ops.STACK}),), allow_any_len=True, name="x"), lambda ctx,x: - UOp(x.op, src=(f2f_rewrite(ctx, x.src[0]), *x.src[1:]), arg=x.arg, tag=ctx[0]) - if x.dtype == ctx[0] else None), - (UPat(Ops.LOAD, dtypes.floats, name="x"), lambda ctx,x: f2f_load(x, *ctx) if x.dtype == ctx[0] else None), + x.replace(src=(f2f_rewrite(ctx, x.src[0]), *x.src[1:])) if x.dtype == ctx.fr else None), + (UPat(Ops.LOAD, dtypes.floats, name="x"), lambda ctx,x: f2f_load(x, ctx.fr, ctx.to) if x.dtype == ctx.fr else None), # bitcasted load should just replace load (UPat(Ops.BITCAST, src=(UPat(Ops.LOAD, name="ld"),), name="bc"), lambda ctx,bc,ld: - f2f_rewrite(ctx, ld.src[0]).load(*ld.src[1:]).bitcast(bc.dtype) if ld.dtype == ctx[0] else None), + f2f_rewrite(ctx, ld.src[0]).load(*ld.src[1:]).bitcast(bc.dtype) if ld.dtype == ctx.fr else None), # bitcast from (UPat(Ops.BITCAST, src=(UPat.var("x", dtypes.floats),), name="bc"), lambda ctx,bc,x: - bc.replace(src=(f2f(x.bitcast(f2f_dt[ctx[1]]), ctx[1], ctx[0]),)) if x.dtype == ctx[1] and bc.dtype.bitsize == ctx[0].bitsize else None), + bc.replace(src=(f2f(x.bitcast(f2f_dt[ctx.to]), ctx.to, ctx.fr),)) if x.dtype == ctx.to and bc.dtype.bitsize == ctx.fr.bitsize else None), # bitcast to (UPat(Ops.BITCAST, src=(UPat.var("x"),), name="bc"), lambda ctx,bc,x: - f2f(x.bitcast(f2f_dt[ctx[0]]), ctx[0], ctx[1]) if bc.dtype == ctx[0] else None), + f2f(x.bitcast(f2f_dt[ctx.fr]), ctx.fr, ctx.to) if bc.dtype == ctx.fr else None), (UPat(Ops.CAST, dtypes.floats, src=(UPat.var("val"),), name="x"), lambda ctx,x,val: - f2f_clamp(val.cast(ctx[1]), ctx[0]) if x.dtype == ctx[0] else None), + f2f(f2f(val.cast(ctx.to).bitcast(f2f_dt[ctx.to]), ctx.to, ctx.fr), ctx.fr, ctx.to) if x.dtype == ctx.fr else None), (UPat(GroupOp.ALU|{Ops.STACK, Ops.INDEX}, dtypes.floats, name="x"), lambda ctx,x: - UOp(x.op, src=tuple(s.cast(ctx[1]) if s.dtype == ctx[0] else s for s in x.src), arg=x.arg, tag=x.tag) if x.dtype == ctx[0] else None), + x.replace(src=tuple(s.cast(ctx.to) if s.dtype == ctx.fr else s for s in x.src)) if x.dtype == ctx.fr else None), (UPat(Ops.STORE, src=(UPat.var("idx"), UPat(Ops.BITCAST, dtypes.floats, name="val")), name='st'), lambda ctx,st,idx,val: - st.replace(src=(idx, val.src[0].bitcast(f2f_dt[ctx[0]]))) if val.dtype == ctx[0] and idx.tag == ctx[0] else None), + st.replace(src=(idx, val.src[0].bitcast(f2f_dt[ctx.fr]))) if val.dtype == ctx.fr and idx.dtype == f2f_dt[ctx.fr] else None), (UPat(Ops.STORE, src=(UPat.var("idx").or_casted(), UPat.var("val", dtypes.floats)), name='st'), lambda ctx,st,idx,val: - f2f_store(st, idx, val, *ctx) if val.dtype == ctx[1] and idx.tag == ctx[0] else None), + f2f_store(st, idx, val, ctx.fr, ctx.to) if val.dtype == ctx.to and idx.dtype == f2f_dt[ctx.fr] else None), ]) -def do_dtype_decomps(sink:UOp, ctx:tuple[set[DType], Renderer]) -> UOp: - def _should_emulate(dt): return dt in EMULATED_DTYPES.tolist(dtypes) or dt not in ctx[1].supported_dtypes() - for fr in sorted(filter(_should_emulate, ctx[0])): +@dataclass +class DTypeDecompCtx: + renderer: Renderer + dtypes: set[DType] = field(default_factory=set) + +def do_dtype_decomps(sink:UOp, ctx:DTypeDecompCtx) -> UOp: + def _should_emulate(dt): return dt in EMULATED_DTYPES.tolist(dtypes) or dt not in ctx.renderer.supported_dtypes() + for fr in sorted(filter(_should_emulate, ctx.dtypes)): to = dtypes.int if fr == dtypes.long else dtypes.float if DEBUG >= 2: print(f"emulating {fr} as {to}") pm = pm_float_decomp if fr in dtypes.floats else pm_long_decomp - sink = graph_rewrite(sink, pm, name=f"decomp {fr} -> {to}", ctx={} if pm is pm_long_decomp else (fr, to), bottom_up=True) - ctx[0].clear() + sink = graph_rewrite(sink, pm, name=f"decomp {fr} -> {to}", ctx=FloatDecompCtx(fr, to), bottom_up=True) + ctx.dtypes.clear() return sink pm_dtype_decomps = PatternMatcher([ # detect dtypes to decompose (UPat(GroupOp.All, (*dtypes.fp8s, dtypes.bfloat16, dtypes.half, dtypes.long, dtypes.ulong), name="x"), lambda x,ctx: - ctx[0].add({dtypes.ulong:dtypes.long}.get(dt:=x.dtype, dt))), + ctx.dtypes.add({dtypes.ulong:dtypes.long}.get(dt:=x.dtype, dt))), # do the rewrites (UPat(Ops.SINK, name="sink"), do_dtype_decomps), ]) diff --git a/tinygrad_repo/tinygrad/codegen/decomp/op.py b/tinygrad_repo/tinygrad/codegen/decomp/op.py index 36907bf9d9..cff34616d4 100644 --- a/tinygrad_repo/tinygrad/codegen/decomp/op.py +++ b/tinygrad_repo/tinygrad/codegen/decomp/op.py @@ -52,12 +52,12 @@ def threefry2x32(x: UOp, key: UOp): # ***** decomposition patterns ***** def floordiv_to_idiv(a:UOp, b:UOp) -> UOp: - if (a.vmin >= 0 and b.vmin >= 0) or (a.vmax <= 0 and b.vmax <= 0): return a.alu(Ops.CDIV, b) - return a.alu(Ops.CDIV, b) - (a.alu(Ops.CMOD, b).ne(0) & (a<0).ne(b<0)) + if (a.vmin >= 0 and b.vmin >= 0) or (a.vmax <= 0 and b.vmax <= 0): return a.alu(Ops.TRUNCDIV, b) + return a.alu(Ops.TRUNCDIV, b) - (a.alu(Ops.TRUNCMOD, b).ne(0) & (a<0).ne(b<0)) def floormod_to_mod(a:UOp, b:UOp) -> UOp: - if (a.vmin >= 0 and b.vmin >= 0) or (a.vmax <= 0 and b.vmax <= 0): return a.alu(Ops.CMOD, b) - r = a.alu(Ops.CMOD, b) + if (a.vmin >= 0 and b.vmin >= 0) or (a.vmax <= 0 and b.vmax <= 0): return a.alu(Ops.TRUNCMOD, b) + r = a.alu(Ops.TRUNCMOD, b) # use where instead of mul to avoid being fused into MULACC (which int64 long-decomp doesn't handle) return r + (r.ne(0) & (a<0).ne(b<0)).where(b, b.const_like(0)) @@ -66,11 +66,11 @@ powers_of_two: dict[int, int] = {2**i:i for i in range(64)} def get_simplifying_rewrite_patterns(ops:tuple[Ops, ...]) -> PatternMatcher: # these are rewrites that make things simpler pat: list[tuple[UPat, Callable]] = [] - # FLOORDIV by 2**y -> x >> y (an arithmetic shift is exactly floor division for any sign); fires before floordiv_to_idiv + # DIV by 2**y -> x >> y (an arithmetic shift is exactly floor division for any sign); fires before floordiv_to_idiv if Ops.SHR in ops: pat.append((UPat.var("x", dtypes.ints)//UPat.cvar("c"), lambda x,c: x >> v if (v:=powers_of_two.get(c.val, 0)) else None)) pat.append((UPat.var("a")//UPat.var("b"), floordiv_to_idiv)) - # FLOORMOD by 2**y -> x & (2**y-1) (correct floor mod for any sign in two's complement); fires before floormod_to_mod + # MOD by 2**y -> x & (2**y-1) (correct floor mod for any sign in two's complement); fires before floormod_to_mod if Ops.AND in ops: pat.append((UPat.var("x", dtypes.ints)%UPat.cvar("c"), lambda x,c: x & (c.val-1) if c.val in powers_of_two else None)) pat.append((UPat.var("a")%UPat.var("b"), floormod_to_mod)) # no real hardware supports THREEFRY, but NullRenderer does @@ -84,37 +84,35 @@ def get_late_rewrite_patterns(ops:tuple[Ops, ...], disable_fast_idiv:bool) -> Pa if Ops.MAX not in ops and Ops.CMPLT in ops: pat += [(UPat(Ops.MAX, name="m"), lambda m: (m.src[0] < m.src[1]).where(m.src[1], m.src[0]))] if Ops.OR in ops: pat += [(UPat.var("x", dtypes.bool).logical_not()&UPat.var("y", dtypes.bool).logical_not(), lambda x,y: (x | y).logical_not())] - # rewrite MUL/CDIV to SHL+SHR: x*(2**y) -> shl(x,y) and x//(2**y) -> shr(x,y) + # rewrite MUL/TRUNCDIV to SHL+SHR: x*(2**y) -> shl(x,y) and x//(2**y) -> shr(x,y) if Ops.SHL in ops: pat += [(UPat.var("x", dtypes.ints)*UPat.cvar("c"), lambda c,x: x << v if (v:=powers_of_two.get(c.val, 0)) else None)] if Ops.SHR in ops: - # uint CDIV by 2**v -> x >> v (FLOORDIV is lowered to CDIV by the rule above before reaching here) - pat += [(UPat(Ops.CDIV, src=(UPat.var("x", dtypes.uints), UPat.cvar("c"))), + # uint TRUNCDIV by 2**v -> x >> v (DIV is lowered to TRUNCDIV by the rule above before reaching here) + pat += [(UPat(Ops.TRUNCDIV, src=(UPat.var("x", dtypes.uints), UPat.cvar("c"))), lambda x,c: x >> v if (v:=powers_of_two.get(c.val, 0)) else None)] - # signed CDIV (trunc) by 2**v -> (x + (x<0 ? c-1 : 0)) >> v - pat += [(UPat(Ops.CDIV, src=(UPat.var("x", dtypes.ints), UPat.cvar("c"))), + # signed TRUNCDIV by 2**v -> (x + (x<0 ? c-1 : 0)) >> v + pat += [(UPat(Ops.TRUNCDIV, src=(UPat.var("x", dtypes.ints), UPat.cvar("c"))), lambda x,c: (x+(l.const_like(l.vmin) if (l:=(x<0)).vmin==l.vmax else l).where(c-1, 0)) >> v if (v:=powers_of_two.get(c.val, 0)) else None)] if not disable_fast_idiv: # fast_idiv handles non-pow2 divisors on non-negative inputs - pat += [(UPat(Ops.CDIV, src=(UPat.var("x", dtypes.ints), UPat.cvar("d"))), lambda ctx, x, d: fast_idiv(ctx, x, d.val))] - # rewrite raw CMOD -> x - d*fast_idiv(x,d), only when fast_idiv can actually divide; - # avoids disturbing floormod_to_mod's general-path output (which uses a trunc Ops.CMOD as an implementation detail) - pat += [(UPat(Ops.CMOD, src=(UPat.var("x", dtypes.ints), UPat.cvar("d"))), + pat += [(UPat(Ops.TRUNCDIV, src=(UPat.var("x", dtypes.ints), UPat.cvar("d"))), lambda ctx, x, d: fast_idiv(ctx, x, d.val))] + # rewrite raw TRUNCMOD -> x - d*fast_idiv(x,d), only when fast_idiv can actually divide; + # avoids disturbing floormod_to_mod's general-path output (which uses Ops.TRUNCMOD as an implementation detail) + pat += [(UPat(Ops.TRUNCMOD, src=(UPat.var("x", dtypes.ints), UPat.cvar("d"))), lambda ctx, x, d: x - d * q if (q:=fast_idiv(ctx, x, d.val)) is not None else None)] if Ops.NEG in ops: - pat += [(UPat.var('x')*-1, lambda ctx,x: x.alu(Ops.NEG))] + pat += [(UPat.var('x')*UPat.cvar('c'), lambda ctx,x,c: x.alu(Ops.NEG) if c.val == x.dtype.const(-1) else None)] if Ops.SUB in ops: pat += [(UPat.var('x')+UPat.var('y').alu(Ops.NEG), lambda ctx,x,y: x.alu(Ops.SUB, y))] if Ops.CMPLT in ops: # These are late rewrites because simplex expects equalities to be a certain format pat += [ - ((UPat.var("x", dtypes.sints) < UPat.cvar("c")).logical_not(), lambda x,c: c-1 x==c ] - if Ops.CMPEQ in ops: pat += [(UPat.var('x').ne(UPat.var('y')).logical_not(), lambda x,y: x.alu(Ops.CMPEQ, y))] + if Ops.CMPNE in ops: pat += [(UPat.var('x').eq(UPat.var('y')).logical_not(), lambda x,y: x.alu(Ops.CMPNE, y))] if Ops.MULACC in ops: pat += [(UPat.var('a')*UPat.var('b')+UPat.var('c'), lambda a,b,c: a.alu(Ops.MULACC, b, c))] # also fuse (x << n) + c → MULACC(x, 2^n, c) since MUL→SHL may run first diff --git a/tinygrad_repo/tinygrad/codegen/late/coalesce.py b/tinygrad_repo/tinygrad/codegen/late/coalesce.py index 7ec5fa2357..dc1dc23a51 100644 --- a/tinygrad_repo/tinygrad/codegen/late/coalesce.py +++ b/tinygrad_repo/tinygrad/codegen/late/coalesce.py @@ -72,8 +72,8 @@ def image_valid_dims(base:DType, size:int, arch:str) -> list[tuple[int,int]]: return [(pxls//ALIGN//k, ALIGN*k) for k in range(ceildiv(pxls//ALIGN, MAXW), min(pxls//ALIGN, MAXW//ALIGN)+1) if (pxls//ALIGN)%k == 0] def transform_to_image(ctx, buf:UOp, x:UOp) -> UOp|None: - shapes, ren = ctx - if not IMAGE or ren.target.device not in {"QCOM", "CL", "PYTHON", "NULL"}: return None + shapes, ren, read_write = ctx + if not IMAGE or ren.target.device not in {"QCOM", "CL", "PYTHON", "NULL"} or buf in read_write: return None valid, x = x.get_valid(), x.get_idx() # search for dims that drop the most valid statements best_drop, cands = -1, [] @@ -89,10 +89,7 @@ def transform_to_image(ctx, buf:UOp, x:UOp) -> UOp|None: # the image dims are stored in the param's arg, the size stays the flat buffer len buf = buf.replace(arg=replace(buf.arg, image=(h, w))) shapes[buf.arg.slot] = (h, w) - if valid.op is not Ops.CONST or valid.val is not True: - return buf.index(cidx.src[1].valid(valid), cidx.src[0].valid(valid)) - else: - return buf.index(cidx.src[1], cidx.src[0]) + return buf.index(cidx.src[1].valid(valid), cidx.src[0].valid(valid)) def store_image(x:UOp, d:UOp) -> UOp: # image load/store is always float, a half image converts on store @@ -131,7 +128,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: # build replacements replacements = {} - for (op,buf,base,valid,ld_arg),offsets in memory.items(): + for (op,buf,_,_,ld_arg),offsets in memory.items(): # allowed lengths (copied in) lengths = [] must_divide = True @@ -148,21 +145,21 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: grouped_offsets = [[x for _,x in group] for _,group in itertools.groupby(enumerate(sorted(offsets.keys())), lambda x: x[1]-x[0])] for full_grp in grouped_offsets: while len(full_grp): - offset = (base+full_grp[0]) if isinstance(base, UOp) else UOp.const(full_grp[0]) - length = [l for l in lengths if l <= len(full_grp) and (not must_divide or offset.divides(l) is not None)][0] + # the group starts at its first access's index, valid included + offset = offsets[full_grp[0]][0].src[0].src[1] + length = [l for l in lengths if l <= len(full_grp) and (not must_divide or offset.get_idx().divides(l) is not None)][0] grp, full_grp = full_grp[:length], full_grp[length:] - # NOTE: we apply the valid again after we determine the length - offset = offset.valid(valid) - idx = UOp(Ops.SHRINK, src=(buf, offset, UOp.const(length))) if length > 1 else buf.index(offset) + assert op is not Ops.STORE or all(len(offsets[g]) == 1 for g in grp), "attempting multiple stores" + if length == 1: continue + idx = UOp(Ops.SHRINK, src=(buf, offset, UOp.const(length))) if op == Ops.STORE: - assert all(len(offsets[g]) == 1 for g in grp), "attempting multiple stores" datas = [offsets[g][0].src[1] for g in grp] - store = idx.store(UOp.stack(*datas) if length > 1 else datas[0]) + store = idx.store(UOp.stack(*datas)) for g in grp: replacements[offsets[g][0]] = store else: ld = idx.load(arg=ld_arg) for i,g in enumerate(grp): - for oo in offsets[g]: replacements[oo] = ld.index(i) if length > 1 else ld + for oo in offsets[g]: replacements[oo] = ld.index(i) # apply return sink.substitute(replacements, name="memory coalescing") diff --git a/tinygrad_repo/tinygrad/codegen/late/gater.py b/tinygrad_repo/tinygrad/codegen/late/gater.py index 319a60c320..02ee798389 100644 --- a/tinygrad_repo/tinygrad/codegen/late/gater.py +++ b/tinygrad_repo/tinygrad/codegen/late/gater.py @@ -2,28 +2,16 @@ from tinygrad.uop.ops import PatternMatcher, UPat, Ops from tinygrad.dtype import Invalid, dtypes -def move_where_load(gate, l, a, w): - return l.replace(src=(l.src[0], l.vconst_like(0) if a.is_invalid else l.const_like(a.val) if a.op is Ops.CONST else - a.src[0] if a.op is Ops.CAST and a.src[0].dtype == l.dtype else a.cast(l.dtype), l.src[2])).cast(w.dtype) +def move_where_load(gate, l, a): return l.replace(src=(l.src[0], a.cast(l.dtype), l.src[2])) +def ungate(x, gate): return x.replace(src=tuple(s.src[1] if s.op is Ops.WHERE and s.src[0] is gate and s.src[2].is_invalid else s for s in x.src)) +gated_mop = UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(), UPat.var("gate").where(UPat(), UPat(arg=Invalid))), name="mop", allow_any_len=True) pm_move_gates_from_index = PatternMatcher([ - # for image idx (must be first) - (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), - UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).load(name="l"), - lambda buf,gate,idx_y,idx_x,l: buf.index(idx_y, idx_x).load(l.vconst_like(0), gate)), - (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), - UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).store(UPat.var("data")), - lambda buf,gate,idx_y,idx_x,data: buf.index(idx_y, idx_x).store(data, gate)), - # here we create the alt value for load to be 0s and remove the where Invalid - (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(), UPat.var("gate").where(UPat.var("idx"), UPat(arg=Invalid)),), name="mop", allow_any_len=True) \ - .load(name="l"), lambda mop,gate,idx,l: mop.replace(src=(mop.src[0],idx)+mop.src[2:]).load(l.vconst_like(0), gate)), - (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(), UPat.var("gate").where(UPat.var("idx"), UPat(arg=Invalid)),), name="mop", allow_any_len=True) \ - .store(UPat.var("data")), lambda mop,gate,idx,data: mop.replace(src=(mop.src[0],idx)+mop.src[2:]).store(data, gate)), + (gated_mop.load(name="l"), lambda mop,gate,l: ungate(mop, gate).load(l.vconst_like(0), gate)), + (gated_mop.store(UPat.var("data")), lambda mop,gate,data: ungate(mop, gate).store(data, gate)), # Where after gated load becomes alt value - (UPat.var("gate").where(UPat().load(UPat(), UPat.var("gate", dtype=dtypes.bool), name="l").or_casted(), UPat.var("a")).named("w"), - move_where_load), - (UPat.var("gate").where(UPat.var("a"), UPat().load(UPat(), ~UPat.var("gate", dtype=dtypes.bool), name="l").or_casted()).named("w"), - move_where_load), + (UPat.var("gate").where(UPat().load(UPat(), UPat.var("gate", dtype=dtypes.bool), name="l"), UPat.var("a")), move_where_load), + (UPat.var("gate").where(UPat.var("a"), UPat().load(UPat(), ~UPat.var("gate", dtype=dtypes.bool), name="l")), move_where_load), ]) diff --git a/tinygrad_repo/tinygrad/codegen/late/regalloc.py b/tinygrad_repo/tinygrad/codegen/late/regalloc.py index 87228bd247..0dadf77654 100644 --- a/tinygrad_repo/tinygrad/codegen/late/regalloc.py +++ b/tinygrad_repo/tinygrad/codegen/late/regalloc.py @@ -1,7 +1,7 @@ import itertools from tinygrad.helpers import dedup from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat -from tinygrad.renderer.isa import ISARenderer, Register, rdef, LinearContext +from tinygrad.renderer.x86 import ISARenderer, Register, rdef, regs, with_regs, LinearContext from typing import Any from dataclasses import replace @@ -18,15 +18,15 @@ class LinearScanRegallocContext: # compute live ranges self.live_range: dict[Register, list[int]] = {} lr = self.live_range - loops: dict[int, int] = {} # the interval of each loop, from its RANGE to the last uop that reads that RANGE + loops: dict[int, int] = {} # the interval of each loop, from its RANGE to the last non pseudo uop that reads that RANGE for idx,u in reversed(list(enumerate(uops))): if u.op in PSEUDO_OPS: continue - defs = u.tag if isinstance(u.tag, tuple) else () + defs = regs(u) for v in defs + tuple(rdef(s) for s in dedup(u.src)): if isinstance(v, Register): lr.setdefault(v, []).insert(0, idx) for v in defs: - if v in lr and (n:=max((e for s,e in loops.items() if s <= lr[v][-1] < e), default=None)): lr[v].append(n) - if u.op is Ops.RANGE: loops[idx] = max(j for j,x in enumerate(uops) if u in x.src) + if (n:=max((e for s,e in loops.items() if s <= lr[v][-1] < e), default=None)): lr[v].append(n) + if u.op is Ops.RANGE: loops[idx] = max(j for j,x in enumerate(uops) if u in x.src and x.op not in PSEUDO_OPS) # allocate registers self.spills: dict[Register, Any] = {} # mapping from virtual to arbitrary spill slot @@ -55,15 +55,13 @@ class LinearScanRegallocContext: if u.op in PSEUDO_OPS: continue # allocate uses for s in u.src: - # HACK: cause of later hacks to lower range - if u.op in (Ops.END, Ops.BACKEDGE): continue if not isinstance(v:=rdef(s), Register): continue if v not in live: live[v] = fill(v, i) self.reals.setdefault(i, {})[v] = live[v] # allocate defs - if isinstance(u.tag, tuple): - for j,v in enumerate(u.tag): + if regs(u): + for j,v in enumerate(regs(u)): if v not in live: cons = v.cons # two address instructions (src is reused by def) can only coalesce reused src. @@ -71,8 +69,7 @@ class LinearScanRegallocContext: if ren.is_two_address(u) and j == 0: uses = tuple(live.get(rdef(s)) for s in u.src) cons = ((uses[0],) if uses[0] in cons else ()) + tuple(r for r in cons if r not in uses) - # HACK: cause the range is missing the comparison - live[v] = alloc(cons, i+1 if u.op is not Ops.RANGE else i) + live[v] = alloc(cons, i+1) self.reals.setdefault(i, {})[v] = live[v] # loop prologue, avoid loading inside the loop @@ -88,8 +85,8 @@ class LinearScanRegallocContext: live_in[v] = live[v] live_ins.append(live_in) - # loop epilogue, reload registers that were live at loop entry - if u.op in (Ops.END, Ops.BACKEDGE): + # loop epilogue at the loop's last reader, reload registers that were live at loop entry + if i in loops.values(): # TODO: if a uop is in a different reg in live out vs live in move between registers instead of loading # TODO: don't reload if first use in loop is a load live_in = live_ins.pop() @@ -104,10 +101,10 @@ def regalloc_rewrite(ctx:LinearScanRegallocContext, x:UOp): if x.op in PSEUDO_OPS: return None nsrc = [] def retag(x:UOp, r:Register) -> UOp: - return x.replace(src=(retag(x.src[0],r),*x.src[1:])) if x.op in {Ops.AFTER, Ops.BITCAST} else x.replace(tag=(r,)) + return x.replace(src=(retag(x.src[0],r),*x.src[1:])) if x.op in {Ops.AFTER, Ops.BITCAST} else with_regs(x, (r,)) for j,s in enumerate(x.src): # v here is the virtual defined by the original s as s is the rewritten version - if i in ctx.reals and (v:=rdef(ctx.uops[i].src[j])) in ctx.reals.get(i, {}): + if (v:=rdef(ctx.uops[i].src[j])) in ctx.reals.get(i, {}): reg = replace(ctx.reals[i][v], size=v.size) if v in ctx.spills: fill = ctx.ren.fill(ctx.spills[v], ctx.vdef(v), reg) @@ -115,14 +112,13 @@ def regalloc_rewrite(ctx:LinearScanRegallocContext, x:UOp): nsrc.append(s.replace(src=(fill,)+s.src[1:]) if s.op is Ops.AFTER else fill) else: nsrc.append(retag(s, reg)) else: nsrc.append(s) - ndefs = tuple(replace(ctx.reals[i][v], size=v.size) for v in x.tag) if isinstance(x.tag, tuple) else x.tag - nx = x.replace(src=tuple(nsrc), tag=ndefs) + nx = with_regs(x.replace(src=tuple(nsrc)), tuple(replace(ctx.reals[i][v], size=v.size) for v in regs(x))) if regs(x) else x.replace(src=tuple(nsrc)) before = [ctx.ren.fill(ctx.spills[v], ctx.vdef(v), replace(r, size=v.size)) for v,r in ctx.insert_before.get(i, [])] - after = [ctx.ren.spill(ctx.spills[v], nx) for v in x.tag if v in ctx.spills] if isinstance(x.tag, tuple) else [] + after = [ctx.ren.spill(ctx.spills[v], nx) for v in regs(x) if v in ctx.spills] return nx, before + [nx] + after pm_regalloc_rewrite = PatternMatcher([ - (UPat({Ops.INS, Ops.RANGE, Ops.END, Ops.BACKEDGE, Ops.BUFFER, Ops.PARAM, Ops.SPECIAL} | PSEUDO_OPS, name="x"), regalloc_rewrite), + (UPat({Ops.INS, Ops.RANGE, Ops.END} | PSEUDO_OPS, name="x"), regalloc_rewrite), ]) diff --git a/tinygrad_repo/tinygrad/codegen/opt/postrange.py b/tinygrad_repo/tinygrad/codegen/opt/postrange.py index 01d6573843..d7f19cd301 100644 --- a/tinygrad_repo/tinygrad/codegen/opt/postrange.py +++ b/tinygrad_repo/tinygrad/codegen/opt/postrange.py @@ -47,7 +47,7 @@ class Scheduler: special_uops = sorted([x for x in self.ast.backward_slice if x.op is Ops.SPECIAL], key=lambda x: x.arg) special_ops = [colored(str(x.vmax+1), "blue" if x.arg[0] == "g" else "cyan") for x in special_uops] name = k_type + colored('_', 'BLACK').join(['']+special_ops+[colored(x.src[0].render(), color) for x,color in zip(self.rngs, self.colors())]) - return self.ast.replace(arg=KernelInfo(name=name, applied_opts=tuple(self.applied_opts)), tag=1) + return self.ast.replace(arg=KernelInfo(name=name, applied_opts=tuple(self.applied_opts), optimized=True)) def convert_loop_to_global(self) -> None: if not self.ren.has_local: return @@ -237,7 +237,7 @@ def args_from_ast(ast:UOp, dname:str) -> tuple[list[Buffer], dict[str, int]]: return [Buffer(dname, x.max_numel() * x.dtype.itemsize) for x in glbls], {k.expr:int(k.vmax+k.vmin)//2 for k in ast.variables()} def apply_opts(ast:UOp, ren:Renderer, beam:int=0) -> UOp: - if ast.tag is not None: return ast + if ast.arg is not None and ast.arg.optimized: return ast k = Scheduler(ast, ren) k.convert_loop_to_global() if ast.arg is not None and ast.arg.opts_to_apply is not None: diff --git a/tinygrad_repo/tinygrad/codegen/simplify.py b/tinygrad_repo/tinygrad/codegen/simplify.py index 46caa869e4..73cd405311 100644 --- a/tinygrad_repo/tinygrad/codegen/simplify.py +++ b/tinygrad_repo/tinygrad/codegen/simplify.py @@ -1,6 +1,6 @@ import itertools from typing import Callable -from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, graph_rewrite, _substitute, range_start, AxisType, identity_element +from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, GroupOp, graph_rewrite, _substitute, range_start, AxisType, identity_element from tinygrad.uop.symbolic import symbolic from tinygrad.helpers import partition, dedup from tinygrad.dtype import dtypes @@ -8,7 +8,6 @@ from tinygrad.dtype import dtypes def flatten_range(r:UOp) -> UOp|None: off = range_start[r.op] rngs = r.src[off:] - if not len(rngs): return None return r.replace(src=r.src[:off]+tuple(dedup(x for s in rngs for x in ((s,) if s.op is Ops.RANGE else s.ranges)))) pm_flatten_range = PatternMatcher([ @@ -16,14 +15,14 @@ pm_flatten_range = PatternMatcher([ (UPat((Ops.REDUCE, Ops.END), name="r"), flatten_range), ]) -# index/range arithmetic uses FLOORDIV/FLOORMOD prior to late rewrite -def count_divmod(x:UOp) -> int: return sum(u.op in {Ops.FLOORDIV, Ops.FLOORMOD} for u in x.backward_slice) +# index/range arithmetic uses DIV/MOD prior to late rewrite +def count_divmod(x:UOp) -> int: return sum(u.op in {Ops.DIV, Ops.MOD} for u in x.backward_slice) def simplify_merge_adjacent(u:UOp) -> UOp|None: reduce_ranges = [x.ranges for x in u.backward_slice_with_self if x.op is Ops.REDUCE] # on END we only want to merge adjacent ranges, on REDUCE we want to try all combinations for r0, r1 in (zip(u.ended_ranges, u.ended_ranges[1:]) if u.op is Ops.END else itertools.permutations(u.ended_ranges, 2)): - # check same type - if r0.axis_type == r1.axis_type: + # check same type and neither bound depends on the other range + if r0.axis_type == r1.axis_type and r0 not in r1.ranges and r1 not in r0.ranges: # check if the ranges to merge are in the same reduces if all((r0 in rngs) == (r1 in rngs) for rngs in reduce_ranges): s0, s1 = r0.src[0], r1.src[0] @@ -79,16 +78,14 @@ pm_split_ranges = PatternMatcher([ def no_range(u:UOp) -> bool: return not u.op_in_backward_slice_with_self(Ops.RANGE) def reduce_unparented(red:UOp) -> UOp|None: - if red.arg[0] not in {Ops.ADD, Ops.MAX, Ops.MUL}: return None assert all(x.op is Ops.RANGE for x in red.src[1:]), "some reduce srcs aren't ranges" reduce_parented, reduce_unparented = partition(red.src[1:], lambda x: x in red.src[0].ranges) if len(reduce_unparented) == 0: return None ret = red.replace(src=(red.src[0],)+tuple(reduce_parented)) if len(reduce_parented) else red.src[0] - for r in reduce_unparented: - # an idempotent op (MAX, and bool ADD/MUL which are OR/AND) applied N times is the op once, or the identity when N is 0 - if red.arg[0] is Ops.MAX or red.dtype is dtypes.bool: ret = (r.src[0] > 0).where(ret, ret.const_like(identity_element(red.arg[0], red.dtype))) - else: ret = ret * r.src[0] if red.arg[0] is Ops.ADD else ret ** r.src[0] - return ret + # the op applied n times, MAX and bool ADD/MUL (OR/AND) are idempotent. an empty range is the identity + n, op = UOp.uprod(*[r.src[0] for r in reduce_unparented]), red.arg[0] + if op is not Ops.MAX and red.dtype is not dtypes.bool: ret = ret*n if op is Ops.ADD else ret**n + return (n > 0).where(ret, ret.const_like(identity_element(op, red.dtype))) pm_reduce_unparented = PatternMatcher([ # remove any ranges from a REDUCE that aren't referenced in the reduce source @@ -97,10 +94,11 @@ pm_reduce_unparented = PatternMatcher([ pm_reduce_collapse = pm_reduce_unparented + PatternMatcher([ # lift x+y out of reduce on lt - ((UPat.var("x")+UPat.var("y")) < UPat.var("c"), lambda x,y,c: (x < (c-y)) if no_range(y) and no_range(c) else None), + (UPat(Ops.ADD, dtypes.sints+dtypes.floats+dtypes.weaks, [UPat.var("x"), UPat.var("y")]) < UPat.var("c"), + lambda x,y,c: (x < (c-y)) if no_range(y) and no_range(c) else None), # lift x*y out of reduce - ((UPat.var("x")*UPat.var("y")) < UPat.var("c"), - lambda x,y,c: (x < ((c+y-1) // y)) if no_range(y) and no_range(c) and dtypes.is_int(y.dtype) and y.vmin > 0 else None), + (UPat(Ops.MUL, dtypes.sints+(dtypes.weakint,), [UPat.var("x"), UPat.var("y")]) < UPat.var("c"), + lambda x,y,c: (x < ((c+y-1) // y)) if no_range(y) and no_range(c) and y.vmin > 0 else None), # sum over r in [0,N) of [lower<=r max(min(upper,N) - max(lower,0), 0) * val (UPat.any( (UPat(Ops.RANGE, name="r") < UPat.var("upper")).where(UPat.var("val"), 0), @@ -119,10 +117,10 @@ pm_reduce_collapse = pm_reduce_unparented + PatternMatcher([ ])+symbolic pm_reduce_load_collapse = pm_reduce_collapse + PatternMatcher([ - # lift x+y out of reduce on ne - ((UPat.var("x")+UPat.var("y")).or_casted() != UPat.var("c"), lambda x,y,c: (x != (c.cast(y.dtype)-y)) if no_range(y) and no_range(c) else None), + # lift x+y out of reduce on eq + ((UPat.var("x")+UPat.var("y")).or_casted().eq(UPat.var("c")), lambda x,y,c: x.eq(c.cast(y.dtype)-y) if no_range(y) and no_range(c) else None), # reduce on gated load becomes can substitute the range and remove the reduce - ((UPat.var("idx")!=(UPat(Ops.RANGE, name="r").or_casted())).where(0, UPat.var("expr")).reduce(UPat.var("r"), arg=Ops.ADD), + (UPat.var("idx").eq(UPat(Ops.RANGE, name="r").or_casted()).where(UPat.var("expr"), 0).reduce(UPat.var("r"), arg=Ops.ADD), lambda r,idx,expr: (v:=(idx.cast(r.dtype) >= 0) & (idx.cast(r.dtype) < r.src[0])).where(expr.substitute({r:idx.cast(r.dtype).valid(v)}),0)), ]) @@ -133,7 +131,7 @@ def reduce_collapse(red:UOp, u:UOp, pm:PatternMatcher=pm_reduce_collapse) -> UOp replaces: dict[UOp, UOp] = {} for u in included: for s in u.src: - if s in included or s in replaces or s.op in {Ops.CONST, Ops.PARAM, Ops.BUFFER, Ops.ALLOC}: continue + if s in included or s in replaces or s.op in GroupOp.Defines|{Ops.CONST}: continue replaces[s] = UOp.variable(f'in{len(replaces)}', s.vmin, s.vmax, s.dtype) collapse_fxn = u.substitute(replaces).reduce(r, arg=Ops.ADD) sink = graph_rewrite(collapse_fxn, pm, name="reduce_collapse") diff --git a/tinygrad_repo/tinygrad/device.py b/tinygrad_repo/tinygrad/device.py index 08e9f45c47..a4a4b137f6 100644 --- a/tinygrad_repo/tinygrad/device.py +++ b/tinygrad_repo/tinygrad/device.py @@ -6,15 +6,20 @@ import importlib, inspect, functools, itertools, pathlib, os, contextlib, re, at from tinygrad.helpers import mv_address, LRU, getenv, diskcache_get, diskcache_put, DEBUG, GlobalCounters, PROFILE, temp, colored from tinygrad.helpers import Context, CCACHE, ALLOW_DEVICE_USAGE, MAX_BUFFER_SIZE, cpu_events, ProfileEvent, ProfilePointEvent, suppress_finalizing from tinygrad.helpers import select_by_name, select_first_inited, DEV, TracingKey, size_to_str, pluralize, Target, unwrap, round_up, is_numpy_ndarray -from tinygrad.helpers import cpu_profile, perf_counter_us, to_name, HCQ_RUNTIME_DEV, to_tuple -from tinygrad.dtype import dtypes, DType -from tinygrad.runtime.support.memory import BumpAllocator, MMIOInterface +from tinygrad.helpers import cpu_profile, perf_counter_us, HCQ_RUNTIME_DEV, to_tuple +from tinygrad.dtype import dtypes, DType, AddrSpace +from tinygrad.runtime.support.memory import MMIOInterface if TYPE_CHECKING: from tinygrad.renderer import Renderer + from tinygrad.uop.ops import PatternMatcher # **************** Device **************** -ALL_DEVICES = ["METAL", "AMD", "NV", "CUDA", "QCOM", "CL", "CPU", "DSP", "WEBGPU"] +ALL_DEVICES = ["METAL", "AMD", "NV", "QCOM", "CUDA", "HIP", "CL", "CPU", "DSP", "WEBGPU"] +HCQ_DEVS = frozenset(("AMD", "NV", "QCOM", "CUDA", "HIP", "NULL", "METAL", "CL")) +HOST_DEVS = frozenset(("CPU", "PYTHON")) +UVM_DEVS = frozenset(("QCOM", "METAL", "CL")) # cl uses host memcpys + class _Device: def __init__(self) -> None: self._devices = [x.stem[len("ops_"):].upper() for x in (pathlib.Path(__file__).parent/"runtime").iterdir() if x.stem.startswith("ops_")] @@ -78,7 +83,7 @@ class ProfileProgramEvent(ProfileEvent): device:str; name:str; lib:bytes|None; b class ProfileGraphEntry: device:str; name:str|TracingKey; st_id:int; en_id:int; profile_key:bytes|None=None # noqa: E702 @dataclass(frozen=True) -class ProfileGraphEvent(ProfileEvent): ents:list[ProfileGraphEntry]; deps:list[list[int]]; sigs:list[decimal.Decimal] # noqa: E702 +class ProfileGraphEvent(ProfileEvent): ents:list[ProfileGraphEntry]; sigs:list[decimal.Decimal] # noqa: E702 # **************** Buffer + Allocators **************** @@ -246,27 +251,34 @@ class Allocator(Generic[DeviceType]): def __init__(self, dev:DeviceType): self.dev: DeviceType = dev - self.default_buffer_spec: BufferSpec = BufferSpec() - self.cache:dict[tuple[int, BufferSpec|None], list[BufferStorage]] = defaultdict(list) + self.cache:dict[tuple[int, BufferSpec|None], list[tuple[BufferStorage, dict[Compiled, int]]]] = defaultdict(list) def alloc(self, size:int, options:BufferSpec|None=None) -> BufferStorage: assert size > 0, f"alloc size must be positive, getting {size}" - if len(c:=self.cache[(size, options)]): return c.pop() - spec = options if options is not None else self.default_buffer_spec + + # check if old most is free + if (c:=self.cache[(size, options)]) and all(dev.timeline.host.view(fmt='Q')[0] >= freed_at for dev, freed_at in c[0][1].items()): + return c.pop(0)[0] + + spec = options if options is not None else BufferSpec() try: return self._alloc(size, spec) except (RuntimeError, MemoryError): self.free_cache() try: return self._alloc(size, spec) except (RuntimeError, MemoryError) as e: raise MemoryError(f"Allocation of {size_to_str(size)} failed on {self.dev.device}. " - f"Used: {size_to_str(GlobalCounters.mem_used_per_device[self.dev.device])}") from e + f"Used: {size_to_str(GlobalCounters.mem_used_per_device[self.dev.device])}") from e def free(self, storage:BufferStorage, size:int, options:BufferSpec|None=None): - spec = options if options is not None else self.default_buffer_spec - if LRU and self.lru and not spec.nolru and spec.external_ptr is None: self.cache[(size, options)].append(storage) - else: self.do_free(storage, spec) + spec = options if options is not None else BufferSpec() + if not LRU or not self.lru or spec.nolru or spec.external_ptr is not None: return self.do_free(storage, spec) + + # add that buf to cache + users:list[Compiled] = [self.dev, *storage.maps] if storage.host is not None else [] + freed_at = {dev:dev.timeline.host.view(fmt='Q')[1] for dev in users if 'timeline' in vars(dev)} + self.cache[(size, options)].append((storage, freed_at)) def free_cache(self): for (_, options), storages in self.cache.items(): - for storage in storages: self.do_free(storage, options if options is not None else self.default_buffer_spec) + for storage, _ in storages: self.do_free(storage, options if options is not None else BufferSpec()) storages.clear() def do_free(self, storage:BufferStorage, options:BufferSpec): @@ -339,20 +351,20 @@ class TinyELF: lib: bytes name: str target: Target - # tuple of (name, slot, dtype, shape) - signature: tuple[tuple[str|None, int, DType, tuple], ...] + # tuple of (name, addrspace, dtype, shape), one per arg in the kernel's order. an ALU arg is a value, the rest are buffers + signature: tuple[tuple[str|None, AddrSpace, DType, tuple], ...] profile_key: bytes|None = None @staticmethod - def iter_sig(signature:tuple[tuple[str|None, int, DType, tuple], ...], offset:int=0) -> Generator[tuple[int, DType], None, None]: - for _,_,dt,_ in signature: + def iter_sig(signature:tuple[tuple[str|None, AddrSpace, DType, tuple], ...], offset:int=0) -> Generator[tuple[int, DType], None, None]: + for _,a,dt,_ in signature: + if a is not AddrSpace.ALU: dt = dtypes.uint64 # a buffer is passed by its address yield (offset:=round_up(offset, dt.itemsize)), dt offset += dt.itemsize class Program(Generic[DeviceType]): - def __init__(self, dev:DeviceType, obj:TinyELF): pass - def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), - wait=False) -> float|None: pass + def __init__(self, dev:DeviceType, obj:TinyELF, links:dict[str,Program[DeviceType]]): pass + def __call__(self, *args, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), wait=False) -> float|None: pass class Compiled: ifaces:list[Callable] = [] @@ -362,14 +374,13 @@ class Compiled: wait_timeout_ms: float = 30000.0 sleep_timeout_ms: int|None = None can_recover:bool = False - rtalloc_size:int = 64<<20 # the pool every per-linear buffer is carved out of var_vals: dict[str, int] = {} # hcq2 - pm_batch:Any = None - pm_encode:Any = None - pm_lower:Any = None - pm_bufferize:Any = None # one for all devices: each adds the rules of the placeholders it owns, its tags start with its name + pm_encode:PatternMatcher + pm_lower:PatternMatcher + pm_link:PatternMatcher + link_bufs:dict[str, Buffer|Callable[..., Buffer]] = {} def __init__(self, device:str, allocator:Allocator, renderers:list[type[Renderer]], runtime:type[Program[Self]]|None, arch=None): from tinygrad.renderer import Renderer @@ -394,16 +405,8 @@ class Compiled: @property def compiler(self) -> Compiler: return self.renderer.compiler - def runtime(self, obj:TinyELF) -> Program[Self]: return unwrap(self.runtime_t)(self, obj) - - @functools.cache - def rt_allocator(self, spec:BufferSpec) -> BumpAllocator: return BumpAllocator(self.rtalloc_size) - - @functools.cache - def rt_buffer(self, spec:BufferSpec) -> Buffer: - return Buffer(self.device, self.rt_allocator(spec).size, options=spec, preallocate=True) - - def tag(self, *parts:str) -> str: return to_name(self.device, *parts) # of the placeholders it owns + def runtime(self, obj:TinyELF, links:dict[str,Program[Self]]={}) -> Program[Self]: + return unwrap(self.runtime_t)(self, obj, links=links) @functools.cached_property def timeline(self) -> Buffer: # [the signal, the value the last submitted batch signals] @@ -476,21 +479,21 @@ class Compiled: def collect_prof(self): div = decimal.Decimal(self.timestamp_divider) for buf, g in itertools.groupby(self.prof_ents.items(), key=lambda x: x[0][0]) if PROFILE else []: - Compiled.profile_events.append(ProfileGraphEvent([e for _, e in g], [], [t/div for t in buf.host.view(fmt='Q')[:]])) + Compiled.profile_events.append(ProfileGraphEvent([e for _, e in g], [t/div for t in buf.host.view(fmt='Q')[:]])) self.prof_ents.clear() def _at_profile_finalize(self): - if not self.pm_encode.patterns: return + if self.device.split(":")[0] not in HCQ_DEVS: return from tinygrad.tensor import Tensor - tdiffs = [] + tdiffs, n = [], len(Compiled.profile_events) for _ in range(5): with Context(DEBUG=0, BEAM=0, TRACK_MATCH_STATS=0): Tensor.ones(1, device=self.device).contiguous().realize() if not (ents:=list(self.prof_ents.items())): return - self.prof_ents.clear() st = perf_counter_us() self.synchronize() gpu = max(buf.host.view(fmt='Q')[e.en_id] for (buf, _), e in ents)/decimal.Decimal(self.timestamp_divider) tdiffs.append((st+perf_counter_us())/2 - gpu) + del Compiled.profile_events[n:] # cut calibration kernels Compiled.profile_events.append(ProfileDeviceEvent(self.device, statistics.median(tdiffs), self.device_props())) if PROFILE: diff --git a/tinygrad_repo/tinygrad/dtype.py b/tinygrad_repo/tinygrad/dtype.py index 1e5dd2cf78..e404abc694 100644 --- a/tinygrad_repo/tinygrad/dtype.py +++ b/tinygrad_repo/tinygrad/dtype.py @@ -78,6 +78,7 @@ class DType(metaclass=DTypeMetaClass): return float("inf") if dtypes.is_float(self) else True def const(self, val: ConstType): if isinstance(val, InvalidType): return val + if self in dtypes.ints: return truncate[self](int(val)) # NOTE: float('nan') != float('nan'), so we canonicalize here if isinstance(val, float) and math.isnan(val): val = math.nan # int is the default. wrap floats in ConstFloat to distinguish -0.0 from 0.0 in cache diff --git a/tinygrad_repo/tinygrad/engine/jit.py b/tinygrad_repo/tinygrad/engine/jit.py index 271d8b61cd..430e451268 100644 --- a/tinygrad_repo/tinygrad/engine/jit.py +++ b/tinygrad_repo/tinygrad/engine/jit.py @@ -9,7 +9,7 @@ from tinygrad.engine.realize import capturing, compile_linear, link_linear, run_ from tinygrad.schedule.memory import memory_plan_rewrite, _collect_bufs from tinygrad.nn.state import get_parameters from tinygrad.uop.movement import mop_cleanup -from dataclasses import dataclass +from dataclasses import dataclass, field def prune_linear(linear:UOp, needed:set[UOp]) -> tuple[UOp, UOp]: kept, onetime = [], [] @@ -21,19 +21,17 @@ def prune_linear(linear:UOp, needed:set[UOp]) -> tuple[UOp, UOp]: else: onetime.append(si) return linear.replace(src=tuple(kept)), linear.replace(src=tuple(onetime)) -def _copy_input(u:UOp) -> UOp: - if u.on_disk(): raise JitError("cannot make an independent copy of a written DISK input") - run_linear(UOp(Ops.LINEAR, src=((new:=UOp.new_buffer(u.device, u.max_numel(), u.dtype)).store_call(u),))) - return new - -@rewrite_group(lambda linear,held_bufs,input_uops,ret=(): f"JIT {pluralize('call', len(linear.src))}") -def jit_lower(linear:UOp, held_bufs:set[UOp], input_uops:list[UOp]) -> UOp: +@rewrite_group(lambda linear,held_bufs,input_uops,input_vars,ret=(): f"JIT {pluralize('call', len(linear.src))}") +def jit_lower(linear:UOp, held_bufs:set[UOp], input_uops:list[UOp], input_vars:dict[str, int]) -> UOp: if VIZ: graph_rewrite(linear, PatternMatcher([]), name="View captured linear") # parametrize input buffers: map each input buffer UOp to a PARAM with the correct slot index linear = linear.substitute({u: UOp.param(i, u.dtype, u.max_numel(), u.device) for i,u in enumerate(input_uops)}, walk=True) + # unbind input variables, they change every call and are read from var_vals. any other binding was fixed at capture and stays + linear = linear.substitute({v: v.unbound() for v in linear.toposort() if v.is_bound_var and v.expr in input_vars}, walk=True) linear = memory_plan_rewrite(linear, held_bufs) - linear = linear.substitute({u: u.rtag("scratch") for u in linear.toposort() if u.op is Ops.BUFFER and u not in held_bufs}, walk=True) + scratch = {u: UOp.alloc((u.max_numel(),), u.dtype, device=u.device) for u in linear.toposort() if u.op is Ops.BUFFER and u not in held_bufs} + linear = linear.substitute(scratch, walk=True) linear = compile_linear(linear, beam=getenv("JITBEAM", BEAM.value), input_uops=input_uops, cache=False) if VIZ: graph_rewrite(linear, PatternMatcher([]), name="View compiled linear") return linear @@ -54,9 +52,7 @@ class CapturedJit(Generic[ReturnType]): _linear: UOp expected_names: list[int|str] expected_input_info: list[tuple[UOp, tuple[Variable, ...], DType, str]] # (view, variables, dtype, device) per input - - @functools.cached_property - def linear(self) -> UOp: return link_linear(self._linear, allow_cache=False) # do not cache jit + _linears: dict[tuple[UOp, ...], tuple[UOp, dict[UOp, UOp]]] = field(default_factory=dict) # aliased inputs -> (linked linear with copies, staging) def __reduce__(self): return self.__class__, (self.ret, self._linear, self.expected_names, self.expected_input_info) @@ -68,17 +64,20 @@ class CapturedJit(Generic[ReturnType]): return [(t, *ub) for t in get_parameters(self.ret) if (ub:=t.uop.unbind_all())[1]] def __call__(self, input_uops:list[UOp], var_vals:dict[str, int]) -> ReturnType: - concrete = tuple(_copy_input(u) if u in self._jit_bufs else u for u in input_uops) - if DEBUG >= 1 and len(self.linear.src) >= 10: print(f"jit execs {len(self.linear.src)} calls") - run_linear(self.linear, var_vals, input_uops=concrete, jit=True) + # aliased inputs + if (aliased:=tuple(u for u in input_uops if u in self._jit_bufs)) not in self._linears: + if any(u.on_disk() for u in aliased): raise JitError("cannot make an independent copy of a written DISK input") + staging = {u: UOp.new_buffer(u.device, u.max_numel(), u.dtype) for u in aliased} + copies = compile_linear(UOp(Ops.LINEAR, src=tuple(s.store_call(u) for u,s in staging.items()))) + self._linears[aliased] = (link_linear(self._linear.replace(src=copies.src+self._linear.src)), staging) + + linear, staging = self._linears[aliased] + if DEBUG >= 1 and len(linear.src) >= 10: print(f"jit execs {len(linear.src)} calls") + run_linear(linear, var_vals, input_uops=[staging.get(u, u) for u in input_uops], jit=True) for t,u,vals in self._symbolic_ret: t.uop = u.substitute({v:v.bind(var_vals.get(v.expr, i)) for v,i in vals.items()}, walk=True) return self.ret - def free_intermediates(self): - bufs = [u.arg.buffer for u in self._jit_bufs if u.tag == "scratch"] - if not (freed:=[b for b in flatten(b.bufs if isinstance(b, MultiBuffer) else [b] for b in bufs) if b.is_allocated()]): return - for b in freed: b.deallocate() - self.__dict__.pop("linear", None) # should be relinked + def free_intermediates(self): self._linears.clear() def _prepare_jit_inputs(args, kwargs): input_tensors: list[tuple[int|str, Tensor]] = [(name,t) for name,t in list(enumerate(args))+sorted(kwargs.items()) if t.__class__ is Tensor] @@ -159,7 +158,7 @@ class _TinyJit(Generic[ReturnType]): def _buf_or_none(u:UOp) -> Buffer|MultiBuffer|None: return u.arg.buffer if u.op is Ops.BUFFER else None held_bufs = {u for tref in list(all_tensors) if (t:=tref()) is not None for u in t.uop.toposort() if _buf_or_none(u) is not None} held_bufs |= {u for u in big_linear.toposort() if (b:=_buf_or_none(u)) is not None and b.is_allocated()} - linear = jit_lower(big_linear, held_bufs, input_buf_uops) + linear = jit_lower(big_linear, held_bufs, input_buf_uops, var_vals) # drop the pre-planning graph: it keeps the whole capture-time working set allocated (big_linear) or referenced (held_bufs). # the planned linear only uses the arena/held buffers, so the intermediates must be freed before linking and first exec del big_linear, held_bufs diff --git a/tinygrad_repo/tinygrad/engine/realize.py b/tinygrad_repo/tinygrad/engine/realize.py index 2a63a328c8..adc1b3d0fa 100644 --- a/tinygrad_repo/tinygrad/engine/realize.py +++ b/tinygrad_repo/tinygrad/engine/realize.py @@ -5,18 +5,21 @@ from dataclasses import dataclass, replace, field from tinygrad.helpers import CAPTURE_PROCESS_REPLAY, colored, DEBUG, GlobalCounters, ansipad, prod, flatten, Context, to_tuple, tqdm, unwrap from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us, cpu_profile from tinygrad.uop.ops import get_process_replay_loc, Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo, KernelInfo -from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry, DEV, HCQ_RUNTIME_DEV +from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry, DEV, HCQ_RUNTIME_DEV, HOST_DEVS +from tinygrad.dtype import AddrSpace from tinygrad.renderer import Estimates, Renderer from tinygrad.codegen import to_program, to_program_cache, to_program_key, to_program_context from tinygrad.engine.worker import get_worker_pool, terminate_worker_pool # **************** Helpers **************** -def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call.src[1:] if not s.is_bound_var) -def get_call_var_uops(call:UOp, prg:UOp) -> list[UOp]: - # a bound value is a bare CONST: the Variable states the width +def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call.src[1:] if not s.is_variable) +def get_call_kernel_args(call:UOp, prg:UOp, devs:tuple[str, ...]) -> list[UOp]: + # one per kernel param in order: a buffer is its address, a value bound in the call is a CONST, any other value is its slotless var read + # from var_vals at every run (the jit unbinds its input variables) bound = {s.expr: UOp.const(s.arg.val) for s in call.src[1:] if s.is_bound_var} - return [bound.get(v.expr, v) for v in prg.arg.vars] + return [bound.get(p.expr, p.replace(arg=replace(p.arg, slot=-1))).ccast(p.dtype) if p.addrspace is AddrSpace.ALU else + call.src[1+p.arg.slot].getaddr(devs) for p in prg.kernel_params] def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: ast = call.body @@ -29,7 +32,7 @@ def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: def get_call_kernels(call:UOp) -> list[tuple[str, UOp, tuple|None]]: if isinstance(call.arg.aux, HCQInfo): # the submitter itself, then every kernel it enqueues kernels:list[tuple[str, UOp, tuple|None]] = [(Device[call.arg.aux.device[0]].host, call, None)] - return kernels + [(d, call, (name, estimates, key, bufs, io)) for devices,name,estimates,_,key,bufs,io in call.arg.aux.kernels for d in devices] + return kernels + [(d, call, (name, estimates, key, bufs, io)) for d,name,estimates,_,key,bufs,io in call.arg.aux.kernels] ast = call.body if ast.op is Ops.CUSTOM_FUNCTION and ast.arg.name == "validate": return [] return [(d, call, None) for d in to_tuple(call.device)] @@ -67,7 +70,8 @@ def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|No kernels = get_call_kernels(call) # everything below is the per kernel display: exec events for the profiler and DEBUG=2 lines args = [] if isinstance(call.arg.aux, HCQInfo) else resolve_params(call, ctx.input_uops) - lanes = list(unwrap_multi(call, [args[g] for g in call.body.arg.globals] if call.body.op is Ops.PROGRAM else args)) if args else [] + prg_args = [_resolve(call.src[1+g], ctx.input_uops) for g in call.body.arg.globals] if call.body.op is Ops.PROGRAM else args + lanes = list(unwrap_multi(call, prg_args)) if args else [] for i, (device, kcall, stats) in enumerate(kernels): et = ets[i] if i < len(ets) else None bufs = lanes[i][0] if i < len(lanes) else [cast(Buffer, ctx.input_uops[s].buffer) for s in (stats[3] if stats else ())] @@ -101,7 +105,9 @@ def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|No runtime_cache: dict[tuple[bytes, str], Any] = {} def get_runtime(device:str, ast:UOp, cache=True): if (runtime:=runtime_cache.get(key:=(ast.key, device))) is None: - runtime = Device[device].runtime(ast.to_elf()) + links = {call.body.src[0].arg.function_name:get_runtime(device, call.body, cache) + for call in ast.src[1].src if call.op is Ops.CALL and call.body.op is Ops.PROGRAM} + runtime = Device[device].runtime(ast.to_elf(), links=links) if cache: runtime_cache[key] = runtime return runtime @@ -130,8 +136,8 @@ def unwrap_multi(call:UOp, resolved:list[UOp]) -> Iterator[tuple[list[Buffer], d if not any(isinstance(b, MultiBuffer) for b in bufs): yield cast(list[Buffer], bufs), {} else: # the DEVICE axis is bound per device at launch: it's a RANGE in the AST and the _device_num variable after codegen - # on archs like x86, stack args (after the 6th) aren't PARAMs in the body, so also check the PROGRAM's vars - has_dnum = ((isinstance(call.body.arg, ProgramInfo) and any(v.arg.name == '_device_num' for v in call.body.arg.vars)) or + # on x86 the params aren't PARAMs in the body, so also check the PROGRAM's kernel_params + has_dnum = ((isinstance(call.body.arg, ProgramInfo) and any(p.arg.name == '_device_num' for p in call.body.kernel_params)) or any((x.op is Ops.RANGE and x.axis_type is AxisType.DEVICE) or (x.op is Ops.PARAM and x.arg.name == '_device_num') for x in call.body.toposort())) lanes = max(len(b.bufs) for b in bufs if isinstance(b, MultiBuffer)) # a single buffer is shared by every lane @@ -150,32 +156,30 @@ def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp, devices=None) -> list[float|None]: ets:list[float|None] = [] - resolved = resolve_params(call, ctx.input_uops) - for device, (bufs, device_vars) in zip(devices or to_tuple(call.src[1].device), unwrap_multi(call, [resolved[i] for i in ast.arg.globals])): + resolved = [_resolve(call.src[1+i], ctx.input_uops) for i in ast.arg.globals] + for device, (bufs, device_vars) in zip(devices or to_tuple(get_call_arg_uops(call)[0].device), unwrap_multi(call, resolved)): if devices is None and device.split(":")[0] in HOST_DEVS: Device[device].synchronize(ctx.timeout) - var_vals = {**ctx.var_vals, **device_vars} - prg_bufs = [b.ensure_allocated() for b in bufs] + var_vals = {**ctx.var_vals, **device_vars, **{s.expr: s.arg.val for s in call.src[1:] if s.is_bound_var}} + prg_bufs = {i:b.ensure_allocated() for i,b in zip(ast.arg.globals, bufs)} rt = get_runtime(device, ast, cache=ctx.cache) global_sz, local_sz = ast.arg.launch_dims(var_vals) - try: vals = tuple(var_vals[v.expr] if v.is_variable else _resolve(call.src[1 + v.arg.slot], ctx.input_uops).val for v in ast.arg.vars) + try: args = [(var_vals[p.expr] if p.is_variable else _resolve(call.src[1+p.arg.slot], ctx.input_uops).val) if p.addrspace is AddrSpace.ALU else + prg_bufs[p.arg.slot].get_buf(device) for p in ast.kernel_params] except KeyError as e: raise RuntimeError(f"unbound Variable {e}") from None - ets.append(rt(*[b.get_buf(device) for b in prg_bufs], global_size=global_sz, local_size=local_sz, vals=vals, wait=ctx.wait, timeout=ctx.timeout)) + ets.append(rt(*args, global_size=global_sz, local_size=local_sz, wait=ctx.wait, timeout=ctx.timeout)) return ets def exec_validate(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: import numpy as np from tinygrad.dtype import _to_np_dtype - resolved = resolve_params(call, ctx.input_uops) Device["CPU"].synchronize(ctx.timeout) - for bufs, device_vars in unwrap_multi(call, resolved): - bufs, dev_bufs = bufs[:len(bufs)//2], bufs[len(bufs)//2:] - var_vals = {**ctx.var_vals, **device_vars} - cpu_rt = get_runtime("CPU", prg:=to_program(ast.src[0], Device["CPU"].renderer)) - global_size, local_size = prg.arg.launch_dims(var_vals) - cpu_rt(*[bufs[i].ensure_allocated()._buf for i in prg.arg.globals], global_size=global_size, local_size=local_size, vals=prg.arg.vals(var_vals)) + exec_kernel(ctx, call, prg:=to_program(ast.src[0], Device["CPU"].renderer)) # the shadows sit at the call's slots, so it runs like any call + pos = [k for k, s in enumerate(call.src[1:]) if not s.is_variable] + for lane, _ in unwrap_multi(call, resolve_params(call, ctx.input_uops)): + bufs, dev_bufs = dict(zip(pos, lane[:len(lane)//2])), dict(zip(pos, lane[len(lane)//2:])) for i in prg.arg.outs: - dt = _to_np_dtype(resolved[i].dtype) - assert dt is not None, f"no np dtype for {resolved[i].dtype}" + dt = _to_np_dtype(call.src[1+i].dtype) + assert dt is not None, f"no np dtype for {call.src[1+i].dtype}" np.testing.assert_allclose(np.frombuffer(dev_bufs[i].ensure_allocated().as_memoryview(), dt), np.frombuffer(bufs[i].as_memoryview(), dt), rtol=1e-3, atol=1e-3) return [] @@ -185,18 +189,19 @@ def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: ctx = replace(ctx, wait=ctx.wait and not info.skip_wait, var_vals={**ctx.var_vals, **{k: v for d in info.device for k, v in cast(Any, Device[d]).var_vals.items()}}) slots = {d: cast(Buffer, call.src[1 + i].buffer) for d, i in info.slots} - for d in {d for devs, _, _, prof, *_ in info.kernels for d in devs if prof and (slots[d], prof[0]) in Device[d].prof_ents}: Device[d].synchronize() + if PROFILE: + for d in {d for d, _, _, prof, *_ in info.kernels if prof and (slots[d], prof[0]) in Device[d].prof_ents}: Device[d].synchronize() ets = exec_kernel(ctx, call, ast, devices=(Device[info.device[0]].host,)) if not (ctx.wait or PROFILE): return ets - for devs, name, _, prof, pkey, *_ in info.kernels: - for d in (devs if prof else ()): cast(Any, Device[d]).prof_ents[(slots[d], prof[0])] = ProfileGraphEntry(d, name, prof[0], prof[1], pkey) + for d, name, _, prof, pkey, *_ in info.kernels: + if prof: cast(Any, Device[d]).prof_ents[(slots[d], prof[0])] = ProfileGraphEntry(d, name, prof[0], prof[1], pkey) if ctx.wait: for device in info.device: cast(Any, Device[device]).synchronize(timeout=ctx.timeout) def _prof_tm(device:str, prof:tuple[int, ...]) -> float: st, en = (slots[device].host.view(fmt='Q')[x] for x in prof) return float(en-st) / cast(Any, Device[device]).timestamp_divider / 1e6 - return ets + [_prof_tm(device, prof) if ctx.wait else None for devices, _, _, prof, *_ in info.kernels if prof for device in devices] + return ets + [_prof_tm(d, prof) if ctx.wait else None for d, _, _, prof, *_ in info.kernels if prof] # flatten LINEAR-in-LINEAR: any nested LINEAR child gets inlined into its parent's src pm_flatten_linear = PatternMatcher([ @@ -205,10 +210,10 @@ pm_flatten_linear = PatternMatcher([ ]) def _validate(call:UOp, sink:UOp) -> UOp: - params = get_call_arg_uops(call) - shadows = tuple(UOp.new_buffer(("CPU",)*len(p.device) if isinstance(p.device, tuple) else "CPU", prod(p.max_shape), p.dtype) for p in params) - copies = tuple(s.store_call(p) for s, p in zip(shadows, params)) - return UOp(Ops.LINEAR, src=copies + (call, UOp.custom_function("validate", sink).call(*shadows, *params))) + shadows = tuple(p if p.is_bound_var else + UOp.new_buffer(("CPU",)*len(p.device) if isinstance(p.device, tuple) else "CPU", prod(p.max_shape), p.dtype) for p in call.src[1:]) + copies = tuple(s.store_call(p) for s, p in zip(shadows, call.src[1:]) if not p.is_bound_var) + return UOp(Ops.LINEAR, src=copies + (call, UOp.custom_function("validate", sink).call(*shadows, *call.src[1:]))) pm_validate = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.SINK, name="sink"),), name="call", allow_any_len=True), _validate)]) + pm_flatten_linear # ctx is beam value @@ -258,7 +263,7 @@ def lower_and_compile(linear:UOp, verbose=True) -> UOp: return linear.substitute({c: c.replace(src=(to_program_cache[keys[c]], *c.src[1:])) for c in ar}, name="precompile kernels") -from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, HCQInfo, HOST_DEVS # noqa: E402 # down here, hcq2 imports realize +from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, HCQInfo # noqa: E402 # down here, hcq2 imports realize pm_exec = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.STORE, name="ast"),), name="call", allow_any_len=True), exec_copy), @@ -274,8 +279,7 @@ def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:li linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile, cache=cache) return linear -def link_linear(linear:UOp, input_uops:list[UOp]|None=None, allow_cache=True) -> UOp: - return hcq_link(linear, input_uops=input_uops, allow_cache=allow_cache) +def link_linear(linear:UOp, input_uops:list[UOp]|None=None) -> UOp: return hcq_link(linear, input_uops=input_uops) def run_linear(linear:UOp, var_vals:dict[str, int]|None=None, input_uops:Sequence[UOp]=(), update_stats=True, jit=False, wait=False): assert linear.op is Ops.LINEAR, f"run_linear calls on {linear.op}" @@ -287,7 +291,7 @@ def run_linear(linear:UOp, var_vals:dict[str, int]|None=None, input_uops:Sequenc def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None, clear_l2:bool=False) -> Iterator[float]: ctx = ExecContext(var_vals or {}, update_stats=False, wait=True, timeout=timeout, cache=False) - linear = link_linear(compile_linear(UOp(Ops.LINEAR, src=(call,)), beam=0, profile=True, cache=False), allow_cache=ctx.cache) + linear = link_linear(compile_linear(UOp(Ops.LINEAR, src=(call,)), beam=0, profile=True, cache=False)) while True: if clear_l2: if hasattr(dev:=Device[call.device], 'invalidate_caches'): dev.invalidate_caches() diff --git a/tinygrad_repo/tinygrad/engine/worker.py b/tinygrad_repo/tinygrad/engine/worker.py index e723e4a249..e8c2ea097f 100644 --- a/tinygrad_repo/tinygrad/engine/worker.py +++ b/tinygrad_repo/tinygrad/engine/worker.py @@ -1,4 +1,4 @@ -import multiprocessing, multiprocessing.util, atexit, signal, sys, os, threading, contextlib, subprocess +import multiprocessing, multiprocessing.util, atexit, signal, sys, os, threading, contextlib, subprocess, importlib from multiprocessing.context import SpawnContext, SpawnProcess from tinygrad.helpers import Context, getenv, PARALLEL @@ -9,22 +9,18 @@ def _init_worker(): Context(ALLOW_DEVICE_USAGE=0, VIZ=0, TRACK_MATCH_STATS=0).__enter__() signal.signal(signal.SIGINT, signal.SIG_IGN) -# spawn normally reimports the user's __main__ before _init_worker. This replays top-level code and can recursively create pools. There is no public -# multiprocessing switch to skip that import, so hide the two attributes used to locate __main__ while each worker (including replacements) starts. -_spawn_lock, _missing = threading.Lock(), object() +# spawn reimports the user's __main__ before _init_worker, replaying top-level code and potentially creating pools recursively. +# A spec named "__main__" makes multiprocessing.spawn skip that import. +_spawn_lock = threading.Lock() @contextlib.contextmanager def _without_main(): - main = sys.modules.get("__main__") - if main is None: - yield - return + main = sys.modules["__main__"] with _spawn_lock: - saved = {name:getattr(main, name, _missing) for name in ("__file__", "__spec__")} + saved = main.__spec__ try: - for name in saved: setattr(main, name, None) + main.__spec__ = importlib.machinery.ModuleSpec("__main__", None) yield - finally: - for name,value in saved.items(): delattr(main, name) if value is _missing else setattr(main, name, value) + finally: main.__spec__ = saved # spawn execs with the live environ, other threads changing os.environ (pytest) can break it: https://github.com/python/cpython/issues/157613 def _spawnv_passfds(path, args, passfds): diff --git a/tinygrad_repo/tinygrad/helpers.py b/tinygrad_repo/tinygrad/helpers.py index 89873031e2..d16ce38ce5 100644 --- a/tinygrad_repo/tinygrad/helpers.py +++ b/tinygrad_repo/tinygrad/helpers.py @@ -14,12 +14,9 @@ U = TypeVar("U") def prod(x:Iterable[T]) -> T|int: return functools.reduce(operator.mul, x, 1) # NOTE: helpers is not allowed to import from anything else in tinygrad -OSX, WIN = sys.platform == "darwin", sys.platform == "win32" +OSX = sys.platform == "darwin" BASEDIR = pathlib.Path(__file__).parent -# fix colors on Windows, https://stackoverflow.com/questions/12492810/python-how-can-i-make-the-ansi-escape-codes-to-work-also-in-windows -if WIN: os.system("") - def dedup(x:Iterable[T]): return list(dict.fromkeys(x)) # retains list order def to_name(*parts:str) -> str: return "_".join(parts).replace(":", "_").lower() def argfix(*x): @@ -240,7 +237,7 @@ class _DEV(ContextVar): DEV, DEBUG, BEAM, NOOPT = _DEV("DEV", ""), ContextVar("DEBUG", 0), ContextVar("BEAM", 0), ContextVar("NOOPT", 0) IMAGE, FLOAT16, OPENPILOT_HACKS = ContextVar("IMAGE", 0), ContextVar("FLOAT16", 0), ContextVar("OPENPILOT_HACKS", 0) JIT = ContextVar("JIT", 1) -WINO, CAPTURING, TRACEMETA, NO_COLOR = ContextVar("WINO", 0), ContextVar("CAPTURING", 1), ContextVar("TRACEMETA", 1), ContextVar("NO_COLOR", 0) +WINO, CAPTURING, NO_COLOR = ContextVar("WINO", 0), ContextVar("CAPTURING", 1), ContextVar("NO_COLOR", 0) TRAINING = ContextVar("TRAINING", 0) USE_TC, TC_SELECT, TC_OPT, TC_MIN_GLOBALS = ContextVar("TC", 1), ContextVar("TC_SELECT", -1), ContextVar("TC_OPT", 0), ContextVar("TC_MIN_GLOBALS", 0) TRANSCENDENTAL = ContextVar("TRANSCENDENTAL", 1) @@ -294,13 +291,6 @@ USE_ATOMICS = ContextVar("USE_ATOMICS", 0) # don't allow broadcast DISALLOW_BROADCAST = ContextVar("DISALLOW_BROADCAST", 0) -@dataclass(frozen=True) -class Metadata: - name: str - backward: bool = False - def __hash__(self): return hash(self.name) - def __str__(self): return self.name + (" bw" if self.backward else "") - # **************** global state Counters **************** class GlobalCounters: diff --git a/tinygrad_repo/tinygrad/llm/cli.py b/tinygrad_repo/tinygrad/llm/cli.py index 2a008564e1..e337fc7923 100644 --- a/tinygrad_repo/tinygrad/llm/cli.py +++ b/tinygrad_repo/tinygrad/llm/cli.py @@ -1,5 +1,5 @@ from __future__ import annotations -import sys, argparse, codecs, itertools, typing, re, unicodedata, json, time +import sys, argparse, codecs, itertools, typing, re, unicodedata, json, time, pathlib from typing import TYPE_CHECKING from tinygrad import nn from tinygrad.uop.ops import UOp, Ops @@ -10,7 +10,7 @@ if TYPE_CHECKING: class SimpleTokenizer: def __init__(self, normal_tokens:dict[str, int], special_tokens:dict[str, int], preset:str="llama3", - bos_id:int|None=None, eos_id:int=0, eot_id:int|None=None, eom_id:int|None=None): + bos_id:int|None=None, eos_id:int=0, eot_id:int|None=None, eom_id:int|None=None, chat_template:str|None=None): preset = {"qwen35":"qwen2","qwen35moe":"qwen2"}.get(preset, preset) if preset not in ("llama3","llama-v3","llama-bpe","qwen2","olmo","kimi-k2","tekken","glm4","gpt-4o"): raise ValueError(f"Invalid tokenizer preset '{preset}'") @@ -30,17 +30,18 @@ class SimpleTokenizer: r_ws, r_p_N, r_p_L = r"\t\n\x0b\x0c\r\x85" + ucat_range("Z"), ucat_range("N"), ucat_range("L") contr, r_l, r_n = "(?i:'s|'t|'re|'ve|'m|'ll|'d)", f"[^\\r\\n{r_p_N}{r_p_L}]?", f"[{r_p_N}]" if preset == "tekken" else f"[{r_p_N}]{{1,3}}" r_p, r_w, r_t = f" ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*", f"{contr}|{r_l}[{r_p_L}]+", f"[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+" - if preset in ("tekken", "gpt-4o"): + if preset in ("tekken", "gpt-4o", "kimi-k2"): r_up, r_lo = ucat_range(("Lu","Lt","Lm","Lo","M")), ucat_range(("Ll","Lm","Lo","M")) - sfx = f"{contr}?" if preset == "gpt-4o" else "" - r_p, r_w = f" ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n/]*", f"{r_l}[{r_up}]*[{r_lo}]+{sfx}|{r_l}[{r_up}]+[{r_lo}]*{sfx}" + sfx = "" if preset == "tekken" else f"{contr}?" + r_w = f"{r_l}[{r_up}]*[{r_lo}]+{sfx}|{r_l}[{r_up}]+[{r_lo}]*{sfx}" + if preset != "kimi-k2": r_p = f" ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n/]*" self._split_to_word = re.compile(f"{r_w}|{r_n}|{r_p}|{r_t}") self._split_to_sentence = re.compile("|".join(re.escape(tok) for tok in special_tokens.keys()) if special_tokens else r"(?!)") self._normal_tokens = {bytes(self._byte_decoder[c] for c in tok): tid for tok, tid in normal_tokens.items()} self._special_tokens = special_tokens self._tok2bytes = {tid: tok for tok, tid in self._normal_tokens.items()} | {tid: tok.encode() for tok, tid in self._special_tokens.items()} - self.preset = preset + self.preset, self.chat_template = preset, chat_template self.bos_id, self.eos_id, self.eot_id, self.eom_id = bos_id, eos_id, eot_id, eom_id @staticmethod @@ -52,7 +53,21 @@ class SimpleTokenizer: return SimpleTokenizer(dict(normal_tokens), special_tokens_dict, kv["tokenizer.ggml.pre"], bos_id=kv.get('tokenizer.ggml.bos_token_id') if kv.get('tokenizer.ggml.add_bos_token', True) else None, eos_id=kv.get('tokenizer.ggml.eos_token_id', 0), eot_id=kv.get('tokenizer.ggml.eot_token_id', special_tokens_dict.get('<|im_end|>')), - eom_id=kv.get('tokenizer.ggml.eom_token_id')) + eom_id=kv.get('tokenizer.ggml.eom_token_id'), chat_template=kv.get('tokenizer.chat_template')) + + @staticmethod + def from_hf(path:str|pathlib.Path, preset:str): + path = pathlib.Path(path) + cfg, tok = (json.loads((path / name).read_text()) for name in ("config.json", "tokenizer.json")) + tok_cfg = json.loads((path / "tokenizer_config.json").read_text()) if (path / "tokenizer_config.json").exists() else {} + added = {t["content"]: t["id"] for t in tok.get("added_tokens", [])} + vocab = tok["model"]["vocab"] + eos = cfg.get("eos_token_id", 0) + eos = eos if isinstance(eos, list) else [eos] + return SimpleTokenizer({t: i for t, i in vocab.items() if t not in added}, added, preset, + bos_id=vocab.get(tok_cfg.get("bos_token")), eos_id=eos[0], + eot_id=eos[1] if len(eos) > 1 else None, eom_id=eos[2] if len(eos) > 2 else None, + chat_template=template.read_text() if (template:=path / "chat_template.jinja").exists() else tok_cfg.get("chat_template")) def _encode_word(self, word:bytes) -> list[int]: if (early_token:=self._normal_tokens.get(word)) is not None: return [early_token] @@ -141,7 +156,8 @@ from tinygrad.llm.serve import LLMServer def main(): parser = argparse.ArgumentParser() - parser.add_argument("--model", "-m", default=list(models.keys())[0], help=f"Model choice ({', '.join(models.keys())}) or path to a local GGUF file") + parser.add_argument("--model", "-m", default=list(models.keys())[0], + help=f"Model choice ({', '.join(models.keys())}) or local GGUF file/HF checkpoint directory") parser.add_argument("--max_context", type=int, default=4096, help="Max Context Length") parser.add_argument("--serve", nargs='?', type=int, const=8000, metavar="PORT", help="Run OpenAI compatible API (optional port, default 8000)") parser.add_argument("--warmup", action="store_true", help="warmup the JIT") @@ -155,19 +171,20 @@ def main(): # load the model st = time.perf_counter() with Context(DEBUG=max(DEBUG.value, 1 if args.serve else 0)): - model, kv = Transformer.from_gguf(fetch(models.get(args.model, args.model)), args.max_context, shard=args.shard) - model_name = kv.get('general.name') or kv.get('general.basename') or args.model + path = fetch(models.get(args.model, args.model)) + model, kv = (Transformer.from_hf if path.is_dir() else Transformer.from_gguf)(path, args.max_context, shard=args.shard) + model_name = kv.get('general.name') or kv.get('general.basename') or kv.get('_name_or_path') or args.model file_sizes = [y.nbytes()*args.shard for y in UOp.sink(*[x.uop for x in nn.state.get_parameters(model)]).toposort() if y.op is Ops.BUFFER] print(f"loaded model \"{model_name}\" at {sum(file_sizes)*1e-9/(time.perf_counter()-st):.2f} GB/s with {sum(file_sizes):,} bytes " f"and {sum(x.numel() for x in nn.state.get_parameters(model)):,} params, " f"max context {args.max_context} on {nn.state.get_parameters(model)[0].device}") # get tokenizer - tok = SimpleTokenizer.from_gguf_kv(kv) + tok = SimpleTokenizer.from_hf(path, preset='glm4') if path.is_dir() else SimpleTokenizer.from_gguf_kv(kv) # use the model's chat template if jinja2 is available (enables model-specific formatting) template: jinja2.Template|FallbackTemplate = FallbackTemplate(tok) - if not args.no_chat_template and (ct := kv.get('tokenizer.chat_template')) is not None: + if not args.no_chat_template and (ct := tok.chat_template) is not None: try: import jinja2 env = jinja2.Environment(extensions=['jinja2.ext.loopcontrols']) # (for {% break %}) diff --git a/tinygrad_repo/tinygrad/llm/hf.py b/tinygrad_repo/tinygrad/llm/hf.py new file mode 100644 index 0000000000..c886060d74 --- /dev/null +++ b/tinygrad_repo/tinygrad/llm/hf.py @@ -0,0 +1,44 @@ +import json, pathlib +from tinygrad import Tensor, dtypes, nn, Device + +def e4m3fn_to_fnuz(weight:Tensor, scale:Tensor) -> tuple[Tensor, Tensor]: + bits = weight.bitcast(dtypes.uint8) + return bits.eq(0x80).where(0, bits).contiguous().bitcast(dtypes.fp8e4m3fnuz), scale * 2 + +def hf_load(path:pathlib.Path, devices:tuple[str, ...], names:dict[str, str|list[str]], shard_map:dict[str, int]) -> dict[str, Tensor]: + index = path / "model.safetensors.index.json" + weights:dict[str, Tensor] = {} + if index.exists(): weight_map = json.loads(index.read_text())["weight_map"] + else: + weights = nn.state.safe_load(path / "model.safetensors") + weight_map = dict.fromkeys(weights, "model.safetensors") + + def pop_tensor(source:str|list[str], axis:int|None) -> Tensor: + sources = source if isinstance(source, list) else [source] + stack = isinstance(source, list) + parts = [] + for s in sources: + if s not in weights: weights.update(nn.state.safe_load(path / weight_map[s])) + parts.append(weights.pop(s)) + dtype, shape = parts[0].dtype, parts[0].shape + if stack: shape = (len(parts), *shape) + assert axis is None or shape[axis] % len(devices) == 0, f"cannot split {sources[0]} on axis {axis}" + if stack: w = Tensor.stack(*[t.bitcast(dtypes.uint8).to(devices[0]) for t in parts]).contiguous().realize() + else: w = parts[0].bitcast(dtypes.uint8) + return w.shard(devices, axis).contiguous().realize().bitcast(dtype) + + use_fnuz = dtypes.fp8e4m3fnuz in Device[devices[0]].renderer.supported_dtypes() + state_dict:dict[str, Tensor] = {} + for name, source in names.items(): + first = source[0] if isinstance(source, list) else source + if first not in weight_map: continue + axis = shard_map.get(name) + w = pop_tensor(source, axis) + scale_source = [s + "_scale_inv" for s in source] if isinstance(source, list) else source + "_scale_inv" + scale = pop_tensor(scale_source, axis) if first + "_scale_inv" in weight_map else None + if scale is not None and use_fnuz: + w, scale = e4m3fn_to_fnuz(w, scale) + Tensor.realize(w, scale) + state_dict[name] = w + if scale is not None: state_dict[name + "_scale_inv"] = scale + return state_dict diff --git a/tinygrad_repo/tinygrad/llm/kernels/amd.py b/tinygrad_repo/tinygrad/llm/kernels/amd.py index d60cd086f6..9edaa6e0bf 100644 --- a/tinygrad_repo/tinygrad/llm/kernels/amd.py +++ b/tinygrad_repo/tinygrad/llm/kernels/amd.py @@ -4,7 +4,7 @@ from typing import Callable, cast from tinygrad import Tensor, UOp, nn, Device, Context from tinygrad.llm.gguf import ggml_data_to_tensor from tinygrad.dtype import AddrSpace, dtypes -from tinygrad.helpers import prod, getenv +from tinygrad.helpers import prod, getenv, unwrap from tinygrad.uop.ops import AxisType, KernelInfo, Ops, resolve from tinygrad.renderer.cstyle import HIPRenderer @@ -14,12 +14,13 @@ WAVES_M, WAVES_N, LANES_PER_WAVE_M, LANES_PER_WAVE_N = 2, 2, 2, 16 WMMA_ACC, THREADS_PER_BLOCK = WMMA_M // LANES_PER_WAVE_M, WARP_SIZE * WAVES_M * WAVES_N LDS_PAD, WMMA_ARG, LOG2E = 4, ((WMMA_N, WMMA_M, WMMA_K), 32), math.log2(math.e) GGML_BLOCK_SIZE, Q8_GROUP_SIZE = 256, 32 -Q2_K, Q3_K, Q4_K, Q5_K, Q6_K = 10, 11, 12, 13, 14 +Q8_0, Q2_K, Q3_K, Q4_K, Q5_K, Q6_K = 8, 10, 11, 12, 13, 14 IQ2_XS, IQ3_XXS, IQ4_NL, IQ3_S, IQ2_S, IQ4_XS = 17, 18, 20, 21, 22, 23 -QUANT_SIZES = {Q2_K: 84, Q3_K: 110, Q4_K: 144, Q5_K: 176, Q6_K: 210, IQ2_XS: 74, +QUANT_SIZES = {Q8_0: 272, Q2_K: 84, Q3_K: 110, Q4_K: 144, Q5_K: 176, Q6_K: 210, IQ2_XS: 74, IQ3_XXS: 98, IQ4_NL: 144, IQ3_S: 110, IQ2_S: 82, IQ4_XS: 136} # bytes per 256 weights -HALFWORD_QUANTS = (Q6_K, Q2_K, Q3_K, IQ2_XS, IQ3_XXS, IQ4_NL, IQ3_S, IQ2_S) -QUANT_NAMES = {Q2_K: "q2_k", Q3_K: "q3_k", Q4_K: "q4_k", Q5_K: "q5_k", Q6_K: "q6", IQ4_XS: "iq4_xs", +HALFWORD_QUANTS = (Q8_0, Q6_K, Q2_K, Q3_K, IQ2_XS, IQ3_XXS, IQ4_NL, IQ3_S, IQ2_S) +SMALL_BLOCK_QUANTS = {Q8_0: 17, IQ4_NL: 9} +QUANT_NAMES = {Q8_0: "q8_0", Q2_K: "q2_k", Q3_K: "q3_k", Q4_K: "q4_k", Q5_K: "q5_k", Q6_K: "q6", IQ4_XS: "iq4_xs", IQ2_XS: "iq2_xs", IQ3_XXS: "iq3_xxs", IQ4_NL: "iq4_nl", IQ3_S: "iq3_s", IQ2_S: "iq2_s"} def _unbind(v:int|UOp) -> int|UOp: return v.unbind_all()[0] if isinstance(v, UOp) else v @@ -27,17 +28,20 @@ def _unbind(v:int|UOp) -> int|UOp: return v.unbind_all()[0] if isinstance(v, UOp @functools.cache def amd_custom_kernels_supported(device:str|tuple[str, ...]|None) -> bool: if getenv("DISABLE_AMD_KERNELS"): return False - # Wave32 and WMMA kernels support RDNA3/4. CDNA's wave64/MFMA is not supported. + # Wave32 and WMMA kernels support RDNA3/4 and CDNA3/4 (CDNA packs the 32-lane groups into wave64s, ds_swizzle stays within 32 lanes). if isinstance(device, tuple): device = device[0] if device is None or device.split(":")[0] != "AMD": return False # @function contexts set ALLOW_DEVICE_USAGE=0 (scheduling must not open devices); the device is always open here with Context(ALLOW_DEVICE_USAGE=1): - return (t:=getattr(Device[device], "target", None)) is not None and t[0] in (11, 12) and isinstance(Device[device].renderer, HIPRenderer) + return (t:=getattr(Device[device], "target", None)) is not None and t[0] in (11, 12, 9) and isinstance(Device[device].renderer, HIPRenderer) @functools.cache -def _wmma_rdna4(device:str|tuple[str, ...]) -> bool: +def _gfx_version(device:str|tuple[str, ...]|None) -> tuple[int, ...]: if isinstance(device, tuple): device = device[0] - with Context(ALLOW_DEVICE_USAGE=1): return getattr(Device[device], "target")[0] == 12 + with Context(ALLOW_DEVICE_USAGE=1): return getattr(Device[unwrap(device)], "target") + +def _is_rdna4(device:str|tuple[str, ...]|None) -> bool: return _gfx_version(device)[0] == 12 +def _is_cdna(device:str|tuple[str, ...]|None) -> bool: return _gfx_version(device)[0] == 9 def warp_reduce(val:UOp, maximum:bool=False, full_wave:bool=False) -> UOp: for offset in ((16, 8, 4, 2, 1) if full_wave else (8, 4, 2, 1)): @@ -51,44 +55,66 @@ def _reg(shape:tuple[int, ...], value:float, dep:UOp|None=None) -> UOp: ret = UOp.alloc(shape, dtypes.float, addrspace=AddrSpace.REG) return ret.after((ret if dep is None else ret.after(dep)).store(ret.const_like(value))) +def fp8_matmul(x:Tensor, weight:Tensor, scale:Tensor, block_size:tuple[int, int]) -> Tensor: + block_n, block_k = block_size + n, k = weight.shape[-2:] + x_blocks = x.float().pad((0, -k % block_k)).unflatten(-1, (-1, block_k)).rearrange("... t g k -> ... g t k") + w_blocks = weight.pad((0, -k % block_k)).unflatten(-1, (-1, block_k)).rearrange("... n g k -> ... g k n") + limit = 224.0 if weight.dtype == dtypes.fp8e4m3fnuz else 448.0 + x_scale = x_blocks.abs().max(-1, keepdim=True).clamp(min_=1e-10) / limit + quantized = (x_blocks / x_scale).clamp(-limit, limit).cast(weight.dtype).contiguous() + partials = (quantized.float() @ w_blocks.float()).contiguous() + w_scale = scale.transpose(-1, -2).repeat_interleave(block_n, -1)[..., None, :n] + return (partials * x_scale * w_scale).sum(-3).cast(x.dtype) + # ******** quant linear: q8-activation kernels over packed ggml weights ******** +def find_packed(decoded:Tensor, in_features:int) -> tuple[int, Tensor]|None: + """(ggml type, packed words) if `decoded` is exactly the dequantization of packed ggml weights, rows of in_features""" + if in_features % Q8_GROUP_SIZE: return None + packed_sizes = {typ: decoded.numel() * type_size // 256 for typ,type_size in QUANT_SIZES.items() + if in_features % GGML_BLOCK_SIZE == 0 or typ in SMALL_BLOCK_QUANTS} + graph = decoded.uop.toposort() + raw = next((u for u in graph if u.op in (Ops.SHRINK, Ops.BUFFER, Ops.UNSHARD) and u.dtype == dtypes.uint8 and + prod(u.shape) in packed_sizes.values()), None) + if raw is None: return None + # Only unwrap storage/order-preserving views, then require the exact dequantization expression. + # This rejects subsequent arithmetic and permutations, including RoPE's concatenated query weights. + # a permute keeps the order if the axes not of size 1 on a device stay sorted (gguf_shard moves the devices) + def unwrapped(u:UOp) -> UOp: + while u.op in (Ops.RESHAPE, Ops.STAGE) or (u.op is Ops.CAST and dtypes.is_float(u.dtype) and dtypes.is_float(u.src[0].dtype)) or \ + (u.op is Ops.PERMUTE and (kept:=[a for a in u.marg if u.src[0].shard_shape[a] != 1]) == sorted(kept)): + u = u.src[0] + return u + # Several formats have the same byte count (Q3_K/IQ3_S and Q4_K/IQ4_NL). Match the expression, not just the size. + for ggml_type, size in packed_sizes.items(): + if size != prod(raw.shape): continue + expected = ggml_data_to_tensor(Tensor(raw).flatten(), int(decoded.numel()), ggml_type) + if unwrapped(decoded.uop).key == unwrapped(expected.uop).key: break + else: return None + # Some blocks are only halfword-aligned; keep all formats as zero-copy views of the GGUF storage. + word_dtype = dtypes.uint16 if ggml_type in HALFWORD_QUANTS else dtypes.uint32 + raw_offset = raw.contiguous_view_offset() + if raw_offset is None or raw_offset % word_dtype.itemsize or raw.buf_uop.dtype != dtypes.uint8: return None + return ggml_type, Tensor(raw).flatten().bitcast(word_dtype).contiguous() + class Linear(nn.Linear): ggml_type:int|None = None use_custom_quant = True shard_axis:int|None = None + weight_scale_inv:Tensor|None = None + weight_block_size:tuple[int, int] = (128, 128) def __init__(self, in_features:int, out_features:int, bias=True): super().__init__(in_features, out_features, bias) self.in_features, self.out_features = in_features, out_features def set_quantized(self, decoded:Tensor): - if self.in_features % GGML_BLOCK_SIZE: return - packed_sizes = {typ: decoded.numel() // 256 * type_size for typ,type_size in QUANT_SIZES.items()} - graph = decoded.uop.toposort() - raw = next((u for u in graph if u.op in (Ops.SHRINK, Ops.BUFFER, Ops.UNSHARD) and u.dtype == dtypes.uint8 and - prod(u.shape) in packed_sizes.values()), None) - if raw is None: return - # Only unwrap storage/order-preserving views, then require the exact dequantization expression. - # This rejects subsequent arithmetic and permutations, including RoPE's concatenated query weights. - # a permute keeps the order if the axes not of size 1 on a device stay sorted (gguf_shard moves the devices) - def unwrapped(u:UOp) -> UOp: - while u.op in (Ops.RESHAPE, Ops.STAGE) or (u.op is Ops.CAST and dtypes.is_float(u.dtype) and dtypes.is_float(u.src[0].dtype)) or \ - (u.op is Ops.PERMUTE and (kept:=[a for a in u.marg if u.src[0].shard_shape[a] != 1]) == sorted(kept)): - u = u.src[0] - return u - # Several formats have the same byte count (Q3_K/IQ3_S and Q4_K/IQ4_NL). Match the expression, not just the size. - for ggml_type, size in packed_sizes.items(): - if size != prod(raw.shape): continue - expected = ggml_data_to_tensor(Tensor(raw).flatten(), self.in_features * self.out_features, ggml_type) - if unwrapped(decoded.uop).key == unwrapped(expected.uop).key: break - else: return - # Some blocks are only halfword-aligned; keep all formats as zero-copy views of the GGUF storage. - word_dtype = dtypes.uint16 if ggml_type in HALFWORD_QUANTS else dtypes.uint32 - raw_offset = raw.contiguous_view_offset() - if raw_offset is None or raw_offset % word_dtype.itemsize or raw.buf_uop.dtype != dtypes.uint8: return - self.ggml_type = ggml_type + if (found:=find_packed(decoded, self.in_features)) is None: return + self.ggml_type, self.weight = found self.shard_axis = decoded.uop.axis - self.weight = Tensor(raw).flatten().bitcast(word_dtype).contiguous() def __call__(self, x:Tensor) -> Tensor: + if self.weight_scale_inv is not None: + out = fp8_matmul(x, self.weight, self.weight_scale_inv, self.weight_block_size) + return out + self.bias if self.bias is not None else out supported = self.use_custom_quant and amd_custom_kernels_supported(self.weight.device) if self.ggml_type is None and supported: self.set_quantized(self.weight) @@ -109,7 +135,33 @@ class Linear(nn.Linear): return out.shrink(tuple((0, s) for s in (*x.shape[:-1], self.out_features))) return super().__call__(x) -def _amd_dp4a(a:UOp, b:UOp, c:UOp) -> UOp: +class ExpertWeights: + """Like Linear but with num_experts dimension. Weight shape: (num_experts, out_features, in_features).""" + ggml_type:int|None = None + use_custom_quant = True + weight_scale_inv:Tensor|None = None + weight_block_size:tuple[int, int] = (128, 128) + def __init__(self, num_experts:int, in_features:int, out_features:int, bias:bool=False): + self.in_features = in_features + self.weight = Tensor.zeros(num_experts, out_features, in_features) + if bias: self.bias = Tensor.zeros(num_experts, out_features) + def set_quantized(self): + if self.weight.uop.axis == 0 or (found:=find_packed(self.weight, self.in_features)) is None: return + self.ggml_type, self.packed = found + def __call__(self, sel:Tensor, x:Tensor) -> Tensor: + # sel: (B, T, k), x: (B, T, 1, in) or (B, T, k, in) -> output: (B, T, k, out) + supported = self.weight_scale_inv is None and self.use_custom_quant and amd_custom_kernels_supported(self.weight.device) + if self.ggml_type is None and supported: + self.set_quantized() + if self.ggml_type is None: self.use_custom_quant = supported = False # not a supported quant format + if self.weight_scale_inv is not None: + ret = fp8_matmul(x.unsqueeze(-2), self.weight[sel], self.weight_scale_inv[sel], self.weight_block_size).squeeze(-2) + elif self.ggml_type is not None and supported and isinstance(x.shape[1], int) and x.shape[1] == 1: ret = moe_linear(self, sel, x) + else: ret = (x.unsqueeze(-2) @ self.weight[sel].transpose(-1, -2)).contiguous().squeeze(-2) + return ret + self.bias[sel] if hasattr(self, 'bias') else ret + +def _amd_dp4a(a:UOp, b:UOp, c:UOp, cdna:bool) -> UOp: + if cdna: return UOp(Ops.CUSTOMI, src=(a, b, c), arg=("__builtin_amdgcn_sdot4({}, {}, {}, false)", dtypes.int32)) # CDNA has no sudot4 (dot8-insts) return UOp(Ops.CUSTOMI, src=(a, b, c), arg=("__builtin_amdgcn_sudot4(true, {}, true, {}, {}, false)", dtypes.int32)) def _amd_byte_perm(a:UOp, b:UOp, selectors:UOp) -> UOp: @@ -225,6 +277,7 @@ def _quant_word(raw:UOp, base:UOp, subgroup:UOp, i:int|UOp, ggml_type:int, grid: # Four packed weight bytes, shared by integer-dot decode and FP16 WMMA prefill. def byte(offset): return _load_byte(raw, base, offset) def word(offset): return _load_u32(raw, base, offset, stream=ggml_type == Q6_K) + if ggml_type == Q8_0: return _load_u32(raw, base, 2+i*4, stream=True) if ggml_type in (Q4_K, Q5_K): offset = (4 if ggml_type == Q4_K else 12) + (subgroup//2)*8 weights = (_amd_load(raw[base+offset], 8)[i] >> ((subgroup&1)*4).cast(dtypes.uint32)) & 0x0f0f0f0f @@ -259,12 +312,19 @@ def _quant_word(raw:UOp, base:UOp, subgroup:UOp, i:int|UOp, ggml_type:int, grid: return _iq_signed_word(grid[index*grid_words+i%grid_words], signs >> (4*(i%2))) @functools.cache -def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, *grids:UOp, - out_features:int, in_features:int, ggml_type:int) -> UOp: +def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, *grids:UOp, out_features:int, in_features:int, ggml_type:int, + cdna:bool=False) -> UOp: + return _quant_decode_body(out, raw, xq, xd, xs, grids, out_features, in_features, ggml_type, cdna) + +def _quant_decode_body(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, grids:tuple[UOp, ...], out_features:int, in_features:int, ggml_type:int, + cdna:bool, gate:Callable|None=None, xtoken:Callable|None=None, name:str="linear_") -> UOp: + # gate(token, base) -> base: moves the weight rows, xtoken(token) -> the activation row of an output row def group_dot(token:UOp, output:UOp, group:UOp) -> UOp: block, subgroup = group//8, group%8 base = (output*in_features//256 + block) * (QUANT_SIZES[ggml_type]//raw.dtype.itemsize) - if ggml_type == IQ4_NL: base = (output*in_features//32 + group)*9 + if ggml_type in SMALL_BLOCK_QUANTS: base = (output*in_features//32 + group)*SMALL_BLOCK_QUANTS[ggml_type] + if gate is not None: base = gate(token, base) + if xtoken is not None: token = xtoken(token) def byte(offset): return _load_byte(raw, base, offset) def word(offset): return _load_u32(raw, base, offset) xwords = _amd_load(xq[token, group, 0], 8) @@ -273,7 +333,7 @@ def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, *grids:UOp, for i in range(8): weights = _quant_word(raw, base, subgroup, i, ggml_type, grids[0] if grids else None) acc = i//(8//len(dots)) - dots[acc] = _amd_dp4a(weights, xwords[i], dots[acc]) + dots[acc] = _amd_dp4a(weights, xwords[i], dots[acc], cdna) if ggml_type in (Q4_K, Q5_K): d, dmin, scale, minimum = _q5_scales(raw, base, subgroup) total = dots[0].float()*d*scale - (xs[token, group, 0].load()+xs[token, group, 1].load())*dmin*minimum @@ -303,24 +363,52 @@ def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, *grids:UOp, if ggml_type == IQ3_S: total *= (1+2*((byte(106+subgroup//2) >> ((subgroup%2)*4)) & 15)).float() if ggml_type == IQ3_XXS: total *= ((word(66+subgroup*4) >> 28).float()+0.5)*0.5 return total * xd[token, group] - return _decode_linear(out, out_features, in_features//32, group_dot, "linear_"+QUANT_NAMES[ggml_type]) + return _decode_linear(out, out_features, in_features//32, group_dot, name+QUANT_NAMES[ggml_type]) -def _wmma_layout(out:UOp, out_features:int, token_tile:int, output_tiles:int): +@functools.cache +def _moe_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, sel:UOp, *grids:UOp, + out_features:int, in_features:int, ggml_type:int, x_share:int, cdna:bool=False) -> UOp: + # out rows are (token, slot): the slot's expert reads its weight rows, x_share slots share an activation row + expert_words = out_features * in_features * QUANT_SIZES[ggml_type] // 256 // raw.dtype.itemsize + def gate(row:UOp, base:UOp) -> UOp: return base + sel[row].cast(dtypes.weakint).maximum(0).minimum(raw.shape[0]//expert_words-1) * expert_words + def xtoken(row:UOp) -> UOp: return row // x_share + return _quant_decode_body(out, raw, xq, xd, xs, grids, out_features, in_features, ggml_type, cdna, gate, xtoken, "moe_") + +def moe_linear(layer:ExpertWeights, sel:Tensor, x:Tensor) -> Tensor: + B, T, slots = cast(tuple[int, int, int], sel.shape[-3:]) # sel: (B, T, k), weight: (E, O, I) + in_features, x_slots = int(x.uop.shard_shape[-1]), int(x.shape[-2]) # gate/up: (B, T, 1, D), down: (B, T, k, H) + assert x_slots in (1, slots), f"x has {x_slots} rows per token, expected 1 or {slots}" + local_out, global_out, ggml_type = int(layer.weight.uop.max_shard_shape[1]), int(layer.weight.shape[1]), unwrap(layer.ggml_type) + splits = layer.in_features // in_features + assert (layer.weight.uop.axis == 2) == (splits > 1), f"input features split over {splits} devices, weight split on axis {layer.weight.uop.axis}" + xq, xd, xs = q8_quantize(x.contiguous(), B*T*x_slots, in_features) + out = Tensor.empty(B*T*slots, global_out, splits*((in_features+1023)//1024), dtype=dtypes.float32, device=x.device, axis=layer.weight.uop.axis) + fxn = functools.partial(_moe_decode_kernel, out_features=local_out, in_features=in_features, ggml_type=ggml_type, x_share=slots//x_slots, + cdna=_is_cdna(x.device)) + extra = (_iq_grid(x.device, ggml_type),) if ggml_type in (IQ2_XS, IQ3_XXS, IQ3_S, IQ2_S) else () + result = Tensor.custom_kernel(out, layer.packed, xq, xd, xs, sel.cast(dtypes.int32).flatten(), *extra, fxn=fxn)[0].sum(-1) + return result.reshape(B, T, slots, global_out) + +# kgroups: groups of 16 lanes that split k in a 16x16x16 fragment, RDNA3 copies a and b in both halves +def _wmma_words(lane_group:UOp, kgroups:int) -> tuple[UOp|int, ...]: + return tuple(range(4)) if kgroups == 1 else tuple(lane_group + kgroups*j for j in range(4//kgroups)) + +def _wmma_layout(out:UOp, out_features:int, token_tile:int, output_tiles:int, lanes:int): if out_features % (16*output_tiles): output_tiles = 1 output_waves = 2 if out_features % (32*output_tiles) == 0 else 1 token_block, output_block = UOp.range(out.shape[0]//token_tile, 0), UOp.range(out_features//(16*output_tiles*output_waves), 1) # lane is a hardware WARP range (like the flash kernel): the fragment math stays visible without being # range-split into nested loops, which would scramble the WMMA fragment layout - lane, wave = UOp.range(WARP_SIZE, -1, axis_type=AxisType.WARP), UOp.range(output_waves, 3, axis_type=AxisType.LOCAL) - col, half = lane % 16, lane // 16 + lane, wave = UOp.range(lanes, -1, axis_type=AxisType.WARP), UOp.range(output_waves, 3, axis_type=AxisType.LOCAL) + col, lane_group, rows = lane % 16, lane // 16, 256 // lanes outputs = tuple((output_block*output_waves+wave)*(16*output_tiles) + tile*16 + col for tile in range(output_tiles)) inputs = tuple(token_block*token_tile + tile*16 + col for tile in range(token_tile//16)) - tokens = tuple(tuple(token_block*token_tile + tile*16 + half*8 + i for i in range(8)) for tile in range(token_tile//16)) - return output_waves, token_block, output_block, lane, wave, half, outputs, inputs, tokens + tokens = tuple(tuple(token_block*token_tile + tile*16 + lane_group*rows + i for i in range(rows)) for tile in range(token_tile//16)) + return output_waves, token_block, output_block, lane, wave, lane_group, outputs, inputs, tokens -def _wmma_stores(out, outputs, tokens, accs, update, half, lane, wave, output_waves, rdna4): - # RDNA4 owns eight consecutive rows per half-wave, matching tokens directly. - if rdna4: +def _wmma_stores(out, outputs, tokens, accs, update, lane, wave, output_waves, kgroups): + # RDNA4 and CDNA (kgroups > 1) own consecutive rows per lane group, matching tokens directly. + if kgroups > 1: return [out[token, output].store(acc.after(update)[i].load()) for output,output_accs in zip(outputs, accs) for tile_tokens,acc in zip(tokens, output_accs) for i,token in enumerate(tile_tokens)] # the accumulator fragment halves are exchanged between lane pairs (l, l^16) through LDS (a ds_swizzle without CUSTOM) @@ -331,26 +419,25 @@ def _wmma_stores(out, outputs, tokens, accs, update, half, lane, wave, output_wa def values(ai:int) -> tuple[UOp, ...]: own = tuple(lds[wave, lane, ai*8+i].load() for i in range(8)) peer = tuple(lds[wave, lane ^ 16, ai*8+i].load() for i in range(8)) - low = half.eq(0) + low = (lane // 16).eq(0) return tuple(low.where(own[i], peer[i+4]) if j == 0 else low.where(peer[i], own[i+4]) for i in range(4) for j in range(2)) tt = len(tokens) return [out[token, output].store(value) for ot,(output,output_accs) in enumerate(zip(outputs, accs)) for tile,(tile_tokens,_acc) in enumerate(zip(tokens, output_accs)) for token,value in zip(tile_tokens, values(ot*tt+tile))] -def _quant_linear_wmma(out, x, out_features, in_features, type_words, layout, dequant, name, rdna4): +def _quant_linear_wmma(out, x, out_features, in_features, type_words, layout, dequant, name, kgroups, lanes): x = x.reshape(out.shape[0], in_features) - output_waves, token_block, output_block, lane, wave, physical_half, outputs, input_tokens, tokens = layout + output_waves, token_block, output_block, lane, wave, lane_group, outputs, input_tokens, tokens = layout token_tile, output_tiles = len(tokens)*16, len(outputs) # IQ4_NL blocks are 32 wide: a device can hold part of a 256 weight row group output_words = in_features * type_words // GGML_BLOCK_SIZE - accs = tuple(tuple(UOp.alloc((8,), dtypes.float32, addrspace=AddrSpace.REG) + accs = tuple(tuple(UOp.alloc((len(tokens[0]),), dtypes.float32, addrspace=AddrSpace.REG) for tile in range(token_tile // 16)) for ot in range(output_tiles)) accs = tuple(tuple(acc.after(acc.store(acc.const_like(0))) for acc in output_accs) for output_accs in accs) group = UOp.range(in_features // Q8_GROUP_SIZE, 4, AxisType.LOOP) block, subgroup = group // 8, group % 8 wmma_accs = [list(output_accs) for output_accs in accs] - # gfx12 moves k2 from the element index to lane bit 4: each lane supplies two groups of four. - ks = tuple(i%4 + physical_half*4 + (i//4)*8 for i in range(8)) if rdna4 else tuple(range(16)) + ks = tuple(word*4 + b for word in _wmma_words(lane_group, kgroups) for b in range(4)) for half in range(2): afrags = tuple(UOp.stack(*(x[input_token, group*32 + half*16 + i].cast(dtypes.float16) for i in ks)) for input_token in input_tokens) @@ -358,18 +445,18 @@ def _quant_linear_wmma(out, x, out_features, in_features, type_words, layout, de bfrag = UOp.stack(*dequant(output*output_words + block*type_words, subgroup, half)) for tile,afrag in enumerate(afrags): previous = accs[output_tile][tile].after(group) if half == 0 else wmma_accs[output_tile][tile] - wmma_accs[output_tile][tile] = UOp.wmma(afrag, bfrag, previous, *WMMA_ARG) + wmma_accs[output_tile][tile] = UOp.wmma(afrag, bfrag, previous, WMMA_ARG[0], lanes) update = UOp.group(*(acc.store(value) for output_accs,output_values in zip(accs, wmma_accs) for acc,value in zip(output_accs, output_values))).end(group) - stores = _wmma_stores(out, outputs, tokens, accs, update, physical_half, lane, wave, output_waves, rdna4) + stores = _wmma_stores(out, outputs, tokens, accs, update, lane, wave, output_waves, kgroups) return UOp.group(*stores).end(token_block, output_block, lane, wave).sink(arg=KernelInfo(name=name, opts_to_apply=())) @functools.cache -def _q5_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, out_features:int, in_features:int, ggml_type:int, rdna4:bool=False) -> UOp: +def _q5_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, out_features:int, in_features:int, ggml_type:int, kgroups:int, lanes:int) -> UOp: token_tile, output_tiles = (64, 1 if out_features <= 1024 else 2) if out.shape[0] % 64 == 0 else \ (32 if out.shape[0] % 32 == 0 else 16, 2) - layout = _wmma_layout(out, out_features, token_tile, output_tiles) - word_indices = (layout[5], layout[5]+2) if rdna4 else tuple(range(4)) + layout = _wmma_layout(out, out_features, token_tile, output_tiles, lanes) + word_indices = _wmma_words(layout[5], kgroups) def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: d, dmin, scale, minimum = _q5_scales(raw, base, subgroup) qs_base = base + (4 if ggml_type == Q4_K else 12) + (subgroup // 2)*8 + half*4 @@ -377,41 +464,41 @@ def _q5_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, out_features:int, in_fea (((raw[base+4+half*4+i] >> subgroup.cast(dtypes.uint32) & 0x01010101) << 4) if ggml_type == Q5_K else 0) for i in word_indices) return tuple(((word >> (byte*8) & 255).float()*d*scale-dmin*minimum).cast(dtypes.float16) for word in words for byte in range(4)) return _quant_linear_wmma(out, x, out_features, in_features, QUANT_SIZES[ggml_type]//4, - layout, dequant, f"linear_q{4 if ggml_type == Q4_K else 5}_k_f16_wmma", rdna4) + layout, dequant, f"linear_q{4 if ggml_type == Q4_K else 5}_k_f16_wmma", kgroups, lanes) @functools.cache -def _iq4_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, lut:UOp, out_features:int, in_features:int, rdna4:bool=False) -> UOp: +def _iq4_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, lut:UOp, out_features:int, in_features:int, kgroups:int, lanes:int) -> UOp: token_tile = 32 if out_features <= 1024 and out.shape[0] % 32 == 0 else 64 if out.shape[0] % 64 == 0 and \ out_features <= 6144 else 128 if out.shape[0] % 128 == 0 else \ 32 if out.shape[0] % 32 == 0 else 16 output_tiles = 1 if out_features <= 1024 else 2 if out_features <= 6144 else 1 if out_features < 8192 else 2 - layout = _wmma_layout(out, out_features, token_tile, output_tiles) - output_waves, _, _, lane, wave, half, _, _, _ = layout - word_indices = (half, half+2) if rdna4 else tuple(range(4)) + layout = _wmma_layout(out, out_features, token_tile, output_tiles, lanes) + output_waves, _, _, lane, wave, lane_group, _, _, _ = layout + word_indices = _wmma_words(lane_group, kgroups) local_lut = UOp.alloc((256,), dtypes.uint32, addrspace=AddrSpace.LOCAL) - tid, lut_items = wave*32+lane, 256//(32*output_waves) + tid, lut_items = wave*lanes+lane, 256//(lanes*output_waves) lut = local_lut.after(*(local_lut[tid*lut_items+i].store(lut[tid*lut_items+i]) for i in range(lut_items))) def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: scale = _iq4_scale(raw, base, subgroup) pairs = tuple(lut[((raw[base + 2 + subgroup*4 + word] >> (byte*8)) & 255).cast(dtypes.weakint)] for word in word_indices for byte in range(4)) return tuple((_half((pair >> (half*16)) & 0xffff)*scale).cast(dtypes.float16) for pair in pairs) - return _quant_linear_wmma(out, x, out_features, in_features, QUANT_SIZES[IQ4_XS]//4, layout, dequant, "linear_iq4_xs_f16_wmma", rdna4) + return _quant_linear_wmma(out, x, out_features, in_features, QUANT_SIZES[IQ4_XS]//4, layout, dequant, "linear_iq4_xs_f16_wmma", kgroups, lanes) @functools.cache def _quant_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, *grids:UOp, - out_features:int, in_features:int, ggml_type:int, rdna4:bool=False) -> UOp: + out_features:int, in_features:int, ggml_type:int, kgroups:int, lanes:int) -> UOp: token_tile = 64 if out.shape[0] % 64 == 0 else 32 if out.shape[0] % 32 == 0 else 16 - layout = _wmma_layout(out, out_features, token_tile, 2) - output_waves, _, _, lane, wave, physical_half, _, _, _ = layout - word_indices = (physical_half, physical_half+2) if rdna4 else tuple(range(4)) + layout = _wmma_layout(out, out_features, token_tile, 2, lanes) + output_waves, _, _, lane, wave, lane_group, _, _, _ = layout + word_indices = _wmma_words(lane_group, kgroups) grid = None if grids: grid = UOp.alloc((int(grids[0].numel()),), dtypes.uint32, addrspace=AddrSpace.LOCAL) - tid, threads = wave*32+lane, output_waves*32 + tid, threads = wave*lanes+lane, output_waves*lanes grid = grid.after(*(grid[tid+i*threads].store(grids[0][tid+i*threads]) for i in range(int(grid.numel())//threads))) def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: - if ggml_type == IQ4_NL: base += subgroup*9 # eight independent 18-byte blocks per 256 weights + if ggml_type in SMALL_BLOCK_QUANTS: base += subgroup*SMALL_BLOCK_QUANTS[ggml_type] # eight independent blocks per 256 weights def byte(offset): return _load_byte(raw, base, offset) zero, minimum = 0, UOp.const(0, dtypes.float) if ggml_type == Q2_K: @@ -434,7 +521,7 @@ def _quant_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, *grids:UOp, return tuple((((word >> (b*8)).cast(dtypes.uint8).bitcast(dtypes.int8).float()-zero)*scale-minimum).cast(dtypes.half) for word in words for b in range(4)) return _quant_linear_wmma(out, x, out_features, in_features, QUANT_SIZES[ggml_type]//raw.dtype.itemsize, - layout, dequant, f"linear_{QUANT_NAMES[ggml_type]}_f16_wmma", rdna4) + layout, dequant, f"linear_{QUANT_NAMES[ggml_type]}_f16_wmma", kgroups, lanes) def q8_linear(layer:Linear, x:Tensor) -> Tensor: assert layer.ggml_type in QUANT_SIZES @@ -447,16 +534,17 @@ def q8_linear(layer:Linear, x:Tensor) -> Tensor: extra = (_iq_grid(x.device, layer.ggml_type),) if layer.ggml_type in (IQ2_XS, IQ3_XXS, IQ3_S, IQ2_S) else () # the kernels write the output features of their own device local_out = out_features // len(dev) if isinstance(dev:=layer.weight.device, tuple) and layer.shard_axis == 0 else out_features + cdna = _is_cdna(x.device) if tokens % 16 == 0 and local_out % 16 == 0: if layer.ggml_type == IQ4_XS: fxn, extra = _iq4_linear_f16_wmma_kernel, (iq4_half_lut(x.device),) else: fxn = functools.partial(_q5_linear_f16_wmma_kernel if layer.ggml_type in (Q4_K, Q5_K) else _quant_linear_f16_wmma_kernel, ggml_type=layer.ggml_type) - fxn = functools.partial(fxn, rdna4=_wmma_rdna4(x.device)) + fxn = functools.partial(fxn, kgroups=4 if cdna else 2 if _is_rdna4(x.device) else 1, lanes=64 if cdna else 32) srcs = (x.cast(dtypes.float16).contiguous(), *extra) else: - fxn = functools.partial(_quant_decode_kernel, ggml_type=layer.ggml_type) + fxn = functools.partial(_quant_decode_kernel, ggml_type=layer.ggml_type, cdna=cdna) srcs = (*q8_quantize(x, tokens, in_features), *extra) out_shape += ((in_features+1023)//1024,) out = Tensor.empty(out_shape, dtype=dtypes.float32, device=x.device, axis=None if layer.shard_axis is None else 1-layer.shard_axis) @@ -736,6 +824,7 @@ def flash_attention(q:Tensor, assigned_kv:Tensor, valid_end:int|UOp) -> Tensor: # Non-power-of-two decode dimensions can lose tail-store masks. Q/P, K, and V use separate LDS allocations. supported = D % 32 == 0 and (D & (D-1) == 0 and N % 64 == 0 and group*((D+LDS_PAD)*2+8) <= 65536 if decode else D >= 64 and 2*(2*BLOCK_M*(D+LDS_PAD) + D*(BLOCK_N+LDS_PAD)) <= 65536 and N % BLOCK_N == 0 and q.max_shape[2] % BLOCK_M == 0) + if not decode and _is_cdna(q.device): supported = False # the prefill kernel is RDNA WMMA only if not supported: k, v = (assigned_kv[i, :, :, :valid_end].float() for i in range(2)) mask = None if decode else Tensor.full((T_real, valid_end), -math.inf, dtype=dtypes.float32, device=q.device).triu(valid_end-T_real+1) @@ -748,7 +837,7 @@ def flash_attention(q:Tensor, assigned_kv:Tensor, valid_end:int|UOp) -> Tensor: q, q_start = q.pad_to((*q.shape[:2], T_pad, q.shape[3])), valid_end - T_real B, H, T, D = q.shape out = Tensor.empty(B, H, T, D, dtype="float32", device=q.device, axis=q.uop.axis).reshape(B*H, T, D) - fxn = functools.partial(_amd_flash_attention, valid_kv_len=valid_end, q_start=q_start, rdna4=_wmma_rdna4(q.device)) + fxn = functools.partial(_amd_flash_attention, valid_kv_len=valid_end, q_start=q_start, rdna4=_is_rdna4(q.device)) if isinstance(valid_end, UOp): assigned_kv = Tensor(assigned_kv.uop.after(valid_end)) out = Tensor.custom_kernel(out, q.half().reshape(B*H, T, D), assigned_kv, fxn=fxn)[0].reshape(B, H, T, D) return out if q_start is None else out[:, :, :T_real] diff --git a/tinygrad_repo/tinygrad/llm/model.py b/tinygrad_repo/tinygrad/llm/model.py index 05447993cb..a59b5413e1 100644 --- a/tinygrad_repo/tinygrad/llm/model.py +++ b/tinygrad_repo/tinygrad/llm/model.py @@ -1,10 +1,13 @@ from __future__ import annotations -import enum, functools, itertools, math, pathlib, re +import enum, functools, itertools, json, math, pathlib, re from dataclasses import dataclass, replace +from typing import Callable from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function, dtypes, Device -from tinygrad.llm.kernels.amd import Linear, gated_delta_prefill, flash_attention, amd_custom_kernels_supported +from tinygrad.llm.kernels.amd import Linear, ExpertWeights, fp8_matmul, gated_delta_prefill, flash_attention, amd_custom_kernels_supported from tinygrad.llm.gguf import gguf_parse, gguf_shard +from tinygrad.llm.hf import hf_load from tinygrad.uop.ops import resolve +from tinygrad.helpers import unwrap, round_up, get_child class ExpertGating(enum.IntEnum): SOFTMAX = 1 @@ -33,26 +36,23 @@ def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0, device:str| freqs = Tensor.arange(end).unsqueeze(dim=1) * freqs.unsqueeze(dim=0) return (freqs.cos() * concentration).cat(freqs.sin() * concentration, dim=-1).clone(device) -class ExpertWeights: - """Like Linear but with num_experts dimension. Weight shape: (num_experts, out_features, in_features).""" - def __init__(self, num_experts:int, in_features:int, out_features:int, bias:bool=False): - self.weight = Tensor.zeros(num_experts, out_features, in_features) - if bias: self.bias = Tensor.zeros(num_experts, out_features) - def __call__(self, sel:Tensor, x:Tensor) -> Tensor: - # sel: (B, T, k), x: (B, T, 1, in) or (B, T, k, in) -> output: (B, T, k, out) - ret = (x.unsqueeze(-2) @ self.weight[sel].transpose(-1, -2)).contiguous().squeeze(-2) - return ret + self.bias[sel] if hasattr(self, 'bias') else ret - def gated_activation(gate:Tensor, up:Tensor, *, alpha:float=1.0, limit:float|None=None, up_bias:float=0.0) -> Tensor: if limit is not None and limit > 0: gate, up = gate.clamp(max_=limit), up.clamp(-limit, limit) - return gate * (gate * alpha).sigmoid() * (up + up_bias) + # TODO: remove the need for this contiguous + return gate * (gate * alpha).sigmoid().contiguous() * (up + up_bias) -def apply_rope(x:Tensor, freqs_cis:Tensor) -> Tensor: +def apply_rope(x:Tensor, freqs_cis:Tensor, interleaved:bool=False) -> Tensor: assert x.shape[-1] % 2 == 0 + if interleaved: x = x.rearrange("b h t (d two) -> b h t (two d)", two=2) cos, sin = freqs_cis.reshape(1, 1, x.shape[2], -1).chunk(2, dim=-1) x1, x2 = x.chunk(2, dim=-1) return (x1 * cos - x2 * sin).cat(x2 * cos + x1 * sin, dim=-1) +def permute_rope(w:Tensor, n_heads:int, rope_dim:int) -> Tensor: + x = w.reshape(n_heads, w.shape[0]//n_heads, -1) + prefix = x.shape[1]-rope_dim + return x[:, :prefix].cat(x[:, prefix:].rearrange("n (h two) d -> n (two h) d", two=2), dim=1).reshape(w.shape) + def pairwise_topk(x: Tensor, k: int) -> tuple[Tensor, Tensor]: n = x.shape[-1] vals = Tensor.arange(n).reshape(1,1,n).cast(x.dtype).expand(x.shape) @@ -69,6 +69,23 @@ class SSMConfig: time_step_rank: int inner_size: int kda: bool = False + gate_lower_bound: float|None = None # kda: bounded decay gate + +@dataclass(frozen=True) +class HCConfig: # manifold-constrained hyper-connections: count residual streams, mixed by a doubly stochastic matrix + count: int + sinkhorn_iters: int + eps: float + +@dataclass(frozen=True) +class IndexerConfig: # select top_k tokens by scoring individual keys or pools of kpool keys + n_heads: int + head_dim: int + top_k: int + kpool: int = 1 + norm_eps: float = 1e-6 + rope_dim: int = 0 + rope_interleave: bool = True @dataclass(frozen=True) class TransformerConfig: @@ -110,6 +127,27 @@ class TransformerConfig: swiglu_up_bias: float = 0.0 sliding_window: int = 0 sliding_window_pattern: int = 0 + hc: HCConfig|None = None + indexer: IndexerConfig|None = None + indexer_shared: tuple[bool, ...] = () + rope_interleave: bool = False + +class HyperConnection: + def __init__(self, config:TransformerConfig): + self.count, self.eps, self.iters, self.norm_eps = (hc:=unwrap(config.hc)).count, hc.eps, hc.sinkhorn_iters, config.norm_eps + self.fn = Linear(self.count * config.dim, (2 + self.count) * self.count, bias=False) + self.base, self.scale = {"weight": Tensor.zeros((2 + self.count) * self.count)}, {"weight": Tensor.zeros(3)} + + def __call__(self, xs:Tensor, sublayer:Callable[[Tensor], tuple[Tensor, Tensor|None]]) -> tuple[Tensor, Tensor|None]: + """mHC: the sublayer sees a pre-weighted sum of the (B, T, hc, D) streams, its output is added back by post onto the streams mixed by comb""" + base, scale, mix = self.base["weight"], self.scale["weight"], self.fn(xs.flatten(2).rmsnorm(eps=self.norm_eps)) + pre = (mix[..., :self.count] * scale[0] + base[:self.count]).sigmoid() + self.eps + post = (mix[..., self.count:2*self.count] * scale[1] + base[self.count:2*self.count]).sigmoid() * 2 + comb = (mix[..., 2*self.count:] * scale[2] + base[2*self.count:]).reshape(*mix.shape[:-1], self.count, self.count).softmax(-1) + self.eps + # sinkhorn: normalize the columns then the rows, alternating, to make comb doubly stochastic + for i in range(2 * self.iters - 1): comb = comb / (comb.sum(-1 if i % 2 else -2, keepdim=True) + self.eps) + out, aux = sublayer((xs * pre.unsqueeze(-1)).sum(2)) + return out.unsqueeze(2) * post.unsqueeze(-1) + (comb.transpose(-1, -2) @ xs), aux class FFNBlock: def __init__(self, config:TransformerConfig): @@ -118,6 +156,7 @@ class FFNBlock: # --- RMSNorms -------------------------------------------------------- self.attn_norm = nn.RMSNorm(config.dim, config.norm_eps) self.ffn_norm = nn.RMSNorm(config.dim, config.norm_eps) + if config.hc is not None: self.hc_attn, self.hc_ffn = HyperConnection(config), HyperConnection(config) # --- feed-forward (MoE or dense) ------------------------------------- if config.num_experts > 0: @@ -165,22 +204,32 @@ class FFNBlock: if hasattr(self, 'ffn_gate_inp_shexp'): shexp = shexp * (x * self.ffn_gate_inp_shexp["weight"]).sum(axis=-1, keepdim=True).sigmoid() out = out + shexp return out - # TODO: remove the need for this contiguous - return self.ffn_down(gated_activation(self.ffn_gate(x), self.ffn_up(x), limit=self.config.swiglu_limit).contiguous()) + return self.ffn_down(gated_activation(self.ffn_gate(x), self.ffn_up(x), limit=self.config.swiglu_limit)) # given the token-prefix match, return how much cached state this block can still reuse def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return prefix_len def _init_state(self, x:Tensor): raise NotImplementedError - def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: raise NotImplementedError + def _attention(self, x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None=None) -> Tensor: raise NotImplementedError + def _indexer_mask(self, x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None) -> Tensor|None: + # a full indexer layer selects the top-k tokens for itself and the shared layers after it (MLATransformerBlock) + return idx_mask - def __call__(self, x: Tensor, start_pos: int|UOp): + def __call__(self, x: Tensor, start_pos: int|UOp, idx_mask: Tensor|None=None): self._init_state(x) # we pass in the weights implicitly so we unpack the GGUF on the fly @function(precompile=True, allow_implicit=True) - def _run(x:Tensor, start_pos:int|UOp): - h = x + self._attention(self.attn_norm(x), start_pos) - return (h + self._feed_forward(self.ffn_norm(h))).contiguous() - return _run(x, start_pos) + def _run(x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None): + def attend(h:Tensor) -> tuple[Tensor, Tensor|None]: + h = self.attn_norm(h) + mask = self._indexer_mask(h, start_pos, idx_mask) + return self._attention(h, start_pos, mask), mask + res_attn = res_ffn = lambda xs, sublayer: (xs + (ret:=sublayer(xs))[0], ret[1]) + if self.config.hc is not None: res_attn, res_ffn = self.hc_attn, self.hc_ffn + x, idx_mask = res_attn(x, attend) + out = res_ffn(x, lambda h: (self._feed_forward(self.ffn_norm(h)), None))[0].contiguous() + return (out, idx_mask) if idx_mask is not None else out + out = _run(x, start_pos, idx_mask) + return out if isinstance(out, tuple) else (out, None) class TransformerBlock(FFNBlock): def __init__(self, config:TransformerConfig): @@ -197,7 +246,7 @@ class TransformerBlock(FFNBlock): if config.qk_norm: self.attn_q_norm, self.attn_k_norm = nn.RMSNorm(config.qk_norm, config.norm_eps), nn.RMSNorm(config.qk_norm, config.norm_eps) if config.attn_sinks: self.attn_sinks = {"weight": Tensor.zeros(config.n_heads)} - def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: + def _attention(self, x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None=None) -> Tensor: q, k, v = self.attn_q(x), self.attn_k(x), self.attn_v(x) if self.config.qk_norm and self.config.qk_norm != self.config.head_dim: q, k = self.attn_q_norm(q), self.attn_k_norm(k) @@ -252,7 +301,45 @@ class TransformerBlock(FFNBlock): self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device, yarn=self.config.yarn) +class Indexer: + def __init__(self, config:TransformerConfig): + self.idx, self.max_context = (idx:=unwrap(config.indexer)), config.max_context + self.attn_q_b, self.attn_k = Linear(config.q_lora_rank, idx.n_heads * idx.head_dim, bias=False), Linear(config.dim, idx.head_dim, bias=False) + self.proj, self.k_norm = Linear(config.dim, idx.n_heads, bias=False), nn.LayerNorm(idx.head_dim, eps=idx.norm_eps) + if idx.kpool > 1: + self.compressor_gate, self.compressor_ape = Linear(config.dim, idx.head_dim, bias=False), {"weight": Tensor.zeros(idx.kpool, idx.head_dim)} + + def _init_state(self, x:Tensor): + if not hasattr(self, "cache_kg"): + self.cache_kg = Tensor.zeros(x.shape[0], round_up(self.max_context, self.idx.kpool), + *((2, self.idx.head_dim) if self.idx.kpool > 1 else (self.idx.head_dim,)), + dtype=dtypes.half, device=x.device) + + def __call__(self, x:Tensor, q:Tensor, start_pos:int|UOp, freqs:Tensor|None=None) -> Tensor: + (B, T, _), idx, S = x.shape, self.idx, start_pos + x.shape[1] + k, iq = self.k_norm(self.attn_k(x)), self.attn_q_b(q).reshape(B, T, idx.n_heads, idx.head_dim) + if idx.rope_dim: + rot = unwrap(freqs)[start_pos:start_pos+T] + k = apply_rope(k[..., :idx.rope_dim].unsqueeze(1), rot, idx.rope_interleave).squeeze(1).cat(k[..., idx.rope_dim:], dim=-1) + iq = apply_rope(iq[..., :idx.rope_dim].transpose(1, 2), rot, idx.rope_interleave).transpose(1, 2).cat(iq[..., idx.rope_dim:], dim=-1) + if idx.kpool > 1: + # cache an indexer key and a pooling gate per token, a pool's key is the gate softmax weighted sum of its keys + kg = k.stack(self.compressor_gate(x), dim=2) # (B, T, kpool, 2, D) + kg = Tensor(self.cache_kg.uop.after(self.cache_kg[:, start_pos:S].uop.store(kg.half().uop))).float().reshape(B, -1, idx.kpool, 2, idx.head_dim) + k = ((kg[..., 1, :] + self.compressor_ape["weight"]).softmax(2) * kg[..., 0, :]).sum(2) # (B, S, D) + else: + k = Tensor(self.cache_kg.uop.after(self.cache_kg[:, start_pos:S].uop.store(k.half().uop))).float() # (B, S, D) + scores = ((iq @ k.transpose(-1, -2).unsqueeze(1)).relu() * (self.proj(x) * (idx.n_heads * idx.head_dim)**-0.5).unsqueeze(-1)).sum(2) + # the query sees the top_k/kpool best complete pools and the tokens of its own pool while incomplete (the tail) + q_pos, k_pos = (Tensor(start_pos) + Tensor.arange(T)).reshape(T, 1), Tensor.arange(S) + _, sel = ((Tensor.arange(scores.shape[-1]) + 1) * idx.kpool <= q_pos + 1).where(scores, -math.inf).topk(idx.top_k // idx.kpool) + # selected incomplete pools add only tail/future tokens, handled by the final tail and causal mask + picked = scores.zeros_like(dtype=dtypes.bool).scatter(-1, sel, True).repeat_interleave(idx.kpool, dim=-1)[..., :S] + return ((picked | (k_pos >= (q_pos + 1) // idx.kpool * idx.kpool)) & (k_pos <= q_pos)).where(0.0, -math.inf).reshape(B, 1, T, S).cast(x.dtype) + + class MLATransformerBlock(FFNBlock): + kv_b_block_size:tuple[int, int] = (128, 128) def __init__(self, config:TransformerConfig): super().__init__(config) qk_nope_head_dim = config.head_dim - config.rope_dim @@ -267,40 +354,56 @@ class MLATransformerBlock(FFNBlock): self.attn_k_b = {"weight": Tensor.zeros(config.n_heads, config.kv_lora_rank, qk_nope_head_dim)} self.attn_v_b = {"weight": Tensor.zeros(config.n_heads, config.v_head_dim, config.kv_lora_rank)} self.attn_output = Linear(config.n_heads * config.v_head_dim, config.dim, bias=False) + self.indexer = Indexer(config) if config.indexer else None - def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: + def _attention(self, x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None=None) -> Tensor: B, T, _ = x.shape q_nope_head_dim = self.config.head_dim - self.config.rope_dim q_proj = self.attn_q_b(self.attn_q_a_norm(self.attn_q_a(x))) if self.config.q_lora_rank > 0 else self.attn_q(x) q = q_proj.reshape(B, T, self.config.n_heads, self.config.head_dim).transpose(1, 2) q_nope, q_rope = q[..., :q_nope_head_dim], q[..., q_nope_head_dim:] - if not self.config.ssm or not self.config.ssm.kda: q_rope = apply_rope(q_rope, self.freqs_cis[start_pos:start_pos+T]) - q = (q_nope @ self.attn_k_b["weight"].transpose(-1, -2)).cat(q_rope, dim=-1) + if not self.config.ssm or not self.config.ssm.kda: q_rope = apply_rope(q_rope, self.freqs_cis[start_pos:start_pos+T], self.config.rope_interleave) + if 'weight_scale_inv' in self.attn_k_b: + q_nope = fp8_matmul(q_nope, self.attn_k_b['weight'], self.attn_k_b['weight_scale_inv'], self.kv_b_block_size[::-1]) + else: q_nope = q_nope @ self.attn_k_b['weight'].transpose(-1, -2) + q = q_nope.cat(q_rope, dim=-1) kv_a = self.attn_kv_a_mqa(x) c_kv = self.attn_kv_a_norm(kv_a[..., :self.config.kv_lora_rank]) k_rope = kv_a[..., self.config.kv_lora_rank:].reshape(B, T, 1, self.config.rope_dim).transpose(1, 2) - if not self.config.ssm or not self.config.ssm.kda: k_rope = apply_rope(k_rope, self.freqs_cis[start_pos:start_pos+T]) + if not self.config.ssm or not self.config.ssm.kda: k_rope = apply_rope(k_rope, self.freqs_cis[start_pos:start_pos+T], self.config.rope_interleave) k_store = c_kv.reshape(B, 1, T, self.config.kv_lora_rank).cat(k_rope.reshape(B, 1, T, self.config.rope_dim), dim=-1) k = Tensor(self.cache_k.uop.after(self.cache_k[:, :, start_pos:start_pos+T, :].uop.store(k_store.uop)))[:, :, 0:start_pos+T, :] v = k[..., :self.config.kv_lora_rank] attn = q @ k.transpose(-1, -2) * (1.0 / self.config.head_dim ** 0.5) - if (mask:=self._attn_mask(x, start_pos)) is not None: attn = attn + mask + if (mask:=self._attn_mask(x, start_pos, idx_mask)) is not None: attn = attn + mask attn = attn.softmax(-1) - attn = ((attn @ v) @ self.attn_v_b["weight"].transpose(-1, -2)).transpose(1, 2).reshape(B, T, -1) + attn = attn @ v + if 'weight_scale_inv' in self.attn_v_b: + attn = fp8_matmul(attn, self.attn_v_b['weight'], self.attn_v_b['weight_scale_inv'], self.kv_b_block_size) + else: attn = attn @ self.attn_v_b['weight'].transpose(-1, -2) + attn = attn.transpose(1, 2).reshape(B, T, -1) return self.attn_output(attn) - def _attn_mask(self, x:Tensor, start_pos:int|UOp) -> Tensor|None: # (1, 1, T, start_pos+T), None: everything is visible + def _attn_mask(self, x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None=None) -> Tensor|None: # (1, 1, T, start_pos+T), None: everything is visible + if idx_mask is not None: return idx_mask T = x.shape[1] return Tensor.full((1, 1, T, start_pos+T), float("-inf"), dtype=x.dtype, buffer=False).triu(start_pos+1) if resolve(T != 1) else None + def _indexer_mask(self, x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None) -> Tensor|None: + # a full indexer layer selects the top-k tokens for itself and the shared layers after it + if self.indexer is not None and self.config.max_context > self.indexer.idx.top_k: + return self.indexer(x, self.attn_q_a_norm(self.attn_q_a(x)), start_pos, self.freqs_cis) + return idx_mask + def _init_state(self, x:Tensor): if not hasattr(self, "cache_k"): self.cache_k = Tensor.empty(x.shape[0], 1, self.config.max_context, self.config.kv_lora_rank + self.config.rope_dim, device=x.device) self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device, yarn=self.config.yarn) + if self.indexer: self.indexer._init_state(x) class GatedDeltaNetBlock(FFNBlock): def __init__(self, config:TransformerConfig, ssm:SSMConfig): @@ -309,25 +412,28 @@ class GatedDeltaNetBlock(FFNBlock): assert self.num_v_heads % self.num_k_heads == 0 self.head_v_dim, self.ssm_conv_kernel = ssm.inner_size // ssm.time_step_rank, ssm.conv_kernel self.conv_channels, self.q_dim = ssm.inner_size + 2*ssm.group_count*ssm.state_size, ssm.state_size*ssm.group_count - self.attn_qkv = Linear(config.dim, self.conv_channels, bias=False) if ssm.kda: + self.qkv_dims = (self.q_dim, self.q_dim, ssm.inner_size) + self.attn_q, self.attn_k, self.attn_v = (Linear(config.dim, d, bias=False) for d in self.qkv_dims) + self.ssm_conv1d_q, self.ssm_conv1d_k, self.ssm_conv1d_v = ({"weight": Tensor.zeros(d, 1, ssm.conv_kernel)} for d in self.qkv_dims) self.ssm_g_a, self.ssm_g_b = Linear(config.dim, self.head_v_dim, bias=False), Linear(self.head_v_dim, ssm.inner_size, bias=False) self.ssm_f_a, self.ssm_f_b = Linear(config.dim, self.head_k_dim, bias=False), Linear(self.head_k_dim, ssm.inner_size, bias=False) else: + self.attn_qkv = Linear(config.dim, self.conv_channels, bias=False) + self.ssm_conv1d = {"weight": Tensor.zeros(self.conv_channels, self.ssm_conv_kernel)} self.attn_gate = Linear(config.dim, ssm.inner_size, bias=False) self.ssm_alpha = Linear(config.dim, self.num_v_heads, bias=False) self.ssm_beta = Linear(config.dim, self.num_v_heads, bias=False) - self.ssm_conv1d = {"weight": Tensor.zeros(self.conv_channels, self.ssm_conv_kernel)} self.ssm_dt = {"bias": Tensor.zeros(ssm.inner_size if ssm.kda else self.num_v_heads)} self.ssm_a = Tensor.zeros(self.num_v_heads, 1) if ssm.kda else Tensor.zeros(self.num_v_heads) self.ssm_norm, self.ssm_out = nn.RMSNorm(self.head_v_dim, config.norm_eps), Linear(ssm.inner_size, config.dim, bias=False) - def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: + def _attention(self, x:Tensor, start_pos:int|UOp, idx_mask:Tensor|None=None) -> Tensor: B, T, _ = x.shape # bind ints to a variable so the reset flag stays a runtime value (it toggles when generation restarts at position 0) start_pos = start_pos if isinstance(start_pos, UOp) else UOp.variable("start_pos", 0, self.config.max_context-1).bind(start_pos) initial = Tensor(start_pos).eq(0) - is_kda = hasattr(self, "ssm_g_a") + is_kda = unwrap(self.config.ssm).kda symbolic = isinstance(T, UOp) T_pad = x.max_shape[1] # symbolic chunks are padded to their max size: one graph serves every size @@ -338,23 +444,26 @@ class GatedDeltaNetBlock(FFNBlock): beta = self.ssm_beta(x).sigmoid().reshape(B, T, self.num_v_heads) alpha = self.ssm_f_b(self.ssm_f_a(x)) if is_kda else self.ssm_alpha(x) g, a = (alpha.float() + self.ssm_dt["bias"]).reshape(B, T, self.num_v_heads, -1), self.ssm_a.reshape(self.num_v_heads, -1) - log_alpha = g.softplus() * a + # the decay: log_alpha = softplus(g) * A, or bounded by gate_lower_bound: sigmoid(g * -A) * lb + log_alpha = g.softplus() * a if (lb:=unwrap(self.config.ssm).gate_lower_bound) is None else (g * -a).sigmoid() * lb # qkv conv, conv_state is reset when starting from position 0 - conv_state = initial.where(0, self.conv_state) - # assemble the conv window in a static-size buffer: [conv_state | qkv rows | zero-pad]. - # padded steps are exact no-ops: beta=0 (delta rule off), log_alpha=0 (decay 1 after exp) - conv_window = conv_state.cat(self.attn_qkv(x).cast(conv_state.dtype), dim=1) - conv_window = conv_window.pad_to((B, self.ssm_conv_kernel-1 + T_pad, self.conv_channels)).contiguous() - # the last conv_kernel-1 columns of the window become the next conv state - conv_state_store = self.conv_state.uop.store(conv_window[:, T:T+self.ssm_conv_kernel-1].cast(self.conv_state.dtype).uop) - - conv_out = functools.reduce(lambda a,b: a+b, - (conv_window[:, i:i+T_pad] * self.ssm_conv1d["weight"][:, i] for i in range(self.ssm_conv_kernel))).silu() + def conv(conv_state:Tensor, proj:Tensor, weight:Tensor) -> tuple[Tensor, UOp]: + # assemble the conv window in a static-size buffer: [conv_state | proj rows | zero-pad]. + # padded steps are exact no-ops: beta=0 (delta rule off), log_alpha=0 (decay 1 after exp) + conv_window = initial.where(0, conv_state).cat(proj.cast(conv_state.dtype), dim=1) + conv_window = conv_window.pad_to((B, self.ssm_conv_kernel-1 + T_pad, proj.shape[-1])).contiguous() + # the last conv_kernel-1 columns of the window become the next conv state + store = conv_state.uop.store(conv_window[:, T:T+self.ssm_conv_kernel-1].cast(conv_state.dtype).uop) + return functools.reduce(lambda a,b: a+b, (conv_window[:, i:i+T_pad] * weight[:, i] for i in range(self.ssm_conv_kernel))).silu(), store + projs = (self.attn_q, self.attn_k, self.attn_v) if is_kda else (self.attn_qkv,) + convs = (self.ssm_conv1d_q, self.ssm_conv1d_k, self.ssm_conv1d_v) if is_kda else (self.ssm_conv1d,) + states = self.conv_states if is_kda else [self.conv_state] + conv_outs, conv_state_stores = zip(*(conv(s, proj(x), w["weight"].flatten(1)) for s, proj, w in zip(states, projs, convs))) if symbolic: out_gate = out_gate.pad_to((B, T_pad, self.num_v_heads, self.head_v_dim)) beta, log_alpha = beta.pad_to((B, T_pad, self.num_v_heads)), log_alpha.pad_to((B, T_pad, *log_alpha.shape[2:])) - q, k, v = conv_out.split([self.q_dim, self.q_dim, self.conv_channels - 2*self.q_dim], dim=-1) + q, k, v = conv_outs if is_kda else conv_outs[0].split([self.q_dim, self.q_dim, self.conv_channels - 2*self.q_dim], dim=-1) q, k = (z.reshape(B, T_pad, self.num_k_heads, self.head_k_dim) for z in (q, k)) # l2 norm x/sqrt(sum(x²)+1e-6) as in the FLA kernels, then the v heads share the k heads q, k = ((z.rmsnorm(eps=1e-6/self.head_k_dim) / self.head_k_dim**0.5).repeat(1, 1, self.num_v_heads//self.num_k_heads, 1) for z in (q, k)) @@ -365,14 +474,14 @@ class GatedDeltaNetBlock(FFNBlock): alpha = log_alpha.transpose(1, 2).exp() # per-channel decay for kda, per-head otherwise (B, H, T, K|1) # recurrent: scan over the (padded) tokens, updating the recurrent state. collect the per-step outputs - state = Tensor(self.recurrent_state.uop.after(conv_state_store)) # carry the conv write into this graph if self.head_k_dim % 32 == 0 and self.head_v_dim % 4 == 0 and amd_custom_kernels_supported(x.device): # one fused kernel for the whole scan; it resets and updates the recurrent state in place (RDNA3/4) + state = Tensor(self.recurrent_state.uop.after(*conv_state_stores)) # carry the conv write into this graph core = gated_delta_prefill(q, k, v, beta, alpha, state, Tensor(start_pos)).transpose(1, 2) else: q, k, v, beta = q.unsqueeze(-2), k.unsqueeze(-2), v.unsqueeze(-1), beta.unsqueeze(-1).unsqueeze(-1) alpha = alpha.unsqueeze(-2) - state = initial.where(0, state.float()) + state = initial.where(0, self.recurrent_state.float()) outs = [] for t in range(T_pad): s1 = state * alpha[:, :, t] # decay the state @@ -380,9 +489,10 @@ class GatedDeltaNetBlock(FFNBlock): state = s1 + delta * k[:, :, t] outs.append((state * q[:, :, t]).sum(-1)) - # store the updated recurrent state in place, then read the stacked outputs after the write - state_store = self.recurrent_state.uop.store(state.cast(self.recurrent_state.dtype).uop) - core = Tensor(outs[0].stack(*outs[1:], dim=1).contiguous().uop.after(state_store)) + # store the updated recurrent state in place, then read the stacked outputs after the state and conv writes + state_store = self.recurrent_state.uop.after(self.recurrent_state.uop.store(state.cast(self.recurrent_state.dtype).uop)) + conv_writes = [c.uop.after(st) for c, st in zip(states, conv_state_stores)] + core = Tensor(outs[0].stack(*outs[1:], dim=1).contiguous().uop.after(state_store, *conv_writes)) # output; undo the padding before the output projection z = (self.ssm_norm(core) * (out_gate.sigmoid() if is_kda else out_gate.silu())).cast(x.dtype).contiguous() @@ -390,19 +500,24 @@ class GatedDeltaNetBlock(FFNBlock): return self.ssm_out(z.reshape(B, T, -1)) def _init_state(self, x): - if not hasattr(self, "conv_state"): - self.conv_state = Tensor.zeros(x.shape[0], self.ssm_conv_kernel-1, self.conv_channels, device=x.device).clone() - self.recurrent_state = Tensor.zeros(x.shape[0], self.num_v_heads, self.head_v_dim, self.head_k_dim, device=x.device).clone() + if not hasattr(self, "recurrent_state"): + if kda:=unwrap(self.config.ssm).kda: + self.conv_states = [Tensor.zeros(x.shape[0], self.ssm_conv_kernel-1, d, device=x.device, axis=-1).clone() for d in self.qkv_dims] + else: self.conv_state = Tensor.zeros(x.shape[0], self.ssm_conv_kernel-1, self.conv_channels, device=x.device).clone() + self.recurrent_state = Tensor.zeros(x.shape[0], self.num_v_heads, self.head_v_dim, self.head_k_dim, device=x.device, + axis=1 if kda else None).clone() class Transformer: def __init__(self, config:TransformerConfig): dense_config = replace(config, num_experts=0, num_experts_per_tok=0, shared_expert_dim=0, hidden_dim=config.dense_hidden_dim or config.hidden_dim) if config.ssm: config = replace(config, qk_norm=config.head_dim) + self.config = config block_cls = MLATransformerBlock if config.kv_lora_rank > 0 else TransformerBlock self.blk:list[FFNBlock] = [] for i in range(config.num_blocks): c = dense_config if i < config.leading_dense_blocks else config if config.sliding_window_pattern != 0 and (i+1) % config.sliding_window_pattern == 0: c = replace(c, sliding_window=0) + if config.indexer_shared and config.indexer_shared[i]: c = replace(c, indexer=None) self.blk.append(GatedDeltaNetBlock(c, config.ssm) if config.ssm and config.ssm_layers[i] else block_cls(c)) self.token_embd = nn.Embedding(config.vocab_size, config.dim) self.output_norm = nn.RMSNorm(config.dim, config.norm_eps) @@ -416,7 +531,10 @@ class Transformer: def forward(self, tokens:Tensor, start_pos:int|UOp, temperature:Tensor) -> Tensor: x = self.token_embd(tokens.to(self.token_embd.weight.device)).float() # (B, T, D) - for block in self.blk: x = block(x, start_pos) + if (hc:=self.config.hc) is not None: x = x.unsqueeze(2).expand(*x.shape[:2], hc.count, x.shape[-1]).contiguous() + idx_mask = None + for block in self.blk: x, idx_mask = block(x, start_pos, idx_mask) + if hc is not None: x = x.mean(2) # only run the output projection on the last token logits = self.output(self.output_norm(x[:, -1:]))[:, -1, :].to(tokens.device) # Gumbel-max trick: argmax(logits/temp - log(-log(uniform))) is equivalent to sampling from softmax(logits/temp) @@ -441,6 +559,8 @@ class Transformer: rules = {**{w: 0 for w in ('token_embd.weight', 'output.weight', 'attn_q.weight', 'attn_k.weight', 'attn_v.weight', 'ffn_gate.weight', 'ffn_up.weight', 'attn_q_b.weight', 'attn_k_b.weight', 'attn_v_b.weight')}, **{w: 1 for w in ('attn_output.weight', 'ffn_down.weight', 'ffn_gate_exps.weight', 'ffn_up_exps.weight')}, 'ffn_down_exps.weight':2} + if arch in ('kimi-linear', 'glm5-next'): rules.update({w: 0 for w in ('ssm_conv1d_q.weight', 'ssm_conv1d_k.weight', 'ssm_conv1d_v.weight', + 'ssm_dt.bias', 'ssm_a', 'ssm_beta.weight', 'ssm_f_b.weight', 'ssm_g_b.weight')}) shard_map = {name: rules[k] for name in entries if (k:=re.sub(r"^blk\.\d+\.", "", name)) in rules} devices = tuple(Device.canonicalize(f'{Device.DEFAULT}:{i}') for i in range(shard)) state_dict = gguf_shard(entries, devices, shard_map) @@ -451,6 +571,15 @@ class Transformer: # some models like Llama 3.2 don't have an output.weight, they just tie to the token_embd.weight if 'output.weight' not in state_dict: state_dict['output.weight'] = state_dict['token_embd.weight'] + hc = HCConfig(kv[f'{arch}.hyper_connection.count'], kv[f'{arch}.hyper_connection.sinkhorn_iterations'], + kv[f'{arch}.hyper_connection.epsilon']) if f'{arch}.hyper_connection.count' in kv else None + if hc is not None: state_dict = {k.replace('.hc_attn_', '.hc_attn.').replace('.hc_ffn_', '.hc_ffn.'):v for k,v in state_dict.items()} + + indexer = IndexerConfig(kv[f'{arch}.attention.indexer.head_count'], kv[f'{arch}.attention.indexer.key_length'], + kv[f'{arch}.attention.indexer.top_k'], kv[f'{arch}.attention.indexer.kpool'], + kv[f'{arch}.attention.layer_norm_epsilon']) if f'{arch}.attention.indexer.kpool' in kv else None + if indexer is not None: state_dict = {k.replace('.indexer_compressor_', '.indexer.compressor_'):v for k,v in state_dict.items()} + max_context = min(max_context, kv[f'{arch}.context_length']) if max_context is not None else kv[f'{arch}.context_length'] num_blocks = kv[f'{arch}.block_count'] - kv.get(f'{arch}.nextn_predict_layers', 0) @@ -459,17 +588,14 @@ class Transformer: if arch in ('qwen35', 'qwen35moe'): ssm = SSMConfig(**{k: kv[f'{arch}.ssm.{k}'] for k in ('conv_kernel','state_size','group_count','time_step_rank','inner_size')}) ssm_layers = tuple((i+1) % kv[f'{arch}.full_attention_interval'] != 0 for i in range(num_blocks)) - elif arch == 'kimi-linear': + elif arch in ('kimi-linear', 'glm5-next'): ssm_layers = tuple(x == 0 for x in n_kv_heads)[:num_blocks] n_kv_heads = max(n_kv_heads) - ssm = SSMConfig(kv[f'{arch}.ssm.conv_kernel'], kv[f'{arch}.kda.head_dim'], n_heads, n_heads, n_heads*kv[f'{arch}.kda.head_dim'], kda=True) + ssm = SSMConfig(kv[f'{arch}.ssm.conv_kernel'], kv[f'{arch}.kda.head_dim'], n_heads, n_heads, n_heads*kv[f'{arch}.kda.head_dim'], kda=True, + gate_lower_bound=kv.get(f'{arch}.kda.gate_lower_bound')) for i, is_ssm in enumerate(ssm_layers): if not is_ssm: continue state_dict[f"blk.{i}.ssm_a"] = state_dict[f"blk.{i}.ssm_a"].reshape(n_heads, 1) - state_dict[f"blk.{i}.attn_qkv.weight"] = state_dict.pop(f"blk.{i}.attn_q.weight").cat( - state_dict.pop(f"blk.{i}.attn_k.weight"), state_dict.pop(f"blk.{i}.attn_v.weight"), dim=0).contiguous() - state_dict[f"blk.{i}.ssm_conv1d.weight"] = state_dict.pop(f"blk.{i}.ssm_conv1d_q.weight").cat( - state_dict.pop(f"blk.{i}.ssm_conv1d_k.weight"), state_dict.pop(f"blk.{i}.ssm_conv1d_v.weight"), dim=0).squeeze(1).contiguous() state_dict[f"blk.{i}.ssm_out.weight"] = state_dict.pop(f"blk.{i}.attn_output.weight") if arch in ('qwen35', 'qwen35moe', 'glm4moe', 'gpt-oss'): state_dict = {k.replace('post_attention_norm', 'ffn_norm'):v for k,v in state_dict.items()} @@ -484,16 +610,13 @@ class Transformer: # Permute RoPE weights from interleaved to half-split layout. for name in state_dict: - if arch == 'kimi-linear': continue + if arch in ('kimi-linear', 'glm5-next'): continue if ('attn_q.weight' in name or 'attn_q_b.weight' in name) and (arch == 'llama' or kv_lora_rank): - w = state_dict[name].reshape(n_heads, state_dict[name].shape[0]//n_heads, -1) - prefix = head_dim-rope_dim - state_dict[name] = w[:, :prefix].cat(w[:, prefix:].rearrange("n (h two) d -> n (two h) d", two=2), dim=1).reshape(-1, w.shape[-1]) + state_dict[name] = permute_rope(state_dict[name], n_heads, rope_dim) elif arch == 'llama' and 'attn_k.weight' in name: - w = state_dict[name].reshape(n_kv_heads, state_dict[name].shape[0]//n_kv_heads, -1) - state_dict[name] = w.rearrange("n (h two) d -> n (two h) d", two=2).reshape(-1, w.shape[-1]) + state_dict[name] = permute_rope(state_dict[name], n_kv_heads, state_dict[name].shape[0]//n_kv_heads) elif kv_lora_rank and 'attn_kv_a_mqa.weight' in name: - state_dict[name] = state_dict[name][:kv_lora_rank].cat(state_dict[name][kv_lora_rank:].rearrange("(h two) d -> (two h) d", two=2), dim=0) + state_dict[name] = permute_rope(state_dict[name], 1, state_dict[name].shape[0]-kv_lora_rank) config = TransformerConfig( num_blocks=num_blocks, dim=kv[f'{arch}.embedding_length'], hidden_dim=kv.get(f'{arch}.expert_feed_forward_length', kv.get(f'{arch}.feed_forward_length', 0)), @@ -523,7 +646,7 @@ class Transformer: swiglu_alpha=1.702 if arch == 'gpt-oss' else 1.0, swiglu_limit=7.0 if arch == 'gpt-oss' else kv.get(f'{arch}.swiglu_clamp_exp', [None])[0], swiglu_up_bias=1.0 if arch == 'gpt-oss' else 0.0, attn_sinks='blk.0.attn_sinks.weight' in state_dict, sliding_window=kv.get(f'{arch}.attention.sliding_window', 0), - sliding_window_pattern=kv.get(f'{arch}.attention.sliding_window_pattern', 2 if arch == 'gpt-oss' else 0)) + sliding_window_pattern=kv.get(f'{arch}.attention.sliding_window_pattern', 2 if arch == 'gpt-oss' else 0), hc=hc, indexer=indexer) model = Transformer(config) for p in (nn.state.get_parameters(model) if shard > 1 else []): p.to_(devices) nn.state.load_state_dict(model, state_dict, verbose=False, consume=True, realize=False) # NOTE: rope_freqs.weight (32,) is unused @@ -533,6 +656,85 @@ class Transformer: Tensor.realize(*params) return model, kv + @staticmethod + def from_hf(path:str|pathlib.Path, max_context:int|None=None, + realize=bool(getenv("REALIZE", 0)), shard:int=1) -> tuple[Transformer, dict]: + path = pathlib.Path(path) + cfg = json.loads((path/'config.json').read_text()) + assert cfg['architectures'][0] == 'GlmMoeDsaForCausalLM', "from_hf only supports glm-dsa checkpoints" + assert shard >= 1 and cfg['num_attention_heads'] % shard == 0, "tensor parallel must divide the attention heads" + devices = tuple(Device.canonicalize(f'{Device.DEFAULT}:{i}') for i in range(shard)) + hf_names = {'token_embd.weight': 'model.embed_tokens.weight', 'output_norm.weight': 'model.norm.weight', 'output.weight': 'lm_head.weight', + 'attn_norm.weight': 'input_layernorm.weight', 'ffn_norm.weight': 'post_attention_layernorm.weight', + 'attn_q_a.weight': 'self_attn.q_a_proj.weight', 'attn_q_a_norm.weight': 'self_attn.q_a_layernorm.weight', + 'attn_q_b.weight': 'self_attn.q_b_proj.weight', 'attn_kv_a_mqa.weight': 'self_attn.kv_a_proj_with_mqa.weight', + 'attn_kv_a_norm.weight': 'self_attn.kv_a_layernorm.weight', 'attn_k_b.weight': 'self_attn.kv_b_proj.weight', + 'attn_output.weight': 'self_attn.o_proj.weight', 'indexer.attn_q_b.weight': 'self_attn.indexer.wq_b.weight', + 'indexer.attn_k.weight': 'self_attn.indexer.wk.weight', 'indexer.proj.weight': 'self_attn.indexer.weights_proj.weight', + 'indexer.k_norm.weight': 'self_attn.indexer.k_norm.weight', 'indexer.k_norm.bias': 'self_attn.indexer.k_norm.bias', + 'ffn_gate_inp.weight': 'mlp.gate.weight', 'exp_probs_b.bias': 'mlp.gate.e_score_correction_bias', + **{f'ffn_{p}.weight': f'mlp.{p}_proj.weight' for p in ('gate', 'up', 'down')}, + **{f'ffn_{p}_shexp.weight': f'mlp.shared_experts.{p}_proj.weight' for p in ('gate', 'up', 'down')}, + **{f'ffn_{p}_exps.weight': f'mlp.experts.{{}}.{p}_proj.weight' for p in ('gate', 'up', 'down')}} + shard_axes = {**{w: 0 for w in ('token_embd.weight', 'output.weight', 'attn_q_b.weight', 'attn_k_b.weight', 'ffn_gate.weight', 'ffn_up.weight')}, + **{w: 1 for w in ('attn_output.weight', 'ffn_down.weight', 'ffn_gate_exps.weight', 'ffn_up_exps.weight')}, + **{w: 2 for w in ('ffn_down_exps.weight',)}} + + global_names = ('token_embd.weight', 'output_norm.weight', 'output.weight') + sources:dict[str, str|list[str]] = {k: hf_names[k] for k in global_names} + for layer in range(cfg['num_hidden_layers']): + for key, source in hf_names.items(): + if key in global_names or (key.startswith('indexer.') and cfg['indexer_types'][layer] != 'full'): continue + source = f'model.layers.{layer}.' + source + sources[f'blk.{layer}.{key}'] = [source.format(i) for i in range(cfg['n_routed_experts'])] if '{}' in source else source + shard_map = {name: shard_axes[key] for name in sources if (key:=re.sub(r'^blk\.\d+\.', '', name)) in shard_axes} + state_dict = hf_load(path, devices, sources, shard_map) + block_size = tuple(cfg['quantization_config']['weight_block_size']) + nope, value = cfg['qk_nope_head_dim'], cfg['v_head_dim'] + step = math.gcd(block_size[0], nope, value) + + config = TransformerConfig( + num_blocks=cfg['num_hidden_layers'], dim=cfg['hidden_size'], hidden_dim=cfg['moe_intermediate_size'], + n_heads=cfg['num_attention_heads'], n_kv_heads=1, norm_eps=cfg['rms_norm_eps'], vocab_size=cfg['vocab_size'], + head_dim=cfg['qk_nope_head_dim'] + cfg['qk_rope_head_dim'], v_head_dim=cfg['v_head_dim'], + rope_theta=cfg['rope_parameters']['rope_theta'], rope_dim=cfg['qk_rope_head_dim'], + max_context=min(max_context, cfg['max_position_embeddings']) if max_context is not None else cfg['max_position_embeddings'], + num_experts=cfg['n_routed_experts'], num_experts_per_tok=cfg['num_experts_per_tok'], norm_topk_prob=cfg['norm_topk_prob'], + expert_gating_func=ExpertGating.SIGMOID if cfg['scoring_func'] == 'sigmoid' else ExpertGating.SOFTMAX, + kv_lora_rank=cfg['kv_lora_rank'], q_lora_rank=cfg['q_lora_rank'], leading_dense_blocks=cfg['first_k_dense_replace'], + shared_expert_dim=cfg['n_shared_experts'] * cfg['moe_intermediate_size'], shared_expert_gate=False, expert_bias=True, + dense_hidden_dim=cfg['intermediate_size'], routed_scaling_factor=cfg['routed_scaling_factor'], + indexer=IndexerConfig(cfg['index_n_heads'], cfg['index_head_dim'], cfg['index_topk'], rope_dim=cfg['qk_rope_head_dim'], + rope_interleave=cfg['indexer_rope_interleave']), + indexer_shared=tuple(t != 'full' for t in cfg['indexer_types']), rope_interleave=cfg['rope_interleave']) + model = Transformer(config) + for p in (nn.state.get_parameters(model) if shard > 1 else []): p.to_(devices) + for i, block in enumerate(model.blk): + assert isinstance(block, MLATransformerBlock) + prefix = f'blk.{i}.' + w = state_dict[prefix+'attn_k_b.weight'] + kv = w.reshape(config.n_heads, nope + value, config.kv_lora_rank) + state_dict[prefix+'attn_k_b.weight'] = kv[:, :nope].transpose(1, 2) + state_dict[prefix+'attn_v_b.weight'] = kv[:, nope:] + if (scale:=state_dict.get(prefix+'attn_k_b.weight_scale_inv')) is not None: + # Repeat scales to align their blocks with each head's K/V slices. + scales = scale.repeat_interleave(block_size[0] // step, -2)[:w.shape[0] // step].reshape(config.n_heads, -1, scale.shape[-1]) + state_dict[prefix+'attn_k_b.weight_scale_inv'] = scales[:, :nope // step].transpose(-1, -2) + state_dict[prefix+'attn_v_b.weight_scale_inv'] = scales[:, nope // step:] + block.kv_b_block_size = (step, block_size[1]) + for name, scale in state_dict.items(): + if not name.endswith('.weight_scale_inv'): continue + projection = get_child(model, name.removesuffix('.weight_scale_inv')) + if isinstance(projection, dict): projection['weight_scale_inv'] = scale + else: projection.weight_scale_inv, projection.weight_block_size = scale, block_size + nn.state.load_state_dict(model, state_dict, verbose=False, consume=True, realize=False) + if realize: + params = nn.state.get_parameters(model) + packed = [s.bitcast(dtypes.uint8).contiguous() for s in params] + Tensor.realize(*packed) + for s, data in zip(params, packed): s.replace(data.bitcast(s.dtype)) + return model, cfg + def warmup(self): for _ in range(2): list(zip(range(2), self.generate([0]))) diff --git a/tinygrad_repo/tinygrad/mixin/creation.py b/tinygrad_repo/tinygrad/mixin/creation.py index ef5f2587c7..bf29cec9a6 100644 --- a/tinygrad_repo/tinygrad/mixin/creation.py +++ b/tinygrad_repo/tinygrad/mixin/creation.py @@ -66,11 +66,12 @@ class CreationMixin(DTypeMixin, MovementMixin): @classmethod def full(cls, shape:'tuple[sint, ...]', fill_value:'ConstType|UOp', dtype:DTypeLike|None=None, - device:str|tuple[str, ...]|None=None, buffer=True) -> Self: + device:str|tuple[str, ...]|None=None, buffer=True, axis:int|None=None) -> Self: """ Creates a tensor with the given shape, filled with the given value. You can pass in `dtype` and `device` keyword arguments to control the data type and device of the tensor. + With multiple devices, optionally specify which `axis` to shard on. Pass `buffer=False` to get a broadcast const value instead of a materialized buffer. ```python exec="true" source="above" session="tensor" result="python" @@ -83,11 +84,13 @@ class CreationMixin(DTypeMixin, MovementMixin): # TODO: enable this check # if not buffer: assert device is None, "buffer=False does not support device specification" from tinygrad.uop.ops import UOp + if axis is not None and (isinstance(fill_value, UOp) or not buffer): raise RuntimeError("axis requires a scalar fill value and buffer=True") new_shape = argfix(shape) dt = to_dtype(dtype) if dtype is not None else fill_value.dtype if isinstance(fill_value, UOp) else dtypes.from_py(fill_value) val = cls.const(fill_value, dt).expand(new_shape) if not buffer: return val - ret = val.empty_like(None if dt in dtypes.weaks else dt, device) + if axis is not None: ret = cls.empty(new_shape, device=device, dtype=val.commit_dtype() if dt in dtypes.weaks else dt, axis=axis) + else: ret = val.empty_like(None if dt in dtypes.weaks else dt, device) return cls._wrap_uop(ret._uop.after(ret._uop.store(val._uop))) def full_like(self, fill_value:ConstType, dtype:DTypeLike|None=None, device:str|tuple[str, ...]|None=None, buffer=True) -> Self: diff --git a/tinygrad_repo/tinygrad/mixin/elementwise.py b/tinygrad_repo/tinygrad/mixin/elementwise.py index b62c1873d8..7dedd08344 100644 --- a/tinygrad_repo/tinygrad/mixin/elementwise.py +++ b/tinygrad_repo/tinygrad/mixin/elementwise.py @@ -54,7 +54,7 @@ class ElementwiseMixin(CreationMixin): print(Tensor([False, True]).logical_not().numpy()) ``` """ - return self.cast(dtypes.bool).ne(True) + return self.cast(dtypes.bool).eq(False) def contiguous(self) -> Self: """ @@ -209,7 +209,7 @@ class ElementwiseMixin(CreationMixin): ``` """ a, b = self._broadcasted(x, reverse) - if dtypes.is_int(a.dtype) and dtypes.is_int(b.dtype): return a.alu(Ops.FLOORMOD, b) + if dtypes.is_int(a.dtype) and dtypes.is_int(b.dtype): return a.alu(Ops.MOD, b) return a - a.div(b, rounding_mode="floor") * b def fmod(self, x: Self | ConstType) -> Self: @@ -222,7 +222,7 @@ class ElementwiseMixin(CreationMixin): ``` """ a, b = self._broadcasted(x) - if dtypes.is_int(a.dtype) and dtypes.is_int(b.dtype): return a.alu(Ops.CMOD, b) + if dtypes.is_int(a.dtype) and dtypes.is_int(b.dtype): return a.alu(Ops.TRUNCMOD, b) return a - a.div(b, rounding_mode="trunc") * b def div(self, x: 'Self|ConstType|UOp', reverse: bool = False, rounding_mode: Literal["trunc", "floor"] | None = None) -> Self: @@ -247,8 +247,8 @@ class ElementwiseMixin(CreationMixin): """ a, b = self._broadcasted(x, reverse) if dtypes.is_int(a.dtype) and dtypes.is_int(b.dtype): - if rounding_mode == "trunc": return a.alu(Ops.CDIV, b) - if rounding_mode == "floor": return a.alu(Ops.FLOORDIV, b) + if rounding_mode == "trunc": return a.alu(Ops.TRUNCDIV, b) + if rounding_mode == "floor": return a.alu(Ops.DIV, b) if dtypes.is_int(a.dtype) or a.dtype == dtypes.bool: a = a.cast(dtypes.default_float) # alu, not *: _broadcasted already promoted these, and a second promote would cast 1/b (only a weak CONST is kept weak) d = a.alu(Ops.MUL, b.reciprocal()) @@ -330,10 +330,10 @@ class ElementwiseMixin(CreationMixin): return (self > x).logical_not() def ne(self, x: Self | ConstType) -> Self: - return self._binop(Ops.CMPNE, x, False) + return self.eq(x).logical_not() def eq(self, x: Self | ConstType) -> Self: - return self.ne(x).logical_not() + return self._binop(Ops.CMPEQ, x, False) def __ne__(self, x: Self | ConstType) -> Self: # type: ignore[override] return self.ne(x) @@ -464,7 +464,7 @@ class ElementwiseMixin(CreationMixin): print(Tensor([1., 2., 3., 4.]).reciprocal().numpy()) ``` """ - return self.alu(Ops.RECIPROCAL) + return self.alu(Ops.RECIP) def trunc(self) -> Self: """ diff --git a/tinygrad_repo/tinygrad/mixin/gradient.py b/tinygrad_repo/tinygrad/mixin/gradient.py index 338815b9a5..f1934ff122 100644 --- a/tinygrad_repo/tinygrad/mixin/gradient.py +++ b/tinygrad_repo/tinygrad/mixin/gradient.py @@ -1,6 +1,6 @@ from typing import cast import math, dataclasses -from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, all_metadata, broadcast_axes +from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, broadcast_axes from tinygrad.helpers import argsort from tinygrad.dtype import dtypes, sum_acc_dtype from tinygrad.function import renumber_invalid_outputs @@ -23,7 +23,7 @@ def call_gradient(ctx:UOp, k:UOp, needed:set[int]) -> tuple[UOp|None, ...]: if st.op is Ops.STORE and st.src[0].unsharded_base.op is Ops.PARAM} if fxn.op is Ops.SINK and fxn.arg is None else {} if k.arg.grad_fxn is not None: real = [g.clone(device=args[i].device) if g.device is None else g - for i,g in enumerate(ctx.src if ctx.op is Ops.SINK else (ctx,)) if g.op is not Ops.NOOP] + for i,g in enumerate(ctx.src) if g.op is not Ops.NOOP] git = iter(k.arg.grad_fxn(*real, call=k) if len(real) > 1 else k.arg.grad_fxn(real[0], k)) return (None,) + tuple(None if i in outputs else next(git) for i in range(len(args))) assert outputs, f"expected a CALL with output STOREs or a grad_fxn, got {fxn.op}" @@ -64,13 +64,13 @@ def partial_store_gradient(ctx:UOp, dest:UOp, view:UOp): # ctx is grad_output pm_gradient = PatternMatcher([ (UPat(Ops.CAST, name="ret"), lambda ctx, ret: (ctx.cast(ret.src[0].dtype),)), - (UPat(Ops.RECIPROCAL, name="ret"), lambda ctx, ret: (-ctx * ret * ret,)), + (UPat(Ops.RECIP, name="ret"), lambda ctx, ret: (-ctx * ret * ret,)), (UPat(Ops.SIN, name="ret"), lambda ctx, ret: ((math.pi/2 - ret.src[0]).sin() * ctx,)), (UPat(Ops.LOG2, name="ret"), lambda ctx, ret: (ctx / (ret.src[0] * math.log(2)),)), (UPat(Ops.EXP2, name="ret"), lambda ctx, ret: (ret * ctx * math.log(2),)), (UPat(Ops.SQRT, name="ret"), lambda ctx, ret: (ctx / (ret*2),)), (UPat(Ops.TRUNC), lambda ctx: (ctx.const_like(0),)), - (UPat((Ops.CMPLT, Ops.CMPNE)), lambda: (None, None)), + (UPat((Ops.CMPLT, Ops.CMPEQ)), lambda: (None, None)), (UPat(Ops.ADD), lambda ctx: (ctx, ctx)), (UPat(Ops.POW, name="ret", src=(UPat.var("b"), UPat.var("e"))), lambda ctx, ret, b, e: (ctx * e.eq(0).where(e, e*b.pow(e-1)), ctx * b.eq(0).where((e<0).where(ret.const_like(-math.inf), 0), ret*b.log2()*math.log(2.0)))), @@ -85,7 +85,7 @@ pm_gradient = PatternMatcher([ (UPat(Ops.EXPAND), lambda ctx: (ctx, None)), (UPat(Ops.PAD, name="ret"), lambda ctx, ret: (ctx.shrink(tuple([(p[0], s+p[0]) for s,p in zip(ret.src[0].shape, ret.marg)])), None, None)), (UPat(Ops.SHRINK, name="ret"), lambda ctx, ret: (ctx._mop(Ops.PAD, tuple((p[0], s) for s,p in zip(ret.src[0].shape, ret.marg))), None, None)), - (UPat(Ops.PERMUTE, name="ret"), lambda ctx, ret: (ctx.permute(argsort(ret.marg)),)), + (UPat(Ops.PERMUTE, name="ret"), lambda ctx, ret: (ctx.permute(argsort(ret.marg)), None)), (UPat(Ops.STACK, name="ret"), lambda ctx, ret: tuple(ctx[i] for i in range(len(ret.src)))), (UPat(Ops.COPY, name="ret"), lambda ctx, ret: (ctx.copy_to_device(ret.src[0].device),) + (None,) * (len(ret.src)-1)), (UPat(Ops.UNSHARD, name="ret"), lambda ctx, ret: ctx.shard(ret.device, ret.axis).src), @@ -135,9 +135,4 @@ def compute_gradient(root:UOp, root_grad:UOp, targets:set[UOp]) -> dict[UOp, UOp n if p.op is Ops.NOOP else p for p, n in zip(grads[k].src, v.src)]) else: grads[k] = grads[k] + v else: grads[k] = v - if len(forward_metadata:=all_metadata.get(t0, ())): - backward_metadata = tuple(dataclasses.replace(x, backward=True) for x in forward_metadata) - # we add the backward metadata to everything new in the graph - for bw_uop in v.toposort(lambda x: x not in (t0, *t0.src, grads[t0])): - all_metadata[bw_uop] = all_metadata.get(bw_uop, ())+backward_metadata return grads diff --git a/tinygrad_repo/tinygrad/mixin/op.py b/tinygrad_repo/tinygrad/mixin/op.py index 0c45d6302a..e508beb6bc 100644 --- a/tinygrad_repo/tinygrad/mixin/op.py +++ b/tinygrad_repo/tinygrad/mixin/op.py @@ -1,10 +1,10 @@ from __future__ import annotations import functools, itertools, math, string -from typing import TYPE_CHECKING, Callable, Self, Sequence, Literal, get_args +from typing import TYPE_CHECKING, Any, Callable, Self, Sequence, Literal, get_args from tinygrad.mixin.elementwise import ElementwiseMixin from tinygrad.mixin.movement import MovementMixin from tinygrad.mixin.reduce import ReduceMixin -from tinygrad.uop import Ops +from tinygrad.uop import Ops, GroupOp from tinygrad.uop.ops import _broadcast_shape, resolve, smax, smin, identity_element from tinygrad.dtype import ConstType, DType, DTypeLike, Invalid, PyConst, dtypes, least_upper_dtype, sum_acc_dtype, to_dtype, commit_int from tinygrad.helpers import all_int, argfix, ceildiv, flatten, flat_to_grouped, fully_flatten, get_shape, make_tuple, merge_dicts, prod @@ -29,7 +29,30 @@ class OpMixin(ElementwiseMixin, ReduceMixin): ``` """ assert self.numel() == 1, "must have one element for item" - return self.data()[(0,) * len(self.shape)] + return self.reshape(()).tolist() # type: ignore[return-value] + + # NOTE: list[Any] because return type is recursive (list[list[...]] for higher dimensions) + def tolist(self) -> PyConst|list[Any]: + """ + Returns the value of this tensor as a nested list. + Returns single value for const tensor. + + ```python exec="true" source="above" session="tensor" result="python" + t = Tensor([1, 2, 3, 4]) + print(t.tolist()) + ``` + ```python exec="true" source="above" session="tensor" result="python" + t = Tensor(5) + print(t.tolist()) + ``` + """ + # TODO: remove half once minimum python supports it + if self.dtype in (dtypes.half, dtypes.bfloat16, *dtypes.fp8s): return self.cast(dtypes.float32).tolist() + if 0 in self.shape: + assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" + def _tolist(shape:tuple[int, ...]): return [_tolist(shape[1:]) for _ in range(shape[0])] + return _tolist(self.shape) + return self.data().tolist() def __getitem__(self, indices) -> Self: """ @@ -750,7 +773,7 @@ class OpMixin(ElementwiseMixin, ReduceMixin): return padded[0].usum(*padded[1:]) def _cumalu(self, axis:int, op:Ops) -> Self: - assert self.shape[axis] != 0 and op in (Ops.ADD, Ops.MAX, Ops.MUL) + assert self.shape[axis] != 0 and op in GroupOp.Reduce pads = (None,)*(self.ndim-1) + ((self.shape[axis]-1, 0),) pooled = self.transpose(axis,-1)._pad_constant(pads, identity_element(op, self.commit_dtype()))._pool((self.shape[axis],)) return getattr(pooled, {Ops.ADD: "sum", Ops.MAX: "max", Ops.MUL: "prod"}[op])(-1).transpose(axis, -1) diff --git a/tinygrad_repo/tinygrad/nn/onnx.py b/tinygrad_repo/tinygrad/nn/onnx.py index 24a3ff4f68..c3381d1e5a 100644 --- a/tinygrad_repo/tinygrad/nn/onnx.py +++ b/tinygrad_repo/tinygrad/nn/onnx.py @@ -1,5 +1,5 @@ # pylint: disable=possibly-unused-variable -from typing import Any, Sequence, cast, Literal, NamedTuple, Generator +from typing import Any, Sequence, cast, Literal, NamedTuple, Generator, Callable import dataclasses, functools, io, math, types, warnings, pathlib, sys, os, struct, enum from tinygrad.nn.state import TensorIO from tinygrad.tensor import Tensor, is_numpy_ndarray @@ -146,46 +146,31 @@ class OnnxPBParser: start_pos = self.reader.tell() return start_pos + str_len + def _parse_fields(self, obj:dict[str, Any], fields:dict[int, tuple[str, Callable[[], Any]]], end_pos:int|None=None) -> dict[str, Any]: + repeated = {k for k,v in obj.items() if isinstance(v, list)} + for fid, wire_type in self._parse_message(self._decode_end_pos() if end_pos is None else end_pos): + if (f := fields.get(fid)) is None: self.reader.skip_field(wire_type) + elif f[0] in repeated: obj[f[0]].append(f[1]()) + else: obj[f[0]] = f[1]() + return obj + def _parse_ModelProto(self) -> dict: """Entry point for parsing the ONNX model.""" - obj: dict[str, Any] = {"opset_import": []} - for fid, wire_type in self._parse_message(self.reader.len): - if fid == 8: obj["opset_import"].append(self._parse_OperatorSetIdProto()) - else: self.reader.skip_field(wire_type) + obj = self._parse_fields({"opset_import": []}, {8: ("opset_import", self._parse_OperatorSetIdProto)}, self.reader.len) self.opset_imports = {Domain.from_onnx(x.get('domain')):x.get('version', 1) for x in obj["opset_import"]} self.reader.seek(0) - for fid, wire_type in self._parse_message(self.reader.len): - match fid: - case 4: obj["domain"] = self.reader.read_string() - case 5: obj["model_version"] = self.reader.read_int64() - case 7: obj["graph"] = self._parse_GraphProto() - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields(obj, {4: ("domain", self.reader.read_string), 5: ("model_version", self.reader.read_int64), + 7: ("graph", self._parse_GraphProto)}, self.reader.len) def _parse_GraphProto(self) -> dict: - obj: dict[str, Any] = {"node": [], "initializer": [], "input": [], "output": []} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["node"].append(self._parse_NodeProto()) - case 2: obj["name"] = self.reader.read_string() - case 5: obj["initializer"].append(self._parse_TensorProto()) - case 11: obj["input"].append(self._parse_ValueInfoProto()) - case 12: obj["output"].append(self._parse_ValueInfoProto()) - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({"node": [], "initializer": [], "input": [], "output": []}, {1: ("node", self._parse_NodeProto), + 2: ("name", self.reader.read_string), 5: ("initializer", self._parse_TensorProto), 11: ("input", self._parse_ValueInfoProto), + 12: ("output", self._parse_ValueInfoProto)}) def _parse_NodeProto(self) -> dict: - obj: dict[str, Any] = {"input": [], "output": [], "attribute": [], "domain": None} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["input"].append(self.reader.read_string()) - case 2: obj["output"].append(self.reader.read_string()) - case 3: obj["name"] = self.reader.read_string() - case 4: obj["op_type"] = self.reader.read_string() - case 5: obj["attribute"].append(self._parse_AttributeProto()) - case 6: obj["doc_string"] = self.reader.read_string() - case 7: obj["domain"] = self.reader.read_string() - case _: self.reader.skip_field(wire_type) + obj = self._parse_fields({"input": [], "output": [], "attribute": [], "domain": None}, {1: ("input", self.reader.read_string), + 2: ("output", self.reader.read_string), 3: ("name", self.reader.read_string), 4: ("op_type", self.reader.read_string), + 5: ("attribute", self._parse_AttributeProto), 6: ("doc_string", self.reader.read_string), 7: ("domain", self.reader.read_string)}) # parse node attributes = {attr_dict["name"]: attr_dict[AttributeType(attr_dict["type"]).to_field_name()] for attr_dict in obj["attribute"]} @@ -194,25 +179,15 @@ class OnnxPBParser: return obj def _parse_TensorProto(self) -> dict: - obj: dict[str, Any] = {"dims": []} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["dims"].append(self.reader.read_int64()) - case 2: obj["data_type"] = self.reader.read_int64() - case 4: obj["float_data"] = self.reader.read_packed_floats() - case 5: obj["int32_data"] = self.reader.read_packed_int64s() - case 7: obj["int64_data"] = self.reader.read_packed_int64s() - case 8: obj["name"] = self.reader.read_string() - case 9: obj["raw_data"] = self.reader.read_bytes() - case 10: obj["double_data"] = self.reader.read_packed_floats() - case 11: obj["uint64_data"] = self.reader.read_packed_int64s() - case 13: obj.setdefault("external_data", []).append(self._parse_StringStringEntryProto()) - case 14: obj["data_location"] = self.reader.read_int64() - case _: self.reader.skip_field(wire_type) + obj = self._parse_fields({"dims": [], "external_data": []}, {1: ("dims", self.reader.read_int64), 2: ("data_type", self.reader.read_int64), + 4: ("float_data", self.reader.read_packed_floats), 5: ("int32_data", self.reader.read_packed_int64s), + 7: ("int64_data", self.reader.read_packed_int64s), 8: ("name", self.reader.read_string), 9: ("raw_data", self.reader.read_bytes), + 10: ("double_data", self.reader.read_packed_floats), 11: ("uint64_data", self.reader.read_packed_int64s), + 13: ("external_data", self._parse_StringStringEntryProto), 14: ("data_location", self.reader.read_int64)}) # load external data if self.load_external_data and obj.get("data_location", 0) == 1: - if "external_data" not in obj: raise ValueError("no external_data") + if not obj["external_data"]: raise ValueError("no external_data") location, length, offset = None, None, 0 for kv in obj["external_data"]: if kv["key"] == "location": location = kv["value"] @@ -250,30 +225,15 @@ class OnnxPBParser: return obj def _parse_AttributeProto(self) -> dict: - obj: dict[str, Any] = {"floats": [], "ints": [], "strings": []} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["name"] = self.reader.read_string() - case 2: obj["f"] = self.reader.read_float() - case 3: obj["i"] = self.reader.read_int64() - case 4: obj["s"] = self.reader.read_bytes().data().tobytes().decode("utf8") - case 5: obj["t"] = self._parse_TensorProto()['parsed_tensor'] - case 6: obj["g"] = OnnxRunner._from_subgraph(self._parse_GraphProto()) - case 7: obj["floats"].append(self.reader.read_float()) - case 8: obj["ints"].append(self.reader.read_int64()) - case 9: obj["strings"].append(self.reader.read_bytes().data().tobytes().decode("utf8")) - case 20: obj["type"] = self.reader.read_int64() - case _: self.reader.skip_field(wire_type) + obj = self._parse_fields({"floats": [], "ints": [], "strings": []}, {1: ("name", self.reader.read_string), 2: ("f", self.reader.read_float), + 3: ("i", self.reader.read_int64), 4: ("s", self.reader.read_string), 5: ("t", lambda: self._parse_TensorProto()['parsed_tensor']), + 6: ("g", lambda: OnnxRunner._from_subgraph(self._parse_GraphProto())), 7: ("floats", self.reader.read_float), + 8: ("ints", self.reader.read_int64), 9: ("strings", self.reader.read_string), 20: ("type", self.reader.read_int64)}) obj["floats"], obj["ints"], obj["strings"] = tuple(obj["floats"]), tuple(obj["ints"]), tuple(obj["strings"]) return obj def _parse_ValueInfoProto(self) -> dict: - obj: dict[str, Any] = {} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["name"] = self.reader.read_string() - case 2: obj["type"] = self._parse_TypeProto() - case _: self.reader.skip_field(wire_type) + obj = self._parse_fields({}, {1: ("name", self.reader.read_string), 2: ("type", self._parse_TypeProto)}) # parse type if "type" not in obj: return {**obj, "parsed_type": None} @@ -287,66 +247,26 @@ class OnnxPBParser: return obj def _parse_TypeProto(self) -> dict: - obj: dict[str, Any] = {} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["tensor_type"] = self._parse_TypeProtoTensor() - case 4: obj["sequence_type"] = self._parse_TypeProtoWrapper() - case 9: obj["optional_type"] = self._parse_TypeProtoWrapper() - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({}, {1: ("tensor_type", self._parse_TypeProtoTensor), 4: ("sequence_type", self._parse_TypeProtoWrapper), + 9: ("optional_type", self._parse_TypeProtoWrapper)}) def _parse_TypeProtoTensor(self) -> dict: - obj: dict[str, Any] = {} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["elem_type"] = self.reader.read_int64() - case 2: obj["shape"] = self._parse_TensorShapeProto() - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({}, {1: ("elem_type", self.reader.read_int64), 2: ("shape", self._parse_TensorShapeProto)}) def _parse_TypeProtoWrapper(self) -> dict: - obj = {} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["elem_type"] = self._parse_TypeProto() - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({}, {1: ("elem_type", self._parse_TypeProto)}) def _parse_TensorShapeProto(self) -> dict: - obj: dict[str, Any] = {"dim": []} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["dim"].append(self._parse_TensorShapeProtoDimension()) - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({"dim": []}, {1: ("dim", self._parse_TensorShapeProtoDimension)}) def _parse_TensorShapeProtoDimension(self) -> dict: - obj: dict[str, Any] = {} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["dim_value"] = self.reader.read_int64() - case 2: obj["dim_param"] = self.reader.read_string() - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({}, {1: ("dim_value", self.reader.read_int64), 2: ("dim_param", self.reader.read_string)}) def _parse_StringStringEntryProto(self) -> dict: - obj: dict[str, Any] = {} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["key"] = self.reader.read_string() - case 2: obj["value"] = self.reader.read_string() - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({}, {1: ("key", self.reader.read_string), 2: ("value", self.reader.read_string)}) def _parse_OperatorSetIdProto(self) -> dict: - obj: dict[str, Any] = {} - for fid, wire_type in self._parse_message(self._decode_end_pos()): - match fid: - case 1: obj["domain"] = self.reader.read_string() - case 2: obj["version"] = self.reader.read_int64() - case _: self.reader.skip_field(wire_type) - return obj + return self._parse_fields({}, {1: ("domain", self.reader.read_string), 2: ("version", self.reader.read_int64)}) # ***** python const ***** required_input_python_consts: dict[str, tuple[int, ...]] = { diff --git a/tinygrad_repo/tinygrad/renderer/__init__.py b/tinygrad_repo/tinygrad/renderer/__init__.py index aaab09fa6f..59a9369537 100644 --- a/tinygrad_repo/tinygrad/renderer/__init__.py +++ b/tinygrad_repo/tinygrad/renderer/__init__.py @@ -37,7 +37,7 @@ class Estimates: for u in uops: if u.op in {Ops.LOAD, Ops.STORE}: buf = u - while len(buf.src) and buf.op not in {Ops.PARAM, Ops.BUFFER, Ops.ALLOC}: buf = buf.src[0] + while len(buf.src) and buf.op not in GroupOp.Defines: buf = buf.src[0] if buf.op is Ops.PARAM: # u.src[0] is INDEX, cap at buffer size for re-reads (e.g. matmul) accessed = mem.get((buf, u.op), 0) + u.src[0].max_numel() * u.src[0].dtype.itemsize * mults @@ -45,7 +45,7 @@ class Estimates: if u.op is Ops.RANGE: mult_stack.append(mults) if u.dtype is not dtypes.void: # unbounded loop, unknown trip count - mults *= cast(sint, u.src[0].ssimplify()) + mults *= cast(sint, u.src[0].without_after.ssimplify()) # SPECIAL are already counted in mults mults = mults.substitute({x:x.const_like(0) for x in mults.toposort() if x.op is Ops.SPECIAL}) if isinstance(mults, UOp) else mults elif u.op in {Ops.END, Ops.BACKEDGE}: mults = mult_stack.pop(-1) @@ -63,6 +63,7 @@ class Estimates: class Renderer: target: Target suffix: str = "" + inline_prefix: str|None = None # TODO: make this generic with a list of supported types supports_float4: bool = True has_local: bool = True diff --git a/tinygrad_repo/tinygrad/renderer/amd/elf.py b/tinygrad_repo/tinygrad/renderer/amd/elf.py index 8c918c063d..d9fddd6f6d 100644 --- a/tinygrad_repo/tinygrad/renderer/amd/elf.py +++ b/tinygrad_repo/tinygrad/renderer/amd/elf.py @@ -2,7 +2,7 @@ import ctypes from tinygrad.dtype import AddrSpace from tinygrad.helpers import ceildiv, round_up -from tinygrad.uop.ops import UOp, Ops +from tinygrad.uop.ops import UOp, Ops, AxisType from tinygrad.runtime.autogen import amdgpu_kd, hsa, libc from tinygrad.renderer.amd.dsl import Reg, FixedBitField from tinygrad.runtime.autogen.amd.common import OpType @@ -35,14 +35,17 @@ def assemble_linear(prg:UOp, lin:UOp, arch:str) -> bytes: elif val.offset < 106: max_sgpr = max(max_sgpr, val.offset + val.sz) # ** scan sink for metadata - sink, param_sizes, lds_size, gids = prg.src[0], {}, 0, set() + sink, param_sizes, lds_size, scratch_size, gids = prg.src[0], {}, 0, 0, set() + arch = next(v for k, v in _arch_map.items() if arch.startswith(k)) + is_cdna, is_rdna4 = arch == "cdna", arch == "rdna4" + wave32 = not is_cdna for u in sink.toposort(): if u.op is Ops.PARAM: param_sizes[u.arg.slot] = u.dtype.itemsize if u.addrspace is AddrSpace.ALU else 8 elif u.op is Ops.BUFFER and u.addrspace is AddrSpace.LOCAL: lds_size += u.max_numel() * u.dtype.itemsize + elif u.op is Ops.ALLOC and u.addrspace is AddrSpace.GLOBAL and u.arg.name == "scratch": scratch_size += u.max_numel() * u.dtype.itemsize elif u.op is Ops.SPECIAL and u.arg.startswith("gidx"): gids.add(int(u.arg[-1])) + elif u.op is Ops.RANGE and u.axis_type is AxisType.WARP and u.src[0].arg == 64: wave32 = True code_bytes = b"".join(inst.to_bytes() for inst in insts) - arch = next(v for k, v in _arch_map.items() if arch.startswith(k)) - is_cdna, is_rdna4 = arch == "cdna", arch == "rdna4" # ** pad text to ISA alignment padding_inst = (s_nop_cdna(0) if is_cdna else s_code_end()).to_bytes() @@ -54,11 +57,12 @@ def assemble_linear(prg:UOp, lin:UOp, arch:str) -> bytes: accum_offset = round_up(max_vgpr, 4) if max_accvgpr > 0 else 0 next_free_vgpr = round_up(accum_offset + max_accvgpr, 8) if max_accvgpr > 0 else round_up(max_vgpr, 8) next_free_sgpr = round_up(max_sgpr, 8) - vgpr_granule = max(0, (next_free_vgpr + 7) // 8 - 1) + vgpr_granule = max(0, (next_free_vgpr - 1) // (8 if is_cdna or wave32 else 4)) # CDNA: add 6 for VCC(2) + FLAT_SCRATCH(2) + XNACK_MASK(2), next_free_sgpr is unused in RDNA. sgpr_granule = max(0, ceildiv(next_free_sgpr + 6, 8) - 1) if is_cdna else 0 desc = amdgpu_kd.llvm_amdhsa_kernel_descriptor_t() desc.group_segment_fixed_size = lds_size + desc.private_segment_fixed_size = scratch_size for sz in (param_sizes[i] for i in sorted(param_sizes)): desc.kernarg_size = round_up(desc.kernarg_size, sz) + sz desc.kernel_code_entry_byte_offset = -len(text) @@ -70,12 +74,13 @@ def assemble_linear(prg:UOp, lin:UOp, arch:str) -> bytes: (0 if is_rdna4 else 1) << amdgpu_kd.COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT | (0 if is_rdna4 else 1) << amdgpu_kd.COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT | (0 if is_cdna else 1) << amdgpu_kd.COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT) - desc.compute_pgm_rsrc2 = (2 << amdgpu_kd.COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT | + desc.compute_pgm_rsrc2 = (int(scratch_size > 0) << amdgpu_kd.COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT | + 2 << amdgpu_kd.COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT | int(0 in gids) << amdgpu_kd.COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT | int(1 in gids) << amdgpu_kd.COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT | int(2 in gids) << amdgpu_kd.COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT) desc.kernel_code_properties = (1 << amdgpu_kd.KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT | - (0 if is_cdna else 1) << amdgpu_kd.KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT) + int(wave32) << amdgpu_kd.KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT) if is_cdna and max_accvgpr > 0: desc.compute_pgm_rsrc3 = max(0, accum_offset // 4 - 1) << amdgpu_kd.COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT rodata = bytes(desc) diff --git a/tinygrad_repo/tinygrad/renderer/amd/generate.py b/tinygrad_repo/tinygrad/renderer/amd/generate.py index 0ad79c6c4a..0923d90eac 100644 --- a/tinygrad_repo/tinygrad/renderer/amd/generate.py +++ b/tinygrad_repo/tinygrad/renderer/amd/generate.py @@ -148,7 +148,7 @@ def parse_xml(filename: str): for fmt, _, otype in op_info.values(): if fmt and fmt not in fmts: fmts[fmt] = 0 if otype: op_types_set.add(otype) - if op_info: types[(name, base_enum)] = op_info + if op_info or (enc.find("Operands") is not None and not enc.findall("Operands/Operand")): types[(name, base_enum)] = op_info # Find opcodes that only exist in a specific variant encoding (no base format version) suffix_only_ops: dict[str, dict[str, set[int]]] = {} # {suffix: {base_fmt: {opcodes}}} for base_fmt, opcodes in opcode_encs.items(): @@ -240,8 +240,9 @@ def extract_pcode(pages: list[list[tuple[float, float, str, str]]], name_to_op: for j in range(1, len(sorted_lines)): prev_page, prev_y, _ = sorted_lines[j-1] curr_page, curr_y, _ = sorted_lines[j] - if depth == 0 and curr_page == prev_page and prev_y - curr_y > 30: break - if depth == 0 and curr_page != prev_page and prev_y > 60 and curr_y < 730: break + if depth == 0 and ((curr_page == prev_page and prev_y - curr_y > 30) or (curr_page != prev_page and prev_y > 60 and curr_y < 730)): + if any(' = ' in t for *_, t in filtered) or not any(' = ' in t for *_, t in sorted_lines[j:]): break + filtered = [] filtered.append(sorted_lines[j]) code = sorted_lines[j][2].split("//")[0] depth += code.count("{") - code.count("}") @@ -462,6 +463,7 @@ def write_ins(encodings, enums, suffix_only_ops, types, arch, path): with open(path, "w") as f: f.write("\n".join(lines)) def write_operands(types: dict, enums: dict, arch: str, path: pathlib.Path) -> None: + types = {k: v for k, v in types.items() if v} valid = {(name, fmt) for fmt, ops in enums.items() for name in ops.values()} # only import enums that are actually used as keys in OPERANDS used_bases = {eb for (nm, eb) in types if (nm, eb) in valid} @@ -536,6 +538,9 @@ if __name__ == "__main__": pages = extract_pdf_text(cfg["pdf"]) name_to_op = {name: op for ops in arch_data[arch]["enums"].values() for op, name in ops.items()} pcode = extract_pcode(pages, name_to_op) + for fmt, ops in arch_data[arch]["enums"].items(): + for op, name in ops.items(): + if arch_data[arch]["types"].get((name, fmt)) == {}: pcode.setdefault((name, op), '') base = autogen_base / arch write_pcode(pcode, arch_data[arch]["enums"], arch, base / "str_pcode.py") print(f" {arch}: {len(pcode)} pcode entries") diff --git a/tinygrad_repo/tinygrad/renderer/cstyle.py b/tinygrad_repo/tinygrad/renderer/cstyle.py index 67ebc3da89..770ae5c355 100644 --- a/tinygrad_repo/tinygrad/renderer/cstyle.py +++ b/tinygrad_repo/tinygrad/renderer/cstyle.py @@ -1,5 +1,5 @@ from typing import Literal, Callable -import math, sys +import math from collections import defaultdict, Counter from tinygrad.renderer import tc from tinygrad.uop.ops import GroupOp, Ops, UOp, PatternMatcher, UPat, range_str, axis_letters @@ -69,13 +69,23 @@ base_rewrite = PatternMatcher([ f"(({ctx.abi}{ctx.render_dtype(x.dtype)}(*)({', '.join(map(ctx.render_type, x.src[1:]))}))({f.arg.name}))" f"({', '.join(f'({ctx.render_type(y)})({ctx[y]})' for y in x.src[1:])})" + (";" if x.dtype is dtypes.void else "")), - (UPat(Ops.CALL, src=(UPat(Ops.LINEAR, name="body"),), allow_any_len=True, name="x"), lambda ctx,x,body: f"{ctx.fn_names[body]}(" - f"{', '.join(f'({ctx.param_type(p)}){ctx[x.src[s+1]]}' for s,p in sorted((u.arg.slot,u) for u in body.src if u.op is Ops.PARAM))});"), + # call to PROGRAM links BINARY at runtime + (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="body"),), allow_any_len=True, name="x"), lambda ctx,x,body: + f"{body.src[0].arg.function_name}(&(struct {body.src[0].arg.function_name}_args){{" + f"{', '.join(f'({ctx.param_type(p)}){ctx[x.src[p.arg.slot+1]]}' for p in body.arg.params)}}});"), + + # call to LINEAR renders the call in SOURCE + (UPat(Ops.CALL, src=(UPat(Ops.LINEAR, name="body"),), allow_any_len=True, name="x"), lambda ctx,x,body: f"{body.arg}(" + f"{', '.join(f'({ctx.param_type(p)}){ctx[x.src[s+1]]}' for s,p in [(u.arg.slot,u) for u in body.src if u.op is Ops.PARAM])});"), # custom passes through with format (UPat((Ops.CUSTOM, Ops.CUSTOMI), name="x"), lambda ctx,x: x.arg[0].format(*[ctx[y] for y in x.src])), ]) +# C promotes uchar and ushort to int, WGSL holds them in u32 and PTX holds uchar in u16, so the ops that wrap compute in uint and narrow after +pm_narrow_uint = PatternMatcher([(UPat((Ops.ADD, Ops.SUB, Ops.MUL, Ops.NEG, Ops.SHL, Ops.MULACC), (dtypes.uint8, dtypes.uint16), name="x"), + lambda x: x.replace(src=tuple(s.cast(dtypes.uint32) for s in x.src)).cast(x.dtype))]) + def create_non_native_float_pats(dts:tuple[DType, ...], casting:bool=True): patterns = PatternMatcher([ # a weak CONST states no width and cannot be restated: commit it at the emulated dtype a sibling src states @@ -129,6 +139,7 @@ class CStyleLanguage(Renderer): var_suffix: str = "" barrier: str = "" code_for_workitem: dict[Literal["g", "l"], Callable] = {} + args_struct_prefix: str|None = None extra_args: list[str] = [] float4: str|None = None float4_style: tuple[str, str] = ('(', ')') @@ -141,12 +152,12 @@ class CStyleLanguage(Renderer): infinity: str = "INFINITY" nan: str = "NAN" code_for_op: dict = { - Ops.SQRT: lambda x,dtype: f"sqrt({x})", Ops.RECIPROCAL: lambda x,dtype: f"(1/{x})", Ops.NEG: lambda x,dtype: f"-{x}", + Ops.SQRT: lambda x,dtype: f"sqrt({x})", Ops.RECIP: lambda x,dtype: f"(1/{x})", Ops.NEG: lambda x,dtype: f"-{x}", Ops.EXP2: lambda x,dtype: f"exp2({x})", Ops.LOG2: lambda x,dtype: f"log2({x})", Ops.SIN: lambda x,dtype: f"sin({x})", Ops.TRUNC: lambda x,dtype: f"trunc({x})", Ops.AND: lambda a,b,dtype: f"({a}&{b})", Ops.XOR: lambda a,b,dtype: f"({a}^{b})", Ops.OR: lambda a,b,dtype: f"({a}|{b})", Ops.ADD: lambda a,b,dtype: f"({a}+{b})", Ops.SUB: lambda a,b,dtype: f"({a}-{b})", Ops.MUL: lambda a,b,dtype: f"({a}*{b})", - Ops.CMOD: lambda a,b,dtype: f"({a}%{b})", Ops.CDIV: lambda a,b,dtype: f"({a}/{b})", Ops.CMPNE: lambda a,b,dtype: f"({a}!={b})", + Ops.TRUNCMOD: lambda a,b,dtype: f"({a}%{b})", Ops.TRUNCDIV: lambda a,b,dtype: f"({a}/{b})", Ops.CMPNE: lambda a,b,dtype: f"({a}!={b})", Ops.SHR: lambda a,b,dtype: f"({a}>>{b})", Ops.SHL: lambda a,b,dtype: f"({a}<<{b})", Ops.CMPLT: lambda a,b,dtype: f"({a}<{b})", Ops.WHERE: lambda a,b,c,dtype: f"({a}?{b}:{c})", Ops.CMPEQ: lambda a,b,dtype: f"({a}=={b})"} @@ -159,6 +170,10 @@ class CStyleLanguage(Renderer): buftypes = [(name, ("volatile " if u.arg.volatile else "")+(self.var_prefix if u.addrspace == AddrSpace.ALU else "")+ self._render_dtype(u.dtype, sz=1, addrspace=u.addrspace, mutable=mutable, shape=u._shape)+ (self.var_suffix if u.addrspace == AddrSpace.ALU else self.buffer_suffix)) for name,(u,mutable) in bufs] + if self.args_struct_prefix is not None: + prefix = (prefix or []) + [f"struct {function_name}_args {{ " + "".join(f"{self.param_type(u)} arg{u.arg.slot}; " for _,(u,_) in bufs) + "};"] + kernel = [f" {self.param_type(u)} {n} = args->arg{u.arg.slot};" for n,(u,_) in bufs] + kernel + buftypes = [("args", f"{self.args_struct_prefix} struct {function_name}_args*")] local_dims = [u.src[0] for u in uops if u.op is Ops.SPECIAL and u.arg[0] == "l"] launch_bounds = prod([d.vmax for d in local_dims]) prg = ''.join([f"{self.kernel_typedef.format(launch_bounds=launch_bounds)} {function_name}(",] + @@ -266,16 +281,15 @@ class CStyleLanguage(Renderer): return (name, kernel, list(bufs.values())) def param_type(self, p:UOp): return "volatile "*p.arg.volatile + self._render_dtype(p.dtype, 1, p.addrspace, True, p.addrspace != AddrSpace.ALU) def render(self, uops:list[UOp]) -> str: - prefix, call_bodies, self.fn_names = [], [], dict[UOp, str]() - topo = UOp.sink(*uops).toposort(enter_calls=True) - prefix += [f"extern void {f}();" for f in dedup(u.arg.name for u in topo if u.op is Ops.CUSTOM_FUNCTION)] # symbols to link - for body in (u for u in topo if u.op is Ops.LINEAR): - self.fn_names[body] = body.arg + (f"_{n}" if (n:=sum(b.arg == body.arg for b in self.fn_names)) else "") # a name traced with other args - _, call, bufs = self._render(body.src) - params = ', '.join(f"{self.param_type(p)} {n}" for n,(p,_) in bufs) - prefix.append(f"static inline void {self.fn_names[body]}({params}) {{\n" + '\n'.join(call) + "\n}") - call_bodies.extend(body.src) - return self.render_kernel(*self._render(uops), call_bodies+list(uops), prefix or None) + bodies = dedup(u.body for u in UOp.sink(*uops).toposort(enter_calls=bool(self.inline_prefix)) if u.op is Ops.CALL) + fns = {b:b.src[0].arg.function_name for b in bodies if b.op is Ops.PROGRAM} + inlines = {b:self._render(list(b.src)) for b in bodies if b.op is Ops.LINEAR} + prefix = [f"extern void {b.arg.name}();" if b.op is Ops.CUSTOM_FUNCTION else + f"struct {fns[b]}_args {{ " + ''.join(f"{self.param_type(p)} arg{p.arg.slot}; " for p in b.arg.params) + + f"}}; extern void {fns[b]}(const struct {fns[b]}_args*);" if b.op is Ops.PROGRAM else + f"static inline {self.inline_prefix} void {b.arg}({', '.join(f'{self.param_type(p)} {n}' for n,(p,_) in inlines[b][2])}) {{\n" + + '\n'.join(inlines[b][1]) + "\n}" for b in bodies] + return self.render_kernel(*self._render(uops), uops+[u for b in inlines for u in b.src], prefix or None) class ClangRenderer(CStyleLanguage): float4 = "(float4)" @@ -283,6 +297,7 @@ class ClangRenderer(CStyleLanguage): gep_arr_threshold = 0 has_local = False global_max = (1, 0, 0) + args_struct_prefix = "const" infinity = "__builtin_inff()" nan = '__builtin_nanf("")' @@ -290,21 +305,18 @@ class ClangRenderer(CStyleLanguage): barrier = "__atomic_thread_fence(__ATOMIC_SEQ_CST);" buffer_suffix = " restrict" type_map = {**CStyleLanguage.type_map, dtypes.bool:"_Bool", dtypes.f16:"__fp16"} - code_for_op = {**({k:v for k,v in CStyleLanguage.code_for_op.items() if k not in [Ops.EXP2, Ops.SIN, Ops.LOG2, Ops.RECIPROCAL]}), + code_for_op = {**({k:v for k,v in CStyleLanguage.code_for_op.items() if k not in [Ops.EXP2, Ops.SIN, Ops.LOG2, Ops.RECIP]}), Ops.SQRT: lambda x,dtype: f"__builtin_sqrt({x})" if dtype == dtypes.float64 else f"__builtin_sqrtf({x})", Ops.TRUNC: lambda x,dtype: f"__builtin_trunc({x})" if dtype == dtypes.float64 else f"__builtin_truncf({x})", Ops.FDIV: lambda a,b,dtype: f"({a}/{b})"} # LLVM legalizes double => half cast on systems that don't support it natively (like x86 cpus without AVX512-FP16) into a compiler-rt libcall. extra_matcher = PatternMatcher([(UPat.var("x", dtypes.float64).cast(dtypes.float16), lambda x: x.cast(dtypes.float32).cast(dtypes.float16))]) \ - + create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast + + pm_narrow_uint + create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast - if sys.platform == 'win32': - abi = "__attribute__((ms_abi)) " - kernel_typedef = abi + "void" def render_vector_prefix(self, dt:DType, count:int) -> str: # round (down) to power of two (this is actually the default clang behavior) - alignment = 2**int(math.log2(dt.itemsize * count)) if getenv("ALIGNED", 1) and not dtypes.is_bool(dt) else 1 + alignment = dt.itemsize if self.target.device == "CPU" else (2**int(math.log2(dt.itemsize * count)) if not dtypes.is_bool(dt) else 1) vec = self._render_dtype(dt, count, AddrSpace.REG) return f"typedef {self.render_dtype(dt)} {vec} __attribute__((aligned({alignment}),ext_vector_type({count})));" @@ -335,7 +347,7 @@ class OpenCLRenderer(CStyleLanguage): code_for_workitem = {"g": lambda x: f"get_group_id({x})", "l": lambda x: f"get_local_id({x})"} type_map = {**CStyleLanguage.type_map, dtypes.i8: "char", dtypes.u8: "uchar", dtypes.u32: "uint", dtypes.u16: "ushort", dtypes.u64: "ulong", dtypes.bf16: "ushort"} - extra_matcher = create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast + extra_matcher = pm_narrow_uint + create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast string_rewrite = pm_bf16_ushort_const + PatternMatcher([ (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))" @@ -371,15 +383,16 @@ class MetalRenderer(CStyleLanguage): barrier = "threadgroup_barrier(mem_flags::mem_threadgroup);" float4 = "float4" code_for_workitem = {"g": lambda x: f"gid.{chr(120+int(x))}", "l": lambda x: f"lid.{chr(120+int(x))}"} + args_struct_prefix = "constant" # uint3 used for gid/lid - TODO: this should probably be `ushort3 lid [[thread_position_in_threadgroup]]` - extra_args = ['constant args_t& args [[buffer(0)]]', 'uint3 gid [[threadgroup_position_in_grid]]', 'uint3 lid [[thread_position_in_threadgroup]]'] + extra_args = ['uint3 gid [[threadgroup_position_in_grid]]', 'uint3 lid [[thread_position_in_threadgroup]]'] type_map = {**CStyleLanguage.type_map, dtypes.u32: "uint", dtypes.bf16: "bfloat"} # precise::sin code_for_op = {**CStyleLanguage.code_for_op, Ops.SIN: lambda x,dtype: f"precise::sin({x})"} # upcast to float32 all the ops that don't support bfloat16 - extra_matcher = PatternMatcher([ + extra_matcher = pm_narrow_uint + PatternMatcher([ # NOTE: this is copied from PTX (UPat((Ops.SQRT, Ops.EXP2, Ops.LOG2, Ops.SIN, Ops.TRUNC), dtype=dtypes.bfloat16, name="x"), lambda x: (UOp(x.op, src=tuple(vv.cast(dtypes.float) for vv in x.src), arg=x.arg).cast(dtypes.bfloat16))), @@ -400,10 +413,7 @@ f"""{dstr_out} __{name}({dstr_in} a, {dstr_in} b, {dstr_out} c){{ mat_a.thread_elements()[0] = a[0]; mat_b.thread_elements()[0] = b[0]; mat_c.thread_elements()[0] = c[0]; mat_a.thread_elements()[1] = a[1]; mat_b.thread_elements()[1] = b[1]; mat_c.thread_elements()[1] = c[1]; simdgroup_multiply_accumulate(mat_c, mat_a, mat_b, mat_c);\n return {dstr_out}(mat_c.thread_elements()[0], mat_c.thread_elements()[1]);\n}}""") - # one argument buffer: a struct of the buffer pointers and the scalars, so a binding is a gpu address (an icb offset keeps 32 bits, an address 64) - args = [(name, self._render_dtype(u.dtype, addrspace=u.addrspace)) for name, (u, _) in bufs] - prefix.append("struct args_t { " + " ".join(f"{t} {n};" for n, t in args) + " };") - return super().render_kernel(function_name, [" " + " ".join(f"{t} {n} = args.{n};" for n, t in args)] + kernel, [], uops, prefix) + return super().render_kernel(function_name, kernel, bufs, uops, prefix) def supported_dtypes(self): return {d for d in super().supported_dtypes() if (d != dtypes.bfloat16 or ((arch:=self.target.arch).startswith("Apple") and int(arch[5:]) >= 6)) @@ -438,10 +448,10 @@ class CUDARenderer(CStyleLanguage): Ops.LOG2: lambda x,dtype: f"hlog2({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"log2({x})", Ops.EXP2: lambda x,dtype: f"hexp2({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"exp2({x})", Ops.SQRT: lambda x,dtype: f"hsqrt({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"sqrt({x})", - Ops.RECIPROCAL: lambda x,dtype: f"hrcp({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"(1/{x})" } + Ops.RECIP: lambda x,dtype: f"hrcp({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"(1/{x})" } type_map = {**CStyleLanguage.type_map, dtypes.u32: "uint", dtypes.bf16: "nv_bfloat16", dtypes.fp8e4m3: "__nv_fp8_e4m3", dtypes.fp8e5m2: "__nv_fp8_e5m2"} - extra_matcher = create_non_native_float_pats(dtypes.fp8s, casting=False) + PatternMatcher([ + extra_matcher = pm_narrow_uint + create_non_native_float_pats(dtypes.fp8s, casting=False) + PatternMatcher([ (UPat(Ops.CAST, dtypes.fp8s, UPat.var("x", dtypes.fp8s), name='y'), lambda x,y: x.cast(dtypes.float).cast(y.dtype) if x.dtype!=y.dtype else None), ]) string_rewrite = PatternMatcher([ @@ -501,16 +511,14 @@ class HIPRenderer(CStyleLanguage): # NOTE: this is only really needed on gfx12, even though gfx11 reports the same limitation global_max = (2147483647, 65535, 65535) global_prod_max = (0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF) + inline_prefix = "__attribute__((device))" @staticmethod def is_cdna(arch): return arch in {"gfx942", "gfx950"} - @staticmethod - def is_cdna4(arch): return arch == "gfx950" def __init__(self, target:Target, use_hipcc=False): # gfx942 => MI300, gfx1100 => RX 7900, gfx1201 => RX 9700 super().__init__(target) from tinygrad.runtime.support.compiler_amd import HIPCompiler, HIPCCCompiler self.compiler, self.tensor_cores = (HIPCCCompiler if use_hipcc else HIPCompiler)(target.arch), tc.get_amd(target.arch) - if not self.is_cdna4(target.arch): self.extra_matcher += pm_manual_bf16_cast if self.is_cdna(target.arch): self.string_rewrite = PatternMatcher([ (UPat(Ops.WMMA, name="x"), lambda ctx,x: f"__{_wmma_name(x)}({ctx[x.src[0]]}, {ctx[x.src[1]]}, {ctx[x.src[2]]}," @@ -531,7 +539,6 @@ class HIPRenderer(CStyleLanguage): # a LOAD flagged nontemporal renders as the cache-bypassing builtin (only used on global loads) self.string_rewrite = PatternMatcher([(UPat(Ops.LOAD, arg="nontemporal", src=(UPat.var("bidx"),)), lambda ctx,bidx: f"__builtin_nontemporal_load({ctx.render_ptr(bidx)})")]) + self.string_rewrite - if not self.is_cdna4(target.arch): self.string_rewrite = pm_bf16_ushort_const + self.string_rewrite # https://clang.llvm.org/docs/AttributeReference.html#amdgpu-flat-work-group-size # NOTE: this makes hlb_cifar10 twice as fast, there may be more gains in tweaking these parameters @@ -544,8 +551,8 @@ class HIPRenderer(CStyleLanguage): barrier = '__builtin_amdgcn_fence(__ATOMIC_RELEASE, "workgroup");' + '__builtin_amdgcn_s_barrier();' + \ '__builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "workgroup");' float4 = "make_float4" - type_map = {**CStyleLanguage.type_map, dtypes.bf16: "hip_bfloat16", **{d: ("hip_fp8", "hip_bf8")[fp8_index(d)] for d in dtypes.fp8s}} - extra_matcher = create_non_native_float_pats((dtypes.bfloat16, *dtypes.fp8s)) + type_map = {**CStyleLanguage.type_map, **{d: ("hip_fp8", "hip_bf8")[fp8_index(d)] for d in dtypes.fp8s}} + extra_matcher = pm_narrow_uint + create_non_native_float_pats((dtypes.bfloat16, *dtypes.fp8s)) def asm(self, prg:UOp, lin:UOp) -> bytes: from tinygrad.renderer.amd.elf import assemble_linear @@ -557,55 +564,53 @@ class HIPRenderer(CStyleLanguage): f"{vec} make_{vec}({', '.join([f'{scal} {x}' for x in _nms[:count]])}) {{ return {{ {', '.join(_nms[:count])} }}; }}" def render_kernel(self, function_name, kernel, bufs, uops, prefix=None) -> str: - prefix, ockl = [], [] + defines, ockl = [], [] type_map = {dtypes.bf16: "bf16", dtypes.f32: "f32", dtypes.f16: "f16", **{d: ("_" if self.is_cdna(self.target.arch) else "") + ("fp8_fp8", "bf8_bf8")[fp8_index(d)] for d in dtypes.fp8s}} used_dtypes = uops_to_dtypes(uops) if any(u.op is Ops.CAST and u.src[0].op is Ops.CONST and not math.isfinite(u.src[0].val) for u in uops): - prefix += ["#define INFINITY (__builtin_inff())", "#define NAN (__builtin_nanf(\"\"))"] + defines += ["#define INFINITY (__builtin_inff())", "#define NAN (__builtin_nanf(\"\"))"] if any(u.op is Ops.SPECIAL for u in uops): - prefix.append("typedef long unsigned int size_t;") + defines.append("typedef long unsigned int size_t;") ockl = [(f"__ockl_get_{name}", "unsigned int", "size_t", "const") for name in ["local_id", "group_id", "local_size"]] ocml_ops = {Ops.EXP2: ("exp2", "pure"), Ops.LOG2: ("log2", "pure"), Ops.SQRT: ("sqrt", "const"), Ops.SIN: ("sin", ""), Ops.TRUNC: ("trunc", "")} ocml = [(f"__ocml_{ocml_ops[op][0]}_f{dt.bitsize}", self.render_dtype(dt), self.render_dtype(dt), ocml_ops[op][1]) for op, dt in dedup((u.op, u.dtype) for u in uops) if op in ocml_ops and dt in (dtypes.half, dtypes.float, dtypes.double)] - if any(dt == dtypes.bfloat16 for dt, _ in used_dtypes): - prefix.append(f"typedef {'__bf16' if self.is_cdna4(self.target.arch) else 'unsigned short'} hip_bfloat16;") - if any(dt == dtypes.half for dt, _ in used_dtypes): prefix.append("#define half _Float16") + if any(dt == dtypes.half for dt, _ in used_dtypes): defines.append("#define half _Float16") if any(dt in dtypes.fp8s for dt, _ in used_dtypes): - prefix += ["typedef unsigned char hip_bf8;", "typedef unsigned char hip_fp8;"] + defines += ["typedef unsigned char hip_bf8;", "typedef unsigned char hip_fp8;"] if any((u.op is Ops.CAST and u.dtype in dtypes.fp8s and u.src[0].dtype == dtypes.float) or (u.op is Ops.CAST and u.src[0].op is Ops.CONST and u.dtype in dtypes.fp8s) for u in uops): fp8_max = amd_fp8s(self.target.arch)[0].max - prefix.append(f"""static inline __attribute__((device)) unsigned char f32_to_fp8(float v, int is_bf8) {{ + defines.append(f"""static inline __attribute__((device)) unsigned char f32_to_fp8(float v, int is_bf8) {{ v = (((*(unsigned*)&v)&0x7F800000)!=0x7F800000)?__builtin_amdgcn_fmed3f(v,is_bf8?57344.0f:{fp8_max}f,is_bf8?-57344.0f:-{fp8_max}f) : v; return (unsigned char)(is_bf8?__builtin_amdgcn_cvt_pk_bf8_f32(v,v,0,false):__builtin_amdgcn_cvt_pk_fp8_f32(v,v,0,false));\n}}""") - prefix += [f'extern "C" __attribute__((device{f", {atr}" if atr else ""})) {dto} {meth}({dti});' for meth,dti,dto,atr in ockl+ocml] - prefix += [self.render_vector_prefix(dt, count) for dt, count in used_dtypes if count > 1] + defines += [f'extern "C" __attribute__((device{f", {atr}" if atr else ""})) {dto} {meth}({dti});' for meth,dti,dto,atr in ockl+ocml] + defines += [self.render_vector_prefix(dt, count) for dt, count in used_dtypes if count > 1] for name, (N, M, K), dtype_in, dtype_out, _ in wmma_args(uops): # TODO: handle TCs f32_bf16 and bf16_bf16 w/ wrapper if self.is_cdna(self.target.arch): if (N, M, K) == (16, 16, 16): type_map[dtypes.bf16] = 'bf16_1k' elif (N, M, K) == (16, 16, 32): type_map = {**type_map, dtypes.bf16: "_bf16", dtypes.f16: "_f16"} elif (N, M, K) == (16, 16, 128): type_map = {**type_map, dtypes.fp8e4m3: "_f8f6f4", dtypes.fp8e5m2: "_f8f6f4"} - prefix.append(f"#define __{name} __builtin_amdgcn_mfma_{'scale_' if K == 128 else ''}f32_{N}x{M}x{K}{type_map[dtype_in]}") + defines.append(f"#define __{name} __builtin_amdgcn_mfma_{'scale_' if K == 128 else ''}f32_{N}x{M}x{K}{type_map[dtype_in]}") elif dtype_out == dtypes.int32: # RDNA4 uses 8 int8 values per lane in 2 VGPRs, RDNA3 uses 16 in 4 VGPRs num_regs = 2 if (is_rdna4:=self.tensor_cores == tc.amd_rdna4) else 4 - prefix.append(f"typedef int wmma_int{num_regs} __attribute__((ext_vector_type({num_regs})));\n"+ + defines.append(f"typedef int wmma_int{num_regs} __attribute__((ext_vector_type({num_regs})));\n"+ f"static inline __attribute__((device)) int8 __{name}"+f"""(signed_char{num_regs*4} a, signed_char{num_regs*4} b, int8 c) {{ return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32{'_gfx12' if is_rdna4 else ''}(true, __builtin_bit_cast(wmma_int{num_regs}, a), true, __builtin_bit_cast(wmma_int{num_regs}, b), c, false);\n}}""") # #define __WMMA_16_16_16_f16_f16 __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12 elif self.tensor_cores == tc.amd_rdna4: - prefix.append(f"#define __{name} __builtin_amdgcn_wmma_{type_map[dtype_out]}_16x16x16_{type_map[dtype_in]}_w32_gfx12") + defines.append(f"#define __{name} __builtin_amdgcn_wmma_{type_map[dtype_out]}_16x16x16_{type_map[dtype_in]}_w32_gfx12") elif dtype_out == dtypes.float: - prefix.append(f"#define __{name} __builtin_amdgcn_wmma_f32_16x16x16_{'f16' if dtype_in == dtypes.half else 'bf16'}_w32") - else: prefix.append(f"static inline __attribute__((device)) half8 __{name}"+"""(half16 a, half16 b, half8 c) { + defines.append(f"#define __{name} __builtin_amdgcn_wmma_f32_16x16x16_{'f16' if dtype_in == dtypes.half else 'bf16'}_w32") + else: defines.append(f"static inline __attribute__((device)) half8 __{name}"+"""(half16 a, half16 b, half8 c) { half16 c_frag = {}; half8 d; for (int n = 0; n < 8; n++) { c_frag[n*2] = c[n]; } c_frag = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(a, b, c_frag, false); for (int n = 0; n < 8; n++) { d[n] = c_frag[n*2]; } return d;\n}""") - return super().render_kernel(function_name, kernel, bufs, uops, prefix) + return super().render_kernel(function_name, kernel, bufs, uops, defines + (prefix or [])) def supported_dtypes(self): return {d for d in super().supported_dtypes() if d not in dtypes.fp8s or d in amd_fp8s(self.target.arch)} @@ -620,8 +625,9 @@ class QCOMCLRenderer(OpenCLRenderer): # QCOM compiler is flaky with half def supported_dtypes(self): - return {d for d in Renderer.supported_dtypes(self) - if (d != dtypes.float16 or (bool(IMAGE) and bool(FLOAT16))) and d not in dtypes.fp8s+(dtypes.bfloat16,dtypes.double)} + unsupported = {*dtypes.fp8s, dtypes.bfloat16, dtypes.double, dtypes.long, dtypes.ulong} + if not (IMAGE and FLOAT16): unsupported.add(dtypes.half) + return Renderer.supported_dtypes(self) - unsupported # QCOM's load vectorizer emits invalid IR for vectorized bool loads ("Range types must match load type"), type bool buffers as uchar def _render_dtype(self, dtype:DType, sz:int=1, addrspace=AddrSpace.ALU, mutable=True, override_ptr=False, shape=None): diff --git a/tinygrad_repo/tinygrad/renderer/isa/__init__.py b/tinygrad_repo/tinygrad/renderer/isa/__init__.py deleted file mode 100644 index dd54137f5f..0000000000 --- a/tinygrad_repo/tinygrad/renderer/isa/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations -import itertools -from dataclasses import dataclass, field -from tinygrad.renderer import Renderer -from tinygrad.uop.ops import PatternMatcher, UOp, Ops -from typing import Any - -@dataclass(frozen=True) -class Register: - name: str - index: int - _cons: tuple[Register, ...] = field(default_factory=tuple) - # vreg size represents the area an instructions output occupies, not necessarily the entire register - size: int = 8 - @property - def cons(self): return self._cons or (self,) - def __repr__(self): return self.name - -class IselContext: - def __init__(self, sink:UOp): - self.reg_n = itertools.count() - self.alloc_n = itertools.count(max((u.arg.slot for u in sink.toposort() if u.op is Ops.ALLOC), default=-1)+1) - def arg_key(u:UOp): return (1, u.arg) if u.op is Ops.SPECIAL else (0, u.arg.slot) - self.func_args = sorted([u for u in sink.toposort() if u.op in {Ops.PARAM, Ops.SPECIAL}], key=arg_key) - - def vreg(self, cons:tuple[Register, ...]|Register, size:int=0): - cons = cons if isinstance(cons, tuple) else (cons,) - return Register(f"v{next(self.reg_n)}", 0, cons, size or cons[0].size) - -def rdef(u:UOp): - if u.op in {Ops.NOOP, Ops.AFTER, Ops.BITCAST} and u.src: return rdef(u.src[0]) - return u.tag[0] if isinstance(u.tag, tuple) else u.tag - -class LinearContext: - def __init__(self, ren:ISARenderer): - self.ren, self.stack_size = ren, 0 - self.loop_label: dict[UOp, str] = {} - def assign_spill_slot(self, r:Register, u:UOp) -> Any: raise NotImplementedError("arch specific") - -class ISARenderer(Renderer): - pre_isel_matcher: PatternMatcher - isel_matcher: PatternMatcher - pre_regalloc_matcher: PatternMatcher - post_regalloc_matcher: PatternMatcher - linear_ctx_type: type = LinearContext - - def is_two_address(self, x:UOp) -> bool: return False - def spill(self, spill_slot:Any, x:UOp) -> UOp: raise NotImplementedError("arch specific") - def fill(self, spill_slot:Any, x:UOp, reg:Register) -> UOp: raise NotImplementedError("arch specific") - def asm_str(self, uops:list[UOp], function_name:str) -> str: raise NotImplementedError("arch specific") diff --git a/tinygrad_repo/tinygrad/renderer/llvmir.py b/tinygrad_repo/tinygrad/renderer/llvmir.py index 530dce9b56..8a9d2ab745 100644 --- a/tinygrad_repo/tinygrad/renderer/llvmir.py +++ b/tinygrad_repo/tinygrad/renderer/llvmir.py @@ -1,11 +1,11 @@ -import math, struct, sys +import struct from tinygrad.renderer import tc from tinygrad.renderer import Renderer from tinygrad.renderer.cstyle import HIPRenderer, create_non_native_float_pats, pm_manual_bf16_cast, fp8_index, amd_fp8s from tinygrad.codegen.decomp.transcendental import xexp2, xlog2 from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, GroupOp, range_str from tinygrad.dtype import dtypes, float_to_fp8, DType, truncate, AddrSpace -from tinygrad.helpers import prod, Target, OSX +from tinygrad.helpers import prod, Target def is_volatile(u:UOp) -> bool: return (buf:=u.buf_uop).op is Ops.PARAM and buf.arg.volatile @@ -21,8 +21,7 @@ def lparam(u:UOp) -> str: return ldt(u.dtype, ptr=u.addrspace not in (None, Addr def lconst(x, dtype:DType): if dtype in dtypes.floats: if dtype in dtypes.fp8s: return float_to_fp8(x, dtype) - if math.isinf(x) or math.isnan(x): return "0x%02X%02X%02X%02X%02X%02X%02X%02X" % tuple(struct.pack("d",x)[::-1]) - return truncate[dtype](x) + return "0x" + struct.pack(">d", truncate[dtype](x)).hex() return int(x) def lcast(input_type:DType, output_type:DType): @@ -68,15 +67,17 @@ def render_wmma_amd(ctx, wmma: UOp, cdna=False, rdna4=False) -> str: f"{dt_map[wmma.arg[1]]}{suffix}(" + ", ".join(args) + ")" # llvm ops, lop[][] -unsigned_lop = { Ops.ADD: "add", Ops.MUL: "mul", Ops.CDIV: "udiv", Ops.CMOD: "urem", +unsigned_lop = { Ops.ADD: "add", Ops.MUL: "mul", Ops.TRUNCDIV: "udiv", Ops.TRUNCMOD: "urem", Ops.CMPLT: "icmp ult", Ops.CMPNE: "icmp ne", Ops.CMPEQ: "icmp eq", Ops.OR: "or", Ops.AND: "and", Ops.XOR: "xor", Ops.SHL: "shl", Ops.SHR: "lshr",} -signed_lop = {**unsigned_lop, Ops.ADD: "add nsw", Ops.CMPLT: "icmp slt", Ops.CDIV: "sdiv", Ops.CMOD: "srem", Ops.SHR: "ashr"} -flags = " nsz arcp contract afn" +signed_lop = {**unsigned_lop, Ops.ADD: "add nsw", Ops.CMPLT: "icmp slt", Ops.TRUNCDIV: "sdiv", Ops.TRUNCMOD: "srem", Ops.SHR: "ashr"} +flags = " contract afn" float_lop = {Ops.ADD: "fadd"+flags, Ops.MUL: "fmul"+flags, Ops.CMPLT: f"fcmp{flags} olt", Ops.CMPNE: f"fcmp{flags} une", Ops.CMPEQ: f"fcmp{flags} oeq", Ops.FDIV: "fdiv"+flags} lop = {**{x:unsigned_lop for x in (dtypes.bool,)+dtypes.uints}, **{x:signed_lop for x in dtypes.sints}, **{x:float_lop for x in dtypes.floats}} +llvm_intrinsics = {Ops.SQRT: "sqrt", Ops.LOG2: "log2", Ops.EXP2: "exp2"} + base_rewrite = PatternMatcher([ # memory load/store (UPat((Ops.INDEX, Ops.SHRINK), name="x"), lambda ctx,x: None if x.src[0].addrspace in (None, AddrSpace.ALU) else @@ -91,15 +92,15 @@ base_rewrite = PatternMatcher([ f" br label {ctx[x]}_entry\n{ctx[x][1:]}_entry:\n" f" br i1 {ctx[mask]}, label {ctx[x]}_load, label {ctx[x]}_exit\n{ctx[x][1:]}_load:\n" f" {ctx[x]}_yes = load {'volatile ' if is_volatile(idx) else ''}{ldt(idx.dtype, idx.max_numel())}, " - f"{ldt(idx.dtype, idx.max_numel(), True)} {ctx[idx]}\n" + f"{ldt(idx.dtype, idx.max_numel(), True)} {ctx[idx]}, align {idx.dtype.itemsize}\n" f" br label {ctx[x]}_exit\n{ctx[x][1:]}_exit:\n" f" {ctx[x]} = phi {ldt(x.dtype, x.max_numel())} [{ctx[x]}_yes, {ctx[x]}_load], [{ctx[alt]}, {ctx[x]}_entry]"), (UPat.var('idx').load(name="x"), lambda ctx,x,idx: f" {ctx[x]} = load {'volatile ' if is_volatile(idx) else ''}{ldt(idx.dtype, idx.max_numel())}, " - f"{ldt(idx.dtype, idx.max_numel(), True)} {ctx[idx]}"), + f"{ldt(idx.dtype, idx.max_numel(), True)} {ctx[idx]}, align {idx.dtype.itemsize}"), (UPat.var('idx').store(UPat.var("var")), lambda ctx,idx,var: f" store {'volatile ' if is_volatile(idx) else ''}{ldt(var.dtype, idx.max_numel())} {ctx[var]}, " - f"{ldt(idx.dtype, idx.max_numel(), True)} {ctx[idx]}"), + f"{ldt(idx.dtype, idx.max_numel(), True)} {ctx[idx]}, align {idx.dtype.itemsize}"), # GEP/VECTORIZE/CAST for float4 support (UPat(Ops.STACK, name="x"), lambda ctx,x: @@ -107,6 +108,8 @@ base_rewrite = PatternMatcher([ f" = insertelement {ldt(x.dtype, x.max_numel())} "+(f"{ctx[x]}_{i-1}" if i != 0 else "poison")+ f", {ldt(u.dtype)} {ctx[u]}, i32 {i}" for i,u in enumerate(x.src)])), # unary/binary/ternary ops + (UPat(tuple(llvm_intrinsics), name="x"), + lambda ctx, x: f" {ctx[x]} = call {ldt(x.dtype)} @llvm.{llvm_intrinsics[x.op]}.{ldt(x.dtype)}({ldt(x.src[0].dtype)} {ctx[x.src[0]]})"), (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f" {ctx[x]} = bitcast {ldt(x.src[0].dtype, x.src[0].max_numel())} {ctx[x.src[0]]} to {ldt(x.dtype, x.max_numel())}"), (UPat(Ops.CAST, name="x"), lambda ctx,x: f" {ctx[x]} = {lcast(x.src[0].dtype, x.dtype)} {ldt(x.src[0].dtype)} {ctx[x.src[0]]} to {ldt(x.dtype)}"), @@ -148,29 +151,37 @@ base_rewrite = PatternMatcher([ # calls (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, name="f"),), allow_any_len=True, name="x"), lambda ctx,x,f: f" {'' if x.dtype == dtypes.void else ctx[x] + ' = '}call {ldt(x.dtype)} @{f.arg.name}({', '.join(f'{lparam(y)} {ctx[y]}' for y in x.src[1:])})"), - (UPat(Ops.CALL, src=(UPat(Ops.LINEAR, name="body"),), allow_any_len=True, name="x"), lambda ctx,x,body: f" call void @{ctx[body]}(" + - ", ".join(f"{ldt(p.dtype, ptr=p.addrspace != AddrSpace.ALU)} {ctx[x.src[p.arg.slot+1]]}" for p in body.src if p.op is Ops.PARAM) + ")"), +]) + +pm_widen_uidx = PatternMatcher([ + # GEP sign-extends indices to pointer width, so unsigned indices must be widened first. + (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(), UPat(dtype=(dtypes.uint8, dtypes.uint16, dtypes.uint32), name="i")), + allow_any_len=True, name="x"), lambda x,i: x.replace(src=(x.src[0], i.cast(dtypes.int64), *x.src[2:]))), ]) class LLVMRenderer(Renderer): abi: str | None string_rewrite: PatternMatcher - code_for_op = {k:lambda:None for v in lop.values() for k in v.keys()} + code_for_op = {**{k:lambda:None for v in lop.values() for k in v.keys()}, **{op: lambda: None for op in llvm_intrinsics}} - extra_matcher = create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast + PatternMatcher([ - # GEP sign-extends indices to pointer width, so unsigned indices must be widened first. - (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(), UPat(dtype=(dtypes.uint8, dtypes.uint16, dtypes.uint32), name="i")), - allow_any_len=True, name="x"), lambda x,i: x.replace(src=(x.src[0], i.cast(dtypes.int64), *x.src[2:]))), - ]) - def _render_fn(self, name:str, args:list[tuple[str,UOp]], kernel:list[str], prefix:list[str]|None=None) -> str: + def __init__(self, target:Target): + super().__init__(target) + self.extra_matcher = create_non_native_float_pats((dtypes.bfloat16,))+ \ + pm_manual_bf16_cast+pm_widen_uidx if target.arch != "gfx950" else pm_widen_uidx + def _render_fn(self, name:str, args:list[tuple[str,UOp]], kernel:list[str], prefix:list[str]|None=None, args_struct=False) -> str: # Buffer views may start at an offset from the aligned allocation. - sargs = ", ".join([f"{ldt(u.dtype, ptr=u.addrspace != AddrSpace.ALU)}{' noalias' if u.addrspace == AddrSpace.GLOBAL else ''} " + \ - name for name,u in args]) + tys = [ldt(u.dtype, ptr=u.addrspace != AddrSpace.ALU) for _,u in args] + sargs = ", ".join(f"{t}{' noalias' if u.addrspace == AddrSpace.GLOBAL else ''} {name}" for t,(name,u) in zip(tys, args)) + if args_struct: + args_t = "{ " + ", ".join(tys) + " }" + kernel = [f" %args_t = load {args_t}, ptr %args"] + [f" {n} = extractvalue {args_t} %args_t, {i}" for i,(n,_) in enumerate(args)] + kernel + sargs = "ptr %args" return "\n".join((prefix or []) + [f"define{' ' + self.abi if self.abi else ''} void @{name}({sargs}) #0", "{"] + kernel + [" ret void\n}"]) - def _render_kernel(self, uops: list[UOp], prefix:list[str]|None=None, name="test", fns:dict[UOp, str]|None=None) -> tuple[tuple[str, ...], str]: - r: dict[UOp, str] = dict(fns or {}) # the functions the kernel calls, by name + def _render_kernel(self, uops: list[UOp], prefix:list[str]|None=None, name="test", args_struct=False) -> tuple[tuple[str, ...], str]: + r: dict[UOp, str] = {} args: list[tuple[str, UOp]] = [] kernel: list[str] = [] + call_args: list[str] = [] vc = -1 local_args: list[str] = [] @@ -197,6 +208,16 @@ class LLVMRenderer(Renderer): else: kernel.append(f" {r[u]} = alloca [{size} x {ldt(u.dtype)}], align 16") elif u.op is Ops.BINARY: r[u] = f"@bin_{u.key.hex()}" + elif u.op is Ops.CALL and u.body.op is Ops.PROGRAM: + vc += 1 + types = [ldt(p.dtype, ptr=p.addrspace != AddrSpace.ALU) for p in u.body.arg.params] + arg_type = "{ " + ", ".join(types) + " }" + r[u] = f"%v{vc}" + call_args.append(f" {r[u]} = alloca {arg_type}") + for i, (p, ty) in enumerate(zip(u.body.arg.params, types)): + kernel.append(f" {r[u]}_{i} = getelementptr inbounds {arg_type}, ptr {r[u]}, i32 0, i32 {i}") + kernel.append(f" store {ty} {r[u.src[p.arg.slot+1]]}, ptr {r[u]}_{i}") + kernel.append(f" call void @{u.body.src[0].arg.function_name}(ptr {r[u]})") elif u.op is Ops.CAST and u.src[0].op is Ops.CONST: r[u] = lconst(u.src[0].val, u.dtype) elif (u.op is Ops.CAST and ldt(u.dtype) == ldt(u.src[0].dtype)) or (u.op is Ops.BITCAST and u.addrspace not in (None, AddrSpace.ALU)): r[u] = r[u.src[0]] # cast from signed to unsigned of the same size is a noop, or pointer cast @@ -209,48 +230,41 @@ class LLVMRenderer(Renderer): if l is None: raise RuntimeError(f"failed to render {u.op} with {u.dtype} srcs {[x.dtype for x in u.src]}") kernel.append(l) - return tuple(local_args), self._render_fn(name, args, kernel, prefix) + return tuple(local_args), self._render_fn(name, args, call_args + kernel, prefix, args_struct) class CPULLVMRenderer(LLVMRenderer): has_local = False global_max = (1, 0, 0) - abi = 'win64cc' if sys.platform == 'win32' else None + abi = None string_rewrite = base_rewrite - def render(self, uops: list[UOp]) -> str: # the kernel is first, it is the entry. its functions follow, a name traced with other args gets a suffix - fns = {b: f"{b.arg}_{i}" for i, b in enumerate(b for b in UOp.sink(*uops).toposort(enter_calls=True) if b.op is Ops.LINEAR)} - defs = [self._render_kernel(b.src, name=n, fns=fns)[1] for b, n in fns.items()] - return "\n".join((k:=self._render_kernel(uops, fns=fns))[0] + (k[1], *defs, self._render_footer(uops))) + def render(self, uops: list[UOp]) -> str: return "\n".join((k:=self._render_kernel(uops, args_struct=True))[0] + (k[1], self._render_footer(uops))) def _render_footer(self, uops: list[UOp]) -> str: decls = {x.src[0].arg.name: f"declare {ldt(x.dtype)} @{x.src[0].arg.name}({', '.join(map(lparam, x.src[1:]))})" - for x in UOp.sink(*uops).toposort(enter_calls=True) if x.op is Ops.CALL and x.src[0].op is Ops.CUSTOM_FUNCTION} + for x in uops if x.op is Ops.CALL and x.src[0].op is Ops.CUSTOM_FUNCTION} + decls |= {x.body.src[0].arg.function_name: f"declare void @{x.body.src[0].arg.function_name}(ptr)" + for x in uops if x.op is Ops.CALL and x.body.op is Ops.PROGRAM} blobs = {x.key: f"@bin_{x.key.hex()} = private constant [{len(x.arg)} x i8] c\"" + "".join("\\%02X" % b for b in x.arg) + '", align 16' - for x in UOp.sink(*uops).toposort(enter_calls=True) if x.op is Ops.BINARY} + for x in uops if x.op is Ops.BINARY} return "\n".join([*decls.values(), *blobs.values(), 'attributes #0 = { alwaysinline nounwind "no-builtins" "no-trapping-math"="true" }']) def __init__(self, target:Target): super().__init__(target) from tinygrad.runtime.support.compiler_llvm import CPULLVMCompiler self.compiler = CPULLVMCompiler(target.arch.split(",")) - # FIXME: fp16 works on non-osx, but only if the cpu supports it def supported_dtypes(self): return {d for d in super().supported_dtypes() if - (d != dtypes.bfloat16 or self.target.arch.startswith(("x86", "arm"))) and (d != dtypes.half or OSX) and d not in dtypes.fp8s} + (d != dtypes.bfloat16 or self.target.arch.startswith(("x86", "arm"))) and d not in dtypes.fp8s} barrier = 'fence syncscope("workgroup") release\ntail call void @llvm.amdgcn.s.barrier()\nfence syncscope("workgroup") acquire\n' code_for_workitem = {"g": lambda x: f"tail call i32 @llvm.amdgcn.workgroup.id.{chr(120+int(x))}()", "l": lambda x: f"tail call i32 @llvm.amdgcn.workitem.id.{chr(120+int(x))}()"} -# https://rocm.docs.amd.com/projects/llvm-project/en/latest/LLVM/llvm/html/AMDGPUUsage.html#llvm-ir-intrinsics -llvm_intrinsics = {Ops.SQRT: "sqrt", Ops.LOG2: "log2", Ops.EXP2: "exp2"} class AMDLLVMRenderer(LLVMRenderer): shared_max = HIPRenderer.shared_max global_max = HIPRenderer.global_max global_prod_max = HIPRenderer.global_prod_max abi = "amdgpu_kernel" - code_for_op = {**LLVMRenderer.code_for_op, **{op: lambda: None for op in llvm_intrinsics}} string_rewrite = PatternMatcher([ (UPat(Ops.SPECIAL, name="x"), lambda ctx, x: f" {ctx[x]} = " + f"{ code_for_workitem[x.arg[0]](x.arg[-1])}; "), - (UPat(tuple(llvm_intrinsics), name="x"), - lambda ctx, x: f" {ctx[x]} = call {ldt(x.dtype)} @llvm.{llvm_intrinsics[x.op]}.{ldt(x.dtype)}({ldt(x.src[0].dtype)} {ctx[x.src[0]]})"), (UPat(Ops.BARRIER), lambda ctx: barrier), (UPat(Ops.CAST, dtypes.fp8s, (UPat(dtype=dtypes.float),), name="x",), lambda ctx,x: f" {ctx[x]} = call i8 @f32_to_fp8({ldt(x.src[0].dtype)} {ctx[x.src[0]]}, i1 {fp8_index(x.dtype)})"), @@ -258,11 +272,6 @@ class AMDLLVMRenderer(LLVMRenderer): f" {ctx[x]}_i32 = zext i8 {ctx[x.src[0]]} to i32\n" f" {ctx[x]} = call float @llvm.amdgcn.cvt.f32.{('fp8', 'bf8')[fp8_index(y.dtype)]}(i32 {ctx[x]}_i32, i32 0)"), ]) + base_rewrite - extra_matcher = LLVMRenderer.extra_matcher + create_non_native_float_pats(dtypes.fp8s) + PatternMatcher([ - # amd llvm intrinsics llvm.log2/llvm.exp2 don't support double - (UPat(Ops.LOG2, dtype=dtypes.double, src=(UPat.var("d"),)), xlog2), - (UPat(Ops.EXP2, dtype=dtypes.double, src=(UPat.var("d"),)), xexp2), - ]) def asm(self, prg: UOp, lin: UOp) -> bytes: from tinygrad.renderer.amd.elf import assemble_linear return assemble_linear(prg, lin, self.target.arch) @@ -297,6 +306,13 @@ exit: %packed = phi i32 [%packed_bf8, %do_bf8], [%packed_fp8, %do_fp8]\n %trunc (UPat(Ops.WMMA, name="wmma"), lambda ctx, wmma, rdna4=AMDLLVMRenderer.is_rdna4(target.arch), cdna=self.is_cdna: render_wmma_amd(ctx, wmma, cdna, rdna4)) ]) + self.extra_matcher = self.extra_matcher + create_non_native_float_pats(dtypes.fp8s) + PatternMatcher([ + # amd llvm intrinsics llvm.log2/llvm.exp2 don't support double + (UPat(Ops.LOG2, dtype=dtypes.double, src=(UPat.var("d"),)), xlog2), + (UPat(Ops.EXP2, dtype=dtypes.double, src=(UPat.var("d"),)), xexp2), + (UPat.var("a", dtype=(dtypes.f16,dtypes.bf16)).cast((dtypes.f16, dtypes.bf16), name="b"), + lambda a,b: a.float().cast(b.dtype) if a.dtype != b.dtype else None), + ]) if self.is_cdna: self.extra_matcher += tc.pm_validate_wmma_cdna if target.arch in {"gfx1100", "gfx1151"}: self.extra_matcher += tc.pm_validate_wmma_rdna3 if target.arch in {"gfx1200", "gfx1201"}: self.extra_matcher += tc.pm_validate_wmma_rdna4 diff --git a/tinygrad_repo/tinygrad/renderer/nir.py b/tinygrad_repo/tinygrad/renderer/nir.py index f6a07b32b4..3c837f2d18 100644 --- a/tinygrad_repo/tinygrad/renderer/nir.py +++ b/tinygrad_repo/tinygrad/renderer/nir.py @@ -16,11 +16,11 @@ def glsl_type(t:DType): return { **{d:g(f"glsl_type_builtin_{'u' * (d in dtypes.uints)}int{str(d.bitsize)+'_t' if d.itemsize != 4 else ''}") for d in dtypes.ints}}[t] # alu ops, aop[][] -u_aop = { Ops.ADD: "iadd", Ops.MUL: "imul", Ops.CDIV: "udiv", Ops.CMOD: "umod", Ops.CMPLT: "ult", Ops.CMPNE: "ine", Ops.CMPEQ: "ieq", Ops.OR: "ior", - Ops.AND: "iand", Ops.XOR: "ixor", Ops.WHERE: "bcsel", Ops.MAX: "umax", Ops.SHL: "ishl", Ops.SHR: "ushr"} -s_aop = {**u_aop, Ops.CMPLT: "ilt", Ops.CDIV: "idiv", Ops.CMOD: "irem", Ops.MAX: "imax", Ops.SHR: "ishr"} -f_aop = { Ops.ADD: "fadd", Ops.MUL: "fmul", Ops.CMPLT: "flt", Ops.CMPNE: "fneu", Ops.CMPEQ: "feq", Ops.FDIV: "fdiv", Ops.RECIPROCAL: "frcp", - Ops.MAX: "fmax", Ops.TRUNC: "ftrunc", Ops.SIN: "fsin", Ops.EXP2: "fexp2", Ops.LOG2: "flog2"} +u_aop = { Ops.ADD: "iadd", Ops.MUL: "imul", Ops.TRUNCDIV: "udiv", Ops.TRUNCMOD: "umod", Ops.CMPLT: "ult", Ops.CMPNE: "ine", Ops.CMPEQ: "ieq", + Ops.OR: "ior", Ops.AND: "iand", Ops.XOR: "ixor", Ops.WHERE: "bcsel", Ops.MAX: "umax", Ops.SHL: "ishl", Ops.SHR: "ushr"} +s_aop = {**u_aop, Ops.CMPLT: "ilt", Ops.TRUNCDIV: "idiv", Ops.TRUNCMOD: "irem", Ops.MAX: "imax", Ops.SHR: "ishr"} +f_aop = { Ops.ADD: "fadd", Ops.MUL: "fmul", Ops.CMPLT: "flt", Ops.CMPNE: "fneu", Ops.CMPEQ: "feq", Ops.FDIV: "fdiv", Ops.RECIP: "frcp", + Ops.MAX: "fmax", Ops.TRUNC: "ftrunc", Ops.SIN: "fsin", Ops.EXP2: "fexp2", Ops.LOG2: "flog2", Ops.SQRT: "fsqrt"} aop = {**{x:u_aop for x in (dtypes.bool,)+dtypes.uints}, **{x:s_aop for x in dtypes.sints}, **{x:f_aop for x in dtypes.floats}} def c(t:DType, u:bool=True) -> str: return "u" if t in dtypes.uints and u else ("i" if t in dtypes.ints else ("f" if t in dtypes.floats else "b")) diff --git a/tinygrad_repo/tinygrad/renderer/ptx.py b/tinygrad_repo/tinygrad/renderer/ptx.py index 249cdc4215..d45c88bbab 100644 --- a/tinygrad_repo/tinygrad/renderer/ptx.py +++ b/tinygrad_repo/tinygrad/renderer/ptx.py @@ -5,7 +5,7 @@ from tinygrad.renderer import tc from tinygrad.uop.ops import Ops, UOp, PatternMatcher, UPat, GroupOp from tinygrad.dtype import dtypes, DType, AddrSpace from tinygrad.renderer import Renderer, with_storage -from tinygrad.renderer.cstyle import CUDARenderer +from tinygrad.renderer.cstyle import CUDARenderer, pm_narrow_uint from tinygrad.helpers import flatten, prod, unwrap, Target def render_val(x, dtype): @@ -16,7 +16,7 @@ def render_val(x, dtype): return str(int(x)) + ("U" if dtypes.is_unsigned(dtype) else "") asm_for_op: dict[Ops, Callable] = { - Ops.RECIPROCAL: lambda d,a,dt,name: f"rcp.approx.{name} {d}, {a};", + Ops.RECIP: lambda d,a,dt,name: f"rcp.approx.{name} {d}, {a};", Ops.EXP2: lambda d,a,dt,name: f"ex2.approx.{name} {d}, {a};", Ops.LOG2: lambda d,a,dt,name: f"lg2.approx.{name} {d}, {a};", Ops.SIN: lambda d,a,dt,name: f"sin.approx.{name} {d}, {a};", Ops.SQRT: lambda d,a,dt,name: f"sqrt.approx.{name} {d}, {a};", Ops.TRUNC: lambda d,a,dt,name: f"cvt.rzi.{name}.{name} {d}, {a};", @@ -26,7 +26,7 @@ asm_for_op: dict[Ops, Callable] = { Ops.XOR: lambda d,a,b,dt,name: f"xor.pred {d}, {a}, {b};" if dt == dtypes.bool else f"xor.b{name[1:]} {d}, {a}, {b};", Ops.AND: lambda d,a,b,dt, name: f"and.pred {d}, {a}, {b};" if dt == dtypes.bool else f"and.b{name[1:]} {d}, {a}, {b};", Ops.OR: lambda d,a,b,dt, name: f"or.pred {d}, {a}, {b};" if dt == dtypes.bool else f"or.b{name[1:]} {d}, {a}, {b};", - Ops.CDIV: lambda d,a,b,dt,name: f"div.{name} {d}, {a}, {b};", Ops.CMOD: lambda d,a,b,dt,name: f"rem.{name} {d}, {a}, {b};", + Ops.TRUNCDIV: lambda d,a,b,dt,name: f"div.{name} {d}, {a}, {b};", Ops.TRUNCMOD: lambda d,a,b,dt,name: f"rem.{name} {d}, {a}, {b};", Ops.MAX: lambda d,a,b,dt,name: f"max.{name} {d}, {a}, {b};", Ops.CMPEQ: lambda d,a,b,dt,name: f"setp.eq.{name} {d}, {a}, {b};", Ops.CMPLT: lambda d,a,b,dt,name: f"setp.lt.{name} {d}, {a}, {b};", Ops.CMPNE: lambda d,a,b,dt,name: f"setp.{'neu' if dtypes.is_float(dt) else 'ne'}.{name} {d}, {a}, {b};", @@ -39,7 +39,7 @@ supports_half = (Ops.EXP2, Ops.ADD, Ops.MUL, Ops.MAX, Ops.CMPLT, Ops.WHERE, Ops. doesnt_support_half: tuple[Ops, ...] = tuple(op for op in asm_for_op.keys() if op not in supports_half) ptx_matcher = PatternMatcher([ # bool CMPNE is XOR, bool CMPLT is XOR+AND (universal makes this slow, this is for renderer only) - (UPat.var('x', dtype=dtypes.bool).ne(UPat.var('y')), lambda x,y: x^y), + (UPat.var('x', dtype=dtypes.bool).alu(Ops.CMPNE, UPat.var('y')), lambda x,y: x^y), (UPat.var('x', dtype=dtypes.bool).alu(Ops.CMPEQ, UPat.var('y')), lambda x,y: (x^y)^True), (UPat.var('x', dtype=dtypes.bool)> UPat.var("y"), lambda x,y: UOp(Ops.SHR, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), - # fix nan check: 'a != a -> is_nan()'. the decomp rewrites (a != a).logical_not() to CMPEQ, so match both forms - (UPat.var("a", dtypes.floats) != UPat.var("a"), is_nan), - (UPat.var("a", dtypes.floats).alu(Ops.CMPEQ, UPat.var("a")), lambda a: is_nan(a).ne(True)), + # fix nan checks after decomp, which can emit either comparison + (UPat.var("a", dtypes.floats).alu(Ops.CMPNE, UPat.var("a")), is_nan), + (UPat.var("a", dtypes.floats).eq(UPat.var("a")), lambda a: is_nan(a).logical_not()), + # WGSL float != can be ordered: explicitly include unordered (NaN) operands. + (UPat.var("a", dtypes.floats).alu(Ops.CMPNE, UPat.var("b")), lambda a,b: (a({c.val})" if c.val < 0 else f"{c.val&0xFFFFFFFF}u"), + (UPat.cvar("c").cast((dtypes.uchar, dtypes.ushort, dtypes.uint32)), lambda c: f"{c.val&0xFFFFFFFF}u"), # a negative const must state its type: contextual conversion of a bare abstract int rejects it in a u32 position (UPat.cvar("c").cast(dtypes.int32, name="x"), lambda ctx,x,c: f"i32({v})" if (v:=truncate[x.dtype](c.val)) < 0 else f"{v}"), (UPat(Ops.BUFFER, name="x"), lambda ctx,x: diff --git a/tinygrad_repo/tinygrad/renderer/x86.py b/tinygrad_repo/tinygrad/renderer/x86.py new file mode 100644 index 0000000000..69c54a8c20 --- /dev/null +++ b/tinygrad_repo/tinygrad/renderer/x86.py @@ -0,0 +1,800 @@ +from __future__ import annotations +# flake8: noqa: E702 +# allow semicolons to put multiple ops on one line +import struct, functools, itertools +from typing import Any, cast +from tinygrad.dtype import dtypes, DType, truncate, AddrSpace +from tinygrad.uop import FastEnum, auto, Ops, GroupOp +from tinygrad.uop.ops import UOp, UPat, PatternMatcher, promo_dtype +from tinygrad.renderer import Renderer +from tinygrad.device import TinyELF +from tinygrad.helpers import unwrap, Target +from dataclasses import dataclass, field, replace + +@dataclass(frozen=True) +class Register: + name: str + index: int + _cons: tuple[Register, ...] = field(default_factory=tuple) + size: int = 8 + @property + def cons(self): return self._cons or (self,) + def __repr__(self): return self.name + +class IselContext: + def __init__(self, sink:UOp): + self.reg_n = itertools.count() + params = sorted([u for u in sink.toposort() if u.op is Ops.PARAM], key=lambda u: u.arg.slot) + self.arg_offsets = {u: o for u, (o, _) in zip(params, TinyELF.iter_sig(tuple((None, u.addrspace, u.dtype, ()) for u in params)))} + + def vreg(self, cons:tuple[Register, ...]|Register, size:int=0): + cons = cons if isinstance(cons, tuple) else (cons,) + return Register(f"v{next(self.reg_n)}", 0, cons, size or cons[0].size) + +# the registers an INS defines follow its opcode and dtype in its arg +def regs(u:UOp) -> tuple[Register, ...]: return u.arg[2:] if u.op is Ops.INS else () +def with_regs(u:UOp, r:tuple[Register, ...]) -> UOp: return u.replace(arg=u.arg[:2]+r) + +def rdef(u:UOp): + if u.op in {Ops.AFTER, Ops.BITCAST}: return rdef(u.src[0]) + return r[0] if (r:=regs(u)) else None + +class LinearContext: + def __init__(self, ren:ISARenderer): + self.ren, self.stack_size = ren, 0 + def insert_linearized(self, uops:list[UOp]) -> list[UOp]: return uops + def assign_spill_slot(self, r:Register, u:UOp) -> Any: raise NotImplementedError("arch specific") + +class ISARenderer(Renderer): + pre_isel_matcher: PatternMatcher + isel_matcher: PatternMatcher + pre_regalloc_matcher: PatternMatcher + post_regalloc_matcher: PatternMatcher + linear_ctx_type: type = LinearContext + + def is_two_address(self, x:UOp) -> bool: return False + def spill(self, spill_slot:Any, x:UOp) -> UOp: raise NotImplementedError("arch specific") + def fill(self, spill_slot:Any, x:UOp, reg:Register) -> UOp: raise NotImplementedError("arch specific") + def asm_str(self, uops:list[UOp], function_name:str) -> str: raise NotImplementedError("arch specific") + +# ***** X86 Ops ***** + +class X86Ops(FastEnum): + # NOTE: X86Ops with i suffix are variants that take an immediate, m suffix are variants that can write to memory instead of read from + # these aren't real instructions + LABEL = auto(); IMM = auto(); REG = auto() + # index + LEA = auto() + # register / memory / immediate moves + MOV = auto(); MOVm = auto(); MOVi = auto(); MOVABS = auto() + VMOVSS = auto(); VMOVSD = auto(); VMOVUPS = auto() + VMOVSSm = auto(); VMOVSDm = auto(); VMOVUPSm = auto() + # casts + MOVZX = auto(); MOVSX = auto(); MOVSXD = auto() + VCVTPH2PS = auto(); VCVTPS2PH = auto() + VCVTSS2SD = auto(); VCVTSD2SS = auto(); VCVTSI2SS = auto(); VCVTSI2SD = auto() + VCVTTSS2SI = auto(); VCVTTSD2SI = auto() + # bitcasts + VMOVD = auto(); VMOVQ = auto(); VMOVDm = auto(); VMOVQm = auto() + # comparisons + VCMPSS = auto(); VCMPSD = auto() + SETNE = auto(); SETE = auto(); SETL = auto(); SETB = auto() + # where + CMOVNE = auto(); CMOVE = auto(); CMOVL = auto(); CMOVB = auto() + VBLENDVPS = auto(); VBLENDVPD = auto() + # jumps + JNE = auto(); JE = auto(); JL = auto(); JB = auto(); JGE = auto(); JMP = auto() + # vectorize / gep + VINSERTPS = auto(); VPSRLDQ = auto() + VPEXTRW = auto(); VPEXTRD = auto() + VPINSRW = auto(); VPINSRD = auto() + # int binary + IDIV = auto(); DIV = auto() + ADD = auto(); ADDi = auto(); SUB = auto(); SUBi = auto(); IMUL = auto(); IMULi = auto() + AND = auto(); ANDi = auto(); XOR = auto(); XORi = auto(); OR = auto(); ORi = auto() + SHL = auto(); SHLi = auto(); SHR = auto(); SHRi = auto(); SAR = auto(); SARi = auto(); CMP = auto(); CMPi = auto() + # float unary (sometimes not unary) + VROUNDSS = auto(); VROUNDSD = auto(); VSQRTSS = auto(); VSQRTSD = auto() + # float binary + VADDSS = auto(); VADDSD = auto(); VSUBSS = auto(); VSUBSD = auto(); VMULSS = auto(); VMULSD = auto(); VDIVSS = auto(); VDIVSD = auto() + # return + RET = auto() + +class X86GroupOp: + # X86Ops whose first src is also the destination + TwoAddress = {X86Ops.ADD, X86Ops.ADDi, X86Ops.AND, X86Ops.ANDi, X86Ops.XOR, X86Ops.XORi, X86Ops.OR, X86Ops.ORi, X86Ops.IMUL, + X86Ops.SUB, X86Ops.SUBi, X86Ops.SHL, X86Ops.SHLi, X86Ops.SHR, X86Ops.SHRi, X86Ops.SAR, X86Ops.SARi, + X86Ops.IDIV, X86Ops.DIV, X86Ops.CMOVNE, X86Ops.CMOVE, X86Ops.CMOVL, X86Ops.CMOVB} + + # X86Ops whose second src is the rm field, so that src is what can be a memory operand + Rm2nd = {X86Ops.ADD, X86Ops.SUB, X86Ops.AND, X86Ops.OR, X86Ops.XOR, X86Ops.IMUL, X86Ops.CMP, + X86Ops.VADDSS, X86Ops.VADDSD, X86Ops.VSUBSS, X86Ops.VSUBSD, X86Ops.VMULSS, X86Ops.VMULSD, X86Ops.VDIVSS, X86Ops.VDIVSD, + X86Ops.VBLENDVPS, X86Ops.VBLENDVPD, X86Ops.VCMPSS, X86Ops.VCMPSD, X86Ops.VROUNDSS, X86Ops.VROUNDSD, X86Ops.VSQRTSS, X86Ops.VSQRTSD, + X86Ops.VINSERTPS, X86Ops.VPINSRW, X86Ops.VPINSRD, X86Ops.CMOVNE, X86Ops.CMOVE, X86Ops.CMOVL, X86Ops.CMOVB, + X86Ops.VCVTSI2SS, X86Ops.VCVTSI2SD, X86Ops.VCVTSS2SD, X86Ops.VCVTSD2SS, X86Ops.IDIV, X86Ops.DIV} + + # X86Ops that can write to memory + WriteMem = {X86Ops.MOVm, X86Ops.MOVi, X86Ops.VMOVSSm, X86Ops.VMOVSDm, X86Ops.VMOVUPSm, X86Ops.VMOVDm, X86Ops.VMOVQm, + X86Ops.ADDi, X86Ops.SUBi, X86Ops.ANDi, X86Ops.ORi, X86Ops.XORi, X86Ops.SHL, X86Ops.SHLi, X86Ops.SHR, X86Ops.SHRi, X86Ops.SAR, + X86Ops.SARi, X86Ops.SETNE, X86Ops.SETE, X86Ops.SETL, X86Ops.SETB, + X86Ops.VCVTPS2PH, X86Ops.VPEXTRW, X86Ops.VPEXTRD} + + # X86Ops that read flags + ReadFlags = {X86Ops.CMOVB, X86Ops.CMOVL, X86Ops.CMOVE, X86Ops.CMOVNE, X86Ops.SETB, X86Ops.SETL, X86Ops.SETE, X86Ops.SETNE, X86Ops.JB, X86Ops.JL, + X86Ops.JE, X86Ops.JNE, X86Ops.JGE} + + # X86Ops that write flags or can modify flags to undefined values + WriteFlags = {X86Ops.CMP, X86Ops.CMPi, X86Ops.ADD, X86Ops.ADDi, X86Ops.SUB, X86Ops.SUBi, X86Ops.IMUL, X86Ops.IMULi, X86Ops.IDIV, X86Ops.DIV, + X86Ops.SHL, X86Ops.SHLi, X86Ops.SHR, X86Ops.SHRi, X86Ops.SAR, X86Ops.SARi, X86Ops.AND, X86Ops.ANDi, X86Ops.XOR, X86Ops.XORi, + X86Ops.OR, X86Ops.ORi} + + # X86Ops that jump, their first src is what the LABEL they jump to marks + Jump = {X86Ops.JL, X86Ops.JB, X86Ops.JE, X86Ops.JNE, X86Ops.JGE, X86Ops.JMP} + + # X86Ops whose first src is the rm field. a TwoAddress op drops its first src post regalloc, so its Rm2nd src ends up first + Rm1st = {X86Ops.MOV, X86Ops.VMOVSS, X86Ops.VMOVSD, X86Ops.VMOVUPS, X86Ops.MOVZX, X86Ops.MOVSX, X86Ops.MOVSXD, X86Ops.VMOVD, X86Ops.VMOVQ, + X86Ops.VCVTTSS2SI, X86Ops.VCVTTSD2SI, X86Ops.VCVTPH2PS, X86Ops.CMPi, X86Ops.IMULi, X86Ops.LEA, X86Ops.VPSRLDQ} | (Rm2nd & TwoAddress) + +# ***** X86 legalization ***** +def is_regbuf(x:UOp) -> bool: return x.without_after.addrspace is AddrSpace.REG and x.max_numel() == 1 + +extra_matcher = PatternMatcher([ + # bool CMPNE is XOR, bool CMPEQ is XOR+XOR, bool CMPLT is XOR+AND + (UPat.var('x', dtypes.bool).alu(Ops.CMPNE, UPat.var('y')), lambda x,y: x^y), + (UPat.var('x', dtypes.bool).alu(Ops.CMPEQ, UPat.var('y')), lambda x,y: (x^y)^True), + (UPat.var('x', dtypes.bool)> 1) | (y & 1)).cast(dtypes.int64).cast(x.dtype) * 2)), + # no int8 mul or cmove, cast to int16 + (UPat.var("a", dtypes.int8s) * UPat.var("b"), lambda a,b: (a.cast(dtypes.int16) * b.cast(dtypes.int16)).cast(a.dtype)), + (UPat.var("m").where(UPat.var("a", (dtypes.bool,)+dtypes.int8s), UPat.var("b")), + lambda m,a,b: m.where(a.cast(dtypes.int16), b.cast(dtypes.int16)).cast(a.dtype)), + # cast to bool is a nonzero test, this cancels the cast-back the where rule emits so it doesn't survive isel + (UPat.var("x", dtypes.ints).cast(dtypes.bool), lambda x: x.alu(Ops.CMPNE, UOp.cconst(0, x.dtype))), + # float16 alus are done in float32 + (UPat(GroupOp.ALU, dtypes.float16, name="x"), lambda x: UOp(x.op, + src=tuple(s.cast(dtypes.float) if s.dtype != dtypes.bool else s for s in x.src)).cast(x.dtype)), + (UPat(GroupOp.Comparison, src=[UPat(dtype=dtypes.float16), UPat()], name="x"), + lambda x: UOp(x.op, src=tuple(s.cast(dtypes.float32) for s in x.src)).cast(x.dtype)), + # a float WHERE blends at the width of its value, so it needs a comparison at that width to make the mask + (UPat.var("m", dtypes.bool).where(UPat.var("a", dtypes.floats+(dtypes.weakfloat,)), UPat.var("b")).named("w"), + lambda m,a,b,w: m.cast(w.dtype).ne(UOp.cconst(0, w.dtype)).where(a, b) + if w.dtype in dtypes.floats and promo_dtype(m.src) is not w.dtype else None), + # rewrite -x -> -0.0 - x + (UPat(Ops.NEG, name="x"), lambda x: UOp(Ops.SUB, src=(x.const_like(-0.0),) + x.src)), + # TODO: add support for mod, requires support for accessing the 2nd+ reg of a multi output instruction + (UPat(Ops.TRUNCMOD, src=(UPat.var("x"), UPat.var("y"))), lambda x,y: x - y * x.alu(Ops.TRUNCDIV, y)), + # scalar reg gated mops become cmovs + (UPat((Ops.INDEX, Ops.SHRINK), name="addr").load(UPat.var("alt"), UPat.var("gate")), + lambda addr,alt,gate: gate.where(addr.load(), alt) if is_regbuf(addr.src[0]) else None), + (UPat((Ops.INDEX, Ops.SHRINK), name="addr").store(UPat.var("val"), UPat.var("gate")), + lambda addr,val,gate: addr.store(gate.where(val, addr.load())) if is_regbuf(addr.src[0]) else None), +]) + +# ***** X86 pre instruction selection ***** +def scratch_buffer(elem_dt:DType, count:int, slot:int) -> UOp: return UOp.placeholder((count,), elem_dt, slot, AddrSpace.LOCAL) + +def gated_load(ctx, addr:UOp, alt:UOp, gate:UOp, x:UOp): + local = scratch_buffer(addr.src[0].dtype, x.max_numel(), next(ctx)) + local_idx = local.index(UOp.cconst(0, dtypes.int32)) + # the AFTER orders the load after the scratch store + sel = gate.where(addr, local_idx) + return UOp(Ops.AFTER, src=(sel, (local_idx if x.max_numel() == 1 else local).store(alt))).load() + +def gated_store(addr:UOp, gate:UOp, val:UOp): + local = scratch_buffer(addr.src[0].dtype, val.max_numel(), -1) + sel = gate.where(addr, local.index(UOp.cconst(0, dtypes.int32))) + return UOp(Ops.AFTER, src=(sel,)).store(val) + +# a gate the flags can be picked with, or the bool compared to zero that replaces one they can't: only an integer +# comparison sets the flags, see cmp. NOTE: the 0 is int so the bool zero-extends and compares as int (a byte compare renders +# different kernels) +def flag_gate(m:UOp) -> UOp|None: + return None if m.op in GroupOp.Comparison and m.src[0].dtype not in dtypes.floats else m.cast(dtypes.int).alu(Ops.CMPNE, UOp.cconst(0, dtypes.int)) + +# legalize the new style graph for isel. NOTE: this runs after the spec is verified, some of these rewrites violate it +pre_isel_matcher = PatternMatcher([ + # vector BUFFERs get modeled in STACK space + (UPat((Ops.BUFFER, Ops.ALLOC), name="x"), lambda x: x.replace(arg=replace(x.arg, addrspace=AddrSpace.LOCAL)) + if x.addrspace is AddrSpace.REG and x.max_numel() > 1 else None), + # widening a uint32 is free, the 32bit write that produced it already zeroed the upper half + (UPat(dtype=dtypes.uint32).cast(dtypes.int64s, name="x"), lambda x: x.replace(op=Ops.BITCAST)), + (UPat.var("y", dtypes.ints+(dtypes.bool,)).cast(dtypes.ints, name="x"), + lambda y,x: x.replace(op=Ops.BITCAST) if x.dtype.itemsize == y.dtype.itemsize else None), + # gated load/store become a conditional move on the address, the load/store are unconditional + (UPat((Ops.INDEX, Ops.SHRINK), name="addr").load(UPat.var("alt"), UPat.var("gate"), name="x"), gated_load), + (UPat((Ops.INDEX, Ops.SHRINK), name="addr").store(UPat.var("val"), UPat.var("gate")), gated_store), + # a conditional backedge picks with the flags, and so does the cmove, which is legalized in isel + (UPat(Ops.BACKEDGE, src=(UPat(), UPat(), UPat.var("m", dtypes.bool)), name="x"), + lambda m,x: x.replace(src=x.src[:2]+(g,)) if (g:=flag_gate(m)) is not None else None), +]) + +# ***** X86 registers ***** +RAX = Register("rax", 0) +RCX = Register("rcx", 1) +RDX = Register("rdx", 2) +RBX = Register("rbx", 3) +RSP = Register("rsp", 4) +RBP = Register("rbp", 5) +RSI = Register("rsi", 6) +RDI = Register("rdi", 7) +GPR = (RAX, RCX, RDX, RBX, RSP, RBP, RSI, RDI) + tuple(Register(f"r{i}", i) for i in range(8, 16)) +XMM = tuple(Register(f"xmm{i}", i, size=16) for i in range(16)) +# gprs you can write to +WGPR = tuple(r for r in GPR if r != RSP) + +CALLEE_SAVED = (RBX, RBP, GPR[12], GPR[13], GPR[14], GPR[15]) + +reg_strs = {"rax": {4:"eax", 2:"ax", 1:"al"}, "rcx": {4:"ecx", 2:"cx", 1:"cl"}, "rdx": {4:"edx", 2:"dx", 1:"dl"}, "rbx": {4:"ebx", 2:"bx", 1:"bl"}, + "rsp": {4:"esp", 2:"sp", 1:"spl"}, "rbp": {4:"ebp", 2:"bp", 1:"bpl"}, "rsi": {4:"esi", 2:"si", 1:"sil"}, "rdi": {4:"edi", 2:"di", 1:"dil"}, + **{f"r{i}": {4:f"r{i}d", 2:f"r{i}w", 1:f"r{i}b"} for i in range(8, 16)}} + +# a register defined outside the instruction stream: the stack pointer, the args pointer, a callee saved register +def reg(dt:DType, r:Register) -> UOp: return UOp(Ops.INS, arg=(X86Ops.REG, dt, r)) +def stack_pointer() -> UOp: return reg(dtypes.uint64, RSP) +def args_pointer() -> UOp: return reg(dtypes.uint64, RDI) + +# ***** X86 instruction selection ***** +# a register buffer is its own virtual register, named by its identity +def regbuf_reg(x:UOp) -> Register: return Register(f"b{x.key.hex()}", 0, *((XMM, 16) if x.dtype in dtypes.floats else (WGPR, x.dtype.itemsize))) +# a value moved into a register constrained to cons +def pin(ctx:IselContext, x:UOp, cons:Register|tuple[Register, ...]) -> UOp: return x.ins(X86Ops.MOV, ctx.vreg(cons, x.dtype.itemsize), src=(x,)) +def base(x:UOp, i:int) -> UOp: return s.src[0] if (s:=x.src[i]).op is Ops.INDEX else s +def lane(x:UOp, i:int) -> int: return s.src[1].src[0].val if (s:=x.src[i]).op is Ops.INDEX else 0 +def to_int(dt:DType): return {dtypes.float16: dtypes.int16, dtypes.float32: dtypes.int32, dtypes.float64: dtypes.int64}[dt] +def imm(dt:DType, v:int) -> UOp: return UOp.cconst(truncate[dt](v), dt).ins(X86Ops.IMM) +def is_imm(x:UOp) -> bool: return x.op is Ops.INS and x.arg[0] is X86Ops.IMM +def to_imm(c:UOp) -> UOp|None: + if not (c.op is Ops.CAST and (v:=c.src[0]).op is Ops.CONST): return None + if c.dtype in dtypes.int64s: return imm(dtypes.int32, v.val) if not v.overflows(dtypes.int32) else None + if c.dtype in dtypes.ints+(dtypes.bool,): return imm(c.dtype, v.val) + return None +# the flag path, which only an integer comparison can take: an x86 float compare sets carry, zero and parity together when an +# operand is NaN, so a NaN reads as "below" and as "equal", and it clears sign and overflow, so nothing reads as "less" +def cmp(x:UOp) -> UOp: + if x.src[0].dtype in dtypes.floats: raise RuntimeError(f"no flag compare for {x.src[0].dtype}, a float gate must be a mask") + return x.ins(X86Ops.CMP, dtype=dtypes.void) if (i:=to_imm(x.src[1])) is None else x.ins(X86Ops.CMPi, dtype=dtypes.void, src=(x.src[0], i)) +# comparisons that produce masks, the mask has the width of the operands +def mask(x:UOp) -> UOp: + dt, v = x.src[0].dtype, imm(dtypes.uint8, {Ops.CMPLT: 1, Ops.CMPNE: 4, Ops.CMPEQ: 0}[x.op]) + return x.ins(X86Ops.VCMPSS if dt is dtypes.float32 else X86Ops.VCMPSD, dtype=dt, src=x.src + (v,)) + +# vinsertps xmm2, xmm0, xmm1, imm +# inserts any 32 bit element in xmm1 into any position in xmm0 according to immm, result is written to xmm2 +# this is the fallback slow case for when you can't match more a powerful shuffle +def vinsertps(x:UOp) -> UOp: + def _insert(ret:UOp, i:int) -> UOp: + s, v = base(x, i), lane(x, i) + return x.ins(X86Ops.VINSERTPS, src=(ret, s, imm(dtypes.uint8, v << 6 | i << 4))) + return functools.reduce(_insert, range(len(x.src)), UOp(Ops.NOOP)) + +# vpinsrd xmm2, xmm0, eax, imm +# inserts the element in eax into any position in xmm0, result is written to xmm2 according to imm +def vpins(x:UOp, srcs:tuple[UOp, ...]) -> UOp: + op = {2: X86Ops.VPINSRW, 4: X86Ops.VPINSRD}[x.dtype.itemsize] + return functools.reduce(lambda ret,i: x.ins(op, src=(ret, srcs[i], imm(dtypes.uint8, i))), range(len(srcs)), UOp(Ops.NOOP)) + +def idiv(ctx:IselContext, x:UOp) -> UOp: + op = X86Ops.DIV if x.dtype in dtypes.uints else X86Ops.IDIV + val = UOp.cconst(0, x.dtype) if x.dtype in dtypes.uints else x.src[0] >> UOp.cconst(x.dtype.itemsize*8-1, x.dtype) + ext = [] if x.dtype in dtypes.int8s else [pin(ctx, val, RDX)] + dividend_dtype = dtypes.int16 if x.dtype in dtypes.int8s else x.dtype + dividend = pin(ctx, x.src[0].cast(dividend_dtype), RAX) + defs = [ctx.vreg(RAX, x.dtype.itemsize), ctx.vreg(RDX, x.dtype.itemsize)][:1+(x.dtype not in dtypes.int8s)] + divisor = pin(ctx, x.src[1], tuple(r for r in WGPR if r not in (RAX, RDX))) + idiv = x.ins(op, *defs, src=(dividend, divisor) + tuple(ext)) + # this move "cleanses" the register constraints (rax/rdx) of idiv + return x.ins(X86Ops.MOV, src=(idiv,)) + +# a variable shift count implicitly reads cl so it goes in rcx, the shifted value can't be in rcx +def shift(ctx:IselContext, x:UOp, op:X86Ops) -> UOp: + val = pin(ctx, x.src[0], tuple(r for r in WGPR if r is not RCX)) + cnt = pin(ctx, x.src[1], RCX) + return x.ins(op, src=(val, cnt)) + +# a memory address operand is (base, index, displacement, size) +def fold_address(x:UOp) -> tuple[UOp, UOp, UOp, UOp]: + def _disp(v:int) -> UOp: return imm(dtypes.int32 if abs(v) > dtypes.int8.max else dtypes.int8, v) + def _cast(v:UOp) -> UOp: return v.cast(dtypes.int64) if v.vmin < 0 else v.cast(dtypes.uint32) if v.dtype.itemsize < 4 else v + if x.op not in {Ops.INDEX, Ops.SHRINK}: return (x, UOp(Ops.NOOP), _disp(0), imm(dtypes.uint32, x.dtype.itemsize)) + base, idx = x.src[0], x.src[1] + # buffers are indexed by element, the stack pointer by byte + scale = 1 if base in (stack_pointer(), args_pointer()) else base.dtype.itemsize + sz = imm(dtypes.uint32, base.dtype.itemsize) + if idx.op is Ops.ADD and (c:=idx.src[1]).op is Ops.CAST and c.src[0].op is Ops.CONST: + return (base, _cast(idx.src[0]), _disp(c.src[0].val * scale), sz) + if idx.op is Ops.CAST and idx.src[0].op is Ops.CONST: return (base, UOp(Ops.NOOP), _disp(idx.src[0].val * scale), sz) + return (base, _cast(idx), _disp(0), sz) + +# the value of a BUFFER is its address, it moves through registers and the stack as a 64bit int +def lea(x:UOp) -> UOp: return x.ins(X86Ops.LEA, src=fold_address(x), dtype=dtypes.uint64) +def abi(ctx:IselContext, x:UOp) -> UOp: + src = fold_address(args_pointer().index(UOp.cconst(ctx.arg_offsets[x], dtypes.uint32))) + return x.ins(X86Ops.MOV, dtype=x.dtype if x.addrspace is AddrSpace.ALU else dtypes.uint64, src=src) + +GPR_DEST_OPS = {X86Ops.VPEXTRW, X86Ops.VPEXTRD, X86Ops.VCVTTSS2SI, X86Ops.VCVTTSD2SI, X86Ops.VMOVDm, X86Ops.VMOVQm} +XMM_OPS = {op for op in X86Ops if op.name.startswith('V')} - GPR_DEST_OPS + +def _is_vec_xmm(y: UOp) -> bool: + return (y.op is Ops.INS and y.arg[0] in XMM_OPS) or (y.op not in (Ops.BUFFER, Ops.ALLOC, Ops.PARAM, Ops.AFTER, Ops.INS) and y.max_numel() > 1) + +def _xmm_sz(x: UOp) -> X86Ops: + bits = x.max_numel() * x.dtype.itemsize + if bits >= 16: return X86Ops.VMOVUPS + if bits >= 8: return X86Ops.VMOVSD + return X86Ops.VMOVSS + +def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: + # register placeholders with real registers + if is_imm(x): return None + # no register definition + if x.dtype is dtypes.void: return None + # already allocated vregs + if regs(x) and regs(x)[0]._cons: return None + # a register buffer is a REG, a stack buffer becomes the LEA of its address before regalloc + if x.op in {Ops.BUFFER, Ops.ALLOC}: return reg(x.dtype, regbuf_reg(x)) if x.addrspace is AddrSpace.REG else None + if regs(x): defs = [ctx.vreg(regs(x), x.dtype.itemsize)] + elif x.dtype in dtypes.floats or x.arg[0] in XMM_OPS: defs = [ctx.vreg(XMM)] + else: defs = [ctx.vreg(WGPR, x.dtype.itemsize)] + # TODO: add this once the scheduler can track register pressure + # if x.arg[0] in X86GroupOp.WriteFlags: defs.append(ctx.vreg(RFLAGS)) + return with_regs(x, tuple(defs)) + +def copy_op(dt:DType) -> X86Ops: + return {dtypes.float16:X86Ops.VMOVSS, dtypes.float32:X86Ops.VMOVSS, dtypes.float64:X86Ops.VMOVSD, dtypes.bool:X86Ops.MOVZX}.get(dt, X86Ops.MOV) + +isel_matcher = PatternMatcher([ + # **** Op -> Op **** + (UPat(Ops.RANGE, src=(UPat.cvar("c").cast(),), name="x"), lambda c,x: x.replace(src=(imm(x.dtype, c.val),))), + # BACKEDGE becomes a conditional jump to its RANGE, it uses flags + (UPat(Ops.BACKEDGE, src=(UPat(), UPat(), UPat(GroupOp.Comparison, name="y")), name="x"), lambda x,y: x.ins({Ops.CMPEQ: X86Ops.JE, + Ops.CMPNE: X86Ops.JNE, Ops.CMPLT: X86Ops.JB if y.src[0].dtype in dtypes.uints else X86Ops.JL}[y.op], src=(x.src[1], x.src[0], cmp(y)))), + # **** Op -> X86Op **** + # function abi constraints + (UPat(Ops.PARAM, name="x"), abi), + # conditional moves between addresses, lea both srcs + (UPat.var("m").where(UPat((Ops.INDEX, Ops.SHRINK), name="a"), UPat((Ops.INDEX, Ops.SHRINK), name="b")), lambda m,a,b: + m.where(lea(a), lea(b)) if not _is_vec_xmm(a.src[0]) else None), + # constants that can't be immediates, move them to registers + (UPat.cvar("c").cast(dtypes.int64s, name="x"), lambda c,x: x.ins(X86Ops.MOVABS, src=(imm(x.dtype, c.val),))), + (UPat.cvar("c").cast(dtypes.ints+(dtypes.bool,), name="x"), lambda c,x: x.ins(X86Ops.MOVi, src=(imm(x.dtype, c.val),))), + (UPat.cvar("c").cast(dtypes.floats, name="x"), lambda c,x: + UOp.cconst(struct.unpack((dt:=to_int(x.dtype)).fmt, struct.pack(x.dtype.fmt, c.val))[0], dt).bitcast(x.dtype)), + # conditional moves that use masks, the mask has the width of the values + (UPat(GroupOp.Comparison, src=(UPat(dtype=dtypes.float32), UPat()), name="m").where(UPat.var("a", dtypes.float32), UPat.var("b")), lambda m,a,b: + a.ins(X86Ops.VBLENDVPS, src=(b, a, mask(m)))), + (UPat(GroupOp.Comparison, src=(UPat(dtype=dtypes.float64), UPat()), name="m").where(UPat.var("a", dtypes.float64), UPat.var("b")), lambda m,a,b: + a.ins(X86Ops.VBLENDVPD, src=(b, a, mask(m)))), + # in this case we have a mask producing comparison whose user expects a bool, so we convert to bool + (UPat(GroupOp.Comparison, src=(UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: + UOp(Ops.AND, src=(mask(x).bitcast(dt:=to_int(y.dtype)), UOp.cconst(1, dt))).bitcast(dtypes.bool)), + # conditional moves that use flags + # TODO: remove this once we allow all flag producing ops in cmove + # the blends took every float gate a mask can serve, so a gate that is still not an integer comparison becomes one here + (UPat.var("m", dtypes.bool).where(UPat.var("a"), UPat.var("b")), lambda m,a,b: g.where(a, b) if (g:=flag_gate(m)) is not None else None), + (UPat(Ops.CMPLT, src=(UPat(dtype=dtypes.sints), UPat()), name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: + a.ins(X86Ops.CMOVL, src=(b, a, cmp(m)))), + (UPat(Ops.CMPLT, name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVB, src=(b, a, cmp(m)))), + (UPat(Ops.CMPEQ, name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVE, src=(b, a, cmp(m)))), + (UPat(Ops.CMPNE, name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVNE, src=(b, a, cmp(m)))), + # comparisons whose user doesn't use the flag, move flag result to register + (UPat(Ops.CMPLT, src=(UPat(dtype=dtypes.uints), UPat()), name="x"), lambda x: x.ins(X86Ops.SETB, src=(cmp(x),))), + (UPat(Ops.CMPLT, name="x"), lambda x: x.ins(X86Ops.SETL, src=(cmp(x),))), + (UPat(Ops.CMPEQ, name="x"), lambda x: x.ins(X86Ops.SETE, src=(cmp(x),))), + (UPat(Ops.CMPNE, name="x"), lambda x: x.ins(X86Ops.SETNE, src=(cmp(x),))), + # float unary + (UPat.var("y", dtypes.float32).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSS, src=(y, y))), + (UPat.var("y", dtypes.float64).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSD, src=(y, y))), + (UPat.var("y", dtypes.float32).trunc().named("x"), lambda y,x: x.ins(X86Ops.VROUNDSS, src=(y, y, imm(dtypes.uint8, 3)))), + (UPat.var("y", dtypes.float64).trunc().named("x"), lambda y,x: x.ins(X86Ops.VROUNDSD, src=(y, y, imm(dtypes.uint8, 3)))), + # for float16 we route the srcs through gprs, this is suboptimal for values in xmms, in that case we want vpunpcklwd + (UPat(Ops.STACK, dtypes.float16, name="x"), lambda x: vpins(x, tuple(s.bitcast(dtypes.int16) for s in x.src))), + (UPat(Ops.STACK, dtypes.float32, name="x"), vinsertps), + (UPat(Ops.STACK, dtypes.int32s, name="x"), lambda x: vpins(x, x.src)), + # INDEX on a vector register value extracts a single element + (UPat.var("y", dtypes.int32s).index(UPat.cvar("c").cast(), name="x"), + lambda y,c,x: x.ins(X86Ops.VPEXTRD, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), + (UPat.var("y", dtypes.floats).index(UPat.cvar("c").cast(), name="x"), + lambda y,c,x: x.ins(X86Ops.VPSRLDQ, src=(y, imm(dtypes.uint8, c.val * x.dtype.itemsize))) if _is_vec_xmm(y) else None), + # int binary + ((UPat(dtype=dtypes.ints).alu(Ops.TRUNCDIV, UPat())).named("x"), idiv), + # int binary with immediate + (UPat.var("a", dtypes.ints) << UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SHLi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.uints) >> UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SHRi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.sints) >> UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SARi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.ints) + UPat.cvar().cast(name="c"), lambda a,c: a.ins(X86Ops.ADDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints) * UPat.cvar().cast(name="c"), lambda a,c: a.ins(X86Ops.IMULi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) & UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.ANDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) | UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.ORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) ^ UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.XORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat(Ops.SUB, dtypes.ints, (UPat.var("a"), UPat.cvar().cast(name="c"))), + lambda a,c: a.ins(X86Ops.SUBi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + # int binary with register + ((UPat(dtype=dtypes.ints) << UPat()).named("x"), lambda ctx,x: shift(ctx, x, X86Ops.SHL)), + ((UPat(dtype=dtypes.uints) >> UPat()).named("x"), lambda ctx,x: shift(ctx, x, X86Ops.SHR)), + ((UPat(dtype=dtypes.sints) >> UPat()).named("x"), lambda ctx,x: shift(ctx, x, X86Ops.SAR)), + (UPat.var("a", dtypes.ints) + UPat.var("b"), lambda a,b: a.ins(X86Ops.ADD, src=(a, b))), + (UPat.var("a", dtypes.ints) * UPat.var("b"), lambda a,b: a.ins(X86Ops.IMUL, src=(a, b))), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) & UPat.var("b"), lambda a,b: a.ins(X86Ops.AND, src=(a, b))), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) | UPat.var("b"), lambda a,b: a.ins(X86Ops.OR, src=(a, b))), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) ^ UPat.var("b"), lambda a,b: a.ins(X86Ops.XOR, src=(a, b))), + (UPat(Ops.SUB, dtypes.ints, (UPat.var("a"), UPat.var("b"))), lambda a,b: a.ins(X86Ops.SUB, src=(a, b))), + # float binary + ((UPat(dtype=dtypes.float32) + UPat()).named("x"), lambda x: x.ins(X86Ops.VADDSS)), + ((UPat(dtype=dtypes.float64) + UPat()).named("x"), lambda x: x.ins(X86Ops.VADDSD)), + ((UPat(dtype=dtypes.float32) * UPat()).named("x"), lambda x: x.ins(X86Ops.VMULSS)), + ((UPat(dtype=dtypes.float64) * UPat()).named("x"), lambda x: x.ins(X86Ops.VMULSD)), + (UPat(Ops.SUB, dtypes.float32, name="x"), lambda x: x.ins(X86Ops.VSUBSS)), + (UPat(Ops.SUB, dtypes.float64, name="x"), lambda x: x.ins(X86Ops.VSUBSD)), + (UPat(Ops.FDIV, dtypes.float32, name="x"), lambda x: x.ins(X86Ops.VDIVSS)), + (UPat(Ops.FDIV, dtypes.float64, name="x"), lambda x: x.ins(X86Ops.VDIVSD)), + # casts + (UPat(dtype=dtypes.float32).cast(dtypes.float16, name="x"), lambda x: x.ins(X86Ops.VCVTPS2PH, src=x.src + (imm(dtypes.uint8, 4),))), + (UPat(dtype=dtypes.float16).cast(dtypes.float32, name="x"), lambda x: x.ins(X86Ops.VCVTPH2PS)), + (UPat(dtype=dtypes.float32).cast(dtypes.int32s+dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.VCVTTSS2SI)), + (UPat(dtype=dtypes.float64).cast(dtypes.int32s+dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.VCVTTSD2SI)), + (UPat.var("y", dtypes.float32).cast(dtypes.float64, name="x"), lambda y,x: x.ins(X86Ops.VCVTSS2SD, src=(y, y))), + (UPat.var("y", dtypes.float64).cast(dtypes.float32, name="x"), lambda y,x: x.ins(X86Ops.VCVTSD2SS, src=(y, y))), + (UPat.var("y", (dtypes.int32, dtypes.int64)).cast(dtypes.float32, name="x"), lambda y,x: x.ins(X86Ops.VCVTSI2SS, src=(UOp(Ops.NOOP), y))), + (UPat.var("y", (dtypes.int32, dtypes.int64)).cast(dtypes.float64, name="x"), lambda y,x: x.ins(X86Ops.VCVTSI2SD, src=(UOp(Ops.NOOP), y))), + (UPat(dtype=(dtypes.uint8, dtypes.uint16, dtypes.bool)).cast(dtypes.ints, name="x"), lambda x: + x.ins(X86Ops.MOVZX) if x.src[0].dtype.itemsize < x.dtype.itemsize else None), + (UPat(dtype=dtypes.int32).cast(dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.MOVSXD)), + (UPat(dtype=dtypes.sints).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOVSX) if x.src[0].dtype.itemsize < x.dtype.itemsize else None), + (UPat(dtype=dtypes.ints).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOV)), + # bitcasts between scalar floats and ints + (UPat.var("y", dtypes.float16).bitcast(dtypes.int16s).named("x"), lambda y,x: x.ins(X86Ops.VPEXTRW, src=(y, imm(dtypes.uint8, 0)))), + (UPat(dtype=dtypes.int16s).bitcast(dtypes.float16).named("x"), lambda x: vpins(x, x.src)), + (UPat(dtype=dtypes.int32s).bitcast(dtypes.float32).named("x"), lambda x: x.ins(X86Ops.VMOVD)), + (UPat(dtype=dtypes.int64s).bitcast(dtypes.float64).named("x"), lambda x: x.ins(X86Ops.VMOVQ)), + (UPat(dtype=dtypes.float32).bitcast(dtypes.int32s).named("x"), lambda x: x.ins(X86Ops.VMOVDm)), + (UPat(dtype=dtypes.float64).bitcast(dtypes.int64s).named("x"), lambda x: x.ins(X86Ops.VMOVQm)), + # lower register mops: a store is just a copy, load is an anon copy to preserve ordering + (UPat.var("a").store(UPat.var("val")), lambda a,val: + UOp(Ops.STACK).after(buf.ins(copy_op(val.dtype), regbuf_reg(buf.without_after), src=(val.after(buf),))) + if is_regbuf((buf := a.src[0] if a.op is Ops.INDEX else a)) else None), + (UPat.var("a").load().named("x"), lambda a,x: + x.ins(copy_op(x.dtype), src=(buf,)) if is_regbuf((buf:=a.src[0] if a.op is Ops.INDEX else a)) else None), + # index on a buffer (or the stack pointer) computes an address, addresses are 64bit values + (UPat((Ops.INDEX, Ops.SHRINK), name="x"), lambda x: lea(x) if not _is_vec_xmm(x.src[0]) and x.addrspace is not AddrSpace.REG else None), + # TODO: fuse stores, very few cases -- store cmp becomes setcc, store gep int becomes vpextr, store bitcast to int becomes vmovd/q + # load, store + (UPat(Ops.LOAD, dtypes.floats, src=(UPat(name="a"),), name="x"), lambda x,a: None if a.addrspace is AddrSpace.REG else + x.ins(X86Ops.VPINSRW, src=(UOp(Ops.NOOP),) + fold_address(a) + (imm(dtypes.uint8, 0),)) if x.max_numel() * x.dtype.itemsize == 2 else + x.ins(_xmm_sz(x), src=fold_address(a))), + (UPat(Ops.LOAD, dtypes.ints+(dtypes.bool,), src=(UPat(name="a"),), name="x"), lambda x,a: None if a.addrspace is AddrSpace.REG else + x.ins(X86Ops.MOV, src=fold_address(a)) if x.max_numel() == 1 else x.ins(_xmm_sz(x), src=fold_address(a))), + (UPat.var("a").store(UPat.var("b", dtypes.floats), name="x"), lambda a,b,x: None if a.addrspace is AddrSpace.REG else + x.ins(X86Ops.VPEXTRW, src=fold_address(a) + (b, imm(dtypes.uint8, 0))) if b.max_numel() * b.dtype.itemsize == 2 else + x.ins(X86Ops[_xmm_sz(b).name+"m"], src=fold_address(a) + (b,))), + (UPat.var("a").store(UPat.var("b", dtypes.ints+(dtypes.bool,)), name="x"), lambda a,b,x: None if a.addrspace is AddrSpace.REG else + x.ins(X86Ops[_xmm_sz(b).name+"m"], src=fold_address(a) + (b,)) if b.max_numel() > 1 else + x.ins(X86Ops.MOVm, src=fold_address(a) + (b,)) if (i:=to_imm(b)) is None else x.ins(X86Ops.MOVi, src=fold_address(a) + (i,))), + # allocate virtual registers + (UPat((Ops.INS, Ops.BUFFER, Ops.ALLOC), name="x"), alloc_vregs), +]) + +# ***** pre register allocation ***** +# the flags belong to the last instruction that wrote them. x86 has no good way to store/restore them (then regalloc would +# handle it), so a consumer that no longer owns its compare re-emits it. Unlike a regalloc rematerialization this is not +# optional, there is no fallback load from stack +def flag_rematerialize(ctx:X86LinearContext, x:UOp): + if x.op in (Ops.RANGE, Ops.END) or x.arg[0] in X86GroupOp.WriteFlags: ctx.lock = x + elif x.arg[0] in X86GroupOp.ReadFlags and ctx.lock is not (flag_def:=x.src[-1]): + ctx.lock = flag_def + return (x, [flag_def, x]) + return None + +# the address of a stack buffer keeps the buffer's dtype so the element size of loads and stores through it is known +def alloc_buffer(ctx:X86LinearContext, x:UOp): + nx = UOp(Ops.INS, fold_address(stack_pointer().index(UOp.cconst(ctx.stack_size, dtypes.uint32))), + (X86Ops.LEA, x.dtype, Register(f"s{ctx.stack_size}", 0, WGPR, 8))) + ctx.stack_size += x.max_numel() * x.dtype.itemsize + return nx, [stack_pointer(), nx] + +# TODO: control flow should be overhauled so that this isn't necessary +# a LABEL marks its src: the start of a loop its RANGE, the exit of a loop the start LABEL. a jump's first src is what its LABEL marks +def label(x:UOp) -> UOp: return UOp(Ops.INS, arg=(X86Ops.LABEL, dtypes.void), src=(x,)) + +# a loop's LABEL follows its RANGE. a bounded RANGE's index is a register named by the RANGE, 0 before the loop, compared to the bound +# after the LABEL and incremented before the END, the body reads it as AFTER(index, RANGE) +def lower_range(x:UOp) -> tuple[UOp, list[UOp]]: + if x.dtype is dtypes.void: return x, [x, label(x)] + bound, deps = (x.src[0].src[0], x.src[0].src[1:]) if x.src[0].op is Ops.AFTER else (x.src[0], ()) + idx = UOp(Ops.INS, arg=(X86Ops.MOVi, x.dtype, regbuf_reg(x)), src=(imm(x.dtype, 0),) + deps) + cmp = UOp(Ops.INS, arg=(X86Ops.CMPi if is_imm(bound) else X86Ops.CMP, dtypes.void), src=(idx, bound)) + return idx.after(x), [idx, x, start:=label(x), cmp, UOp(Ops.INS, arg=(X86Ops.JGE, dtypes.void), src=(start, cmp))] + +def lower_end(x:UOp, c:UOp) -> tuple[UOp, list[UOp]]: + return (end:=x.replace(src=(x.src[0], c.src[1]))), [c.ins(X86Ops.ADDi, *regs(c.src[0]), src=(c, imm(c.dtype, 1))), end] + +pre_regalloc_matcher = PatternMatcher([ + (UPat(Ops.BUFFER, name="x"), lambda ctx,x: alloc_buffer(ctx,x) if x.addrspace is not AddrSpace.REG else None), + (UPat((Ops.INS, Ops.RANGE, Ops.END), name="x"), flag_rematerialize), + (UPat(Ops.RANGE, name="x"), lower_range), + (UPat(Ops.END, src=(UPat(), UPat(Ops.AFTER, name="c")), name="x"), lower_end), +]) + +# ***** post register allocation ***** +# final rewrite to match the isa spec +post_regalloc_matcher = PatternMatcher([ + (UPat(Ops.SINK, name="x"), lambda x: (x, [x.ins(X86Ops.RET)])), + # an END jumps back to its loop's LABEL and labels the loop's exit + (UPat(Ops.END, name="x"), lambda x: (x, [UOp(Ops.INS, arg=(X86Ops.JMP, dtypes.void), src=(x.src[1],)), label(label(x.src[1]))])), + # rewrite two address instructions to two address form, if reused src wasn't coalesced insert a move + (UPat(Ops.INS, name="x"), lambda ctx,x: (nx:=x.replace(src=x.src[1:]), + [ctx.ren.copy(x.src[0], rdef(x)), nx] if rdef(x) != rdef(x.src[0]) else [nx]) if x.arg[0] in X86GroupOp.TwoAddress else None), +]) + +# ***** X86 instruction encoding ***** + +def encode(x:UOp, opc:int, reg:int|None=None, pp:int=0, sel:int=0, we:int=0) -> bytes|None: + def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|None=None, scale_uop:UOp|None=None, + vvvv_uop:UOp|None=None, imm_uop:UOp|None=None) -> bytes: + nonlocal reg, opc + # get the encoding values of the different fields + reg = cast(int, cast(Register, rdef(reg_uop)).index if reg_uop is not None else reg) + rm = cast(Register, rdef(rm_uop)).index + idx = cast(Register, rdef(idx_uop)).index if idx_uop is not None and rdef(idx_uop) is not None else 4 + # TODO: shouldn't rely on bitcast to convey encoding info + def _sz(x:UOp): return x.dtype.itemsize if x.op is Ops.BITCAST else rdef(x).size + # for a memory operand the rm size is the element size carried by the address, otherwise it's the size of the value in the register + rm_sz = scale_uop.src[0].val if scale_uop is not None else _sz(rm_uop) + reg_sz = _sz(reg_uop) if reg_uop is not None else 0 + sz = reg_sz or rm_sz + + # encode instruction + inst = bytes([]) + assert 0 <= reg <= 15 and 0 <= idx <= 15 and 0 <= rm <= 15 + # r extends reg field, x extends index field, b extends rm or base field + r, _x, b = reg >> 3, idx >> 3, rm >> 3 + if sel: # VEX bytes + vvvv = (vd.index if isinstance(vd := rdef(vvvv_uop), Register) else reg) if vvvv_uop is not None else 0 + if sel == 1 and _x == b == we == 0: inst += bytes([0xC5, (~r & 0b1) << 7 | (~vvvv & 0b1111) << 3 | pp]) + else: inst += bytes([0xC4, (~r & 0b1) << 7 | (~_x & 0b1) << 6 | (~b & 0b1) << 5 | sel, we << 7 | (~vvvv & 0b1111) << 3 | pp]) + else: # optional PREFIX and REX bytes + # PREFIX byte signaling 16 bit variant of instruction + if sz == 2: inst += bytes([0x66]) + # bit signaling 64 bit variant of instruction + w = sz == 8 + # legacy 8bit opcode is 1 less than 16-64bit variants + demote = (rm_sz == 1 or reg_sz == 1) and x.arg[0] not in X86GroupOp.ReadFlags | {X86Ops.LEA} + # REX byte is required when 64 bit or an extended reg is used (index 8 - 15) or lower 8 bits of (rsp, rbp, rsi, rdi) are accessed + if w | r | _x | b | (reg_sz == 1 & reg >> 2) | (rm_sz == 1 & rm >> 2) | (demote and disp_uop is None and rm >= 4): + inst += bytes([0b0100 << 4 | w << 3 | r << 2 | _x << 1 | b]) + if demote: opc -= 1 + # OPCODE byte + inst += opc.to_bytes((opc.bit_length() + 7) // 8, 'big') + # MODRM byte + # now we only care about the lower 3 bits + idx, rm, reg = idx & 0b111, rm & 0b111, reg & 0b111 + # 0b00 -- signals memory access with no displacement + # 0b01 -- signals memory access with 8bit displacement + # 0b10 -- signals memory access with 32bit displacement + # 0b11 -- signals no memory access + if disp_uop is not None: + assert is_imm(disp_uop), "displacement must be an immediate" + assert disp_uop.dtype in (dtypes.int8, dtypes.int32), "displacement can only be 1 or 4 byte signed int" + # rbp/r13 always require a displacement + if disp_uop.src[0].val != 0 or rm == 0b101: mod = 0b01 if disp_uop.dtype.itemsize == 1 else 0b10 + else: mod = 0b00 + else: mod = 0b11 + # x 0b0 and idx 0b100 means rsp which means no index exists + # rm 0b100 (rsp/r12) signals a sib byte is required, rm then is encoded in the base field of SIB + _rm = rm if idx == 0b100 and _x == 0b0 else 0b100 + inst += bytes([mod << 6 | reg << 3 | _rm]) + # SIB byte + if _rm == 0b100 and mod != 0b11: + scale = {1: 0b00, 2: 0b01, 4: 0b10, 8: 0b11}[1 if idx == 0b100 and _x == 0b0 else rm_sz] + inst += bytes([scale << 6 | idx << 3 | rm]) + # DISP byte + if mod == 0b01 or mod == 0b10: + assert disp_uop is not None + inst += struct.pack(unwrap(disp_uop.dtype.fmt), disp_uop.src[0].val) + # IMM byte + if imm_uop is not None: + if is_imm(imm_uop): inst += struct.pack(unwrap(imm_uop.dtype.fmt), imm_uop.src[0].val) + elif isinstance(rdef(imm_uop), Register): inst += bytes([(rdef(imm_uop).index & 0b1111) << 4 | 0b0000]) + return inst + + # get the encoding structure of the uop + # when a uop writes to memory it takes the form of a store, dtype is void, no definition + address:tuple[UOp|None, ...] + if x.arg[0] in X86GroupOp.WriteMem: + if len(x.src) > 4: address, rest = x.src[:4], x.src[4:] + else: address, rest = (x, None, None, None), x.src + imm_uop = rest[:1] if rest and is_imm(rest[0]) else (None,) + return _encode(rest[0], *address, *(None, *rest[1:])) if reg is None else _encode(None, *address, *(None, *imm_uop)) + + if x.arg[0] in X86GroupOp.Rm1st: + if len(x.src) > 3: address, rest = x.src[:4], x.src[4:] + else: address, rest = (x.src[0], None, None, None), x.src[1:] + imm_uop = rest[:1] if rest and is_imm(rest[0]) else (None,) + return _encode(x, *address, *(None, *imm_uop)) if reg is None else _encode(None, *address, *(x if sel else None, *imm_uop)) + + if x.arg[0] in X86GroupOp.Rm2nd: + if len(x.src) > 4: address, rest = x.src[1:5], x.src[:1] + x.src[5:] + else: address, rest = (x.src[1], None, None, None), x.src[:1] + x.src[2:] + # cmp reg, rm doesn't define a new register + return _encode(x, *address, *rest) if x.dtype is not dtypes.void else _encode(rest[0], *address) + + return None + +# https://www.felixcloutier.com/x86/ +# legacy version -> VEX version +# prefix field: None -> 0 | 66 -> 1 | F3 -> 2 | F2 -> 3 +# opcode map select: 0F -> 1 | 0F38 -> 2 | 0F3A -> 3 +encodings = { + # moves + X86Ops.MOVABS: lambda x: + bytes([0b0100 << 4 | 0b1 << 3 | 0b00 << 2 | rdef(x).index >> 3, 0xB8 + (rdef(x).index & 0b111)]) + struct.pack(x.dtype.fmt, x.src[0].src[0].val), + X86Ops.MOV: lambda x: encode(x, 0x8B), X86Ops.MOVi: lambda x: encode(x, 0xC7, reg=0), + X86Ops.MOVm: lambda x: encode(x, 0x89), X86Ops.LEA: lambda x: encode(x, 0x8D), + X86Ops.VMOVSS: lambda x: encode(x, 0x10, pp=2, sel=1), X86Ops.VMOVSSm: lambda x: encode(x, 0x11, pp=2, sel=1), + X86Ops.VMOVSD: lambda x: encode(x, 0x10, pp=3, sel=1), X86Ops.VMOVSDm: lambda x: encode(x, 0x11, pp=3, sel=1), + X86Ops.VMOVUPS: lambda x: encode(x, 0x10, pp=0, sel=1), X86Ops.VMOVUPSm: lambda x: encode(x, 0x11, pp=0, sel=1), + X86Ops.VMOVD: lambda x: encode(x, 0x6E, pp=1, sel=1), X86Ops.VMOVQ: lambda x: encode(x, 0x6E, pp=1, sel=1, we=1), + X86Ops.VMOVDm: lambda x: encode(x, 0x7E, pp=1, sel=1), X86Ops.VMOVQm: lambda x: encode(x, 0x7E, pp=1, sel=1, we=1), + # casts + X86Ops.MOVZX: lambda x: encode(x, 0x0FB7), + X86Ops.MOVSX: lambda x: encode(x, 0x0FBF), X86Ops.MOVSXD: lambda x: encode(x, 0x63), + X86Ops.VCVTSS2SD: lambda x: encode(x, 0x5A, pp=2, sel=1), X86Ops.VCVTSD2SS: lambda x: encode(x, 0x5A, pp=3, sel=1), + X86Ops.VCVTPH2PS: lambda x: encode(x, 0x13, pp=1, sel=2), X86Ops.VCVTPS2PH: lambda x: encode(x, 0x1D, pp=1, sel=3), + # the int src is the 2nd src (the rm field), its width picks the 32 or 64 bit form + X86Ops.VCVTSI2SS: lambda x: encode(x, 0x2A, pp=2, sel=1, we=x.src[1].dtype.itemsize == 8), + X86Ops.VCVTSI2SD: lambda x: encode(x, 0x2A, pp=3, sel=1, we=x.src[1].dtype.itemsize == 8), + X86Ops.VCVTTSS2SI: lambda x: encode(x, 0x2C, pp=2, sel=1, we=x.dtype.itemsize == 8), + X86Ops.VCVTTSD2SI: lambda x: encode(x, 0x2C, pp=3, sel=1, we=x.dtype.itemsize == 8), + # int division + X86Ops.IDIV: lambda x: encode(x, 0xF7, reg=7), X86Ops.DIV: lambda x: encode(x, 0xF7, reg=6), + # scalar int binary + X86Ops.SHL: lambda x: encode(x, 0xD3, reg=4), X86Ops.SHLi: lambda x: encode(x, 0xC1, reg=4), + X86Ops.SHR: lambda x: encode(x, 0xD3, reg=5), X86Ops.SHRi: lambda x: encode(x, 0xC1, reg=5), + X86Ops.SAR: lambda x: encode(x, 0xD3, reg=7), X86Ops.SARi: lambda x: encode(x, 0xC1, reg=7), + X86Ops.ADD: lambda x: encode(x, 0x03), X86Ops.ADDi: lambda x: encode(x, 0x81, reg=0), + X86Ops.SUB: lambda x: encode(x, 0x2B), X86Ops.SUBi: lambda x: encode(x, 0x81, reg=5), + X86Ops.AND: lambda x: encode(x, 0x23), X86Ops.ANDi: lambda x: encode(x, 0x81, reg=4), + X86Ops.XOR: lambda x: encode(x, 0x33), X86Ops.XORi: lambda x: encode(x, 0x81, reg=6), + X86Ops.OR: lambda x: encode(x, 0x0B), X86Ops.ORi: lambda x: encode(x, 0x81, reg=1), + X86Ops.CMP: lambda x: encode(x, 0x3B), X86Ops.CMPi: lambda x: encode(x, 0x81, reg=7), + X86Ops.IMUL: lambda x: encode(x, 0x0FAF), X86Ops.IMULi: lambda x: encode(x, 0x69), + X86Ops.SETB: lambda x: encode(x, 0x0F92, reg=0), X86Ops.SETL: lambda x: encode(x, 0x0F9C, reg=0), + X86Ops.SETE: lambda x: encode(x, 0x0F94, reg=0), X86Ops.SETNE: lambda x: encode(x, 0x0F95, reg=0), + # unary + X86Ops.VSQRTSS: lambda x: encode(x, 0x51, pp=2, sel=1), X86Ops.VSQRTSD: lambda x: encode(x, 0x51, pp=3, sel=1), + X86Ops.VROUNDSS: lambda x: encode(x, 0x0A, pp=1, sel=3), X86Ops.VROUNDSD: lambda x: encode(x, 0x0B, pp=1, sel=3), + # float binary + X86Ops.VADDSS: lambda x: encode(x, 0x58, pp=2, sel=1), X86Ops.VADDSD: lambda x: encode(x, 0x58, pp=3, sel=1), + X86Ops.VSUBSS: lambda x: encode(x, 0x5C, pp=2, sel=1), X86Ops.VSUBSD: lambda x: encode(x, 0x5C, pp=3, sel=1), + X86Ops.VMULSS: lambda x: encode(x, 0x59, pp=2, sel=1), X86Ops.VMULSD: lambda x: encode(x, 0x59, pp=3, sel=1), + X86Ops.VDIVSS: lambda x: encode(x, 0x5E, pp=2, sel=1), X86Ops.VDIVSD: lambda x: encode(x, 0x5E, pp=3, sel=1), + X86Ops.VCMPSS: lambda x: encode(x, 0xC2, pp=2, sel=1), X86Ops.VCMPSD: lambda x: encode(x, 0xC2, pp=3, sel=1), + # ternary + X86Ops.CMOVB: lambda x: encode(x, 0x0F42), X86Ops.CMOVL: lambda x: encode(x, 0x0F4C), + X86Ops.CMOVE: lambda x: encode(x, 0x0F44), X86Ops.CMOVNE: lambda x: encode(x, 0x0F45), + X86Ops.VBLENDVPS: lambda x: encode(x, 0x4A, pp=1, sel=3), X86Ops.VBLENDVPD: lambda x: encode(x, 0x4B, pp=1, sel=3), + # shuffles + X86Ops.VPSRLDQ: lambda x: encode(x, 0x73, reg=3, pp=1, sel=1), + X86Ops.VPINSRW: lambda x: encode(x, 0xC4, pp=1, sel=1), X86Ops.VPINSRD: lambda x: encode(x, 0x22, pp=1, sel=3), + X86Ops.VINSERTPS: lambda x: encode(x, 0x21, pp=1, sel=3), + # extract + X86Ops.VPEXTRW: lambda x: encode(x, 0x15, pp=1, sel=3), X86Ops.VPEXTRD: lambda x: encode(x, 0x16, pp=1, sel=3), + # jumps are encoded with a placeholder which gets patched later once the real offset is known + X86Ops.JE: lambda x: bytes([0x0F, 0x84]) + int(0).to_bytes(4), + X86Ops.JNE: lambda x: bytes([0x0F, 0x85]) + int(0).to_bytes(4), + X86Ops.JL: lambda x: bytes([0x0F, 0x8C]) + int(0).to_bytes(4), + X86Ops.JB: lambda x: bytes([0x0F, 0x82]) + int(0).to_bytes(4), + X86Ops.JGE: lambda x: bytes([0x0F, 0x8D]) + int(0).to_bytes(4), + X86Ops.JMP: lambda x: bytes([0xE9]) + int(0).to_bytes(4), + X86Ops.RET: lambda x: bytes([0xC3]), +} + +class X86LinearContext(LinearContext): + def __init__(self, ren:X86Renderer): + super().__init__(ren) + self.lock: UOp|None = None + + # stack alloc/dealloc and callee saved handling + def insert_linearized(self, uops:list[UOp]) -> list[UOp]: + before, after, used = [], [], {r.name for u in uops if isinstance((r:=rdef(u)), Register)} + for r in CALLEE_SAVED: + if r.name in used: + a = reg(dtypes.uint64, r) + slot = self.assign_spill_slot(r,a) + before.append(self.ren.spill(slot,a)) + after.append(self.ren.fill(slot,a,r)) + + if self.stack_size > 0: + before.insert(0, stack_pointer().ins(X86Ops.SUBi, RSP, src=(stack_pointer(),imm(dtypes.int32, self.stack_size)))) + after.append(stack_pointer().ins(X86Ops.ADDi, RSP, src=(stack_pointer(),imm(dtypes.int32, self.stack_size)))) + return before + uops[:-1] + after + [uops[-1]] + + def assign_spill_slot(self, r:Register, u:UOp) -> int: + sz = r.cons[0].size + offset = self.stack_size + (sz - self.stack_size % sz) %sz + self.stack_size = offset + sz + return offset + +class X86Renderer(ISARenderer): + device = "CPU" + has_local = False + global_max = (1, 0, 0) + extra_matcher = extra_matcher + pre_isel_matcher = pre_isel_matcher + isel_matcher = isel_matcher + pre_regalloc_matcher = pre_regalloc_matcher + post_regalloc_matcher = post_regalloc_matcher + code_for_op = {x: lambda: None for x in (Ops.SQRT, Ops.AND, Ops.OR, Ops.SHL, Ops.SHR, Ops.NEG, Ops.SUB, Ops.FDIV, Ops.CMPLT, Ops.CMPNE)} + linear_ctx_type = X86LinearContext + def __init__(self, target:Target): + if target.arch.split(",")[0] != "x86_64": raise RuntimeError(f"X86Renderer only supports x86_64, got {target.arch}") + super().__init__(target) + from tinygrad.runtime.support.compiler_cpu import X86Compiler + self.compiler = X86Compiler() + def is_two_address(self, x:UOp) -> bool: return x.op is Ops.INS and x.arg[0] in X86GroupOp.TwoAddress + def copy(self, x:UOp, reg:Register) -> UOp: return x.ins(X86Ops.MOV, reg, src=(x,)) + + def spill(self, spill_slot:int, x:UOp) -> UOp: + op = X86Ops.VMOVUPSm if rdef(x).cons[0] in XMM else X86Ops.MOVm + disp = UOp.cconst(spill_slot, dtypes.int32) + return UOp(Ops.INS, src=fold_address(stack_pointer().index(disp)) + (x,), arg=(op, dtypes.void)) + + def fill(self, spill_slot:int, x:UOp, reg:Register) -> UOp: + disp = UOp.cconst(spill_slot, dtypes.int32) + return UOp(Ops.INS, src=fold_address(stack_pointer().index(disp)), arg=(X86Ops.VMOVUPS if reg.cons[0] in XMM else X86Ops.MOV, x.dtype, reg)) + + def asm_str(self, uops:list[UOp], function_name:str) -> str: + def _format_op(x:UOp) -> str: return f" {(o[7:-1] if (o:=str(x.arg[0]))[-1] in ('i', 'm') else o[7:]).lower():7s}" + def _format_operands(x:UOp) -> str: + def _format(src:tuple[UOp, ...]) -> list[str]: + return [str(s.src[0].val) if is_imm(s) else reg_strs[o].get(rdef(s).size, o) if \ + (o:=str(rdef(s))) in reg_strs else o for s in src if is_imm(s) or rdef(s) is not None] + def _mem_adress(base:UOp, idx:UOp, disp:UOp, scale:UOp) -> list[str]: + return [f"[{rdef(base)}" + (f" + {rdef(idx)}*{scale.src[0].val}" if rdef(idx) else "") + (f" + {d}" if (d:=disp.src[0].val) else "") + "]"] + + if len(x.src) > 4 and x.arg[0] in X86GroupOp.WriteMem: ret = _mem_adress(*x.src[:4]) + _format(x.src[4:]) + elif len(x.src) > 3 and x.arg[0] in X86GroupOp.Rm1st: ret = _format((x,)) + _mem_adress(*x.src[:4]) + _format(x.src[4:]) + elif len(x.src) > 4 and x.arg[0] in X86GroupOp.Rm2nd: ret = _format((x, x.src[0])) + _mem_adress(*x.src[1:5]) + _format(x.src[5:]) + else: ret = _format((x,) + x.src) + return ", ".join(ret) + + asm = [f".{function_name}:"] + for u in uops: + if u.op is not Ops.INS or u.arg[0] in {X86Ops.IMM, X86Ops.REG}: continue + if u.arg[0] is X86Ops.LABEL: asm.append(f".L{uops.index(u)}:") + elif u.arg[0] is X86Ops.RET: asm.append(_format_op(u)) + elif u.arg[0] in X86GroupOp.Jump: asm.append(_format_op(u) + f" .L{uops.index(label(u.src[0]))}") + else: asm.append(_format_op(u) + " " + _format_operands(u)) + return "\n".join(asm) + + def render(self, uops:list[UOp]) -> str: + targets: dict[UOp, int] = {} + jumps: dict[UOp, int] = {} + binary = bytearray() + for u in uops: + if u.op is not Ops.INS or u.arg[0] in {X86Ops.IMM, X86Ops.REG}: continue + if u.arg[0] is X86Ops.LABEL: + targets[u.src[0]] = len(binary) + continue + if u.arg[0] not in encodings or (l:=encodings[u.arg[0]](u)) is None: + raise RuntimeError(f"failed to encode {u.arg[0]} with {u.dtype} srcs {[x.dtype for x in u.src]}") + binary.extend(l) + if u.arg[0] in X86GroupOp.Jump: jumps[u] = len(binary) + # fixup jump targets now that encoding size is known + for u in uops: + if (t:=jumps.get(u)) is not None: binary[t-4:t] = (targets[u.src[0]] - t).to_bytes(4, 'little', signed=True) + return binary.hex() + + def supported_dtypes(self): return {d for d in super().supported_dtypes() if d not in dtypes.fp8s+(dtypes.bfloat16,)} diff --git a/tinygrad_repo/tinygrad/runtime/autogen/__init__.py b/tinygrad_repo/tinygrad/runtime/autogen/__init__.py index c29e1e76bd..91dd87f413 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/__init__.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/__init__.py @@ -4,7 +4,8 @@ from tinygrad.helpers import fetch, flatten, system, getenv root = (here:=pathlib.Path(__file__).parent).parents[2] nv_src = {"nv_570": "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/81fe4fb417c8ac3b9bdcc1d56827d116743892a5.tar.gz", "nv_580": "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/2af9f1f0f7de4988432d4ae875b5858ffdb09cc2.tar.gz", - "nv_610": "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/refs/tags/610.43.03.tar.gz"} + "nv_610": "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/refs/tags/610.43.03.tar.gz", + "nv_615": "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/refs/tags/615.78.08.tar.gz"} ffmpeg_src = "https://ffmpeg.org/releases/ffmpeg-8.0.1.tar.gz" rocr_src = "https://github.com/ROCm/rocm-systems/archive/refs/tags/rocm-7.1.1.tar.gz" linux_headers_deb = "https://snapshot.debian.org/archive/debian/20260207T145350Z/pool/main/l/linux/linux-libc-dev_6.18.9-1_all.deb" @@ -21,12 +22,11 @@ comgr_2_src = "https://repo.radeon.com/rocm/apt/6.2/pool/main/c/comgr/comgr_2.8. def macossdk(): return system("xcrun --show-sdk-path") llvm_lib = ( - (win_llvm:=r"'C:\\Program Files\\LLVM\\bin\\LLVM-C.dll' if WIN else ") + (mac_llvm:=repr([f'/opt/homebrew/opt/llvm@{i}/lib/libLLVM.dylib' for i in reversed(range(14, 21+1))]) + " if OSX else ") + (other_llvm:=repr([f'LLVM-{i}' for i in reversed(range(14, 21+1))] + ['LLVM']))) -clang_lib = win_llvm.replace("LLVM-C", "libclang") + (mac_llvm + other_llvm).replace("LLVM", "clang") +clang_lib = (mac_llvm + other_llvm).replace("LLVM", "clang") -webgpu_lib = "os.path.join(sysconfig.get_paths()['purelib'], 'pydawn', 'lib', 'libwebgpu_dawn.dll') if WIN else 'webgpu_dawn'" +webgpu_lib = "'webgpu_dawn'" tinymesa_path = "os.path.join(sysconfig.get_paths()['platlib'], 'tinymesa')" nv_lib_path = ("[f'/{pre}/cuda/targets/{tgt}/lib' for pre in ['opt', 'usr/local'] for tgt in " "[sysconfig.get_config_vars().get(\"MULTIARCH\", \"\").rsplit(\"-\", 1)[0], 'sbsa-linux']]") @@ -65,11 +65,11 @@ def __getattr__(nm): args=["-D__USE_GNU", "-D_GNU_SOURCE"], dll="'c'", errno=True, recsym=True, rules=[(r'([a-z]+) = \1', '')]) # removes stdin = stdin case "avcodec": return load("avcodec", ["{}/libavcodec/hevc/hevc.h", "{}/libavcodec/cbs_h265.h"], srcs=ffmpeg_src) case "opencl": return load("opencl", ["{}/CL/cl.h"], dll="'OpenCL'", args=["-I{}"], srcs=opencl_src) - case "cuda": return load("cuda", ["{}/include/cuda.h"], dll="'nvcuda' if WIN else 'cuda'", args=["-D__CUDA_API_VERSION_INTERNAL"], srcs=cudart_src, macros=False, prolog=["from tinygrad.helpers import WIN"]) + case "cuda": return load("cuda", ["{}/include/cuda.h"], dll="'cuda'", args=["-D__CUDA_API_VERSION_INTERNAL"], srcs=cudart_src, macros=False) case "nvrtc": return load("nvrtc", ["{}/include/nvrtc.h"], dll="'nvrtc'", paths=nv_lib_path, srcs=nvrtc_src, prolog=["import sysconfig"]) case "nvjitlink": load("nvjitlink", [root/"extra/nvJitLink.h"], dll="'nvJitLink'", paths=nv_lib_path, prolog=["import sysconfig"]) case "kfd": return load("kfd", [root/"extra/hip_gpu_driver/kfd_ioctl.h"]) - case "nv_570" | "nv_580" | "nv_610": + case "nv_570" | "nv_580" | "nv_610" | "nv_615": return load(nm, [ *[root/"extra/nv_gpu_driver"/s for s in ["clc9b0.h", "clc6c0qmd.h","clcec0qmd.h", "nvdec_drv.h"]], "{}/kernel-open/common/inc/nvmisc.h", *[f"{{}}/src/common/sdk/nvidia/inc/class/cl{s}.h" for s in ["0000", "0070", "0080", "2080", "2080_notification", "c56f", "c86f", "c96f", "c761", @@ -107,12 +107,12 @@ def __getattr__(nm): return load("io_uring", ["{}/liburing.h", "{}/usr/include/linux/io_uring.h", "{}/usr/include/asm-generic/unistd.h"], args=["-I{}/usr/include"], srcs=[linux_headers_deb, liburing_src], rules=[('__NR', 'NR')], preprocess=_extract_deb) case "llvm": return load("llvm", lambda: [system("llvm-config-20 --includedir")+"/llvm-c/**/*.h"], dll=llvm_lib, - args=lambda: system("llvm-config-20 --cflags").split(), recsym=True, prolog=["from tinygrad.helpers import WIN, OSX"]) + args=lambda: system("llvm-config-20 --cflags").split(), recsym=True, prolog=["from tinygrad.helpers import OSX"]) case "pci": return load("pci", ["{}/usr/include/linux/pci_regs.h"], srcs=linux_headers_deb, preprocess=_extract_deb) case "vfio": return load("vfio", ["{}/usr/include/linux/vfio.h"], args=["-I{}/usr/include"], srcs=linux_headers_deb, preprocess=_extract_deb) # could add rule: WGPU_COMMA -> ',' case "webgpu": return load("webgpu", [root/"extra/webgpu/webgpu.h"], dll=webgpu_lib, - prolog=["from tinygrad.helpers import WIN, OSX", "import sysconfig, os"]) + prolog=["from tinygrad.helpers import OSX", "import sysconfig, os"]) case "libusb": return load("libusb", ["/usr/include/libusb-1.0/libusb.h"], dll="'usb-1.0'") case "hip": return load("hip", ["/opt/rocm/include/hip/hip_ext.h", "/opt/rocm/include/hip/hiprtc.h", "/opt/rocm/include/hip/hip_runtime_api.h", "/opt/rocm/include/hip/driver_types.h"], @@ -167,7 +167,7 @@ def __getattr__(nm): case "libclang": return load("libclang", lambda: [f"{system('llvm-config-20 --includedir')}/clang-c/{s}.h" for s in ["Index", "CXString", "CXSourceLocation", "CXFile"]], - dll=clang_lib, prolog=["from tinygrad.helpers import WIN, OSX"], args=lambda: system("llvm-config-20 --cflags").split()) + dll=clang_lib, prolog=["from tinygrad.helpers import OSX"], args=lambda: system("llvm-config-20 --cflags").split()) case "metal": return load("metal", lambda: [f"{macossdk()}/System/Library/Frameworks/Metal.framework/Headers/MTL{s}.h" for s in ["ComputeCommandEncoder", "ComputePipeline", "CommandQueue", "Device", "IndirectCommandBuffer", "Resource", "CommandEncoder"]], diff --git a/tinygrad_repo/tinygrad/runtime/autogen/amd/cdna/str_pcode.py b/tinygrad_repo/tinygrad/runtime/autogen/amd/cdna/str_pcode.py index a47a8dbeaf..b5226f5131 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/amd/cdna/str_pcode.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/amd/cdna/str_pcode.py @@ -23,6 +23,7 @@ PCODE = { DSOp.DS_CMPST_F32: 'addr = CalcDsAddr(ADDR.b32, OFFSET0.b32, OFFSET1.b32);\ntmp = MEM[addr].f32;\nsrc = DATA2.f32;\ncmp = DATA.f32;\nMEM[addr].f32 = tmp == cmp ? src : tmp;\nRETURN_DATA.f32 = tmp', DSOp.DS_MIN_F32: 'addr = CalcDsAddr(ADDR.b32, OFFSET0.b32, OFFSET1.b32);\ntmp = MEM[addr].f32;\nsrc = DATA.f32;\nMEM[addr].f32 = src < tmp ? src : tmp;\nRETURN_DATA.f32 = tmp', DSOp.DS_MAX_F32: 'addr = CalcDsAddr(ADDR.b32, OFFSET0.b32, OFFSET1.b32);\ntmp = MEM[addr].f32;\nsrc = DATA.f32;\nMEM[addr].f32 = src > tmp ? src : tmp;\nRETURN_DATA.f32 = tmp', + DSOp.DS_NOP: '', DSOp.DS_ADD_F32: 'addr = CalcDsAddr(ADDR.b32, OFFSET0.b32, OFFSET1.b32);\ntmp = MEM[addr].f32;\nMEM[addr].f32 += DATA.f32;\nRETURN_DATA.f32 = tmp', DSOp.DS_PK_ADD_F16: 'tmp = MEM[ADDR];\nsrc = DATA;\ndst[31 : 16].f16 = tmp[31 : 16].f16 + src[31 : 16].f16;\ndst[15 : 0].f16 = tmp[15 : 0].f16 + src[15 : 0].f16;\nMEM[ADDR] = dst.b32;\nRETURN_DATA = tmp', DSOp.DS_PK_ADD_BF16: 'tmp = MEM[ADDR];\nsrc = DATA;\ndst[31 : 16].bf16 = tmp[31 : 16].bf16 + src[31 : 16].bf16;\ndst[15 : 0].bf16 = tmp[15 : 0].bf16 + src[15 : 0].bf16;\nMEM[ADDR] = dst.b32;\nRETURN_DATA = tmp', @@ -58,7 +59,7 @@ PCODE = { DSOp.DS_READ_U8: "RETURN_DATA.u32 = 32'U({ 24'0U, MEM[ADDR].u8 })", DSOp.DS_READ_I16: "RETURN_DATA.i32 = 32'I(signext(MEM[ADDR].i16))", DSOp.DS_READ_U16: "RETURN_DATA.u32 = 32'U({ 16'0U, MEM[ADDR].u16 })", - DSOp.DS_SWIZZLE_B32: 'Offset[4:0]: Swizzle\n0x00: {1,11,9,19,5,15,d,1d,3,13,b,1b,7,17,f,1f,2,12,a,1a,6,16,e,1e,4,14,c,1c,8,18,10,20}\n0x10: {1,9,5,d,3,b,7,f,2,a,6,e,4,c,8,10,11,19,15,1d,13,1b,17,1f,12,1a,16,1e,14,1c,18,20}\n0x1f: No swizzle', + DSOp.DS_SWIZZLE_B32: 'offset = offset1:offset0;\nif (offset >= 0xe000) {\n// FFT decomposition\nmask = offset[4:0];\nfor (i = 0; i < 64; i++) {\nj = reverse_bits(i & 0x1f);\nj = (j >> count_ones(mask));\nj |= (i & mask);\nj |= i & 0x20;\nthread_out[i] = thread_valid[j] ? thread_in[j] : 0;\n}\n} elsif (offset >= 0xc000) {\n// rotate\nrotate = offset[9:5];\nmask = offset[4:0];\nif (offset[10]) {\nrotate = -rotate;\n}\nfor (i = 0; i < 64; i++) {\nj = (i & mask) | ((i + rotate) & ~mask);\nj |= i & 0x20;\nthread_out[i] = thread_valid[j] ? thread_in[j] : 0;\n}\n} elsif (offset[15]) {\n// full data sharing within 4 consecutive threads\nfor (i = 0; i < 64; i+=4) {\nthread_out[i+0] = thread_valid[i+offset[1:0]]?thread_in[i+offset[1:0]]:0;\nthread_out[i+1] = thread_valid[i+offset[3:2]]?thread_in[i+offset[3:2]]:0;\nthread_out[i+2] = thread_valid[i+offset[5:4]]?thread_in[i+offset[5:4]]:0;\nthread_out[i+3] = thread_valid[i+offset[7:6]]?thread_in[i+offset[7:6]]:0;\n}\n} else { // offset[15] == 0\n// limited data sharing within 32 consecutive threads\nxor_mask = offset[14:10];\nor_mask = offset[9:5];\nand_mask = offset[4:0];\nfor (i = 0; i < 64; i++) {\nj = (((i & 0x1f) & and_mask) | or_mask) ^ xor_mask;\nj |= (i & 0x20); // which group of 32\nthread_out[i] = thread_valid[j] ? thread_in[j] : 0;\n}\n}', DSOp.DS_PERMUTE_B32: "// VGPR[laneId][index] is the VGPR RAM\n// VDST, ADDR and DATA0 are from the microcode DS encoding\ndeclare tmp : 32'B[64];\ndeclare OFFSET : 16'U;\ndeclare DATA0 : 32'U;\ndeclare VDST : 32'U;\nfor i in 0 : 63 do\ntmp[i] = 0x0\nendfor;\nfor i in 0 : 63 do\n// If a source thread is disabled, it does not propagate data.\nif EXEC[i].u1 then\n// ADDR needs to be divided by 4.\n// High-order bits are ignored.\ndst_lane = (VGPR[i][ADDR].u32 + OFFSET.u32) / 4U % 64U;\ntmp[dst_lane] = VGPR[i][DATA0]\nendif\nendfor;\n// Copy data into destination VGPRs. If multiple sources\n// select the same destination thread, the highest-numbered\n// source thread wins.\nfor i in 0 : 63 do\nif EXEC[i].u1 then\nVGPR[i][VDST] = tmp[i]\nendif\nendfor", DSOp.DS_BPERMUTE_B32: "// VGPR[laneId][index] is the VGPR RAM\n// VDST, ADDR and DATA0 are from the microcode DS encoding\ndeclare tmp : 32'B[64];\ndeclare OFFSET : 16'U;\ndeclare DATA0 : 32'U;\ndeclare VDST : 32'U;\nfor i in 0 : 63 do\ntmp[i] = 0x0\nendfor;\nfor i in 0 : 63 do\n// ADDR needs to be divided by 4.\n// High-order bits are ignored.\nsrc_lane = (VGPR[i][ADDR].u32 + OFFSET.u32) / 4U % 64U;\n// EXEC is applied to the source VGPR reads.\nif EXEC[src_lane].u1 then\ntmp[i] = VGPR[src_lane][DATA0]\nendif\nendfor;\n// Copy data into destination VGPRs. Some source\n// data may be broadcast to multiple lanes.\nfor i in 0 : 63 do\nif EXEC[i].u1 then\nVGPR[i][VDST] = tmp[i]\nendif\nendfor", DSOp.DS_ADD_U64: 'addr = CalcDsAddr(ADDR.b32, OFFSET0.b32, OFFSET1.b32);\ntmp = MEM[addr].u64;\nMEM[addr].u64 += DATA.u64;\nRETURN_DATA.u64 = tmp', @@ -286,6 +287,8 @@ PCODE = { MUBUFOp.BUFFER_LOAD_SHORT_D16_HI: 'addr = CalcBufferAddr(VADDR.b32, SRSRC.b32, SOFFSET.b32, OFFSET.b32);\nVDATA[31 : 16].b16 = MEM[addr].b16;\n// VDATA[15:0] is preserved.', MUBUFOp.BUFFER_LOAD_FORMAT_D16_HI_X: "addr = CalcBufferAddr(VADDR.b32, SRSRC.b32, SOFFSET.b32, OFFSET.b32);\nVDATA[31 : 16].b16 = 16'B(ConvertFromFormat(MEM[addr + ChannelOffsetX()]));\n// Mem access size depends on format\n// VDATA[15:0].b16 is preserved.", MUBUFOp.BUFFER_STORE_FORMAT_D16_HI_X: "addr = CalcBufferAddr(VADDR.b32, SRSRC.b32, SOFFSET.b32, OFFSET.b32);\nMEM[addr + ChannelOffsetX()] = ConvertToFormat(32'B(VDATA[31 : 16].b16));\n// Mem access size depends on format", + MUBUFOp.BUFFER_WBL2: '', + MUBUFOp.BUFFER_INV: '', MUBUFOp.BUFFER_ATOMIC_SWAP: 'addr = CalcBufferAddr(VADDR.b32, SRSRC.b32, SOFFSET.b32, OFFSET.b32);\ntmp = MEM[addr].b32;\nMEM[addr].b32 = DATA.b32;\nRETURN_DATA.b32 = tmp', MUBUFOp.BUFFER_ATOMIC_CMPSWAP: 'addr = CalcBufferAddr(VADDR.b32, SRSRC.b32, SOFFSET.b32, OFFSET.b32);\ntmp = MEM[addr].u32;\nsrc = DATA[31 : 0].u32;\ncmp = DATA[63 : 32].u32;\nMEM[addr].u32 = tmp == cmp ? src : tmp;\nRETURN_DATA.u32 = tmp', MUBUFOp.BUFFER_ATOMIC_ADD: 'addr = CalcBufferAddr(VADDR.b32, SRSRC.b32, SOFFSET.b32, OFFSET.b32);\ntmp = MEM[addr].u32;\nMEM[addr].u32 += DATA.u32;\nRETURN_DATA.u32 = tmp', @@ -362,6 +365,10 @@ PCODE = { SMEMOp.S_BUFFER_STORE_DWORD: 'addr = CalcScalarBufferAddr(SBASE.b32, SOFFSET.b32, OFFSET.i32);\nMEM[addr].b32 = SDATA[31 : 0]', SMEMOp.S_BUFFER_STORE_DWORDX2: 'addr = CalcScalarBufferAddr(SBASE.b32, SOFFSET.b32, OFFSET.i32);\nMEM[addr].b32 = SDATA[31 : 0];\nMEM[addr + 4U].b32 = SDATA[63 : 32]', SMEMOp.S_BUFFER_STORE_DWORDX4: 'addr = CalcScalarBufferAddr(SBASE.b32, SOFFSET.b32, OFFSET.i32);\nMEM[addr].b32 = SDATA[31 : 0];\nMEM[addr + 4U].b32 = SDATA[63 : 32];\nMEM[addr + 8U].b32 = SDATA[95 : 64];\nMEM[addr + 12U].b32 = SDATA[127 : 96]', + SMEMOp.S_DCACHE_INV: '', + SMEMOp.S_DCACHE_WB: '', + SMEMOp.S_DCACHE_INV_VOL: '', + SMEMOp.S_DCACHE_WB_VOL: '', SMEMOp.S_BUFFER_ATOMIC_SWAP: 'addr = CalcScalarBufferAddr(SBASE.b32, SOFFSET.b32, OFFSET.i32);\ntmp = MEM[addr].b32;\nMEM[addr].b32 = DATA.b32;\nRETURN_DATA.b32 = tmp', SMEMOp.S_BUFFER_ATOMIC_CMPSWAP: 'addr = CalcScalarBufferAddr(SBASE.b32, SOFFSET.b32, OFFSET.i32);\ntmp = MEM[addr].u32;\nsrc = DATA[31 : 0].u32;\ncmp = DATA[63 : 32].u32;\nMEM[addr].u32 = tmp == cmp ? src : tmp;\nRETURN_DATA.u32 = tmp', SMEMOp.S_BUFFER_ATOMIC_ADD: 'addr = CalcScalarBufferAddr(SBASE.b32, SOFFSET.b32, OFFSET.i32);\ntmp = MEM[addr].u32;\nMEM[addr].u32 += DATA.u32;\nRETURN_DATA.u32 = tmp', @@ -562,22 +569,29 @@ PCODE = { SOPKOp.S_SETREG_IMM32_B32: "hwRegId = SIMM16.u16[5 : 0];\noffset = SIMM16.u16[10 : 6];\nsize = SIMM16.u16[15 : 11].u32 + 1U;\n// logical size is in range 1:32\nmask = (1 << size) - 1;\nmask = (mask << offset.u32);\nmask = (mask & HwRegWriteMask(hwRegId, WAVE_STATUS.PRIV));\n// Mask of bits that can be modified\nvalue = ((SIMM32.u32 << offset.u32) & mask.u32);\nvalue = (value | 32'U(HW_REGISTERS[hwRegId].i32 & ~mask));\nHW_REGISTERS[hwRegId] = value.b32;\n// Side-effects may trigger here if certain bits are modified", SOPKOp.S_CALL_B64: "D0.i64 = PC + 4LL;\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL", SOPPOp.S_NOP: 'for i in 0U : SIMM16.u16[3 : 0].u32 do\nnop()\nendfor', + SOPPOp.S_ENDPGM: '', SOPPOp.S_BRANCH: "PC = PC + signext(SIMM16.i16 * 16'4) + 4LL;\n// short jump.", + SOPPOp.S_WAKEUP: '', SOPPOp.S_CBRANCH_SCC0: "if SCC == 1'0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_SCC1: "if SCC == 1'1U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_VCCZ: "if VCCZ.u1 == 1'1U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_VCCNZ: "if VCCZ.u1 == 1'0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_EXECZ: "if EXECZ.u1 == 1'1U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_EXECNZ: "if EXECZ.u1 == 1'0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", + SOPPOp.S_BARRIER: '', SOPPOp.S_WAITCNT: 'SIMM16[3:0] = vmcount (vector memory operations) lower bits [3:0],', SOPPOp.S_SLEEP: 's_sleep 0 // Wait for 0 clocks.\ns_sleep 1 // Wait for 1-64 clocks.\ns_sleep 2 // Wait for 65-128 clocks.', SOPPOp.S_TRAP: 'TrapID = SIMM16.u16[7 : 0];\n"Wait for all instructions to complete";\n// PC passed into trap handler points to S_TRAP itself,\n// *not* to the next instruction.\n{ TTMP[1], TTMP[0] } = { 3\'0, PCRewind[3 : 0], HT[0], TrapID[7 : 0], PC[47 : 0] };\nPC = TBA.i64;\n// trap base address\nWAVE_STATUS.PRIV = 1\'1U', + SOPPOp.S_ICACHE_INV: '', SOPPOp.S_CBRANCH_CDBGSYS: "if WAVE_STATUS.COND_DBG_SYS.u32 != 0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_CDBGUSER: "if WAVE_STATUS.COND_DBG_USER.u32 != 0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_CDBGSYS_OR_USER: "if (WAVE_STATUS.COND_DBG_SYS || WAVE_STATUS.COND_DBG_USER) then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_CDBGSYS_AND_USER: "if (WAVE_STATUS.COND_DBG_SYS && WAVE_STATUS.COND_DBG_USER) then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", + SOPPOp.S_ENDPGM_SAVED: '', SOPPOp.S_SET_GPR_IDX_OFF: "WAVE_MODE.GPR_IDX_EN = 1'0U", SOPPOp.S_SET_GPR_IDX_MODE: 'M0[15 : 12] = SIMM16.u16[3 : 0].b4', + SOPPOp.S_ENDPGM_ORDERED_PS_DONE: '', + VOP1Op.V_NOP_E32: '', VOP1Op.V_MOV_B32_E32: 'D0.b32 = S0.b32', VOP1Op.V_READFIRSTLANE_B32_E32: "declare lane : 32'I;\nif EXEC == 0x0LL then\nlane = 0;\n// Force lane 0 if all lanes are disabled\nelse\nlane = s_ff1_i32_b64(EXEC);\n// Lowest active lane\nendif;\nD0.b32 = VGPR[lane][SRC0.u32]", VOP1Op.V_CVT_I32_F64_E32: 'D0.i32 = f64_to_i32(S0.f64)', @@ -629,6 +643,7 @@ PCODE = { VOP1Op.V_FRACT_F64_E32: 'D0.f64 = S0.f64 + -floor(S0.f64)', VOP1Op.V_FREXP_EXP_I32_F32_E32: "if ((64'F(S0.f32) == +INF) || (64'F(S0.f32) == -INF) || isNAN(64'F(S0.f32))) then\nD0.i32 = 0\nelse\nD0.i32 = exponent(S0.f32) - 127 + 1\nendif", VOP1Op.V_FREXP_MANT_F32_E32: "if ((64'F(S0.f32) == +INF) || (64'F(S0.f32) == -INF) || isNAN(64'F(S0.f32))) then\nD0.f32 = S0.f32\nelse\nD0.f32 = mantissa(S0.f32)\nendif", + VOP1Op.V_CLREXCP_E32: '', VOP1Op.V_MOV_B64_E32: 'D0.b64 = S0.b64', VOP1Op.V_CVT_F16_U16_E32: 'D0.f16 = u16_to_f16(S0.u16)', VOP1Op.V_CVT_F16_I16_E32: 'D0.f16 = i16_to_f16(S0.i16)', @@ -972,6 +987,8 @@ PCODE = { VOP3Op.V_FMAC_F32_E64: 'D0.f32 = fma(S0.f32, S1.f32, D0.f32)', VOP3Op.V_PK_FMAC_F16_E64: 'D0[15 : 0].f16 = fma(S0[15 : 0].f16, S1[15 : 0].f16, D0[15 : 0].f16);\nD0[31 : 16].f16 = fma(S0[31 : 16].f16, S1[31 : 16].f16, D0[31 : 16].f16)', VOP3Op.V_XNOR_B32_E64: 'D0.u32 = ~(S0.u32 ^ S1.u32)', + VOP3Op.V_NOP_E64: '', + VOP3Op.V_CLREXCP_E64: '', VOP3Op.V_MAD_I32_I24_E64: "D0.i32 = 32'I(S0.i24) * 32'I(S1.i24) + S2.i32", VOP3Op.V_MAD_U32_U24_E64: "D0.u32 = 32'U(S0.u24) * 32'U(S1.u24) + S2.u32", VOP3Op.V_CUBEID_F32_E64: '// Set D0.f = cubemap face ID ({0.0, 1.0, ..., 5.0}).\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nif S2.f32 < 0.0F then\nD0.f32 = 5.0F\nelse\nD0.f32 = 4.0F\nendif\nelsif abs(S1.f32) >= abs(S0.f32) then\nif S1.f32 < 0.0F then\nD0.f32 = 3.0F\nelse\nD0.f32 = 2.0F\nendif\nelse\nif S0.f32 < 0.0F then\nD0.f32 = 1.0F\nelse\nD0.f32 = 0.0F\nendif\nendif', diff --git a/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna3/str_pcode.py b/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna3/str_pcode.py index e60d89e2a2..e2a2478a87 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna3/str_pcode.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna3/str_pcode.py @@ -23,7 +23,11 @@ PCODE = { DSOp.DS_CMPSTORE_F32: 'tmp = MEM[ADDR].f32;\nsrc = DATA.f32;\ncmp = DATA2.f32;\nMEM[ADDR].f32 = tmp == cmp ? src : tmp;\nRETURN_DATA.f32 = tmp', DSOp.DS_MIN_F32: 'tmp = MEM[ADDR].f32;\nsrc = DATA.f32;\nMEM[ADDR].f32 = src < tmp ? src : tmp;\nRETURN_DATA.f32 = tmp', DSOp.DS_MAX_F32: 'tmp = MEM[ADDR].f32;\nsrc = DATA.f32;\nMEM[ADDR].f32 = src > tmp ? src : tmp;\nRETURN_DATA.f32 = tmp', + DSOp.DS_NOP: '', DSOp.DS_ADD_F32: 'tmp = MEM[ADDR].f32;\nMEM[ADDR].f32 += DATA.f32;\nRETURN_DATA.f32 = tmp', + DSOp.DS_GWS_SEMA_RELEASE_ALL: '', + DSOp.DS_GWS_SEMA_V: '', + DSOp.DS_GWS_SEMA_P: '', DSOp.DS_STORE_B8: 'MEM[ADDR].b8 = DATA[7 : 0]', DSOp.DS_STORE_B16: 'MEM[ADDR].b16 = DATA[15 : 0]', DSOp.DS_ADD_RTN_U32: 'tmp = MEM[ADDR].u32;\nMEM[ADDR].u32 += DATA.u32;\nRETURN_DATA.u32 = tmp', @@ -301,6 +305,8 @@ PCODE = { MUBUFOp.BUFFER_STORE_D16_HI_B16: 'MEM[ADDR].b16 = VDATA[31 : 16]', MUBUFOp.BUFFER_LOAD_D16_HI_FORMAT_X: "VDATA[31 : 16].b16 = 16'B(ConvertFromFormat(MEM[TADDR.X]));\n// Mem access size depends on format\n// VDATA[15:0].b16 is preserved.", MUBUFOp.BUFFER_STORE_D16_HI_FORMAT_X: "MEM[TADDR.X] = ConvertToFormat(32'B(VDATA[31 : 16].b16));\n// Mem access size depends on format", + MUBUFOp.BUFFER_GL0_INV: '', + MUBUFOp.BUFFER_GL1_INV: '', MUBUFOp.BUFFER_ATOMIC_SWAP_B32: 'tmp = MEM[ADDR].b32;\nMEM[ADDR].b32 = DATA.b32;\nRETURN_DATA.b32 = tmp', MUBUFOp.BUFFER_ATOMIC_CMPSWAP_B32: 'tmp = MEM[ADDR].u32;\nsrc = DATA[31 : 0].u32;\ncmp = DATA[63 : 32].u32;\nMEM[ADDR].u32 = tmp == cmp ? src : tmp;\nRETURN_DATA.u32 = tmp', MUBUFOp.BUFFER_ATOMIC_ADD_U32: 'tmp = MEM[ADDR].u32;\nMEM[ADDR].u32 += DATA.u32;\nRETURN_DATA.u32 = tmp', @@ -364,6 +370,8 @@ PCODE = { SMEMOp.S_BUFFER_LOAD_B128: 'SDATA[31 : 0] = MEM[ADDR].b32;\nSDATA[63 : 32] = MEM[ADDR + 4U].b32;\nSDATA[95 : 64] = MEM[ADDR + 8U].b32;\nSDATA[127 : 96] = MEM[ADDR + 12U].b32', SMEMOp.S_BUFFER_LOAD_B256: 'SDATA[31 : 0] = MEM[ADDR].b32;\nSDATA[63 : 32] = MEM[ADDR + 4U].b32;\nSDATA[95 : 64] = MEM[ADDR + 8U].b32;\nSDATA[127 : 96] = MEM[ADDR + 12U].b32;\nSDATA[159 : 128] = MEM[ADDR + 16U].b32;\nSDATA[191 : 160] = MEM[ADDR + 20U].b32;\nSDATA[223 : 192] = MEM[ADDR + 24U].b32;\nSDATA[255 : 224] = MEM[ADDR + 28U].b32', SMEMOp.S_BUFFER_LOAD_B512: 'SDATA[31 : 0] = MEM[ADDR].b32;\nSDATA[63 : 32] = MEM[ADDR + 4U].b32;\nSDATA[95 : 64] = MEM[ADDR + 8U].b32;\nSDATA[127 : 96] = MEM[ADDR + 12U].b32;\nSDATA[159 : 128] = MEM[ADDR + 16U].b32;\nSDATA[191 : 160] = MEM[ADDR + 20U].b32;\nSDATA[223 : 192] = MEM[ADDR + 24U].b32;\nSDATA[255 : 224] = MEM[ADDR + 28U].b32;\nSDATA[287 : 256] = MEM[ADDR + 32U].b32;\nSDATA[319 : 288] = MEM[ADDR + 36U].b32;\nSDATA[351 : 320] = MEM[ADDR + 40U].b32;\nSDATA[383 : 352] = MEM[ADDR + 44U].b32;\nSDATA[415 : 384] = MEM[ADDR + 48U].b32;\nSDATA[447 : 416] = MEM[ADDR + 52U].b32;\nSDATA[479 : 448] = MEM[ADDR + 56U].b32;\nSDATA[511 : 480] = MEM[ADDR + 60U].b32', + SMEMOp.S_GL1_INV: '', + SMEMOp.S_DCACHE_INV: '', SOP1Op.S_MOV_B32: 'D0.b32 = S0.b32', SOP1Op.S_MOV_B64: 'D0.b64 = S0.b64', SOP1Op.S_CMOV_B32: 'if SCC then\nD0.b32 = S0.b32\nendif', @@ -584,6 +592,7 @@ PCODE = { SOPPOp.S_SLEEP: 's_sleep 0 // Wait for 0 clocks.\ns_sleep 1 // Wait for 1-64 clocks.\ns_sleep 2 // Wait for 65-128 clocks.', SOPPOp.S_DELAY_ALU: 'v_mov_b32 v3, v0\nv_lshlrev_b32 v30, 1, v31\nv_lshlrev_b32 v24, 1, v25\ns_delay_alu instid0(INSTID_VALU_DEP_3) | instskip(INSTSKIP_SKIP_1) | instid1(INSTID_VALU_DEP_1)\n// 1 cycle delay here\nv_add_f32 v0, v1, v3\nv_sub_f32 v11, v9, v9\n// 2 cycles delay here\nv_mul_f32 v10, v13, v11', SOPPOp.S_WAITCNT: 'expcnt <= WaitEXPCNT\nlgkmcnt <= WaitLGKMCNT\nvmcnt <= WaitVMCNT', + SOPPOp.S_WAIT_IDLE: '', SOPPOp.S_TRAP: 'TrapID = SIMM16.u16[7 : 0];\n"Wait for all instructions to complete";\n// PC passed into trap handler points to S_TRAP itself,\n// *not* to the next instruction.\n{ TTMP[1], TTMP[0] } = { 7\'0, HT[0], TrapID[7 : 0], PC[47 : 0] };\nPC = TBA.i64;\n// trap base address\nWAVE_STATUS.PRIV = 1\'1U', SOPPOp.S_CODE_END: '...\ns_endpgm // last real instruction in shader buffer\ns_code_end // 1\ns_code_end // 2\ns_code_end // 3\ns_code_end // 4\ns_code_end // done!', SOPPOp.S_BRANCH: "PC = PC + signext(SIMM16.i16 * 16'4) + 4LL;\n// short jump.", @@ -597,13 +606,20 @@ PCODE = { SOPPOp.S_CBRANCH_CDBGUSER: "if WAVE_STATUS.COND_DBG_USER.u32 != 0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_CDBGSYS_OR_USER: "if (WAVE_STATUS.COND_DBG_SYS || WAVE_STATUS.COND_DBG_USER) then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_CDBGSYS_AND_USER: "if (WAVE_STATUS.COND_DBG_SYS && WAVE_STATUS.COND_DBG_USER) then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", + SOPPOp.S_ENDPGM: '', + SOPPOp.S_ENDPGM_SAVED: '', + SOPPOp.S_ENDPGM_ORDERED_PS_DONE: '', + SOPPOp.S_WAKEUP: '', SOPPOp.S_SETPRIO: 'Priority = {SysUserPrio[1:0], WaveAge[3:0]}\nSysUserPrio = MIN(3, SysPrio[1:0] + UserPrio[1:0]).', + SOPPOp.S_ICACHE_INV: '', + SOPPOp.S_BARRIER: '', VINTERPOp.V_INTERP_P10_F32: 'D0.f32 = fma(VGPR[(laneId.u32 & 0xfffffffcU) + 1U][SRC0.u32].f32, S1.f32, VGPR[laneId.u32 &\n0xfffffffcU][SRC2.u32].f32)', VINTERPOp.V_INTERP_P2_F32: 'D0.f32 = fma(VGPR[(laneId.u32 & 0xfffffffcU) + 2U][SRC0.u32].f32, S1.f32, S2.f32)', VINTERPOp.V_INTERP_P10_F16_F32: "D0.f32 = fma(32'F(VGPR[(laneId.u32 & 0xfffffffcU) + 1U][SRC0.u32].f16), S1.f32, 32'F(VGPR[laneId.u32 &\n0xfffffffcU][SRC2.u32].f16))", VINTERPOp.V_INTERP_P2_F16_F32: "D0.f16 = 16'F(fma(32'F(VGPR[(laneId.u32 & 0xfffffffcU) + 2U][SRC0.u32].f16), S1.f32, S2.f32))", VINTERPOp.V_INTERP_P10_RTZ_F16_F32: "D0.f32 = fma(32'F(VGPR[(laneId.u32 & 0xfffffffcU) + 1U][SRC0.u32].f16), S1.f32, 32'F(VGPR[laneId.u32 &\n0xfffffffcU][SRC2.u32].f16))", VINTERPOp.V_INTERP_P2_RTZ_F16_F32: "D0.f32 = fma(32'F(VGPR[(laneId.u32 & 0xfffffffcU) + 2U][SRC0.u32].f16), S1.f32, S2.f32)", + VOP1Op.V_NOP_E32: '', VOP1Op.V_MOV_B32_E32: 'D0.b32 = S0.b32', VOP1Op.V_READFIRSTLANE_B32_E32: "declare lane : 32'U;\nif WAVE64 then\n// 64 lanes\nif EXEC == 0x0LL then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b64(EXEC));\n// Lowest active lane\nendif\nelse\n// 32 lanes\nif EXEC_LO.i32 == 0 then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b32(EXEC_LO));\n// Lowest active lane\nendif\nendif;\nD0.b32 = VGPR[lane][SRC0.u32]", VOP1Op.V_CVT_I32_F64_E32: 'D0.i32 = f64_to_i32(S0.f64)', @@ -629,6 +645,7 @@ PCODE = { VOP1Op.V_CEIL_F64_E32: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 > 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += 1.0\nendif', VOP1Op.V_RNDNE_F64_E32: 'D0.f64 = floor(S0.f64 + 0.5);\nif (isEven(floor(S0.f64)) && (fract(S0.f64) == 0.5)) then\nD0.f64 -= 1.0\nendif', VOP1Op.V_FLOOR_F64_E32: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 < 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += -1.0\nendif', + VOP1Op.V_PIPEFLUSH_E32: '', VOP1Op.V_MOV_B16_E32: 'D0.b16 = S0.b16', VOP1Op.V_FRACT_F32_E32: 'D0.f32 = S0.f32 + -floor(S0.f32)', VOP1Op.V_TRUNC_F32_E32: 'D0.f32 = trunc(S0.f32)', @@ -961,6 +978,7 @@ PCODE = { VOP3Op.V_MAX_F16_E64: "GT_NEG_ZERO = lambda(a, b) (\n((a > b) || ((64'F(abs(a)) == 0.0) && (64'F(abs(b)) == 0.0) && !sign(a) && sign(b))));\n// Version of comparison where +0.0 > -0.0, differs from IEEE\nif WAVE_MODE.IEEE then\nif isSignalNAN(64'F(S0.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S0.f16)))\nelsif isSignalNAN(64'F(S1.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif isQuietNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif isQuietNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif GT_NEG_ZERO(S0.f16, S1.f16) then\n// NOTE: +0>-0 is TRUE in this comparison\nD0.f16 = S0.f16\nelse\nD0.f16 = S1.f16\nendif\nelse\nif isNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif isNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif GT_NEG_ZERO(S0.f16, S1.f16) then\n// NOTE: +0>-0 is TRUE in this comparison\nD0.f16 = S0.f16\nelse\nD0.f16 = S1.f16\nendif\nendif;\n// Inequalities in the above pseudocode behave differently from IEEE\n// when both inputs are +-0.", VOP3Op.V_MIN_F16_E64: "LT_NEG_ZERO = lambda(a, b) (\n((a < b) || ((64'F(abs(a)) == 0.0) && (64'F(abs(b)) == 0.0) && sign(a) && !sign(b))));\n// Version of comparison where -0.0 < +0.0, differs from IEEE\nif WAVE_MODE.IEEE then\nif isSignalNAN(64'F(S0.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S0.f16)))\nelsif isSignalNAN(64'F(S1.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif isQuietNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif isQuietNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif LT_NEG_ZERO(S0.f16, S1.f16) then\n// NOTE: -0<+0 is TRUE in this comparison\nD0.f16 = S0.f16\nelse\nD0.f16 = S1.f16\nendif\nelse\nif isNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif isNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif LT_NEG_ZERO(S0.f16, S1.f16) then\n// NOTE: -0<+0 is TRUE in this comparison\nD0.f16 = S0.f16\nelse\nD0.f16 = S1.f16\nendif\nendif;\n// Inequalities in the above pseudocode behave differently from IEEE\n// when both inputs are +-0.", VOP3Op.V_LDEXP_F16_E64: "D0.f16 = S0.f16 * 16'F(2.0F ** 32'I(S1.i16))", + VOP3Op.V_NOP_E64: '', VOP3Op.V_MOV_B32_E64: 'D0.b32 = S0.b32', VOP3Op.V_READFIRSTLANE_B32_E64: "declare lane : 32'U;\nif WAVE64 then\n// 64 lanes\nif EXEC == 0x0LL then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b64(EXEC));\n// Lowest active lane\nendif\nelse\n// 32 lanes\nif EXEC_LO.i32 == 0 then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b32(EXEC_LO));\n// Lowest active lane\nendif\nendif;\nD0.b32 = VGPR[lane][SRC0.u32]", VOP3Op.V_CVT_I32_F64_E64: 'D0.i32 = f64_to_i32(S0.f64)', @@ -986,6 +1004,7 @@ PCODE = { VOP3Op.V_CEIL_F64_E64: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 > 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += 1.0\nendif', VOP3Op.V_RNDNE_F64_E64: 'D0.f64 = floor(S0.f64 + 0.5);\nif (isEven(floor(S0.f64)) && (fract(S0.f64) == 0.5)) then\nD0.f64 -= 1.0\nendif', VOP3Op.V_FLOOR_F64_E64: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 < 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += -1.0\nendif', + VOP3Op.V_PIPEFLUSH_E64: '', VOP3Op.V_MOV_B16_E64: 'D0.b16 = S0.b16', VOP3Op.V_FRACT_F32_E64: 'D0.f32 = S0.f32 + -floor(S0.f32)', VOP3Op.V_TRUNC_F32_E64: 'D0.f32 = trunc(S0.f32)', diff --git a/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna4/str_pcode.py b/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna4/str_pcode.py index 2cf5ddc7f4..9abb3ebd02 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna4/str_pcode.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/amd/rdna4/str_pcode.py @@ -22,6 +22,7 @@ PCODE = { DSOp.DS_CMPSTORE_B32: 'addr = CalcDsAddr(vgpr_a.b32, offset.b32);\ntmp = MEM[addr].b32;\nsrc = DATA.b32;\ncmp = DATA2.b32;\nMEM[addr].b32 = tmp == cmp ? src : tmp;\nRETURN_DATA.b32 = tmp', DSOp.DS_MIN_NUM_F32: "tmp = MEM[ADDR].f32;\nsrc = DATA.f32;\nif (isNAN(64'F(src.f32)) && isNAN(64'F(tmp.f32))) then\nMEM[ADDR].f32 = 32'F(cvtToQuietNAN(64'F(src.f32)))\nelsif isNAN(64'F(src.f32)) then\nMEM[ADDR].f32 = tmp.f32\nelsif isNAN(64'F(tmp.f32)) then\nMEM[ADDR].f32 = src.f32\nelsif ((src.f32 < tmp.f32) || ((abs(src.f32) == 0.0F) && (abs(tmp.f32) == 0.0F) && sign(src.f32) &&\n!sign(tmp.f32))) then\n// NOTE: -0<+0 is TRUE in this comparison\nMEM[ADDR].f32 = src.f32\nelse\nMEM[ADDR].f32 = tmp.f32\nendif;\nRETURN_DATA.f32 = tmp", DSOp.DS_MAX_NUM_F32: "tmp = MEM[ADDR].f32;\nsrc = DATA.f32;\nif (isNAN(64'F(src.f32)) && isNAN(64'F(tmp.f32))) then\nMEM[ADDR].f32 = 32'F(cvtToQuietNAN(64'F(src.f32)))\nelsif isNAN(64'F(src.f32)) then\nMEM[ADDR].f32 = tmp.f32\nelsif isNAN(64'F(tmp.f32)) then\nMEM[ADDR].f32 = src.f32\nelsif ((src.f32 > tmp.f32) || ((abs(src.f32) == 0.0F) && (abs(tmp.f32) == 0.0F) && !sign(src.f32) &&\nsign(tmp.f32))) then\n// NOTE: +0>-0 is TRUE in this comparison\nMEM[ADDR].f32 = src.f32\nelse\nMEM[ADDR].f32 = tmp.f32\nendif;\nRETURN_DATA.f32 = tmp", + DSOp.DS_NOP: '', DSOp.DS_ADD_F32: 'addr = CalcDsAddr(vgpr_a.b32, offset.b32);\ntmp = MEM[addr].f32;\nMEM[addr].f32 += DATA.f32;\nRETURN_DATA.f32 = tmp', DSOp.DS_STORE_B8: 'MEM[ADDR].b8 = DATA[7 : 0]', DSOp.DS_STORE_B16: 'MEM[ADDR].b16 = DATA[15 : 0]', @@ -145,6 +146,7 @@ PCODE = { SMEMOp.S_BUFFER_LOAD_U8: "SDATA.u32 = 32'U({ 24'0U, MEM[ADDR].u8 })", SMEMOp.S_BUFFER_LOAD_I16: "SDATA.i32 = 32'I(signext(MEM[ADDR].i16))", SMEMOp.S_BUFFER_LOAD_U16: "SDATA.u32 = 32'U({ 16'0U, MEM[ADDR].u16 })", + SMEMOp.S_DCACHE_INV: '', SMEMOp.S_PREFETCH_INST: "if MODE.SCALAR_PREFETCH_EN.u1 then\nmem_addr = (64'U(S0[63 : 0].i64 + 64'I(IOFFSET.i24)) & 0xffffffffffffff80ULL);\n// Force 128B alignment\nlength = S2.u32;\n// SGPR or M0\nlength += SDATA.u32;\n// SDATA is an immediate\nlength = (length & 31U);\n// Length restricted to 0..31\nlength = (length + 1U) * 128U;\n// Prefetch 1-32 cachelines, units of 128B\nPrefetchScalarInst(mem_addr, length)\nendif", SMEMOp.S_PREFETCH_INST_PC_REL: "if MODE.SCALAR_PREFETCH_EN.u1 then\nmem_addr = (64'U(PC[63 : 0].i64 + 8LL + 64'I(IOFFSET.i24)) & 0xffffffffffffff80ULL);\n// Force 128B alignment\nlength = S1.u32;\n// SGPR or M0\nlength += SDATA.u32;\n// SDATA is an immediate\nlength = (length & 31U);\n// Length restricted to 0..31\nlength = (length + 1U) * 128U;\n// Prefetch 1-32 cachelines, units of 128B\nPrefetchScalarInst(mem_addr, length)\nendif", SMEMOp.S_PREFETCH_DATA: "if MODE.SCALAR_PREFETCH_EN.u1 then\nmem_addr = (64'U(S0[63 : 0].i64 + 64'I(IOFFSET.i24)) & 0xffffffffffffff80ULL);\n// Force 128B alignment\nlength = S2.u32;\n// SGPR or M0\nlength += SDATA.u32;\n// SDATA is an immediate\nlength = (length & 31U);\n// Length restricted to 0..31\nlength = (length + 1U) * 128U;\n// Prefetch 1-32 cachelines, units of 128B\nPrefetchScalarData(mem_addr, length)\nendif", @@ -364,6 +366,7 @@ PCODE = { SOPPOp.S_NOP: 'for i in 0U : SIMM16.u16[3 : 0].u32 do\nnop()\nendfor', SOPPOp.S_SLEEP: 's_sleep { duration: 0 } // Wait for 0 clocks.\ns_sleep { duration: 1 } // Wait for 1-64 clocks.\ns_sleep { duration: 2 } // Wait for 65-128 clocks.\ns_sleep { sleep_forever: 1 } // Wait until an event occurs.', SOPPOp.S_DELAY_ALU: 'v_mov_b32 v3, v0\nv_lshlrev_b32 v30, 1, v31\nv_lshlrev_b32 v24, 1, v25\ns_delay_alu { instid0: INSTID_VALU_DEP_3, instskip: INSTSKIP_SKIP_1, instid1: INSTID_VALU_DEP_1 }\n// 1 cycle delay here\nv_add_f32 v0, v1, v3\nv_sub_f32 v11, v9, v9\n// 2 cycles delay here\nv_mul_f32 v10, v13, v11', + SOPPOp.S_WAIT_IDLE: '', SOPPOp.S_TRAP: 'TrapID = SIMM16.u16[3 : 0].u8;\n"Wait for all instructions to complete";\n// PC passed into trap handler points to S_TRAP itself,\n// *not* to the next instruction.\n{ TTMP[1], TTMP[0] } = { TrapID[3 : 0], 12\'0, PC[47 : 0] };\nPC = TBA.i64;\n// trap base address\nWAVE_STATUS.PRIV = 1\'1U', SOPPOp.S_BARRIER_WAIT: ";\n// barrierBit 0: reserved\n// barrierBit 1: workgroup\n// barrierBit 2: trap\nbarrierBit = SIMM16.i32 >= 0 ? 0 : SIMM16.i32 == -1 ? 1 : 2;\nwhile !WAVE_BARRIER_COMPLETE[barrierBit] do\n// Implemented as a power-saving idle\ns_nop(16'0U)\nendwhile;\nWAVE_BARRIER_COMPLETE[barrierBit] = 1'0U", SOPPOp.S_CODE_END: '...\ns_endpgm // last real instruction in shader buffer\ns_code_end // 1\ns_code_end // 2\ns_code_end // 3\ns_code_end // 4\ns_code_end // done!', @@ -374,7 +377,11 @@ PCODE = { SOPPOp.S_CBRANCH_VCCNZ: "if VCCZ.u1 == 1'0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_EXECZ: "if EXECZ.u1 == 1'1U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", SOPPOp.S_CBRANCH_EXECNZ: "if EXECZ.u1 == 1'0U then\nPC = PC + signext(SIMM16.i16 * 16'4) + 4LL\nelse\nPC = PC + 4LL\nendif", + SOPPOp.S_ENDPGM: '', + SOPPOp.S_ENDPGM_SAVED: '', + SOPPOp.S_WAKEUP: '', SOPPOp.S_SETPRIO: 'SysUserPrio = MIN(3, SysPrio[1:0] + UserPrio[1:0]). Priority = {SysUserPrio[1:0], WaveAge[3:0]}', + SOPPOp.S_ICACHE_INV: '', VBUFFEROp.BUFFER_LOAD_FORMAT_X: 'addr = CalcBufferAddr(vgpr_a.b64, sgpr_o.b64);\nVDATA[31 : 0].b32 = ConvertFromFormat(MEM[addr + ChannelOffsetX()]);\n// Mem access size depends on format', VBUFFEROp.BUFFER_LOAD_FORMAT_XY: 'addr = CalcBufferAddr(vgpr_a.b64, sgpr_o.b64);\nVDATA[31 : 0].b32 = ConvertFromFormat(MEM[addr + ChannelOffsetX()]);\n// Mem access size depends on format\nVDATA[63 : 32].b32 = ConvertFromFormat(MEM[addr + ChannelOffsetY()])', VBUFFEROp.BUFFER_LOAD_FORMAT_XYZ: 'addr = CalcBufferAddr(vgpr_a.b64, sgpr_o.b64);\nVDATA[31 : 0].b32 = ConvertFromFormat(MEM[addr + ChannelOffsetX()]);\n// Mem access size depends on format\nVDATA[63 : 32].b32 = ConvertFromFormat(MEM[addr + ChannelOffsetY()]);\nVDATA[95 : 64].b32 = ConvertFromFormat(MEM[addr + ChannelOffsetZ()])', @@ -545,6 +552,8 @@ PCODE = { VGLOBALOp.GLOBAL_STORE_D16_HI_B16: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\nMEM[addr].b16 = VDATA[31 : 16]', VGLOBALOp.GLOBAL_LOAD_ADDTID_B32: "RETURN_DATA.u32 = MEM[SGPR_ADDR[63 : 0] + INST_OFFSET[11 : 0].b64 + 64'B(laneID.i32 * 4)].u32", VGLOBALOp.GLOBAL_STORE_ADDTID_B32: "MEM[SGPR_ADDR[63 : 0] + INST_OFFSET[11 : 0].b64 + 64'B(laneID.i32 * 4)].u32 = DATA.u32", + VGLOBALOp.GLOBAL_INV: '', + VGLOBALOp.GLOBAL_WB: '', VGLOBALOp.GLOBAL_ATOMIC_SWAP_B32: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\ntmp = MEM[addr].b32;\nMEM[addr].b32 = DATA.b32;\nRETURN_DATA.b32 = tmp', VGLOBALOp.GLOBAL_ATOMIC_CMPSWAP_B32: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\ntmp = MEM[addr].u32;\nsrc = DATA[31 : 0].u32;\ncmp = DATA[63 : 32].u32;\nMEM[addr].u32 = tmp == cmp ? src : tmp;\nRETURN_DATA.u32 = tmp', VGLOBALOp.GLOBAL_ATOMIC_ADD_U32: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\ntmp = MEM[addr].u32;\nMEM[addr].u32 += DATA.u32;\nRETURN_DATA.u32 = tmp', @@ -572,6 +581,7 @@ PCODE = { VGLOBALOp.GLOBAL_ATOMIC_XOR_B64: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\ntmp = MEM[addr].b64;\nMEM[addr].b64 = (tmp ^ DATA.b64);\nRETURN_DATA.b64 = tmp', VGLOBALOp.GLOBAL_ATOMIC_INC_U64: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\ntmp = MEM[addr].u64;\nsrc = DATA.u64;\nMEM[addr].u64 = tmp >= src ? 0ULL : tmp + 1ULL;\nRETURN_DATA.u64 = tmp', VGLOBALOp.GLOBAL_ATOMIC_DEC_U64: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\ntmp = MEM[addr].u64;\nsrc = DATA.u64;\nMEM[addr].u64 = ((tmp == 0ULL) || (tmp > src)) ? src : tmp - 1ULL;\nRETURN_DATA.u64 = tmp', + VGLOBALOp.GLOBAL_WBINV: '', VGLOBALOp.GLOBAL_ATOMIC_COND_SUB_U32: 'addr = CalcGlobalAddr(v_addr.b64, s_saddr.b64);\ntmp = MEM[addr].u32;\nsrc = DATA.u32;\nMEM[ADDR].u32 = tmp >= src ? tmp - src : tmp;\nRETURN_DATA.u32 = tmp', VGLOBALOp.GLOBAL_ATOMIC_MIN_NUM_F32: "tmp = MEM[ADDR].f32;\nsrc = DATA.f32;\nif (isNAN(64'F(src.f32)) && isNAN(64'F(tmp.f32))) then\nMEM[ADDR].f32 = 32'F(cvtToQuietNAN(64'F(src.f32)))\nelsif isNAN(64'F(src.f32)) then\nMEM[ADDR].f32 = tmp.f32\nelsif isNAN(64'F(tmp.f32)) then\nMEM[ADDR].f32 = src.f32\nelsif ((src.f32 < tmp.f32) || ((abs(src.f32) == 0.0F) && (abs(tmp.f32) == 0.0F) && sign(src.f32) &&\n!sign(tmp.f32))) then\n// NOTE: -0<+0 is TRUE in this comparison\nMEM[ADDR].f32 = src.f32\nelse\nMEM[ADDR].f32 = tmp.f32\nendif;\nRETURN_DATA.f32 = tmp", VGLOBALOp.GLOBAL_ATOMIC_MAX_NUM_F32: "tmp = MEM[ADDR].f32;\nsrc = DATA.f32;\nif (isNAN(64'F(src.f32)) && isNAN(64'F(tmp.f32))) then\nMEM[ADDR].f32 = 32'F(cvtToQuietNAN(64'F(src.f32)))\nelsif isNAN(64'F(src.f32)) then\nMEM[ADDR].f32 = tmp.f32\nelsif isNAN(64'F(tmp.f32)) then\nMEM[ADDR].f32 = src.f32\nelsif ((src.f32 > tmp.f32) || ((abs(src.f32) == 0.0F) && (abs(tmp.f32) == 0.0F) && !sign(src.f32) &&\nsign(tmp.f32))) then\n// NOTE: +0>-0 is TRUE in this comparison\nMEM[ADDR].f32 = src.f32\nelse\nMEM[ADDR].f32 = tmp.f32\nendif;\nRETURN_DATA.f32 = tmp", @@ -605,6 +615,7 @@ PCODE = { VINTERPOp.V_INTERP_P2_F16_F32: "D0.f16 = 16'F(fma(32'F(VGPR[(laneId.u32 & 0xfffffffcU) + 2U][SRC0.u32].f16), S1.f32, S2.f32))", VINTERPOp.V_INTERP_P10_RTZ_F16_F32: "D0.f32 = fma(32'F(VGPR[(laneId.u32 & 0xfffffffcU) + 1U][SRC0.u32].f16), S1.f32, 32'F(VGPR[laneId.u32 &\n0xfffffffcU][SRC2.u32].f16))", VINTERPOp.V_INTERP_P2_RTZ_F16_F32: "D0.f32 = fma(32'F(VGPR[(laneId.u32 & 0xfffffffcU) + 2U][SRC0.u32].f16), S1.f32, S2.f32)", + VOP1Op.V_NOP_E32: '', VOP1Op.V_MOV_B32_E32: 'D0.b32 = S0.b32', VOP1Op.V_READFIRSTLANE_B32_E32: "declare lane : 32'U;\nif WAVE64 then\n// 64 lanes\nif EXEC == 0x0LL then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b64(EXEC));\n// Lowest active lane\nendif\nelse\n// 32 lanes\nif EXEC_LO.i32 == 0 then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b32(EXEC_LO));\n// Lowest active lane\nendif\nendif;\nD0.b32 = VGPR[lane][SRC0.u32]", VOP1Op.V_CVT_I32_F64_E32: 'D0.i32 = f64_to_i32(S0.f64)', @@ -630,6 +641,7 @@ PCODE = { VOP1Op.V_CEIL_F64_E32: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 > 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += 1.0\nendif', VOP1Op.V_RNDNE_F64_E32: 'D0.f64 = floor(S0.f64 + 0.5);\nif (isEven(floor(S0.f64)) && (fract(S0.f64) == 0.5)) then\nD0.f64 -= 1.0\nendif', VOP1Op.V_FLOOR_F64_E32: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 < 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += -1.0\nendif', + VOP1Op.V_PIPEFLUSH_E32: '', VOP1Op.V_MOV_B16_E32: 'D0.b16 = S0.b16', VOP1Op.V_FRACT_F32_E32: 'D0.f32 = S0.f32 + -floor(S0.f32)', VOP1Op.V_TRUNC_F32_E32: 'D0.f32 = trunc(S0.f32)', @@ -945,6 +957,7 @@ PCODE = { VOP3Op.V_MUL_F16_E64: 'D0.f16 = S0.f16 * S1.f16', VOP3Op.V_FMAC_F16_E64: 'D0.f16 = fma(S0.f16, S1.f16, D0.f16)', VOP3Op.V_LDEXP_F16_E64: "D0.f16 = S0.f16 * 16'F(2.0F ** 32'I(S1.i16))", + VOP3Op.V_NOP_E64: '', VOP3Op.V_MOV_B32_E64: 'D0.b32 = S0.b32', VOP3Op.V_READFIRSTLANE_B32_E64: "declare lane : 32'U;\nif WAVE64 then\n// 64 lanes\nif EXEC == 0x0LL then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b64(EXEC));\n// Lowest active lane\nendif\nelse\n// 32 lanes\nif EXEC_LO.i32 == 0 then\nlane = 0U;\n// Force lane 0 if all lanes are disabled\nelse\nlane = 32'U(s_ff1_i32_b32(EXEC_LO));\n// Lowest active lane\nendif\nendif;\nD0.b32 = VGPR[lane][SRC0.u32]", VOP3Op.V_CVT_I32_F64_E64: 'D0.i32 = f64_to_i32(S0.f64)', @@ -970,6 +983,7 @@ PCODE = { VOP3Op.V_CEIL_F64_E64: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 > 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += 1.0\nendif', VOP3Op.V_RNDNE_F64_E64: 'D0.f64 = floor(S0.f64 + 0.5);\nif (isEven(floor(S0.f64)) && (fract(S0.f64) == 0.5)) then\nD0.f64 -= 1.0\nendif', VOP3Op.V_FLOOR_F64_E64: 'D0.f64 = trunc(S0.f64);\nif ((S0.f64 < 0.0) && (S0.f64 != D0.f64)) then\nD0.f64 += -1.0\nendif', + VOP3Op.V_PIPEFLUSH_E64: '', VOP3Op.V_MOV_B16_E64: 'D0.b16 = S0.b16', VOP3Op.V_FRACT_F32_E64: 'D0.f32 = S0.f32 + -floor(S0.f32)', VOP3Op.V_TRUNC_F32_E64: 'D0.f32 = trunc(S0.f32)', diff --git a/tinygrad_repo/tinygrad/runtime/autogen/cuda.py b/tinygrad_repo/tinygrad/runtime/autogen/cuda.py index c6c805369e..07b0abd2eb 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/cuda.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/cuda.py @@ -4,8 +4,7 @@ import ctypes from typing import Literal, TypeAlias from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR from tinygrad.runtime.support import c -from tinygrad.helpers import WIN -dll = c.DLL('cuda', 'nvcuda' if WIN else 'cuda') +dll = c.DLL('cuda', 'cuda') cuuint32_t: TypeAlias = ctypes.c_uint32 cuuint64_t: TypeAlias = ctypes.c_uint64 CUdeviceptr_v2: TypeAlias = ctypes.c_uint64 diff --git a/tinygrad_repo/tinygrad/runtime/autogen/libclang.py b/tinygrad_repo/tinygrad/runtime/autogen/libclang.py index e59c5ef911..25dd1bf7a6 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/libclang.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/libclang.py @@ -4,8 +4,8 @@ import ctypes from typing import Literal, TypeAlias from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR from tinygrad.runtime.support import c -from tinygrad.helpers import WIN, OSX -dll = c.DLL('libclang', 'C:\\Program Files\\LLVM\\bin\\libclang.dll' if WIN else ['/opt/homebrew/opt/llvm@21/lib/libclang.dylib', '/opt/homebrew/opt/llvm@20/lib/libclang.dylib', '/opt/homebrew/opt/llvm@19/lib/libclang.dylib', '/opt/homebrew/opt/llvm@18/lib/libclang.dylib', '/opt/homebrew/opt/llvm@17/lib/libclang.dylib', '/opt/homebrew/opt/llvm@16/lib/libclang.dylib', '/opt/homebrew/opt/llvm@15/lib/libclang.dylib', '/opt/homebrew/opt/llvm@14/lib/libclang.dylib'] if OSX else ['clang-21', 'clang-20', 'clang-19', 'clang-18', 'clang-17', 'clang-16', 'clang-15', 'clang-14', 'clang']) +from tinygrad.helpers import OSX +dll = c.DLL('libclang', ['/opt/homebrew/opt/llvm@21/lib/libclang.dylib', '/opt/homebrew/opt/llvm@20/lib/libclang.dylib', '/opt/homebrew/opt/llvm@19/lib/libclang.dylib', '/opt/homebrew/opt/llvm@18/lib/libclang.dylib', '/opt/homebrew/opt/llvm@17/lib/libclang.dylib', '/opt/homebrew/opt/llvm@16/lib/libclang.dylib', '/opt/homebrew/opt/llvm@15/lib/libclang.dylib', '/opt/homebrew/opt/llvm@14/lib/libclang.dylib'] if OSX else ['clang-21', 'clang-20', 'clang-19', 'clang-18', 'clang-17', 'clang-16', 'clang-15', 'clang-14', 'clang']) CXIndex: TypeAlias = ctypes.c_void_p class struct_CXTargetInfoImpl(c.Struct): pass CXTargetInfo: TypeAlias = c.POINTER[struct_CXTargetInfoImpl] diff --git a/tinygrad_repo/tinygrad/runtime/autogen/llvm.py b/tinygrad_repo/tinygrad/runtime/autogen/llvm.py index 4cdda85733..057df5dc64 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/llvm.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/llvm.py @@ -4,8 +4,8 @@ import ctypes from typing import Literal, TypeAlias from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR from tinygrad.runtime.support import c -from tinygrad.helpers import WIN, OSX -dll = c.DLL('llvm', 'C:\\Program Files\\LLVM\\bin\\LLVM-C.dll' if WIN else ['/opt/homebrew/opt/llvm@21/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@20/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@19/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@18/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@17/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@16/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@15/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@14/lib/libLLVM.dylib'] if OSX else ['LLVM-21', 'LLVM-20', 'LLVM-19', 'LLVM-18', 'LLVM-17', 'LLVM-16', 'LLVM-15', 'LLVM-14', 'LLVM']) +from tinygrad.helpers import OSX +dll = c.DLL('llvm', ['/opt/homebrew/opt/llvm@21/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@20/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@19/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@18/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@17/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@16/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@15/lib/libLLVM.dylib', '/opt/homebrew/opt/llvm@14/lib/libLLVM.dylib'] if OSX else ['LLVM-21', 'LLVM-20', 'LLVM-19', 'LLVM-18', 'LLVM-17', 'LLVM-16', 'LLVM-15', 'LLVM-14', 'LLVM']) intmax_t: TypeAlias = ctypes.c_int64 @dll.bind(intmax_t, intmax_t) def imaxabs(__n:intmax_t) -> intmax_t: ... diff --git a/tinygrad_repo/tinygrad/runtime/autogen/nv_615.py b/tinygrad_repo/tinygrad/runtime/autogen/nv_615.py new file mode 100644 index 0000000000..2164713b81 --- /dev/null +++ b/tinygrad_repo/tinygrad/runtime/autogen/nv_615.py @@ -0,0 +1,26824 @@ +# mypy: disable-error-code="empty-body" +from __future__ import annotations +import ctypes +from typing import Literal, TypeAlias +from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR +from tinygrad.runtime.support import c +_anonenum0: dict[int, str] = {(AES128_NONE:=0): 'AES128_NONE', (AES128_CTR:=1): 'AES128_CTR', (AES128_CBC:=2): 'AES128_CBC', (AES128_ECB:=3): 'AES128_ECB', (AES128_OFB:=4): 'AES128_OFB', (AES128_CTR_LSB16B:=5): 'AES128_CTR_LSB16B', (AES128_CLR_AS_ENCRYPT:=6): 'AES128_CLR_AS_ENCRYPT', (AES128_RESERVED:=7): 'AES128_RESERVED'} +_anonenum1: dict[int, str] = {(AES128_CTS_DISABLE:=0): 'AES128_CTS_DISABLE', (AES128_CTS_ENABLE:=1): 'AES128_CTS_ENABLE'} +_anonenum2: dict[int, str] = {(AES128_PADDING_NONE:=0): 'AES128_PADDING_NONE', (AES128_PADDING_CARRY_OVER:=1): 'AES128_PADDING_CARRY_OVER', (AES128_PADDING_RFC2630:=2): 'AES128_PADDING_RFC2630', (AES128_PADDING_RESERVED:=7): 'AES128_PADDING_RESERVED'} +ENCR_MODE: dict[int, str] = {(ENCR_MODE_CTR64:=0): 'ENCR_MODE_CTR64', (ENCR_MODE_CBC:=1): 'ENCR_MODE_CBC', (ENCR_MODE_ECB:=2): 'ENCR_MODE_ECB', (ENCR_MODE_ECB_PARTIAL:=3): 'ENCR_MODE_ECB_PARTIAL', (ENCR_MODE_CBC_PARTIAL:=4): 'ENCR_MODE_CBC_PARTIAL', (ENCR_MODE_CLEAR_INTO_VPR:=5): 'ENCR_MODE_CLEAR_INTO_VPR', (ENCR_MODE_FORCE_INTO_VPR:=6): 'ENCR_MODE_FORCE_INTO_VPR'} +_anonenum3: dict[int, str] = {(DRM_MS_PIFF_CTR:=17): 'DRM_MS_PIFF_CTR'} +_anonenum4: dict[int, str] = {(DRM_MS_PIFF_CBC:=2): 'DRM_MS_PIFF_CBC'} +_anonenum5: dict[int, str] = {(DRM_MARLIN_CTR:=1): 'DRM_MARLIN_CTR'} +_anonenum6: dict[int, str] = {(DRM_MARLIN_CBC:=34): 'DRM_MARLIN_CBC'} +_anonenum7: dict[int, str] = {(DRM_WIDEVINE:=10): 'DRM_WIDEVINE'} +_anonenum8: dict[int, str] = {(DRM_WIDEVINE_CTR:=17): 'DRM_WIDEVINE_CTR'} +_anonenum9: dict[int, str] = {(DRM_ULTRA_VIOLET:=5): 'DRM_ULTRA_VIOLET'} +_anonenum10: dict[int, str] = {(DRM_NONE:=0): 'DRM_NONE'} +_anonenum11: dict[int, str] = {(DRM_CLR_AS_ENCRYPT:=6): 'DRM_CLR_AS_ENCRYPT'} +@c.record +class struct__nvdec_ssm_s(c.Struct): + SIZE = 8 + bytes_of_protected_data: int + bytes_of_clear_data: int + skip_byte_blk: int + crypt_byte_blk: int + skip: int + last: int + pad: int + mode: int + entry_type: int + reserved: int +struct__nvdec_ssm_s.register_fields([('bytes_of_protected_data', ctypes.c_uint32, 0), ('bytes_of_clear_data', ctypes.c_uint32, 4, 16, 0), ('skip_byte_blk', ctypes.c_uint32, 6, 4, 0), ('crypt_byte_blk', ctypes.c_uint32, 6, 4, 4), ('skip', ctypes.c_uint32, 7, 1, 0), ('last', ctypes.c_uint32, 7, 1, 1), ('pad', ctypes.c_uint32, 7, 1, 2), ('mode', ctypes.c_uint32, 7, 1, 3), ('entry_type', ctypes.c_uint32, 7, 1, 4), ('reserved', ctypes.c_uint32, 7, 3, 5)]) +nvdec_ssm_s: TypeAlias = struct__nvdec_ssm_s +@c.record +class struct__nvdec_pass2_otf_ext_s(c.Struct): + SIZE = 12 + ssm_entry_num: int + ssm_iv_num: int + real_stream_length: int + non_slice_data: int + drm_mode: int + reserved: int +struct__nvdec_pass2_otf_ext_s.register_fields([('ssm_entry_num', ctypes.c_uint32, 0, 16, 0), ('ssm_iv_num', ctypes.c_uint32, 2, 16, 0), ('real_stream_length', ctypes.c_uint32, 4), ('non_slice_data', ctypes.c_uint32, 8, 16, 0), ('drm_mode', ctypes.c_uint32, 10, 7, 0), ('reserved', ctypes.c_uint32, 10, 9, 7)]) +nvdec_pass2_otf_ext_s: TypeAlias = struct__nvdec_pass2_otf_ext_s +@c.record +class struct__nvdec_substream_entry_s(c.Struct): + SIZE = 12 + substream_start_offset: int + substream_length: int + substream_first_tile_idx: int + substream_last_tile_idx: int + last_substream_entry_in_frame: int + reserved: int +struct__nvdec_substream_entry_s.register_fields([('substream_start_offset', ctypes.c_uint32, 0), ('substream_length', ctypes.c_uint32, 4), ('substream_first_tile_idx', ctypes.c_uint32, 8, 8, 0), ('substream_last_tile_idx', ctypes.c_uint32, 9, 8, 0), ('last_substream_entry_in_frame', ctypes.c_uint32, 10, 1, 0), ('reserved', ctypes.c_uint32, 10, 15, 1)]) +nvdec_substream_entry_s: TypeAlias = struct__nvdec_substream_entry_s +@c.record +class struct__nvdec_ecdma_config_s(c.Struct): + SIZE = 24 + ecdma_enable: int + ecdma_blk_x_src: int + ecdma_blk_y_src: int + ecdma_blk_x_dst: int + ecdma_blk_y_dst: int + ref_pic_idx: int + boundary0_top: int + boundary0_bottom: int + boundary1_left: int + boundary1_right: int + blk_copy_flag: int + ctb_size: int +struct__nvdec_ecdma_config_s.register_fields([('ecdma_enable', ctypes.c_uint32, 0), ('ecdma_blk_x_src', ctypes.c_uint16, 4), ('ecdma_blk_y_src', ctypes.c_uint16, 6), ('ecdma_blk_x_dst', ctypes.c_uint16, 8), ('ecdma_blk_y_dst', ctypes.c_uint16, 10), ('ref_pic_idx', ctypes.c_uint16, 12), ('boundary0_top', ctypes.c_uint16, 14), ('boundary0_bottom', ctypes.c_uint16, 16), ('boundary1_left', ctypes.c_uint16, 18), ('boundary1_right', ctypes.c_uint16, 20), ('blk_copy_flag', ctypes.c_ubyte, 22), ('ctb_size', ctypes.c_ubyte, 23)]) +nvdec_ecdma_config_s: TypeAlias = struct__nvdec_ecdma_config_s +@c.record +class struct__nvdec_status_hevc_s(c.Struct): + SIZE = 36 + frame_status_intra_cnt: int + frame_status_inter_cnt: int + frame_status_skip_cnt: int + frame_status_fwd_mvx_cnt: int + frame_status_fwd_mvy_cnt: int + frame_status_bwd_mvx_cnt: int + frame_status_bwd_mvy_cnt: int + error_ctb_pos: int + error_slice_pos: int +struct__nvdec_status_hevc_s.register_fields([('frame_status_intra_cnt', ctypes.c_uint32, 0), ('frame_status_inter_cnt', ctypes.c_uint32, 4), ('frame_status_skip_cnt', ctypes.c_uint32, 8), ('frame_status_fwd_mvx_cnt', ctypes.c_uint32, 12), ('frame_status_fwd_mvy_cnt', ctypes.c_uint32, 16), ('frame_status_bwd_mvx_cnt', ctypes.c_uint32, 20), ('frame_status_bwd_mvy_cnt', ctypes.c_uint32, 24), ('error_ctb_pos', ctypes.c_uint32, 28), ('error_slice_pos', ctypes.c_uint32, 32)]) +nvdec_status_hevc_s: TypeAlias = struct__nvdec_status_hevc_s +@c.record +class struct__nvdec_status_vp9_s(c.Struct): + SIZE = 36 + frame_status_intra_cnt: int + frame_status_inter_cnt: int + frame_status_skip_cnt: int + frame_status_fwd_mvx_cnt: int + frame_status_fwd_mvy_cnt: int + frame_status_bwd_mvx_cnt: int + frame_status_bwd_mvy_cnt: int + error_ctb_pos: int + error_slice_pos: int +struct__nvdec_status_vp9_s.register_fields([('frame_status_intra_cnt', ctypes.c_uint32, 0), ('frame_status_inter_cnt', ctypes.c_uint32, 4), ('frame_status_skip_cnt', ctypes.c_uint32, 8), ('frame_status_fwd_mvx_cnt', ctypes.c_uint32, 12), ('frame_status_fwd_mvy_cnt', ctypes.c_uint32, 16), ('frame_status_bwd_mvx_cnt', ctypes.c_uint32, 20), ('frame_status_bwd_mvy_cnt', ctypes.c_uint32, 24), ('error_ctb_pos', ctypes.c_uint32, 28), ('error_slice_pos', ctypes.c_uint32, 32)]) +nvdec_status_vp9_s: TypeAlias = struct__nvdec_status_vp9_s +@c.record +class struct__nvdec_status_s(c.Struct): + SIZE = 56 + mbs_correctly_decoded: int + mbs_in_error: int + cycle_count: int + error_status: int + hevc: struct__nvdec_status_hevc_s + vp9: struct__nvdec_status_vp9_s + slice_header_error_code: int +struct__nvdec_status_s.register_fields([('mbs_correctly_decoded', ctypes.c_uint32, 0), ('mbs_in_error', ctypes.c_uint32, 4), ('cycle_count', ctypes.c_uint32, 8), ('error_status', ctypes.c_uint32, 12), ('hevc', nvdec_status_hevc_s, 16), ('vp9', nvdec_status_vp9_s, 16), ('slice_header_error_code', ctypes.c_uint32, 52)]) +nvdec_status_s: TypeAlias = struct__nvdec_status_s +@c.record +class struct__external_mv_s(c.Struct): + SIZE = 4 + mvx: int + mvy: int + refidx: int +struct__external_mv_s.register_fields([('mvx', ctypes.c_int32, 0, 14, 0), ('mvy', ctypes.c_int32, 1, 14, 6), ('refidx', ctypes.c_uint32, 3, 4, 4)]) +external_mv_s: TypeAlias = struct__external_mv_s +@c.record +class struct__nvdec_hevc_main10_444_ext_s(c.Struct): + SIZE = 28 + transformSkipRotationEnableFlag: int + transformSkipContextEnableFlag: int + intraBlockCopyEnableFlag: int + implicitRdpcmEnableFlag: int + explicitRdpcmEnableFlag: int + extendedPrecisionProcessingFlag: int + intraSmoothingDisabledFlag: int + highPrecisionOffsetsEnableFlag: int + fastRiceAdaptationEnableFlag: int + cabacBypassAlignmentEnableFlag: int + sps_444_extension_reserved: int + log2MaxTransformSkipSize: int + crossComponentPredictionEnableFlag: int + chromaQpAdjustmentEnableFlag: int + diffCuChromaQpAdjustmentDepth: int + chromaQpAdjustmentTableSize: int + log2SaoOffsetScaleLuma: int + log2SaoOffsetScaleChroma: int + pps_444_extension_reserved: int + cb_qp_adjustment: c.Array[ctypes.c_char, Literal[6]] + cr_qp_adjustment: c.Array[ctypes.c_char, Literal[6]] + HevcFltAboveOffset: int + HevcSaoAboveOffset: int +struct__nvdec_hevc_main10_444_ext_s.register_fields([('transformSkipRotationEnableFlag', ctypes.c_uint32, 0, 1, 0), ('transformSkipContextEnableFlag', ctypes.c_uint32, 0, 1, 1), ('intraBlockCopyEnableFlag', ctypes.c_uint32, 0, 1, 2), ('implicitRdpcmEnableFlag', ctypes.c_uint32, 0, 1, 3), ('explicitRdpcmEnableFlag', ctypes.c_uint32, 0, 1, 4), ('extendedPrecisionProcessingFlag', ctypes.c_uint32, 0, 1, 5), ('intraSmoothingDisabledFlag', ctypes.c_uint32, 0, 1, 6), ('highPrecisionOffsetsEnableFlag', ctypes.c_uint32, 0, 1, 7), ('fastRiceAdaptationEnableFlag', ctypes.c_uint32, 1, 1, 0), ('cabacBypassAlignmentEnableFlag', ctypes.c_uint32, 1, 1, 1), ('sps_444_extension_reserved', ctypes.c_uint32, 1, 22, 2), ('log2MaxTransformSkipSize', ctypes.c_uint32, 4, 4, 0), ('crossComponentPredictionEnableFlag', ctypes.c_uint32, 4, 1, 4), ('chromaQpAdjustmentEnableFlag', ctypes.c_uint32, 4, 1, 5), ('diffCuChromaQpAdjustmentDepth', ctypes.c_uint32, 4, 2, 6), ('chromaQpAdjustmentTableSize', ctypes.c_uint32, 5, 3, 0), ('log2SaoOffsetScaleLuma', ctypes.c_uint32, 5, 3, 3), ('log2SaoOffsetScaleChroma', ctypes.c_uint32, 5, 3, 6), ('pps_444_extension_reserved', ctypes.c_uint32, 6, 15, 1), ('cb_qp_adjustment', c.Array[ctypes.c_char, Literal[6]], 8), ('cr_qp_adjustment', c.Array[ctypes.c_char, Literal[6]], 14), ('HevcFltAboveOffset', ctypes.c_uint32, 20), ('HevcSaoAboveOffset', ctypes.c_uint32, 24)]) +nvdec_hevc_main10_444_ext_s: TypeAlias = struct__nvdec_hevc_main10_444_ext_s +@c.record +class struct__nvdec_hevc_pic_v1_s(c.Struct): + SIZE = 32 + hevc_main10_444_ext: struct__nvdec_hevc_main10_444_ext_s + sw_skip_start_length: int + external_ref_mem_dis: int + error_recovery_start_pos: int + error_external_mv_en: int + reserved0: int +struct__nvdec_hevc_pic_v1_s.register_fields([('hevc_main10_444_ext', nvdec_hevc_main10_444_ext_s, 0), ('sw_skip_start_length', ctypes.c_uint32, 28, 14, 0), ('external_ref_mem_dis', ctypes.c_uint32, 29, 1, 6), ('error_recovery_start_pos', ctypes.c_uint32, 29, 2, 7), ('error_external_mv_en', ctypes.c_uint32, 30, 1, 1), ('reserved0', ctypes.c_uint32, 30, 14, 2)]) +nvdec_hevc_pic_v1_s: TypeAlias = struct__nvdec_hevc_pic_v1_s +@c.record +class struct__nvdec_hevc_pic_v2_s(c.Struct): + SIZE = 4 + mv_hevc_enable: int + nuh_layer_id: int + default_ref_layers_active_flag: int + NumDirectRefLayers: int + max_one_active_ref_layer_flag: int + NumActiveRefLayerPics: int + poc_lsb_not_present_flag: int + reserved0: int +struct__nvdec_hevc_pic_v2_s.register_fields([('mv_hevc_enable', ctypes.c_uint32, 0, 1, 0), ('nuh_layer_id', ctypes.c_uint32, 0, 6, 1), ('default_ref_layers_active_flag', ctypes.c_uint32, 0, 1, 7), ('NumDirectRefLayers', ctypes.c_uint32, 1, 6, 0), ('max_one_active_ref_layer_flag', ctypes.c_uint32, 1, 1, 6), ('NumActiveRefLayerPics', ctypes.c_uint32, 1, 6, 7), ('poc_lsb_not_present_flag', ctypes.c_uint32, 2, 1, 5), ('reserved0', ctypes.c_uint32, 2, 10, 6)]) +nvdec_hevc_pic_v2_s: TypeAlias = struct__nvdec_hevc_pic_v2_s +@c.record +class struct__nvdec_hevc_pic_v3_s(c.Struct): + SIZE = 8 + slice_decoding_enable: int + slice_ec_enable: int + slice_ec_mv_type: int + err_detected_sw: int + slice_ec_slice_type: int + slice_strm_recfg_en: int + reserved: int + HevcSliceEdgeOffset: int +struct__nvdec_hevc_pic_v3_s.register_fields([('slice_decoding_enable', ctypes.c_uint32, 0, 1, 0), ('slice_ec_enable', ctypes.c_uint32, 0, 1, 1), ('slice_ec_mv_type', ctypes.c_uint32, 0, 2, 2), ('err_detected_sw', ctypes.c_uint32, 0, 1, 4), ('slice_ec_slice_type', ctypes.c_uint32, 0, 2, 5), ('slice_strm_recfg_en', ctypes.c_uint32, 0, 1, 7), ('reserved', ctypes.c_uint32, 1, 24, 0), ('HevcSliceEdgeOffset', ctypes.c_uint32, 4)]) +nvdec_hevc_pic_v3_s: TypeAlias = struct__nvdec_hevc_pic_v3_s +@c.record +class struct__nvdec_hevc_pic_s(c.Struct): + SIZE = 276 + wrapped_session_key: c.Array[ctypes.c_uint32, Literal[4]] + wrapped_content_key: c.Array[ctypes.c_uint32, Literal[4]] + initialization_vector: c.Array[ctypes.c_uint32, Literal[4]] + stream_len: int + enable_encryption: int + key_increment: int + encryption_mode: int + key_slot_index: int + ssm_en: int + enable_histogram: int + enable_substream_decoding: int + reserved0: int + gptimer_timeout_value: int + tileformat: int + gob_height: int + reserverd_surface_format: int + sw_start_code_e: int + disp_output_mode: int + reserved1: int + framestride: c.Array[ctypes.c_uint32, Literal[2]] + colMvBuffersize: int + HevcSaoBufferOffset: int + HevcBsdCtrlOffset: int + pic_width_in_luma_samples: int + pic_height_in_luma_samples: int + chroma_format_idc: int + bit_depth_luma: int + bit_depth_chroma: int + log2_min_luma_coding_block_size: int + log2_max_luma_coding_block_size: int + log2_min_transform_block_size: int + log2_max_transform_block_size: int + reserved2: int + max_transform_hierarchy_depth_inter: int + max_transform_hierarchy_depth_intra: int + scalingListEnable: int + amp_enable_flag: int + sample_adaptive_offset_enabled_flag: int + pcm_enabled_flag: int + pcm_sample_bit_depth_luma: int + pcm_sample_bit_depth_chroma: int + log2_min_pcm_luma_coding_block_size: int + log2_max_pcm_luma_coding_block_size: int + pcm_loop_filter_disabled_flag: int + sps_temporal_mvp_enabled_flag: int + strong_intra_smoothing_enabled_flag: int + reserved3: int + dependent_slice_segments_enabled_flag: int + output_flag_present_flag: int + num_extra_slice_header_bits: int + sign_data_hiding_enabled_flag: int + cabac_init_present_flag: int + num_ref_idx_l0_default_active: int + num_ref_idx_l1_default_active: int + init_qp: int + constrained_intra_pred_flag: int + transform_skip_enabled_flag: int + cu_qp_delta_enabled_flag: int + diff_cu_qp_delta_depth: int + reserved4: int + pps_cb_qp_offset: bytes + pps_cr_qp_offset: bytes + pps_beta_offset: bytes + pps_tc_offset: bytes + pps_slice_chroma_qp_offsets_present_flag: int + weighted_pred_flag: int + weighted_bipred_flag: int + transquant_bypass_enabled_flag: int + tiles_enabled_flag: int + entropy_coding_sync_enabled_flag: int + num_tile_columns: int + num_tile_rows: int + loop_filter_across_tiles_enabled_flag: int + loop_filter_across_slices_enabled_flag: int + deblocking_filter_control_present_flag: int + deblocking_filter_override_enabled_flag: int + pps_deblocking_filter_disabled_flag: int + lists_modification_present_flag: int + log2_parallel_merge_level: int + slice_segment_header_extension_present_flag: int + reserved5: int + num_ref_frames: int + reserved6: int + longtermflag: int + initreflistidxl0: c.Array[ctypes.c_ubyte, Literal[16]] + initreflistidxl1: c.Array[ctypes.c_ubyte, Literal[16]] + RefDiffPicOrderCnts: c.Array[ctypes.c_int16, Literal[16]] + IDR_picture_flag: int + RAP_picture_flag: int + curr_pic_idx: int + pattern_id: int + sw_hdr_skip_length: int + reserved7: int + ecdma_cfg: struct__nvdec_ecdma_config_s + separate_colour_plane_flag: int + log2_max_pic_order_cnt_lsb_minus4: int + num_short_term_ref_pic_sets: int + num_long_term_ref_pics_sps: int + bBitParsingDisable: int + num_delta_pocs_of_rps_idx: int + long_term_ref_pics_present_flag: int + reserved_dxva: int + num_bits_short_term_ref_pics_in_slice: int + v1: struct__nvdec_hevc_pic_v1_s + v2: struct__nvdec_hevc_pic_v2_s + v3: struct__nvdec_hevc_pic_v3_s + ssm: struct__nvdec_pass2_otf_ext_s +struct__nvdec_hevc_pic_s.register_fields([('wrapped_session_key', c.Array[ctypes.c_uint32, Literal[4]], 0), ('wrapped_content_key', c.Array[ctypes.c_uint32, Literal[4]], 16), ('initialization_vector', c.Array[ctypes.c_uint32, Literal[4]], 32), ('stream_len', ctypes.c_uint32, 48), ('enable_encryption', ctypes.c_uint32, 52), ('key_increment', ctypes.c_uint32, 56, 6, 0), ('encryption_mode', ctypes.c_uint32, 56, 4, 6), ('key_slot_index', ctypes.c_uint32, 57, 4, 2), ('ssm_en', ctypes.c_uint32, 57, 1, 6), ('enable_histogram', ctypes.c_uint32, 57, 1, 7), ('enable_substream_decoding', ctypes.c_uint32, 58, 1, 0), ('reserved0', ctypes.c_uint32, 58, 15, 1), ('gptimer_timeout_value', ctypes.c_uint32, 60), ('tileformat', ctypes.c_ubyte, 64, 2, 0), ('gob_height', ctypes.c_ubyte, 64, 3, 2), ('reserverd_surface_format', ctypes.c_ubyte, 64, 3, 5), ('sw_start_code_e', ctypes.c_ubyte, 65), ('disp_output_mode', ctypes.c_ubyte, 66), ('reserved1', ctypes.c_ubyte, 67), ('framestride', c.Array[ctypes.c_uint32, Literal[2]], 68), ('colMvBuffersize', ctypes.c_uint32, 76), ('HevcSaoBufferOffset', ctypes.c_uint32, 80), ('HevcBsdCtrlOffset', ctypes.c_uint32, 84), ('pic_width_in_luma_samples', ctypes.c_uint16, 88), ('pic_height_in_luma_samples', ctypes.c_uint16, 90), ('chroma_format_idc', ctypes.c_uint32, 92, 4, 0), ('bit_depth_luma', ctypes.c_uint32, 92, 4, 4), ('bit_depth_chroma', ctypes.c_uint32, 93, 4, 0), ('log2_min_luma_coding_block_size', ctypes.c_uint32, 93, 4, 4), ('log2_max_luma_coding_block_size', ctypes.c_uint32, 94, 4, 0), ('log2_min_transform_block_size', ctypes.c_uint32, 94, 4, 4), ('log2_max_transform_block_size', ctypes.c_uint32, 95, 4, 0), ('reserved2', ctypes.c_uint32, 95, 4, 4), ('max_transform_hierarchy_depth_inter', ctypes.c_uint32, 96, 3, 0), ('max_transform_hierarchy_depth_intra', ctypes.c_uint32, 96, 3, 3), ('scalingListEnable', ctypes.c_uint32, 96, 1, 6), ('amp_enable_flag', ctypes.c_uint32, 96, 1, 7), ('sample_adaptive_offset_enabled_flag', ctypes.c_uint32, 97, 1, 0), ('pcm_enabled_flag', ctypes.c_uint32, 97, 1, 1), ('pcm_sample_bit_depth_luma', ctypes.c_uint32, 97, 4, 2), ('pcm_sample_bit_depth_chroma', ctypes.c_uint32, 97, 4, 6), ('log2_min_pcm_luma_coding_block_size', ctypes.c_uint32, 98, 4, 2), ('log2_max_pcm_luma_coding_block_size', ctypes.c_uint32, 98, 4, 6), ('pcm_loop_filter_disabled_flag', ctypes.c_uint32, 99, 1, 2), ('sps_temporal_mvp_enabled_flag', ctypes.c_uint32, 99, 1, 3), ('strong_intra_smoothing_enabled_flag', ctypes.c_uint32, 99, 1, 4), ('reserved3', ctypes.c_uint32, 99, 3, 5), ('dependent_slice_segments_enabled_flag', ctypes.c_uint32, 100, 1, 0), ('output_flag_present_flag', ctypes.c_uint32, 100, 1, 1), ('num_extra_slice_header_bits', ctypes.c_uint32, 100, 3, 2), ('sign_data_hiding_enabled_flag', ctypes.c_uint32, 100, 1, 5), ('cabac_init_present_flag', ctypes.c_uint32, 100, 1, 6), ('num_ref_idx_l0_default_active', ctypes.c_uint32, 100, 4, 7), ('num_ref_idx_l1_default_active', ctypes.c_uint32, 101, 4, 3), ('init_qp', ctypes.c_uint32, 101, 7, 7), ('constrained_intra_pred_flag', ctypes.c_uint32, 102, 1, 6), ('transform_skip_enabled_flag', ctypes.c_uint32, 102, 1, 7), ('cu_qp_delta_enabled_flag', ctypes.c_uint32, 103, 1, 0), ('diff_cu_qp_delta_depth', ctypes.c_uint32, 103, 2, 1), ('reserved4', ctypes.c_uint32, 103, 5, 3), ('pps_cb_qp_offset', ctypes.c_char, 104), ('pps_cr_qp_offset', ctypes.c_char, 105), ('pps_beta_offset', ctypes.c_char, 106), ('pps_tc_offset', ctypes.c_char, 107), ('pps_slice_chroma_qp_offsets_present_flag', ctypes.c_uint32, 108, 1, 0), ('weighted_pred_flag', ctypes.c_uint32, 108, 1, 1), ('weighted_bipred_flag', ctypes.c_uint32, 108, 1, 2), ('transquant_bypass_enabled_flag', ctypes.c_uint32, 108, 1, 3), ('tiles_enabled_flag', ctypes.c_uint32, 108, 1, 4), ('entropy_coding_sync_enabled_flag', ctypes.c_uint32, 108, 1, 5), ('num_tile_columns', ctypes.c_uint32, 108, 5, 6), ('num_tile_rows', ctypes.c_uint32, 109, 5, 3), ('loop_filter_across_tiles_enabled_flag', ctypes.c_uint32, 110, 1, 0), ('loop_filter_across_slices_enabled_flag', ctypes.c_uint32, 110, 1, 1), ('deblocking_filter_control_present_flag', ctypes.c_uint32, 110, 1, 2), ('deblocking_filter_override_enabled_flag', ctypes.c_uint32, 110, 1, 3), ('pps_deblocking_filter_disabled_flag', ctypes.c_uint32, 110, 1, 4), ('lists_modification_present_flag', ctypes.c_uint32, 110, 1, 5), ('log2_parallel_merge_level', ctypes.c_uint32, 110, 3, 6), ('slice_segment_header_extension_present_flag', ctypes.c_uint32, 111, 1, 1), ('reserved5', ctypes.c_uint32, 111, 6, 2), ('num_ref_frames', ctypes.c_ubyte, 112), ('reserved6', ctypes.c_ubyte, 113), ('longtermflag', ctypes.c_uint16, 114), ('initreflistidxl0', c.Array[ctypes.c_ubyte, Literal[16]], 116), ('initreflistidxl1', c.Array[ctypes.c_ubyte, Literal[16]], 132), ('RefDiffPicOrderCnts', c.Array[ctypes.c_int16, Literal[16]], 148), ('IDR_picture_flag', ctypes.c_ubyte, 180), ('RAP_picture_flag', ctypes.c_ubyte, 181), ('curr_pic_idx', ctypes.c_ubyte, 182), ('pattern_id', ctypes.c_ubyte, 183), ('sw_hdr_skip_length', ctypes.c_uint16, 184), ('reserved7', ctypes.c_uint16, 186), ('ecdma_cfg', nvdec_ecdma_config_s, 188), ('separate_colour_plane_flag', ctypes.c_uint32, 212, 1, 0), ('log2_max_pic_order_cnt_lsb_minus4', ctypes.c_uint32, 212, 4, 1), ('num_short_term_ref_pic_sets', ctypes.c_uint32, 212, 7, 5), ('num_long_term_ref_pics_sps', ctypes.c_uint32, 213, 6, 4), ('bBitParsingDisable', ctypes.c_uint32, 214, 1, 2), ('num_delta_pocs_of_rps_idx', ctypes.c_uint32, 214, 8, 3), ('long_term_ref_pics_present_flag', ctypes.c_uint32, 215, 1, 3), ('reserved_dxva', ctypes.c_uint32, 215, 4, 4), ('num_bits_short_term_ref_pics_in_slice', ctypes.c_uint32, 216), ('v1', nvdec_hevc_pic_v1_s, 220), ('v2', nvdec_hevc_pic_v2_s, 252), ('v3', nvdec_hevc_pic_v3_s, 256), ('ssm', nvdec_pass2_otf_ext_s, 264)]) +nvdec_hevc_pic_s: TypeAlias = struct__nvdec_hevc_pic_s +@c.record +class struct__hevc_slice_info_s(c.Struct): + SIZE = 16 + first_flag: int + err_flag: int + last_flag: int + conceal_partial_slice: int + available: int + reserved0: int + ctb_count: int + bs_offset: int + bs_length: int + start_ctbx: int + start_ctby: int +struct__hevc_slice_info_s.register_fields([('first_flag', ctypes.c_uint32, 0, 1, 0), ('err_flag', ctypes.c_uint32, 0, 1, 1), ('last_flag', ctypes.c_uint32, 0, 1, 2), ('conceal_partial_slice', ctypes.c_uint32, 0, 1, 3), ('available', ctypes.c_uint32, 0, 1, 4), ('reserved0', ctypes.c_uint32, 0, 7, 5), ('ctb_count', ctypes.c_uint32, 1, 20, 4), ('bs_offset', ctypes.c_uint32, 4), ('bs_length', ctypes.c_uint32, 8), ('start_ctbx', ctypes.c_uint16, 12), ('start_ctby', ctypes.c_uint16, 14)]) +hevc_slice_info_s: TypeAlias = struct__hevc_slice_info_s +@c.record +class struct__slice_edge_ctb_pos_ctx_s(c.Struct): + SIZE = 8 + next_slice_pos_ctbxy: int + next_slice_segment_addr: int +struct__slice_edge_ctb_pos_ctx_s.register_fields([('next_slice_pos_ctbxy', ctypes.c_uint32, 0), ('next_slice_segment_addr', ctypes.c_uint32, 4)]) +slice_edge_ctb_pos_ctx_s: TypeAlias = struct__slice_edge_ctb_pos_ctx_s +@c.record +class struct__slice_edge_tile_ctx_s(c.Struct): + SIZE = 12 + tileInfo1: int + tileInfo2: int + tileInfo3: int +struct__slice_edge_tile_ctx_s.register_fields([('tileInfo1', ctypes.c_uint32, 0), ('tileInfo2', ctypes.c_uint32, 4), ('tileInfo3', ctypes.c_uint32, 8)]) +slice_edge_tile_ctx_s: TypeAlias = struct__slice_edge_tile_ctx_s +@c.record +class struct__slice_edge_stats_ctx_s(c.Struct): + SIZE = 32 + frame_status_intra_cnt: int + frame_status_inter_cnt: int + frame_status_skip_cnt: int + frame_status_fwd_mvx_cnt: int + frame_status_fwd_mvy_cnt: int + frame_status_bwd_mvx_cnt: int + frame_status_bwd_mvy_cnt: int + frame_status_mv_cnt_ext: int +struct__slice_edge_stats_ctx_s.register_fields([('frame_status_intra_cnt', ctypes.c_uint32, 0), ('frame_status_inter_cnt', ctypes.c_uint32, 4), ('frame_status_skip_cnt', ctypes.c_uint32, 8), ('frame_status_fwd_mvx_cnt', ctypes.c_uint32, 12), ('frame_status_fwd_mvy_cnt', ctypes.c_uint32, 16), ('frame_status_bwd_mvx_cnt', ctypes.c_uint32, 20), ('frame_status_bwd_mvy_cnt', ctypes.c_uint32, 24), ('frame_status_mv_cnt_ext', ctypes.c_uint32, 28)]) +slice_edge_stats_ctx_s: TypeAlias = struct__slice_edge_stats_ctx_s +@c.record +class struct__slice_vpc_edge_ctx_s(c.Struct): + SIZE = 4 + reserved: int +struct__slice_vpc_edge_ctx_s.register_fields([('reserved', ctypes.c_uint32, 0)]) +slice_vpc_edge_ctx_s: TypeAlias = struct__slice_vpc_edge_ctx_s +@c.record +class struct__slice_vpc_main_ctx_s(c.Struct): + SIZE = 4 + reserved: int +struct__slice_vpc_main_ctx_s.register_fields([('reserved', ctypes.c_uint32, 0)]) +slice_vpc_main_ctx_s: TypeAlias = struct__slice_vpc_main_ctx_s +@c.record +class struct__slice_edge_ctx_s(c.Struct): + SIZE = 60 + slice_ctb_pos_ctx: struct__slice_edge_ctb_pos_ctx_s + slice_stats_ctx: struct__slice_edge_stats_ctx_s + slice_tile_ctx: struct__slice_edge_tile_ctx_s + slice_vpc_edge_ctx: struct__slice_vpc_edge_ctx_s + slice_vpc_main_ctx: struct__slice_vpc_main_ctx_s +struct__slice_edge_ctx_s.register_fields([('slice_ctb_pos_ctx', slice_edge_ctb_pos_ctx_s, 0), ('slice_stats_ctx', slice_edge_stats_ctx_s, 8), ('slice_tile_ctx', slice_edge_tile_ctx_s, 40), ('slice_vpc_edge_ctx', slice_vpc_edge_ctx_s, 52), ('slice_vpc_main_ctx', slice_vpc_main_ctx_s, 56)]) +slice_edge_ctx_s: TypeAlias = struct__slice_edge_ctx_s +@c.record +class struct__nvdec_vp9_pic_v1_s(c.Struct): + SIZE = 8 + Vp9FltAboveOffset: int + external_ref_mem_dis: int + bit_depth: int + error_recovery_start_pos: int + error_external_mv_en: int + Reserved0: int +struct__nvdec_vp9_pic_v1_s.register_fields([('Vp9FltAboveOffset', ctypes.c_uint32, 0), ('external_ref_mem_dis', ctypes.c_uint32, 4, 1, 0), ('bit_depth', ctypes.c_uint32, 4, 4, 1), ('error_recovery_start_pos', ctypes.c_uint32, 4, 2, 5), ('error_external_mv_en', ctypes.c_uint32, 4, 1, 7), ('Reserved0', ctypes.c_uint32, 5, 24, 0)]) +nvdec_vp9_pic_v1_s: TypeAlias = struct__nvdec_vp9_pic_v1_s +enum_VP9_FRAME_SFC_ID: dict[int, str] = {(VP9_LAST_FRAME_SFC:=0): 'VP9_LAST_FRAME_SFC', (VP9_GOLDEN_FRAME_SFC:=1): 'VP9_GOLDEN_FRAME_SFC', (VP9_ALTREF_FRAME_SFC:=2): 'VP9_ALTREF_FRAME_SFC', (VP9_CURR_FRAME_SFC:=3): 'VP9_CURR_FRAME_SFC'} +@c.record +class struct__nvdec_vp9_pic_s(c.Struct): + SIZE = 256 + wrapped_session_key: c.Array[ctypes.c_uint32, Literal[4]] + wrapped_content_key: c.Array[ctypes.c_uint32, Literal[4]] + initialization_vector: c.Array[ctypes.c_uint32, Literal[4]] + stream_len: int + enable_encryption: int + key_increment: int + encryption_mode: int + sw_hdr_skip_length: int + key_slot_index: int + ssm_en: int + enable_histogram: int + reserved0: int + gptimer_timeout_value: int + tileformat: int + gob_height: int + reserverd_surface_format: int + reserved1: c.Array[ctypes.c_ubyte, Literal[3]] + Vp9BsdCtrlOffset: int + ref0_width: int + ref0_height: int + ref0_stride: c.Array[ctypes.c_uint16, Literal[2]] + ref1_width: int + ref1_height: int + ref1_stride: c.Array[ctypes.c_uint16, Literal[2]] + ref2_width: int + ref2_height: int + ref2_stride: c.Array[ctypes.c_uint16, Literal[2]] + width: int + height: int + framestride: c.Array[ctypes.c_uint16, Literal[2]] + keyFrame: int + prevIsKeyFrame: int + resolutionChange: int + errorResilient: int + prevShowFrame: int + intraOnly: int + reserved2: int + reserved3: c.Array[ctypes.c_ubyte, Literal[3]] + refFrameSignBias: c.Array[ctypes.c_ubyte, Literal[4]] + loopFilterLevel: bytes + loopFilterSharpness: bytes + qpYAc: int + qpYDc: bytes + qpChAc: bytes + qpChDc: bytes + lossless: bytes + transform_mode: bytes + allow_high_precision_mv: bytes + mcomp_filter_type: bytes + comp_pred_mode: bytes + comp_fixed_ref: bytes + comp_var_ref: c.Array[ctypes.c_char, Literal[2]] + log2_tile_columns: bytes + log2_tile_rows: bytes + segmentEnabled: int + segmentMapUpdate: int + segmentMapTemporalUpdate: int + segmentFeatureMode: int + segmentFeatureEnable: c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[8]] + segmentFeatureData: c.Array[c.Array[ctypes.c_int16, Literal[4]], Literal[8]] + modeRefLfEnabled: bytes + mbRefLfDelta: c.Array[ctypes.c_char, Literal[4]] + mbModeLfDelta: c.Array[ctypes.c_char, Literal[2]] + reserved5: bytes + v1: struct__nvdec_vp9_pic_v1_s + ssm: struct__nvdec_pass2_otf_ext_s +struct__nvdec_vp9_pic_s.register_fields([('wrapped_session_key', c.Array[ctypes.c_uint32, Literal[4]], 0), ('wrapped_content_key', c.Array[ctypes.c_uint32, Literal[4]], 16), ('initialization_vector', c.Array[ctypes.c_uint32, Literal[4]], 32), ('stream_len', ctypes.c_uint32, 48), ('enable_encryption', ctypes.c_uint32, 52), ('key_increment', ctypes.c_uint32, 56, 6, 0), ('encryption_mode', ctypes.c_uint32, 56, 4, 6), ('sw_hdr_skip_length', ctypes.c_uint32, 57, 14, 2), ('key_slot_index', ctypes.c_uint32, 59, 4, 0), ('ssm_en', ctypes.c_uint32, 59, 1, 4), ('enable_histogram', ctypes.c_uint32, 59, 1, 5), ('reserved0', ctypes.c_uint32, 59, 2, 6), ('gptimer_timeout_value', ctypes.c_uint32, 60), ('tileformat', ctypes.c_ubyte, 64, 2, 0), ('gob_height', ctypes.c_ubyte, 64, 3, 2), ('reserverd_surface_format', ctypes.c_ubyte, 64, 3, 5), ('reserved1', c.Array[ctypes.c_ubyte, Literal[3]], 65), ('Vp9BsdCtrlOffset', ctypes.c_uint32, 68), ('ref0_width', ctypes.c_uint16, 72), ('ref0_height', ctypes.c_uint16, 74), ('ref0_stride', c.Array[ctypes.c_uint16, Literal[2]], 76), ('ref1_width', ctypes.c_uint16, 80), ('ref1_height', ctypes.c_uint16, 82), ('ref1_stride', c.Array[ctypes.c_uint16, Literal[2]], 84), ('ref2_width', ctypes.c_uint16, 88), ('ref2_height', ctypes.c_uint16, 90), ('ref2_stride', c.Array[ctypes.c_uint16, Literal[2]], 92), ('width', ctypes.c_uint16, 96), ('height', ctypes.c_uint16, 98), ('framestride', c.Array[ctypes.c_uint16, Literal[2]], 100), ('keyFrame', ctypes.c_ubyte, 104, 1, 0), ('prevIsKeyFrame', ctypes.c_ubyte, 104, 1, 1), ('resolutionChange', ctypes.c_ubyte, 104, 1, 2), ('errorResilient', ctypes.c_ubyte, 104, 1, 3), ('prevShowFrame', ctypes.c_ubyte, 104, 1, 4), ('intraOnly', ctypes.c_ubyte, 104, 1, 5), ('reserved2', ctypes.c_ubyte, 104, 2, 6), ('reserved3', c.Array[ctypes.c_ubyte, Literal[3]], 105), ('refFrameSignBias', c.Array[ctypes.c_ubyte, Literal[4]], 108), ('loopFilterLevel', ctypes.c_char, 112), ('loopFilterSharpness', ctypes.c_char, 113), ('qpYAc', ctypes.c_ubyte, 114), ('qpYDc', ctypes.c_char, 115), ('qpChAc', ctypes.c_char, 116), ('qpChDc', ctypes.c_char, 117), ('lossless', ctypes.c_char, 118), ('transform_mode', ctypes.c_char, 119), ('allow_high_precision_mv', ctypes.c_char, 120), ('mcomp_filter_type', ctypes.c_char, 121), ('comp_pred_mode', ctypes.c_char, 122), ('comp_fixed_ref', ctypes.c_char, 123), ('comp_var_ref', c.Array[ctypes.c_char, Literal[2]], 124), ('log2_tile_columns', ctypes.c_char, 126), ('log2_tile_rows', ctypes.c_char, 127), ('segmentEnabled', ctypes.c_ubyte, 128), ('segmentMapUpdate', ctypes.c_ubyte, 129), ('segmentMapTemporalUpdate', ctypes.c_ubyte, 130), ('segmentFeatureMode', ctypes.c_ubyte, 131), ('segmentFeatureEnable', c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[8]], 132), ('segmentFeatureData', c.Array[c.Array[ctypes.c_int16, Literal[4]], Literal[8]], 164), ('modeRefLfEnabled', ctypes.c_char, 228), ('mbRefLfDelta', c.Array[ctypes.c_char, Literal[4]], 229), ('mbModeLfDelta', c.Array[ctypes.c_char, Literal[2]], 233), ('reserved5', ctypes.c_char, 235), ('v1', nvdec_vp9_pic_v1_s, 236), ('ssm', nvdec_pass2_otf_ext_s, 244)]) +nvdec_vp9_pic_s: TypeAlias = struct__nvdec_vp9_pic_s +@c.record +class nvdec_nmv_context(c.Struct): + SIZE = 69 + joints: c.Array[ctypes.c_ubyte, Literal[3]] + sign: c.Array[ctypes.c_ubyte, Literal[2]] + class0: c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[2]] + fp: c.Array[c.Array[ctypes.c_ubyte, Literal[3]], Literal[2]] + class0_hp: c.Array[ctypes.c_ubyte, Literal[2]] + hp: c.Array[ctypes.c_ubyte, Literal[2]] + classes: c.Array[c.Array[ctypes.c_ubyte, Literal[10]], Literal[2]] + class0_fp: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[3]], Literal[2]], Literal[2]] + bits: c.Array[c.Array[ctypes.c_ubyte, Literal[10]], Literal[2]] +nvdec_nmv_context.register_fields([('joints', c.Array[ctypes.c_ubyte, Literal[3]], 0), ('sign', c.Array[ctypes.c_ubyte, Literal[2]], 3), ('class0', c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[2]], 5), ('fp', c.Array[c.Array[ctypes.c_ubyte, Literal[3]], Literal[2]], 7), ('class0_hp', c.Array[ctypes.c_ubyte, Literal[2]], 13), ('hp', c.Array[ctypes.c_ubyte, Literal[2]], 15), ('classes', c.Array[c.Array[ctypes.c_ubyte, Literal[10]], Literal[2]], 17), ('class0_fp', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[3]], Literal[2]], Literal[2]], 37), ('bits', c.Array[c.Array[ctypes.c_ubyte, Literal[10]], Literal[2]], 49)]) +@c.record +class nvdec_nmv_context_counts(c.Struct): + SIZE = 424 + joints: c.Array[ctypes.c_uint32, Literal[4]] + sign: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]] + classes: c.Array[c.Array[ctypes.c_uint32, Literal[11]], Literal[2]] + class0: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]] + bits: c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[10]], Literal[2]] + class0_fp: c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[2]], Literal[2]] + fp: c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[2]] + class0_hp: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]] + hp: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]] +nvdec_nmv_context_counts.register_fields([('joints', c.Array[ctypes.c_uint32, Literal[4]], 0), ('sign', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]], 16), ('classes', c.Array[c.Array[ctypes.c_uint32, Literal[11]], Literal[2]], 32), ('class0', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]], 120), ('bits', c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[10]], Literal[2]], 136), ('class0_fp', c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[2]], Literal[2]], 296), ('fp', c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[2]], 360), ('class0_hp', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]], 392), ('hp', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]], 408)]) +@c.record +class struct_nvdec_vp9AdaptiveEntropyProbs_s(c.Struct): + SIZE = 2720 + inter_mode_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[7]] + intra_inter_prob: c.Array[ctypes.c_ubyte, Literal[4]] + uv_mode_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[10]] + tx8x8_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[2]] + tx16x16_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[2]], Literal[2]] + tx32x32_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[3]], Literal[2]] + sb_ymode_probB: c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[4]] + sb_ymode_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[4]] + partition_prob: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[16]], Literal[2]] + uv_mode_probB: c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[10]] + switchable_interp_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[2]], Literal[4]] + comp_inter_prob: c.Array[ctypes.c_ubyte, Literal[5]] + mbskip_probs: c.Array[ctypes.c_ubyte, Literal[3]] + pad1: c.Array[ctypes.c_ubyte, Literal[1]] + nmvc: nvdec_nmv_context + single_ref_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[2]], Literal[5]] + comp_ref_prob: c.Array[ctypes.c_ubyte, Literal[5]] + pad2: c.Array[ctypes.c_ubyte, Literal[17]] + probCoeffs: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] + probCoeffs8x8: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] + probCoeffs16x16: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] + probCoeffs32x32: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] +struct_nvdec_vp9AdaptiveEntropyProbs_s.register_fields([('inter_mode_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[7]], 0), ('intra_inter_prob', c.Array[ctypes.c_ubyte, Literal[4]], 28), ('uv_mode_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[10]], 32), ('tx8x8_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[2]], 112), ('tx16x16_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[2]], Literal[2]], 114), ('tx32x32_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[3]], Literal[2]], 118), ('sb_ymode_probB', c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[4]], 124), ('sb_ymode_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[4]], 128), ('partition_prob', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[16]], Literal[2]], 160), ('uv_mode_probB', c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[10]], 288), ('switchable_interp_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[2]], Literal[4]], 298), ('comp_inter_prob', c.Array[ctypes.c_ubyte, Literal[5]], 306), ('mbskip_probs', c.Array[ctypes.c_ubyte, Literal[3]], 311), ('pad1', c.Array[ctypes.c_ubyte, Literal[1]], 314), ('nmvc', nvdec_nmv_context, 315), ('single_ref_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[2]], Literal[5]], 384), ('comp_ref_prob', c.Array[ctypes.c_ubyte, Literal[5]], 394), ('pad2', c.Array[ctypes.c_ubyte, Literal[17]], 399), ('probCoeffs', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 416), ('probCoeffs8x8', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 992), ('probCoeffs16x16', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 1568), ('probCoeffs32x32', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 2144)]) +nvdec_vp9AdaptiveEntropyProbs_t: TypeAlias = struct_nvdec_vp9AdaptiveEntropyProbs_s +@c.record +class struct_nvdec_vp9EntropyProbs_s(c.Struct): + SIZE = 3744 + kf_bmode_prob: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[10]], Literal[10]] + kf_bmode_probB: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[10]], Literal[10]] + ref_pred_probs: c.Array[ctypes.c_ubyte, Literal[3]] + mb_segment_tree_probs: c.Array[ctypes.c_ubyte, Literal[7]] + segment_pred_probs: c.Array[ctypes.c_ubyte, Literal[3]] + ref_scores: c.Array[ctypes.c_ubyte, Literal[4]] + prob_comppred: c.Array[ctypes.c_ubyte, Literal[2]] + pad1: c.Array[ctypes.c_ubyte, Literal[9]] + kf_uv_mode_prob: c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[10]] + kf_uv_mode_probB: c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[10]] + pad2: c.Array[ctypes.c_ubyte, Literal[6]] + a: struct_nvdec_vp9AdaptiveEntropyProbs_s +struct_nvdec_vp9EntropyProbs_s.register_fields([('kf_bmode_prob', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[10]], Literal[10]], 0), ('kf_bmode_probB', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[10]], Literal[10]], 800), ('ref_pred_probs', c.Array[ctypes.c_ubyte, Literal[3]], 900), ('mb_segment_tree_probs', c.Array[ctypes.c_ubyte, Literal[7]], 903), ('segment_pred_probs', c.Array[ctypes.c_ubyte, Literal[3]], 910), ('ref_scores', c.Array[ctypes.c_ubyte, Literal[4]], 913), ('prob_comppred', c.Array[ctypes.c_ubyte, Literal[2]], 917), ('pad1', c.Array[ctypes.c_ubyte, Literal[9]], 919), ('kf_uv_mode_prob', c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[10]], 928), ('kf_uv_mode_probB', c.Array[c.Array[ctypes.c_ubyte, Literal[1]], Literal[10]], 1008), ('pad2', c.Array[ctypes.c_ubyte, Literal[6]], 1018), ('a', nvdec_vp9AdaptiveEntropyProbs_t, 1024)]) +nvdec_vp9EntropyProbs_t: TypeAlias = struct_nvdec_vp9EntropyProbs_s +@c.record +class struct_nvdec_vp9EntropyCounts_s(c.Struct): + SIZE = 13264 + inter_mode_counts: c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[3]], Literal[7]] + sb_ymode_counts: c.Array[c.Array[ctypes.c_uint32, Literal[10]], Literal[4]] + uv_mode_counts: c.Array[c.Array[ctypes.c_uint32, Literal[10]], Literal[10]] + partition_counts: c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[16]] + switchable_interp_counts: c.Array[c.Array[ctypes.c_uint32, Literal[3]], Literal[4]] + intra_inter_count: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[4]] + comp_inter_count: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[5]] + single_ref_count: c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]], Literal[5]] + comp_ref_count: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[5]] + tx32x32_count: c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[2]] + tx16x16_count: c.Array[c.Array[ctypes.c_uint32, Literal[3]], Literal[2]] + tx8x8_count: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]] + mbskip_count: c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[3]] + nmvcount: nvdec_nmv_context_counts + countCoeffs: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] + countCoeffs8x8: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] + countCoeffs16x16: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] + countCoeffs32x32: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]] + countEobs: c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[6]], Literal[6]], Literal[2]], Literal[2]], Literal[4]] +struct_nvdec_vp9EntropyCounts_s.register_fields([('inter_mode_counts', c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[3]], Literal[7]], 0), ('sb_ymode_counts', c.Array[c.Array[ctypes.c_uint32, Literal[10]], Literal[4]], 168), ('uv_mode_counts', c.Array[c.Array[ctypes.c_uint32, Literal[10]], Literal[10]], 328), ('partition_counts', c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[16]], 728), ('switchable_interp_counts', c.Array[c.Array[ctypes.c_uint32, Literal[3]], Literal[4]], 984), ('intra_inter_count', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[4]], 1032), ('comp_inter_count', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[5]], 1064), ('single_ref_count', c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]], Literal[5]], 1104), ('comp_ref_count', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[5]], 1184), ('tx32x32_count', c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[2]], 1224), ('tx16x16_count', c.Array[c.Array[ctypes.c_uint32, Literal[3]], Literal[2]], 1256), ('tx8x8_count', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[2]], 1280), ('mbskip_count', c.Array[c.Array[ctypes.c_uint32, Literal[2]], Literal[3]], 1296), ('nmvcount', nvdec_nmv_context_counts, 1320), ('countCoeffs', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 1744), ('countCoeffs8x8', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 4048), ('countCoeffs16x16', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 6352), ('countCoeffs32x32', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[6]], Literal[6]], Literal[2]], Literal[2]], 8656), ('countEobs', c.Array[c.Array[c.Array[c.Array[c.Array[ctypes.c_uint32, Literal[6]], Literal[6]], Literal[2]], Literal[2]], Literal[4]], 10960)]) +nvdec_vp9EntropyCounts_t: TypeAlias = struct_nvdec_vp9EntropyCounts_s +@c.record +class struct__nvdec_pass2_otf_s(c.Struct): + SIZE = 52 + wrapped_session_key: c.Array[ctypes.c_uint32, Literal[4]] + wrapped_content_key: c.Array[ctypes.c_uint32, Literal[4]] + initialization_vector: c.Array[ctypes.c_uint32, Literal[4]] + enable_encryption: int + key_increment: int + encryption_mode: int + key_slot_index: int + ssm_en: int + reserved1: int +struct__nvdec_pass2_otf_s.register_fields([('wrapped_session_key', c.Array[ctypes.c_uint32, Literal[4]], 0), ('wrapped_content_key', c.Array[ctypes.c_uint32, Literal[4]], 16), ('initialization_vector', c.Array[ctypes.c_uint32, Literal[4]], 32), ('enable_encryption', ctypes.c_uint32, 48, 1, 0), ('key_increment', ctypes.c_uint32, 48, 6, 1), ('encryption_mode', ctypes.c_uint32, 48, 4, 7), ('key_slot_index', ctypes.c_uint32, 49, 4, 3), ('ssm_en', ctypes.c_uint32, 49, 1, 7), ('reserved1', ctypes.c_uint32, 50, 16, 0)]) +nvdec_pass2_otf_s: TypeAlias = struct__nvdec_pass2_otf_s +@c.record +class struct__nvdec_display_param_s(c.Struct): + SIZE = 28 + enableTFOutput: int + VC1MapYFlag: int + MapYValue: int + VC1MapUVFlag: int + MapUVValue: int + OutStride: int + TilingFormat: int + OutputStructure: int + reserved0: int + OutputTop: c.Array[ctypes.c_int32, Literal[2]] + OutputBottom: c.Array[ctypes.c_int32, Literal[2]] + enableHistogram: int + HistogramStartX: int + HistogramStartY: int + reserved1: int + HistogramEndX: int + HistogramEndY: int + reserved2: int +struct__nvdec_display_param_s.register_fields([('enableTFOutput', ctypes.c_uint32, 0, 1, 0), ('VC1MapYFlag', ctypes.c_uint32, 0, 1, 1), ('MapYValue', ctypes.c_uint32, 0, 3, 2), ('VC1MapUVFlag', ctypes.c_uint32, 0, 1, 5), ('MapUVValue', ctypes.c_uint32, 0, 3, 6), ('OutStride', ctypes.c_uint32, 1, 8, 1), ('TilingFormat', ctypes.c_uint32, 2, 3, 1), ('OutputStructure', ctypes.c_uint32, 2, 1, 4), ('reserved0', ctypes.c_uint32, 2, 11, 5), ('OutputTop', c.Array[ctypes.c_int32, Literal[2]], 4), ('OutputBottom', c.Array[ctypes.c_int32, Literal[2]], 12), ('enableHistogram', ctypes.c_uint32, 20, 1, 0), ('HistogramStartX', ctypes.c_uint32, 20, 12, 1), ('HistogramStartY', ctypes.c_uint32, 21, 12, 5), ('reserved1', ctypes.c_uint32, 23, 7, 1), ('HistogramEndX', ctypes.c_uint32, 24, 12, 0), ('HistogramEndY', ctypes.c_uint32, 25, 12, 4), ('reserved2', ctypes.c_uint32, 27, 8, 0)]) +nvdec_display_param_s: TypeAlias = struct__nvdec_display_param_s +@c.record +class struct__nvdec_dpb_entry_s(c.Struct): + SIZE = 16 + index: int + col_idx: int + state: int + is_long_term: int + not_existing: int + is_field: int + top_field_marking: int + bottom_field_marking: int + output_memory_layout: int + reserved: int + FieldOrderCnt: c.Array[ctypes.c_uint32, Literal[2]] + FrameIdx: int +struct__nvdec_dpb_entry_s.register_fields([('index', ctypes.c_uint32, 0, 7, 0), ('col_idx', ctypes.c_uint32, 0, 5, 7), ('state', ctypes.c_uint32, 1, 2, 4), ('is_long_term', ctypes.c_uint32, 1, 1, 6), ('not_existing', ctypes.c_uint32, 1, 1, 7), ('is_field', ctypes.c_uint32, 2, 1, 0), ('top_field_marking', ctypes.c_uint32, 2, 4, 1), ('bottom_field_marking', ctypes.c_uint32, 2, 4, 5), ('output_memory_layout', ctypes.c_uint32, 3, 1, 1), ('reserved', ctypes.c_uint32, 3, 6, 2), ('FieldOrderCnt', c.Array[ctypes.c_uint32, Literal[2]], 4), ('FrameIdx', ctypes.c_int32, 12)]) +nvdec_dpb_entry_s: TypeAlias = struct__nvdec_dpb_entry_s +@c.record +class struct__nvdec_h264_pic_s(c.Struct): + SIZE = 764 + encryption_params: struct__nvdec_pass2_otf_s + eos: c.Array[ctypes.c_ubyte, Literal[16]] + explicitEOSPresentFlag: int + hint_dump_en: int + reserved0: c.Array[ctypes.c_ubyte, Literal[2]] + stream_len: int + slice_count: int + mbhist_buffer_size: int + gptimer_timeout_value: int + log2_max_pic_order_cnt_lsb_minus4: int + delta_pic_order_always_zero_flag: int + frame_mbs_only_flag: int + PicWidthInMbs: int + FrameHeightInMbs: int + tileFormat: int + gob_height: int + reserverd_surface_format: int + entropy_coding_mode_flag: int + pic_order_present_flag: int + num_ref_idx_l0_active_minus1: int + num_ref_idx_l1_active_minus1: int + deblocking_filter_control_present_flag: int + redundant_pic_cnt_present_flag: int + transform_8x8_mode_flag: int + pitch_luma: int + pitch_chroma: int + luma_top_offset: int + luma_bot_offset: int + luma_frame_offset: int + chroma_top_offset: int + chroma_bot_offset: int + chroma_frame_offset: int + HistBufferSize: int + MbaffFrameFlag: int + direct_8x8_inference_flag: int + weighted_pred_flag: int + constrained_intra_pred_flag: int + ref_pic_flag: int + field_pic_flag: int + bottom_field_flag: int + second_field: int + log2_max_frame_num_minus4: int + chroma_format_idc: int + pic_order_cnt_type: int + pic_init_qp_minus26: int + chroma_qp_index_offset: int + second_chroma_qp_index_offset: int + weighted_bipred_idc: int + CurrPicIdx: int + CurrColIdx: int + frame_num: int + frame_surfaces: int + output_memory_layout: int + CurrFieldOrderCnt: c.Array[ctypes.c_int32, Literal[2]] + dpb: c.Array[struct__nvdec_dpb_entry_s, Literal[16]] + WeightScale: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[4]], Literal[6]] + WeightScale8x8: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[8]], Literal[2]] + num_inter_view_refs_lX: c.Array[ctypes.c_ubyte, Literal[2]] + reserved1: c.Array[ctypes.c_char, Literal[14]] + inter_view_refidx_lX: c.Array[c.Array[ctypes.c_byte, Literal[16]], Literal[2]] + lossless_ipred8x8_filter_enable: int + qpprime_y_zero_transform_bypass_flag: int + reserved2: int + displayPara: struct__nvdec_display_param_s + ssm: struct__nvdec_pass2_otf_ext_s +struct__nvdec_h264_pic_s.register_fields([('encryption_params', nvdec_pass2_otf_s, 0), ('eos', c.Array[ctypes.c_ubyte, Literal[16]], 52), ('explicitEOSPresentFlag', ctypes.c_ubyte, 68), ('hint_dump_en', ctypes.c_ubyte, 69), ('reserved0', c.Array[ctypes.c_ubyte, Literal[2]], 70), ('stream_len', ctypes.c_uint32, 72), ('slice_count', ctypes.c_uint32, 76), ('mbhist_buffer_size', ctypes.c_uint32, 80), ('gptimer_timeout_value', ctypes.c_uint32, 84), ('log2_max_pic_order_cnt_lsb_minus4', ctypes.c_int32, 88), ('delta_pic_order_always_zero_flag', ctypes.c_int32, 92), ('frame_mbs_only_flag', ctypes.c_int32, 96), ('PicWidthInMbs', ctypes.c_int32, 100), ('FrameHeightInMbs', ctypes.c_int32, 104), ('tileFormat', ctypes.c_uint32, 108, 2, 0), ('gob_height', ctypes.c_uint32, 108, 3, 2), ('reserverd_surface_format', ctypes.c_uint32, 108, 27, 5), ('entropy_coding_mode_flag', ctypes.c_int32, 112), ('pic_order_present_flag', ctypes.c_int32, 116), ('num_ref_idx_l0_active_minus1', ctypes.c_int32, 120), ('num_ref_idx_l1_active_minus1', ctypes.c_int32, 124), ('deblocking_filter_control_present_flag', ctypes.c_int32, 128), ('redundant_pic_cnt_present_flag', ctypes.c_int32, 132), ('transform_8x8_mode_flag', ctypes.c_int32, 136), ('pitch_luma', ctypes.c_uint32, 140), ('pitch_chroma', ctypes.c_uint32, 144), ('luma_top_offset', ctypes.c_uint32, 148), ('luma_bot_offset', ctypes.c_uint32, 152), ('luma_frame_offset', ctypes.c_uint32, 156), ('chroma_top_offset', ctypes.c_uint32, 160), ('chroma_bot_offset', ctypes.c_uint32, 164), ('chroma_frame_offset', ctypes.c_uint32, 168), ('HistBufferSize', ctypes.c_uint32, 172), ('MbaffFrameFlag', ctypes.c_uint32, 176, 1, 0), ('direct_8x8_inference_flag', ctypes.c_uint32, 176, 1, 1), ('weighted_pred_flag', ctypes.c_uint32, 176, 1, 2), ('constrained_intra_pred_flag', ctypes.c_uint32, 176, 1, 3), ('ref_pic_flag', ctypes.c_uint32, 176, 1, 4), ('field_pic_flag', ctypes.c_uint32, 176, 1, 5), ('bottom_field_flag', ctypes.c_uint32, 176, 1, 6), ('second_field', ctypes.c_uint32, 176, 1, 7), ('log2_max_frame_num_minus4', ctypes.c_uint32, 177, 4, 0), ('chroma_format_idc', ctypes.c_uint32, 177, 2, 4), ('pic_order_cnt_type', ctypes.c_uint32, 177, 2, 6), ('pic_init_qp_minus26', ctypes.c_int32, 178, 6, 0), ('chroma_qp_index_offset', ctypes.c_int32, 178, 5, 6), ('second_chroma_qp_index_offset', ctypes.c_int32, 179, 5, 3), ('weighted_bipred_idc', ctypes.c_uint32, 180, 2, 0), ('CurrPicIdx', ctypes.c_uint32, 180, 7, 2), ('CurrColIdx', ctypes.c_uint32, 181, 5, 1), ('frame_num', ctypes.c_uint32, 181, 16, 6), ('frame_surfaces', ctypes.c_uint32, 183, 1, 6), ('output_memory_layout', ctypes.c_uint32, 183, 1, 7), ('CurrFieldOrderCnt', c.Array[ctypes.c_int32, Literal[2]], 184), ('dpb', c.Array[nvdec_dpb_entry_s, Literal[16]], 192), ('WeightScale', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[4]], Literal[6]], 448), ('WeightScale8x8', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[8]], Literal[2]], 544), ('num_inter_view_refs_lX', c.Array[ctypes.c_ubyte, Literal[2]], 672), ('reserved1', c.Array[ctypes.c_char, Literal[14]], 674), ('inter_view_refidx_lX', c.Array[c.Array[ctypes.c_byte, Literal[16]], Literal[2]], 688), ('lossless_ipred8x8_filter_enable', ctypes.c_uint32, 720, 1, 0), ('qpprime_y_zero_transform_bypass_flag', ctypes.c_uint32, 720, 1, 1), ('reserved2', ctypes.c_uint32, 720, 30, 2), ('displayPara', nvdec_display_param_s, 724), ('ssm', nvdec_pass2_otf_ext_s, 752)]) +nvdec_h264_pic_s: TypeAlias = struct__nvdec_h264_pic_s +enum__vc1_fcm_e: dict[int, str] = {(FCM_PROGRESSIVE:=0): 'FCM_PROGRESSIVE', (FCM_FRAME_INTERLACE:=2): 'FCM_FRAME_INTERLACE', (FCM_FIELD_INTERLACE:=3): 'FCM_FIELD_INTERLACE'} +vc1_fcm_e: TypeAlias = ctypes.c_uint32 +enum__syntax_vc1_ptype_e: dict[int, str] = {(PTYPE_I:=0): 'PTYPE_I', (PTYPE_P:=1): 'PTYPE_P', (PTYPE_B:=2): 'PTYPE_B', (PTYPE_BI:=3): 'PTYPE_BI', (PTYPE_SKIPPED:=4): 'PTYPE_SKIPPED'} +syntax_vc1_ptype_e: TypeAlias = ctypes.c_uint32 +enum_vc1_mvmode_e: dict[int, str] = {(MVMODE_MIXEDMV:=0): 'MVMODE_MIXEDMV', (MVMODE_1MV:=1): 'MVMODE_1MV', (MVMODE_1MV_HALFPEL:=2): 'MVMODE_1MV_HALFPEL', (MVMODE_1MV_HALFPEL_BILINEAR:=3): 'MVMODE_1MV_HALFPEL_BILINEAR', (MVMODE_INTENSITY_COMPENSATION:=4): 'MVMODE_INTENSITY_COMPENSATION'} +enum__vc1_fptype_e: dict[int, str] = {(FPTYPE_I_I:=0): 'FPTYPE_I_I', (FPTYPE_I_P:=1): 'FPTYPE_I_P', (FPTYPE_P_I:=2): 'FPTYPE_P_I', (FPTYPE_P_P:=3): 'FPTYPE_P_P', (FPTYPE_B_B:=4): 'FPTYPE_B_B', (FPTYPE_B_BI:=5): 'FPTYPE_B_BI', (FPTYPE_BI_B:=6): 'FPTYPE_BI_B', (FPTYPE_BI_BI:=7): 'FPTYPE_BI_BI'} +vc1_fptype_e: TypeAlias = ctypes.c_uint32 +enum__vc1_dqprofile_e: dict[int, str] = {(DQPROFILE_ALL_FOUR_EDGES:=0): 'DQPROFILE_ALL_FOUR_EDGES', (DQPROFILE_DOUBLE_EDGE:=1): 'DQPROFILE_DOUBLE_EDGE', (DQPROFILE_SINGLE_EDGE:=2): 'DQPROFILE_SINGLE_EDGE', (DQPROFILE_ALL_MACROBLOCKS:=3): 'DQPROFILE_ALL_MACROBLOCKS'} +vc1_dqprofile_e: TypeAlias = ctypes.c_uint32 +@c.record +class struct__nvdec_vc1_pic_s(c.Struct): + SIZE = 404 + encryption_params: struct__nvdec_pass2_otf_s + eos: c.Array[ctypes.c_ubyte, Literal[16]] + prefixStartCode: c.Array[ctypes.c_ubyte, Literal[4]] + bitstream_offset: int + explicitEOSPresentFlag: int + reserved0: c.Array[ctypes.c_ubyte, Literal[3]] + stream_len: int + slice_count: int + scratch_pic_buffer_size: int + gptimer_timeout_value: int + FrameWidth: int + FrameHeight: int + profile: int + postprocflag: int + pulldown: int + interlace: int + tfcntrflag: int + finterpflag: int + psf: int + tileFormat: int + gob_height: int + reserverd_surface_format: int + multires: int + syncmarker: int + rangered: int + maxbframes: int + dquant: int + panscan_flag: int + refdist_flag: int + quantizer: int + extended_mv: int + extended_dmv: int + overlap: int + vstransform: int + refdist: bytes + reserved1: c.Array[ctypes.c_char, Literal[3]] + fcm: int + ptype: int + tfcntr: int + rptfrm: int + tff: int + rndctrl: int + pqindex: int + halfqp: int + pquantizer: int + postproc: int + condover: int + transacfrm: int + transacfrm2: int + transdctab: int + pqdiff: int + abspq: int + dquantfrm: int + dqprofile: int + dqsbedge: int + dqdbedge: int + dqbilevel: int + mvrange: int + mvmode: int + mvmode2: int + lumscale: int + lumshift: int + mvtab: int + cbptab: int + ttmbf: int + ttfrm: int + bfraction: int + fptype: int + numref: int + reffield: int + dmvrange: int + intcompfield: int + lumscale1: int + lumshift1: int + lumscale2: int + lumshift2: int + mbmodetab: int + imvtab: int + icbptab: int + fourmvbptab: int + fourmvswitch: int + intcomp: int + twomvbptab: int + rangeredfrm: int + HistBufferSize: int + FrameStride: c.Array[ctypes.c_uint32, Literal[2]] + luma_top_offset: int + luma_bot_offset: int + luma_frame_offset: int + chroma_top_offset: int + chroma_bot_offset: int + chroma_frame_offset: int + CodedWidth: int + CodedHeight: int + loopfilter: int + fastuvmc: int + output_memory_layout: int + ref_memory_layout: c.Array[ctypes.c_ubyte, Literal[2]] + reserved3: c.Array[ctypes.c_ubyte, Literal[3]] + displayPara: struct__nvdec_display_param_s + ssm: struct__nvdec_pass2_otf_ext_s +struct__nvdec_vc1_pic_s.register_fields([('encryption_params', nvdec_pass2_otf_s, 0), ('eos', c.Array[ctypes.c_ubyte, Literal[16]], 52), ('prefixStartCode', c.Array[ctypes.c_ubyte, Literal[4]], 68), ('bitstream_offset', ctypes.c_uint32, 72), ('explicitEOSPresentFlag', ctypes.c_ubyte, 76), ('reserved0', c.Array[ctypes.c_ubyte, Literal[3]], 77), ('stream_len', ctypes.c_uint32, 80), ('slice_count', ctypes.c_uint32, 84), ('scratch_pic_buffer_size', ctypes.c_uint32, 88), ('gptimer_timeout_value', ctypes.c_uint32, 92), ('FrameWidth', ctypes.c_uint16, 96), ('FrameHeight', ctypes.c_uint16, 98), ('profile', ctypes.c_ubyte, 100), ('postprocflag', ctypes.c_ubyte, 101), ('pulldown', ctypes.c_ubyte, 102), ('interlace', ctypes.c_ubyte, 103), ('tfcntrflag', ctypes.c_ubyte, 104), ('finterpflag', ctypes.c_ubyte, 105), ('psf', ctypes.c_ubyte, 106), ('tileFormat', ctypes.c_ubyte, 107, 2, 0), ('gob_height', ctypes.c_ubyte, 107, 3, 2), ('reserverd_surface_format', ctypes.c_ubyte, 107, 3, 5), ('multires', ctypes.c_ubyte, 108), ('syncmarker', ctypes.c_ubyte, 109), ('rangered', ctypes.c_ubyte, 110), ('maxbframes', ctypes.c_ubyte, 111), ('dquant', ctypes.c_ubyte, 112), ('panscan_flag', ctypes.c_ubyte, 113), ('refdist_flag', ctypes.c_ubyte, 114), ('quantizer', ctypes.c_ubyte, 115), ('extended_mv', ctypes.c_ubyte, 116), ('extended_dmv', ctypes.c_ubyte, 117), ('overlap', ctypes.c_ubyte, 118), ('vstransform', ctypes.c_ubyte, 119), ('refdist', ctypes.c_char, 120), ('reserved1', c.Array[ctypes.c_char, Literal[3]], 121), ('fcm', vc1_fcm_e, 124), ('ptype', syntax_vc1_ptype_e, 128), ('tfcntr', ctypes.c_int32, 132), ('rptfrm', ctypes.c_int32, 136), ('tff', ctypes.c_int32, 140), ('rndctrl', ctypes.c_int32, 144), ('pqindex', ctypes.c_int32, 148), ('halfqp', ctypes.c_int32, 152), ('pquantizer', ctypes.c_int32, 156), ('postproc', ctypes.c_int32, 160), ('condover', ctypes.c_int32, 164), ('transacfrm', ctypes.c_int32, 168), ('transacfrm2', ctypes.c_int32, 172), ('transdctab', ctypes.c_int32, 176), ('pqdiff', ctypes.c_int32, 180), ('abspq', ctypes.c_int32, 184), ('dquantfrm', ctypes.c_int32, 188), ('dqprofile', vc1_dqprofile_e, 192), ('dqsbedge', ctypes.c_int32, 196), ('dqdbedge', ctypes.c_int32, 200), ('dqbilevel', ctypes.c_int32, 204), ('mvrange', ctypes.c_int32, 208), ('mvmode', ctypes.c_uint32, 212), ('mvmode2', ctypes.c_uint32, 216), ('lumscale', ctypes.c_int32, 220), ('lumshift', ctypes.c_int32, 224), ('mvtab', ctypes.c_int32, 228), ('cbptab', ctypes.c_int32, 232), ('ttmbf', ctypes.c_int32, 236), ('ttfrm', ctypes.c_int32, 240), ('bfraction', ctypes.c_int32, 244), ('fptype', vc1_fptype_e, 248), ('numref', ctypes.c_int32, 252), ('reffield', ctypes.c_int32, 256), ('dmvrange', ctypes.c_int32, 260), ('intcompfield', ctypes.c_int32, 264), ('lumscale1', ctypes.c_int32, 268), ('lumshift1', ctypes.c_int32, 272), ('lumscale2', ctypes.c_int32, 276), ('lumshift2', ctypes.c_int32, 280), ('mbmodetab', ctypes.c_int32, 284), ('imvtab', ctypes.c_int32, 288), ('icbptab', ctypes.c_int32, 292), ('fourmvbptab', ctypes.c_int32, 296), ('fourmvswitch', ctypes.c_int32, 300), ('intcomp', ctypes.c_int32, 304), ('twomvbptab', ctypes.c_int32, 308), ('rangeredfrm', ctypes.c_int32, 312), ('HistBufferSize', ctypes.c_uint32, 316), ('FrameStride', c.Array[ctypes.c_uint32, Literal[2]], 320), ('luma_top_offset', ctypes.c_uint32, 328), ('luma_bot_offset', ctypes.c_uint32, 332), ('luma_frame_offset', ctypes.c_uint32, 336), ('chroma_top_offset', ctypes.c_uint32, 340), ('chroma_bot_offset', ctypes.c_uint32, 344), ('chroma_frame_offset', ctypes.c_uint32, 348), ('CodedWidth', ctypes.c_uint16, 352), ('CodedHeight', ctypes.c_uint16, 354), ('loopfilter', ctypes.c_ubyte, 356), ('fastuvmc', ctypes.c_ubyte, 357), ('output_memory_layout', ctypes.c_ubyte, 358), ('ref_memory_layout', c.Array[ctypes.c_ubyte, Literal[2]], 359), ('reserved3', c.Array[ctypes.c_ubyte, Literal[3]], 361), ('displayPara', nvdec_display_param_s, 364), ('ssm', nvdec_pass2_otf_ext_s, 392)]) +nvdec_vc1_pic_s: TypeAlias = struct__nvdec_vc1_pic_s +@c.record +class struct__nvdec_mpeg2_pic_s(c.Struct): + SIZE = 344 + encryption_params: struct__nvdec_pass2_otf_s + eos: c.Array[ctypes.c_ubyte, Literal[16]] + explicitEOSPresentFlag: int + reserved0: c.Array[ctypes.c_ubyte, Literal[3]] + stream_len: int + slice_count: int + gptimer_timeout_value: int + FrameWidth: int + FrameHeight: int + picture_structure: int + picture_coding_type: int + intra_dc_precision: int + frame_pred_frame_dct: bytes + concealment_motion_vectors: bytes + intra_vlc_format: bytes + tileFormat: int + gob_height: int + reserverd_surface_format: int + reserved1: bytes + f_code: c.Array[ctypes.c_char, Literal[4]] + PicWidthInMbs: int + FrameHeightInMbs: int + pitch_luma: int + pitch_chroma: int + luma_top_offset: int + luma_bot_offset: int + luma_frame_offset: int + chroma_top_offset: int + chroma_bot_offset: int + chroma_frame_offset: int + HistBufferSize: int + output_memory_layout: int + alternate_scan: int + secondfield: int + rounding_type: int + MbInfoSizeInBytes: int + q_scale_type: int + top_field_first: int + full_pel_fwd_vector: int + full_pel_bwd_vector: int + quant_mat_8x8intra: c.Array[ctypes.c_ubyte, Literal[64]] + quant_mat_8x8nonintra: c.Array[ctypes.c_ubyte, Literal[64]] + ref_memory_layout: c.Array[ctypes.c_uint32, Literal[2]] + displayPara: struct__nvdec_display_param_s + ssm: struct__nvdec_pass2_otf_ext_s +struct__nvdec_mpeg2_pic_s.register_fields([('encryption_params', nvdec_pass2_otf_s, 0), ('eos', c.Array[ctypes.c_ubyte, Literal[16]], 52), ('explicitEOSPresentFlag', ctypes.c_ubyte, 68), ('reserved0', c.Array[ctypes.c_ubyte, Literal[3]], 69), ('stream_len', ctypes.c_uint32, 72), ('slice_count', ctypes.c_uint32, 76), ('gptimer_timeout_value', ctypes.c_uint32, 80), ('FrameWidth', ctypes.c_int16, 84), ('FrameHeight', ctypes.c_int16, 86), ('picture_structure', ctypes.c_ubyte, 88), ('picture_coding_type', ctypes.c_ubyte, 89), ('intra_dc_precision', ctypes.c_ubyte, 90), ('frame_pred_frame_dct', ctypes.c_char, 91), ('concealment_motion_vectors', ctypes.c_char, 92), ('intra_vlc_format', ctypes.c_char, 93), ('tileFormat', ctypes.c_ubyte, 94, 2, 0), ('gob_height', ctypes.c_ubyte, 94, 3, 2), ('reserverd_surface_format', ctypes.c_ubyte, 94, 3, 5), ('reserved1', ctypes.c_char, 95), ('f_code', c.Array[ctypes.c_char, Literal[4]], 96), ('PicWidthInMbs', ctypes.c_uint16, 100), ('FrameHeightInMbs', ctypes.c_uint16, 102), ('pitch_luma', ctypes.c_uint32, 104), ('pitch_chroma', ctypes.c_uint32, 108), ('luma_top_offset', ctypes.c_uint32, 112), ('luma_bot_offset', ctypes.c_uint32, 116), ('luma_frame_offset', ctypes.c_uint32, 120), ('chroma_top_offset', ctypes.c_uint32, 124), ('chroma_bot_offset', ctypes.c_uint32, 128), ('chroma_frame_offset', ctypes.c_uint32, 132), ('HistBufferSize', ctypes.c_uint32, 136), ('output_memory_layout', ctypes.c_uint16, 140), ('alternate_scan', ctypes.c_uint16, 142), ('secondfield', ctypes.c_uint16, 144), ('rounding_type', ctypes.c_uint16, 146), ('MbInfoSizeInBytes', ctypes.c_uint32, 148), ('q_scale_type', ctypes.c_uint32, 152), ('top_field_first', ctypes.c_uint32, 156), ('full_pel_fwd_vector', ctypes.c_uint32, 160), ('full_pel_bwd_vector', ctypes.c_uint32, 164), ('quant_mat_8x8intra', c.Array[ctypes.c_ubyte, Literal[64]], 168), ('quant_mat_8x8nonintra', c.Array[ctypes.c_ubyte, Literal[64]], 232), ('ref_memory_layout', c.Array[ctypes.c_uint32, Literal[2]], 296), ('displayPara', nvdec_display_param_s, 304), ('ssm', nvdec_pass2_otf_ext_s, 332)]) +nvdec_mpeg2_pic_s: TypeAlias = struct__nvdec_mpeg2_pic_s +@c.record +class struct__nvdec_mpeg4_pic_s(c.Struct): + SIZE = 368 + encryption_params: struct__nvdec_pass2_otf_s + eos: c.Array[ctypes.c_ubyte, Literal[16]] + explicitEOSPresentFlag: int + reserved2: c.Array[ctypes.c_ubyte, Literal[3]] + stream_len: int + slice_count: int + scratch_pic_buffer_size: int + gptimer_timeout_value: int + FrameWidth: int + FrameHeight: int + vop_time_increment_bitcount: bytes + resync_marker_disable: bytes + tileFormat: int + gob_height: int + reserverd_surface_format: int + reserved3: bytes + width: int + height: int + FrameStride: c.Array[ctypes.c_uint32, Literal[2]] + luma_top_offset: int + luma_bot_offset: int + luma_frame_offset: int + chroma_top_offset: int + chroma_bot_offset: int + chroma_frame_offset: int + HistBufferSize: int + trd: c.Array[ctypes.c_int32, Literal[2]] + trb: c.Array[ctypes.c_int32, Literal[2]] + divx_flags: int + vop_fcode_forward: int + vop_fcode_backward: int + interlaced: int + quant_type: int + quarter_sample: int + short_video_header: int + curr_output_memory_layout: int + ptype: int + rnd: int + alternate_vertical_scan_flag: int + top_field_flag: int + reserved0: c.Array[ctypes.c_ubyte, Literal[3]] + intra_quant_mat: c.Array[ctypes.c_ubyte, Literal[64]] + nonintra_quant_mat: c.Array[ctypes.c_ubyte, Literal[64]] + ref_memory_layout: c.Array[ctypes.c_ubyte, Literal[2]] + reserved1: c.Array[ctypes.c_ubyte, Literal[34]] + displayPara: struct__nvdec_display_param_s +struct__nvdec_mpeg4_pic_s.register_fields([('encryption_params', nvdec_pass2_otf_s, 0), ('eos', c.Array[ctypes.c_ubyte, Literal[16]], 52), ('explicitEOSPresentFlag', ctypes.c_ubyte, 68), ('reserved2', c.Array[ctypes.c_ubyte, Literal[3]], 69), ('stream_len', ctypes.c_uint32, 72), ('slice_count', ctypes.c_uint32, 76), ('scratch_pic_buffer_size', ctypes.c_uint32, 80), ('gptimer_timeout_value', ctypes.c_uint32, 84), ('FrameWidth', ctypes.c_int16, 88), ('FrameHeight', ctypes.c_int16, 90), ('vop_time_increment_bitcount', ctypes.c_char, 92), ('resync_marker_disable', ctypes.c_char, 93), ('tileFormat', ctypes.c_ubyte, 94, 2, 0), ('gob_height', ctypes.c_ubyte, 94, 3, 2), ('reserverd_surface_format', ctypes.c_ubyte, 94, 3, 5), ('reserved3', ctypes.c_char, 95), ('width', ctypes.c_int32, 96), ('height', ctypes.c_int32, 100), ('FrameStride', c.Array[ctypes.c_uint32, Literal[2]], 104), ('luma_top_offset', ctypes.c_uint32, 112), ('luma_bot_offset', ctypes.c_uint32, 116), ('luma_frame_offset', ctypes.c_uint32, 120), ('chroma_top_offset', ctypes.c_uint32, 124), ('chroma_bot_offset', ctypes.c_uint32, 128), ('chroma_frame_offset', ctypes.c_uint32, 132), ('HistBufferSize', ctypes.c_uint32, 136), ('trd', c.Array[ctypes.c_int32, Literal[2]], 140), ('trb', c.Array[ctypes.c_int32, Literal[2]], 148), ('divx_flags', ctypes.c_int32, 156), ('vop_fcode_forward', ctypes.c_int16, 160), ('vop_fcode_backward', ctypes.c_int16, 162), ('interlaced', ctypes.c_ubyte, 164), ('quant_type', ctypes.c_ubyte, 165), ('quarter_sample', ctypes.c_ubyte, 166), ('short_video_header', ctypes.c_ubyte, 167), ('curr_output_memory_layout', ctypes.c_ubyte, 168), ('ptype', ctypes.c_ubyte, 169), ('rnd', ctypes.c_ubyte, 170), ('alternate_vertical_scan_flag', ctypes.c_ubyte, 171), ('top_field_flag', ctypes.c_ubyte, 172), ('reserved0', c.Array[ctypes.c_ubyte, Literal[3]], 173), ('intra_quant_mat', c.Array[ctypes.c_ubyte, Literal[64]], 176), ('nonintra_quant_mat', c.Array[ctypes.c_ubyte, Literal[64]], 240), ('ref_memory_layout', c.Array[ctypes.c_ubyte, Literal[2]], 304), ('reserved1', c.Array[ctypes.c_ubyte, Literal[34]], 306), ('displayPara', nvdec_display_param_s, 340)]) +nvdec_mpeg4_pic_s: TypeAlias = struct__nvdec_mpeg4_pic_s +enum_VP8_FRAME_TYPE: dict[int, str] = {(VP8_KEYFRAME:=0): 'VP8_KEYFRAME', (VP8_INTERFRAME:=1): 'VP8_INTERFRAME'} +enum_VP8_FRAME_SFC_ID: dict[int, str] = {(VP8_GOLDEN_FRAME_SFC:=0): 'VP8_GOLDEN_FRAME_SFC', (VP8_ALTREF_FRAME_SFC:=1): 'VP8_ALTREF_FRAME_SFC', (VP8_LAST_FRAME_SFC:=2): 'VP8_LAST_FRAME_SFC', (VP8_CURR_FRAME_SFC:=3): 'VP8_CURR_FRAME_SFC'} +@c.record +class struct__nvdec_vp8_pic_s(c.Struct): + SIZE = 192 + encryption_params: struct__nvdec_pass2_otf_s + gptimer_timeout_value: int + FrameWidth: int + FrameHeight: int + keyFrame: int + version: int + tileFormat: int + gob_height: int + reserverd_surface_format: int + errorConcealOn: int + firstPartSize: int + HistBufferSize: int + VLDBufferSize: int + FrameStride: c.Array[ctypes.c_uint32, Literal[2]] + luma_top_offset: int + luma_bot_offset: int + luma_frame_offset: int + chroma_top_offset: int + chroma_bot_offset: int + chroma_frame_offset: int + displayPara: struct__nvdec_display_param_s + current_output_memory_layout: bytes + output_memory_layout: c.Array[ctypes.c_char, Literal[3]] + segmentation_feature_data_update: int + reserved1: c.Array[ctypes.c_ubyte, Literal[3]] + resultValue: int + partition_offset: c.Array[ctypes.c_uint32, Literal[8]] + ssm: struct__nvdec_pass2_otf_ext_s +struct__nvdec_vp8_pic_s.register_fields([('encryption_params', nvdec_pass2_otf_s, 0), ('gptimer_timeout_value', ctypes.c_uint32, 52), ('FrameWidth', ctypes.c_uint16, 56), ('FrameHeight', ctypes.c_uint16, 58), ('keyFrame', ctypes.c_ubyte, 60), ('version', ctypes.c_ubyte, 61), ('tileFormat', ctypes.c_ubyte, 62, 2, 0), ('gob_height', ctypes.c_ubyte, 62, 3, 2), ('reserverd_surface_format', ctypes.c_ubyte, 62, 3, 5), ('errorConcealOn', ctypes.c_ubyte, 63), ('firstPartSize', ctypes.c_uint32, 64), ('HistBufferSize', ctypes.c_uint32, 68), ('VLDBufferSize', ctypes.c_uint32, 72), ('FrameStride', c.Array[ctypes.c_uint32, Literal[2]], 76), ('luma_top_offset', ctypes.c_uint32, 84), ('luma_bot_offset', ctypes.c_uint32, 88), ('luma_frame_offset', ctypes.c_uint32, 92), ('chroma_top_offset', ctypes.c_uint32, 96), ('chroma_bot_offset', ctypes.c_uint32, 100), ('chroma_frame_offset', ctypes.c_uint32, 104), ('displayPara', nvdec_display_param_s, 108), ('current_output_memory_layout', ctypes.c_char, 136), ('output_memory_layout', c.Array[ctypes.c_char, Literal[3]], 137), ('segmentation_feature_data_update', ctypes.c_ubyte, 140), ('reserved1', c.Array[ctypes.c_ubyte, Literal[3]], 141), ('resultValue', ctypes.c_uint32, 144), ('partition_offset', c.Array[ctypes.c_uint32, Literal[8]], 148), ('ssm', nvdec_pass2_otf_ext_s, 180)]) +nvdec_vp8_pic_s: TypeAlias = struct__nvdec_vp8_pic_s +@c.record +class struct__bytes_of_data_s(c.Struct): + SIZE = 8 + clear_bytes: int + encypted_bytes: int +struct__bytes_of_data_s.register_fields([('clear_bytes', ctypes.c_uint32, 0), ('encypted_bytes', ctypes.c_uint32, 4)]) +bytes_of_data_s: TypeAlias = struct__bytes_of_data_s +@c.record +class struct__nvdec_pass1_input_data_s(c.Struct): + SIZE = 836 + sample_size: c.Array[struct__bytes_of_data_s, Literal[32]] + initialization_vector: c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[32]] + IvValid: c.Array[ctypes.c_ubyte, Literal[32]] + stream_len: int + clearBufferSize: int + reencryptBufferSize: int + vp8coeffPartitonBufferSize: int + PrevWidth: int + num_nals: int + drm_mode: int + key_sel: int + codec: int + TotalSizeOfClearData: int + SliceHdrOffset: int + EncryptBlkCnt: int + SkipBlkCnt: int +struct__nvdec_pass1_input_data_s.register_fields([('sample_size', c.Array[bytes_of_data_s, Literal[32]], 0), ('initialization_vector', c.Array[c.Array[ctypes.c_uint32, Literal[4]], Literal[32]], 256), ('IvValid', c.Array[ctypes.c_ubyte, Literal[32]], 768), ('stream_len', ctypes.c_uint32, 800), ('clearBufferSize', ctypes.c_uint32, 804), ('reencryptBufferSize', ctypes.c_uint32, 808), ('vp8coeffPartitonBufferSize', ctypes.c_uint32, 812), ('PrevWidth', ctypes.c_uint32, 816), ('num_nals', ctypes.c_uint32, 820, 16, 0), ('drm_mode', ctypes.c_uint32, 822, 8, 0), ('key_sel', ctypes.c_uint32, 823, 4, 0), ('codec', ctypes.c_uint32, 823, 4, 4), ('TotalSizeOfClearData', ctypes.c_uint32, 824), ('SliceHdrOffset', ctypes.c_uint32, 828), ('EncryptBlkCnt', ctypes.c_uint32, 832, 16, 0), ('SkipBlkCnt', ctypes.c_uint32, 834, 16, 0)]) +nvdec_pass1_input_data_s: TypeAlias = struct__nvdec_pass1_input_data_s +@c.record +class struct__nvdec_pass1_output_data_s(c.Struct): + SIZE = 128 + clear_header_size: int + reencrypt_data_size: int + clear_token_data_size: int + key_increment: int + encryption_mode: int + bReEncrypted: int + bvp9SuperFrame: int + vp9NumFramesMinus1: int + reserved1: int + wrapped_session_key: c.Array[ctypes.c_uint32, Literal[4]] + wrapped_content_key: c.Array[ctypes.c_uint32, Literal[4]] + initialization_vector: c.Array[ctypes.c_uint32, Literal[4]] + partition_size: c.Array[ctypes.c_uint32, Literal[8]] + vp9_frame_sizes: c.Array[ctypes.c_uint32, Literal[8]] + vp9_clear_hdr_size: c.Array[ctypes.c_uint32, Literal[8]] +struct__nvdec_pass1_output_data_s.register_fields([('clear_header_size', ctypes.c_uint32, 0), ('reencrypt_data_size', ctypes.c_uint32, 4), ('clear_token_data_size', ctypes.c_uint32, 8), ('key_increment', ctypes.c_uint32, 12, 6, 0), ('encryption_mode', ctypes.c_uint32, 12, 4, 6), ('bReEncrypted', ctypes.c_uint32, 13, 1, 2), ('bvp9SuperFrame', ctypes.c_uint32, 13, 1, 3), ('vp9NumFramesMinus1', ctypes.c_uint32, 13, 3, 4), ('reserved1', ctypes.c_uint32, 13, 17, 7), ('wrapped_session_key', c.Array[ctypes.c_uint32, Literal[4]], 16), ('wrapped_content_key', c.Array[ctypes.c_uint32, Literal[4]], 32), ('initialization_vector', c.Array[ctypes.c_uint32, Literal[4]], 48), ('partition_size', c.Array[ctypes.c_uint32, Literal[8]], 64), ('vp9_frame_sizes', c.Array[ctypes.c_uint32, Literal[8]], 64), ('vp9_clear_hdr_size', c.Array[ctypes.c_uint32, Literal[8]], 96)]) +nvdec_pass1_output_data_s: TypeAlias = struct__nvdec_pass1_output_data_s +@c.record +class struct__scale_factors_reference_s(c.Struct): + SIZE = 4 + x_scale_fp: int + y_scale_fp: int +struct__scale_factors_reference_s.register_fields([('x_scale_fp', ctypes.c_int16, 0), ('y_scale_fp', ctypes.c_int16, 2)]) +scale_factors_reference_s: TypeAlias = struct__scale_factors_reference_s +@c.record +class struct__frame_info_t(c.Struct): + SIZE = 12 + width: int + height: int + stride: c.Array[ctypes.c_uint16, Literal[2]] + frame_buffer_idx: int +struct__frame_info_t.register_fields([('width', ctypes.c_uint16, 0), ('height', ctypes.c_uint16, 2), ('stride', c.Array[ctypes.c_uint16, Literal[2]], 4), ('frame_buffer_idx', ctypes.c_uint32, 8)]) +frame_info_t: TypeAlias = struct__frame_info_t +@c.record +class struct__ref_frame_struct_s(c.Struct): + SIZE = 24 + info: struct__frame_info_t + sf: struct__scale_factors_reference_s + sign_bias: int + wmtype: int + reserved_rf: int + frame_off: int + roffset: int +struct__ref_frame_struct_s.register_fields([('info', frame_info_t, 0), ('sf', scale_factors_reference_s, 12), ('sign_bias', ctypes.c_ubyte, 16, 1, 0), ('wmtype', ctypes.c_ubyte, 16, 2, 1), ('reserved_rf', ctypes.c_ubyte, 16, 5, 3), ('frame_off', ctypes.c_int16, 18), ('roffset', ctypes.c_int16, 20)]) +ref_frame_struct_s: TypeAlias = struct__ref_frame_struct_s +@c.record +class struct__av1_fgs_cfg_t(c.Struct): + SIZE = 12 + apply_grain: int + overlap_flag: int + clip_to_restricted_range: int + chroma_scaling_from_luma: int + num_y_points_b: int + num_cb_points_b: int + num_cr_points_b: int + scaling_shift: int + reserved_fgs: int + sw_random_seed: int + cb_offset: int + cr_offset: int + cb_mult: bytes + cb_luma_mult: bytes + cr_mult: bytes + cr_luma_mult: bytes +struct__av1_fgs_cfg_t.register_fields([('apply_grain', ctypes.c_uint16, 0, 1, 0), ('overlap_flag', ctypes.c_uint16, 0, 1, 1), ('clip_to_restricted_range', ctypes.c_uint16, 0, 1, 2), ('chroma_scaling_from_luma', ctypes.c_uint16, 0, 1, 3), ('num_y_points_b', ctypes.c_uint16, 0, 1, 4), ('num_cb_points_b', ctypes.c_uint16, 0, 1, 5), ('num_cr_points_b', ctypes.c_uint16, 0, 1, 6), ('scaling_shift', ctypes.c_uint16, 0, 4, 7), ('reserved_fgs', ctypes.c_uint16, 1, 5, 3), ('sw_random_seed', ctypes.c_uint16, 2), ('cb_offset', ctypes.c_int16, 4), ('cr_offset', ctypes.c_int16, 6), ('cb_mult', ctypes.c_char, 8), ('cb_luma_mult', ctypes.c_char, 9), ('cr_mult', ctypes.c_char, 10), ('cr_luma_mult', ctypes.c_char, 11)]) +av1_fgs_cfg_t: TypeAlias = struct__av1_fgs_cfg_t +@c.record +class struct__nvdec_av1_pic_s(c.Struct): + SIZE = 572 + encryption_params: struct__nvdec_pass2_otf_s + ssm: struct__nvdec_pass2_otf_ext_s + fgs_cfg: struct__av1_fgs_cfg_t + gptimer_timeout_value: int + stream_len: int + reserved12: int + use_128x128_superblock: int + chroma_format: int + bit_depth: int + enable_filter_intra: int + enable_intra_edge_filter: int + enable_interintra_compound: int + enable_masked_compound: int + enable_dual_filter: int + reserved10: int + reserved0: int + enable_jnt_comp: int + reserved1: int + enable_cdef: int + reserved11: int + enable_fgs: int + enable_substream_decoding: int + reserved2: int + frame_type: int + show_frame: int + reserved13: int + disable_cdf_update: int + allow_screen_content_tools: int + cur_frame_force_integer_mv: int + scale_denom_minus9: int + allow_intrabc: int + allow_high_precision_mv: int + interp_filter: int + switchable_motion_mode: int + use_ref_frame_mvs: int + refresh_frame_context: int + delta_q_present_flag: int + delta_q_res: int + delta_lf_present_flag: int + delta_lf_res: int + delta_lf_multi: int + reserved3: int + coded_lossless: int + tile_enabled: int + reserved4: int + superres_is_scaled: int + reserved_fh: int + tile_cols: int + tile_rows: int + context_update_tile_id: int + cdef_damping_minus_3: int + cdef_bits: int + frame_tx_mode: int + frame_reference_mode: int + skip_mode_flag: int + skip_ref0: int + skip_ref1: int + allow_warp: int + reduced_tx_set_used: int + ref_scaling_enable: int + reserved5: int + reserved6: int + superres_upscaled_width: int + superres_luma_step: int + superres_chroma_step: int + superres_init_luma_subpel_x: int + superres_init_chroma_subpel_x: int + base_qindex: int + y_dc_delta_q: bytes + u_dc_delta_q: bytes + v_dc_delta_q: bytes + u_ac_delta_q: bytes + v_ac_delta_q: bytes + qm_y: int + qm_u: int + qm_v: int + cdef_y_pri_strength: int + cdef_uv_pri_strength: int + cdef_y_sec_strength: int + cdef_uv_sec_strength: int + segment_enabled: int + segment_update_map: int + reserved7: int + segment_temporal_update: int + segment_feature_data: c.Array[c.Array[ctypes.c_int16, Literal[8]], Literal[8]] + last_active_segid: int + segid_preskip: int + prevsegid_flag: int + segment_quant_sign: int + filter_level: c.Array[ctypes.c_ubyte, Literal[2]] + filter_level_u: int + filter_level_v: int + lf_sharpness_level: int + lf_ref_deltas: c.Array[ctypes.c_char, Literal[8]] + lf_mode_deltas: c.Array[ctypes.c_char, Literal[2]] + lr_type: int + lr_unit_size: int + current_frame: struct__frame_info_t + ref_frame: c.Array[struct__ref_frame_struct_s, Literal[7]] + use_temporal0_mvs: int + use_temporal1_mvs: int + use_temporal2_mvs: int + mf1_type: int + mf2_type: int + mf3_type: int + reserved_mfmv: int + mfmv_offset: c.Array[c.Array[ctypes.c_int16, Literal[7]], Literal[3]] + mfmv_side: c.Array[c.Array[ctypes.c_char, Literal[7]], Literal[3]] + tileformat: int + gob_height: int + errorConcealOn: int + reserver8: int + stream_error_detection: int + mv_error_detection: int + coeff_error_detection: int + reserved_eh: int + Av1FltTopOffset: int + Av1FltVertOffset: int + Av1CdefVertOffset: int + Av1LrVertOffset: int + Av1HusVertOffset: int + Av1FgsVertOffset: int + enable_histogram: int + sw_skip_start_length: int + reserved_stat: int +struct__nvdec_av1_pic_s.register_fields([('encryption_params', nvdec_pass2_otf_s, 0), ('ssm', nvdec_pass2_otf_ext_s, 52), ('fgs_cfg', av1_fgs_cfg_t, 64), ('gptimer_timeout_value', ctypes.c_uint32, 76), ('stream_len', ctypes.c_uint32, 80), ('reserved12', ctypes.c_uint32, 84), ('use_128x128_superblock', ctypes.c_uint32, 88, 1, 0), ('chroma_format', ctypes.c_uint32, 88, 2, 1), ('bit_depth', ctypes.c_uint32, 88, 4, 3), ('enable_filter_intra', ctypes.c_uint32, 88, 1, 7), ('enable_intra_edge_filter', ctypes.c_uint32, 89, 1, 0), ('enable_interintra_compound', ctypes.c_uint32, 89, 1, 1), ('enable_masked_compound', ctypes.c_uint32, 89, 1, 2), ('enable_dual_filter', ctypes.c_uint32, 89, 1, 3), ('reserved10', ctypes.c_uint32, 89, 1, 4), ('reserved0', ctypes.c_uint32, 89, 3, 5), ('enable_jnt_comp', ctypes.c_uint32, 90, 1, 0), ('reserved1', ctypes.c_uint32, 90, 1, 1), ('enable_cdef', ctypes.c_uint32, 90, 1, 2), ('reserved11', ctypes.c_uint32, 90, 1, 3), ('enable_fgs', ctypes.c_uint32, 90, 1, 4), ('enable_substream_decoding', ctypes.c_uint32, 90, 1, 5), ('reserved2', ctypes.c_uint32, 90, 10, 6), ('frame_type', ctypes.c_uint32, 92, 2, 0), ('show_frame', ctypes.c_uint32, 92, 1, 2), ('reserved13', ctypes.c_uint32, 92, 1, 3), ('disable_cdf_update', ctypes.c_uint32, 92, 1, 4), ('allow_screen_content_tools', ctypes.c_uint32, 92, 1, 5), ('cur_frame_force_integer_mv', ctypes.c_uint32, 92, 1, 6), ('scale_denom_minus9', ctypes.c_uint32, 92, 3, 7), ('allow_intrabc', ctypes.c_uint32, 93, 1, 2), ('allow_high_precision_mv', ctypes.c_uint32, 93, 1, 3), ('interp_filter', ctypes.c_uint32, 93, 3, 4), ('switchable_motion_mode', ctypes.c_uint32, 93, 1, 7), ('use_ref_frame_mvs', ctypes.c_uint32, 94, 1, 0), ('refresh_frame_context', ctypes.c_uint32, 94, 1, 1), ('delta_q_present_flag', ctypes.c_uint32, 94, 1, 2), ('delta_q_res', ctypes.c_uint32, 94, 2, 3), ('delta_lf_present_flag', ctypes.c_uint32, 94, 1, 5), ('delta_lf_res', ctypes.c_uint32, 94, 2, 6), ('delta_lf_multi', ctypes.c_uint32, 95, 1, 0), ('reserved3', ctypes.c_uint32, 95, 1, 1), ('coded_lossless', ctypes.c_uint32, 95, 1, 2), ('tile_enabled', ctypes.c_uint32, 95, 1, 3), ('reserved4', ctypes.c_uint32, 95, 2, 4), ('superres_is_scaled', ctypes.c_uint32, 95, 1, 6), ('reserved_fh', ctypes.c_uint32, 95, 1, 7), ('tile_cols', ctypes.c_uint32, 96, 8, 0), ('tile_rows', ctypes.c_uint32, 97, 8, 0), ('context_update_tile_id', ctypes.c_uint32, 98, 16, 0), ('cdef_damping_minus_3', ctypes.c_uint32, 100, 2, 0), ('cdef_bits', ctypes.c_uint32, 100, 2, 2), ('frame_tx_mode', ctypes.c_uint32, 100, 3, 4), ('frame_reference_mode', ctypes.c_uint32, 100, 2, 7), ('skip_mode_flag', ctypes.c_uint32, 101, 1, 1), ('skip_ref0', ctypes.c_uint32, 101, 4, 2), ('skip_ref1', ctypes.c_uint32, 101, 4, 6), ('allow_warp', ctypes.c_uint32, 102, 1, 2), ('reduced_tx_set_used', ctypes.c_uint32, 102, 1, 3), ('ref_scaling_enable', ctypes.c_uint32, 102, 1, 4), ('reserved5', ctypes.c_uint32, 102, 1, 5), ('reserved6', ctypes.c_uint32, 102, 10, 6), ('superres_upscaled_width', ctypes.c_uint16, 104), ('superres_luma_step', ctypes.c_uint16, 106), ('superres_chroma_step', ctypes.c_uint16, 108), ('superres_init_luma_subpel_x', ctypes.c_uint16, 110), ('superres_init_chroma_subpel_x', ctypes.c_uint16, 112), ('base_qindex', ctypes.c_ubyte, 114), ('y_dc_delta_q', ctypes.c_char, 115), ('u_dc_delta_q', ctypes.c_char, 116), ('v_dc_delta_q', ctypes.c_char, 117), ('u_ac_delta_q', ctypes.c_char, 118), ('v_ac_delta_q', ctypes.c_char, 119), ('qm_y', ctypes.c_ubyte, 120), ('qm_u', ctypes.c_ubyte, 121), ('qm_v', ctypes.c_ubyte, 122), ('cdef_y_pri_strength', ctypes.c_uint32, 124), ('cdef_uv_pri_strength', ctypes.c_uint32, 128), ('cdef_y_sec_strength', ctypes.c_uint32, 132, 16, 0), ('cdef_uv_sec_strength', ctypes.c_uint32, 134, 16, 0), ('segment_enabled', ctypes.c_ubyte, 136), ('segment_update_map', ctypes.c_ubyte, 137), ('reserved7', ctypes.c_ubyte, 138), ('segment_temporal_update', ctypes.c_ubyte, 139), ('segment_feature_data', c.Array[c.Array[ctypes.c_int16, Literal[8]], Literal[8]], 140), ('last_active_segid', ctypes.c_ubyte, 268), ('segid_preskip', ctypes.c_ubyte, 269), ('prevsegid_flag', ctypes.c_ubyte, 270), ('segment_quant_sign', ctypes.c_ubyte, 271, 8, 0), ('filter_level', c.Array[ctypes.c_ubyte, Literal[2]], 272), ('filter_level_u', ctypes.c_ubyte, 274), ('filter_level_v', ctypes.c_ubyte, 275), ('lf_sharpness_level', ctypes.c_ubyte, 276), ('lf_ref_deltas', c.Array[ctypes.c_char, Literal[8]], 277), ('lf_mode_deltas', c.Array[ctypes.c_char, Literal[2]], 285), ('lr_type', ctypes.c_ubyte, 287), ('lr_unit_size', ctypes.c_ubyte, 288), ('current_frame', frame_info_t, 292), ('ref_frame', c.Array[ref_frame_struct_s, Literal[7]], 304), ('use_temporal0_mvs', ctypes.c_uint32, 472, 1, 0), ('use_temporal1_mvs', ctypes.c_uint32, 472, 1, 1), ('use_temporal2_mvs', ctypes.c_uint32, 472, 1, 2), ('mf1_type', ctypes.c_uint32, 472, 3, 3), ('mf2_type', ctypes.c_uint32, 472, 3, 6), ('mf3_type', ctypes.c_uint32, 473, 3, 1), ('reserved_mfmv', ctypes.c_uint32, 473, 20, 4), ('mfmv_offset', c.Array[c.Array[ctypes.c_int16, Literal[7]], Literal[3]], 476), ('mfmv_side', c.Array[c.Array[ctypes.c_char, Literal[7]], Literal[3]], 518), ('tileformat', ctypes.c_ubyte, 539, 2, 0), ('gob_height', ctypes.c_ubyte, 539, 3, 2), ('errorConcealOn', ctypes.c_ubyte, 539, 1, 5), ('reserver8', ctypes.c_ubyte, 539, 2, 6), ('stream_error_detection', ctypes.c_ubyte, 540, 1, 0), ('mv_error_detection', ctypes.c_ubyte, 540, 1, 1), ('coeff_error_detection', ctypes.c_ubyte, 540, 1, 2), ('reserved_eh', ctypes.c_ubyte, 540, 5, 3), ('Av1FltTopOffset', ctypes.c_uint32, 544), ('Av1FltVertOffset', ctypes.c_uint32, 548), ('Av1CdefVertOffset', ctypes.c_uint32, 552), ('Av1LrVertOffset', ctypes.c_uint32, 556), ('Av1HusVertOffset', ctypes.c_uint32, 560), ('Av1FgsVertOffset', ctypes.c_uint32, 564), ('enable_histogram', ctypes.c_uint32, 568, 1, 0), ('sw_skip_start_length', ctypes.c_uint32, 568, 14, 1), ('reserved_stat', ctypes.c_uint32, 569, 17, 7)]) +nvdec_av1_pic_s: TypeAlias = struct__nvdec_av1_pic_s +@c.record +class struct__AV1FilmGrainMemory(c.Struct): + SIZE = 13056 + scaling_lut_y: c.Array[ctypes.c_ubyte, Literal[256]] + scaling_lut_cb: c.Array[ctypes.c_ubyte, Literal[256]] + scaling_lut_cr: c.Array[ctypes.c_ubyte, Literal[256]] + cropped_luma_grain_block: c.Array[ctypes.c_int16, Literal[4096]] + cropped_cb_grain_block: c.Array[ctypes.c_int16, Literal[1024]] + cropped_cr_grain_block: c.Array[ctypes.c_int16, Literal[1024]] +struct__AV1FilmGrainMemory.register_fields([('scaling_lut_y', c.Array[ctypes.c_ubyte, Literal[256]], 0), ('scaling_lut_cb', c.Array[ctypes.c_ubyte, Literal[256]], 256), ('scaling_lut_cr', c.Array[ctypes.c_ubyte, Literal[256]], 512), ('cropped_luma_grain_block', c.Array[ctypes.c_int16, Literal[4096]], 768), ('cropped_cb_grain_block', c.Array[ctypes.c_int16, Literal[1024]], 8960), ('cropped_cr_grain_block', c.Array[ctypes.c_int16, Literal[1024]], 11008)]) +AV1FilmGrainMemory: TypeAlias = struct__AV1FilmGrainMemory +@c.record +class struct__AV1TileInfo_OLD(c.Struct): + SIZE = 16 + width_in_sb: int + height_in_sb: int + tile_start_b0: int + tile_start_b1: int + tile_start_b2: int + tile_start_b3: int + tile_end_b0: int + tile_end_b1: int + tile_end_b2: int + tile_end_b3: int + padding: c.Array[ctypes.c_ubyte, Literal[6]] +struct__AV1TileInfo_OLD.register_fields([('width_in_sb', ctypes.c_ubyte, 0), ('height_in_sb', ctypes.c_ubyte, 1), ('tile_start_b0', ctypes.c_ubyte, 2), ('tile_start_b1', ctypes.c_ubyte, 3), ('tile_start_b2', ctypes.c_ubyte, 4), ('tile_start_b3', ctypes.c_ubyte, 5), ('tile_end_b0', ctypes.c_ubyte, 6), ('tile_end_b1', ctypes.c_ubyte, 7), ('tile_end_b2', ctypes.c_ubyte, 8), ('tile_end_b3', ctypes.c_ubyte, 9), ('padding', c.Array[ctypes.c_ubyte, Literal[6]], 10)]) +AV1TileInfo_OLD: TypeAlias = struct__AV1TileInfo_OLD +@c.record +class struct__AV1TileInfo(c.Struct): + SIZE = 4 + width_in_sb: int + padding_w: int + height_in_sb: int + padding_h: int +struct__AV1TileInfo.register_fields([('width_in_sb', ctypes.c_ubyte, 0), ('padding_w', ctypes.c_ubyte, 1), ('height_in_sb', ctypes.c_ubyte, 2), ('padding_h', ctypes.c_ubyte, 3)]) +AV1TileInfo: TypeAlias = struct__AV1TileInfo +@c.record +class struct__AV1TileStreamInfo(c.Struct): + SIZE = 16 + tile_start: int + tile_end: int + padding: c.Array[ctypes.c_ubyte, Literal[8]] +struct__AV1TileStreamInfo.register_fields([('tile_start', ctypes.c_uint32, 0), ('tile_end', ctypes.c_uint32, 4), ('padding', c.Array[ctypes.c_ubyte, Literal[8]], 8)]) +AV1TileStreamInfo: TypeAlias = struct__AV1TileStreamInfo +@c.record +class struct__nvdec_new_h264_pic_s(c.Struct): + SIZE = 764 + encryption_params: struct__nvdec_pass2_otf_s + eos: c.Array[ctypes.c_ubyte, Literal[16]] + explicitEOSPresentFlag: int + hint_dump_en: int + reserved0: c.Array[ctypes.c_ubyte, Literal[2]] + stream_len: int + slice_count: int + mbhist_buffer_size: int + gptimer_timeout_value: int + log2_max_pic_order_cnt_lsb_minus4: int + delta_pic_order_always_zero_flag: int + frame_mbs_only_flag: int + PicWidthInMbs: int + FrameHeightInMbs: int + tileFormat: int + gob_height: int + reserverd_surface_format: int + entropy_coding_mode_flag: int + pic_order_present_flag: int + num_ref_idx_l0_active_minus1: int + num_ref_idx_l1_active_minus1: int + deblocking_filter_control_present_flag: int + redundant_pic_cnt_present_flag: int + transform_8x8_mode_flag: int + pitch_luma: int + pitch_chroma: int + luma_top_offset: int + luma_bot_offset: int + luma_frame_offset: int + chroma_top_offset: int + chroma_bot_offset: int + chroma_frame_offset: int + HistBufferSize: int + MbaffFrameFlag: int + direct_8x8_inference_flag: int + weighted_pred_flag: int + constrained_intra_pred_flag: int + ref_pic_flag: int + field_pic_flag: int + bottom_field_flag: int + second_field: int + log2_max_frame_num_minus4: int + chroma_format_idc: int + pic_order_cnt_type: int + pic_init_qp_minus26: int + chroma_qp_index_offset: int + second_chroma_qp_index_offset: int + weighted_bipred_idc: int + CurrPicIdx: int + CurrColIdx: int + frame_num: int + frame_surfaces: int + output_memory_layout: int + CurrFieldOrderCnt: c.Array[ctypes.c_int32, Literal[2]] + dpb: c.Array[struct__nvdec_dpb_entry_s, Literal[16]] + WeightScale: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[4]], Literal[6]] + WeightScale8x8: c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[8]], Literal[2]] + num_inter_view_refs_lX: c.Array[ctypes.c_ubyte, Literal[2]] + reserved1: c.Array[ctypes.c_char, Literal[14]] + inter_view_refidx_lX: c.Array[c.Array[ctypes.c_byte, Literal[16]], Literal[2]] + lossless_ipred8x8_filter_enable: int + qpprime_y_zero_transform_bypass_flag: int + reserved2: int + displayPara: struct__nvdec_display_param_s + ssm: struct__nvdec_pass2_otf_ext_s +struct__nvdec_new_h264_pic_s.register_fields([('encryption_params', nvdec_pass2_otf_s, 0), ('eos', c.Array[ctypes.c_ubyte, Literal[16]], 52), ('explicitEOSPresentFlag', ctypes.c_ubyte, 68), ('hint_dump_en', ctypes.c_ubyte, 69), ('reserved0', c.Array[ctypes.c_ubyte, Literal[2]], 70), ('stream_len', ctypes.c_uint32, 72), ('slice_count', ctypes.c_uint32, 76), ('mbhist_buffer_size', ctypes.c_uint32, 80), ('gptimer_timeout_value', ctypes.c_uint32, 84), ('log2_max_pic_order_cnt_lsb_minus4', ctypes.c_int32, 88), ('delta_pic_order_always_zero_flag', ctypes.c_int32, 92), ('frame_mbs_only_flag', ctypes.c_int32, 96), ('PicWidthInMbs', ctypes.c_int32, 100), ('FrameHeightInMbs', ctypes.c_int32, 104), ('tileFormat', ctypes.c_uint32, 108, 2, 0), ('gob_height', ctypes.c_uint32, 108, 3, 2), ('reserverd_surface_format', ctypes.c_uint32, 108, 27, 5), ('entropy_coding_mode_flag', ctypes.c_int32, 112), ('pic_order_present_flag', ctypes.c_int32, 116), ('num_ref_idx_l0_active_minus1', ctypes.c_int32, 120), ('num_ref_idx_l1_active_minus1', ctypes.c_int32, 124), ('deblocking_filter_control_present_flag', ctypes.c_int32, 128), ('redundant_pic_cnt_present_flag', ctypes.c_int32, 132), ('transform_8x8_mode_flag', ctypes.c_int32, 136), ('pitch_luma', ctypes.c_uint32, 140), ('pitch_chroma', ctypes.c_uint32, 144), ('luma_top_offset', ctypes.c_uint32, 148), ('luma_bot_offset', ctypes.c_uint32, 152), ('luma_frame_offset', ctypes.c_uint32, 156), ('chroma_top_offset', ctypes.c_uint32, 160), ('chroma_bot_offset', ctypes.c_uint32, 164), ('chroma_frame_offset', ctypes.c_uint32, 168), ('HistBufferSize', ctypes.c_uint32, 172), ('MbaffFrameFlag', ctypes.c_uint32, 176, 1, 0), ('direct_8x8_inference_flag', ctypes.c_uint32, 176, 1, 1), ('weighted_pred_flag', ctypes.c_uint32, 176, 1, 2), ('constrained_intra_pred_flag', ctypes.c_uint32, 176, 1, 3), ('ref_pic_flag', ctypes.c_uint32, 176, 1, 4), ('field_pic_flag', ctypes.c_uint32, 176, 1, 5), ('bottom_field_flag', ctypes.c_uint32, 176, 1, 6), ('second_field', ctypes.c_uint32, 176, 1, 7), ('log2_max_frame_num_minus4', ctypes.c_uint32, 177, 4, 0), ('chroma_format_idc', ctypes.c_uint32, 177, 2, 4), ('pic_order_cnt_type', ctypes.c_uint32, 177, 2, 6), ('pic_init_qp_minus26', ctypes.c_int32, 178, 6, 0), ('chroma_qp_index_offset', ctypes.c_int32, 178, 5, 6), ('second_chroma_qp_index_offset', ctypes.c_int32, 179, 5, 3), ('weighted_bipred_idc', ctypes.c_uint32, 180, 2, 0), ('CurrPicIdx', ctypes.c_uint32, 180, 7, 2), ('CurrColIdx', ctypes.c_uint32, 181, 5, 1), ('frame_num', ctypes.c_uint32, 181, 16, 6), ('frame_surfaces', ctypes.c_uint32, 183, 1, 6), ('output_memory_layout', ctypes.c_uint32, 183, 1, 7), ('CurrFieldOrderCnt', c.Array[ctypes.c_int32, Literal[2]], 184), ('dpb', c.Array[nvdec_dpb_entry_s, Literal[16]], 192), ('WeightScale', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[4]], Literal[4]], Literal[6]], 448), ('WeightScale8x8', c.Array[c.Array[c.Array[ctypes.c_ubyte, Literal[8]], Literal[8]], Literal[2]], 544), ('num_inter_view_refs_lX', c.Array[ctypes.c_ubyte, Literal[2]], 672), ('reserved1', c.Array[ctypes.c_char, Literal[14]], 674), ('inter_view_refidx_lX', c.Array[c.Array[ctypes.c_byte, Literal[16]], Literal[2]], 688), ('lossless_ipred8x8_filter_enable', ctypes.c_uint32, 720, 1, 0), ('qpprime_y_zero_transform_bypass_flag', ctypes.c_uint32, 720, 1, 1), ('reserved2', ctypes.c_uint32, 720, 30, 2), ('displayPara', nvdec_display_param_s, 724), ('ssm', nvdec_pass2_otf_ext_s, 752)]) +nvdec_new_h264_pic_s: TypeAlias = struct__nvdec_new_h264_pic_s +@c.record +class nvdec_crc_s(c.Struct): + SIZE = 128 + dbg_crc_enable_partb: int + dbg_crc_enable_partc: int + dbg_crc_enable_partd: int + dbg_crc_enable_parte: int + dbg_crc_intf_partb: int + dbg_crc_intf_partc: int + dbg_crc_intf_partd: int + dbg_crc_intf_parte: int + reserved0: int + dbg_crc_partb_golden: c.Array[ctypes.c_uint32, Literal[4]] + dbg_crc_partc_golden: c.Array[ctypes.c_uint32, Literal[4]] + dbg_crc_partd_golden: c.Array[ctypes.c_uint32, Literal[4]] + dbg_crc_parte_golden: c.Array[ctypes.c_uint32, Literal[4]] + dbg_crc_comp_partb: int + dbg_crc_comp_partc: int + dbg_crc_comp_partd: int + dbg_crc_comp_parte: int + reserved1: int + reserved2: c.Array[ctypes.c_ubyte, Literal[56]] +nvdec_crc_s.register_fields([('dbg_crc_enable_partb', ctypes.c_uint32, 0, 1, 0), ('dbg_crc_enable_partc', ctypes.c_uint32, 0, 1, 1), ('dbg_crc_enable_partd', ctypes.c_uint32, 0, 1, 2), ('dbg_crc_enable_parte', ctypes.c_uint32, 0, 1, 3), ('dbg_crc_intf_partb', ctypes.c_uint32, 0, 6, 4), ('dbg_crc_intf_partc', ctypes.c_uint32, 1, 6, 2), ('dbg_crc_intf_partd', ctypes.c_uint32, 2, 6, 0), ('dbg_crc_intf_parte', ctypes.c_uint32, 2, 6, 6), ('reserved0', ctypes.c_uint32, 3, 4, 4), ('dbg_crc_partb_golden', c.Array[ctypes.c_uint32, Literal[4]], 4), ('dbg_crc_partc_golden', c.Array[ctypes.c_uint32, Literal[4]], 20), ('dbg_crc_partd_golden', c.Array[ctypes.c_uint32, Literal[4]], 36), ('dbg_crc_parte_golden', c.Array[ctypes.c_uint32, Literal[4]], 52), ('dbg_crc_comp_partb', ctypes.c_uint32, 68, 4, 0), ('dbg_crc_comp_partc', ctypes.c_uint32, 68, 4, 4), ('dbg_crc_comp_partd', ctypes.c_uint32, 69, 4, 0), ('dbg_crc_comp_parte', ctypes.c_uint32, 69, 4, 4), ('reserved1', ctypes.c_uint32, 70, 16, 0), ('reserved2', c.Array[ctypes.c_ubyte, Literal[56]], 72)]) +@c.record +class struct_NV0000_ALLOC_PARAMETERS(c.Struct): + SIZE = 120 + hClient: int + processID: int + processName: c.Array[ctypes.c_char, Literal[100]] + pOsPidInfo: ctypes.c_void_p +NvHandle: TypeAlias = ctypes.c_uint32 +NvU32: TypeAlias = ctypes.c_uint32 +NvP64: TypeAlias = ctypes.c_void_p +struct_NV0000_ALLOC_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('processID', NvU32, 4), ('processName', c.Array[ctypes.c_char, Literal[100]], 8), ('pOsPidInfo', NvP64, 112)]) +NV0000_ALLOC_PARAMETERS: TypeAlias = struct_NV0000_ALLOC_PARAMETERS +@c.record +class struct_NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS(c.Struct): + SIZE = 24 + offset: int + limit: int + hVASpace: int +NvU64: TypeAlias = ctypes.c_uint64 +struct_NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS.register_fields([('offset', NvU64, 0), ('limit', NvU64, 8), ('hVASpace', NvHandle, 16)]) +NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS: TypeAlias = struct_NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS +@c.record +class struct_NV0080_ALLOC_PARAMETERS(c.Struct): + SIZE = 56 + deviceId: int + hClientShare: int + hTargetClient: int + hTargetDevice: int + flags: int + vaSpaceSize: int + vaStartInternal: int + vaLimitInternal: int + vaMode: int +NvV32: TypeAlias = ctypes.c_uint32 +struct_NV0080_ALLOC_PARAMETERS.register_fields([('deviceId', NvU32, 0), ('hClientShare', NvHandle, 4), ('hTargetClient', NvHandle, 8), ('hTargetDevice', NvHandle, 12), ('flags', NvV32, 16), ('vaSpaceSize', NvU64, 24), ('vaStartInternal', NvU64, 32), ('vaLimitInternal', NvU64, 40), ('vaMode', NvV32, 48)]) +NV0080_ALLOC_PARAMETERS: TypeAlias = struct_NV0080_ALLOC_PARAMETERS +@c.record +class struct_NV2080_ALLOC_PARAMETERS(c.Struct): + SIZE = 4 + subDeviceId: int +struct_NV2080_ALLOC_PARAMETERS.register_fields([('subDeviceId', NvU32, 0)]) +NV2080_ALLOC_PARAMETERS: TypeAlias = struct_NV2080_ALLOC_PARAMETERS +@c.record +class struct__cl2080_tag0(c.Struct): + SIZE = 7936 + Reserved00: c.Array[ctypes.c_uint32, Literal[1984]] +struct__cl2080_tag0.register_fields([('Reserved00', c.Array[NvV32, Literal[1984]], 0)]) +@c.record +class volatile_struct__cl2080_tag0(c.Struct): + SIZE = 7936 + Reserved00: c.Array[ctypes.c_uint32, Literal[1984]] +Nv2080Typedef: TypeAlias = volatile_struct__cl2080_tag0 +volatile_struct__cl2080_tag0.register_fields([('Reserved00', c.Array[NvV32, Literal[1984]], 0)]) +Nv20Subdevice0: TypeAlias = volatile_struct__cl2080_tag0 +@c.record +class struct_Nv2080HdcpStatusChangeNotificationRec(c.Struct): + SIZE = 8 + displayId: int + hdcpStatusChangeNotif: int +struct_Nv2080HdcpStatusChangeNotificationRec.register_fields([('displayId', NvU32, 0), ('hdcpStatusChangeNotif', NvU32, 4)]) +Nv2080HdcpStatusChangeNotification: TypeAlias = struct_Nv2080HdcpStatusChangeNotificationRec +@c.record +class struct_Nv2080PStateChangeNotificationRec(c.Struct): + SIZE = 12 + timeStamp: struct_Nv2080PStateChangeNotificationRec_timeStamp + NewPstate: int +@c.record +class struct_Nv2080PStateChangeNotificationRec_timeStamp(c.Struct): + SIZE = 8 + nanoseconds: c.Array[ctypes.c_uint32, Literal[2]] +struct_Nv2080PStateChangeNotificationRec_timeStamp.register_fields([('nanoseconds', c.Array[NvU32, Literal[2]], 0)]) +struct_Nv2080PStateChangeNotificationRec.register_fields([('timeStamp', struct_Nv2080PStateChangeNotificationRec_timeStamp, 0), ('NewPstate', NvU32, 8)]) +Nv2080PStateChangeNotification: TypeAlias = struct_Nv2080PStateChangeNotificationRec +@c.record +class struct_Nv2080ClocksChangeNotificationRec(c.Struct): + SIZE = 8 + timeStamp: struct_Nv2080ClocksChangeNotificationRec_timeStamp +@c.record +class struct_Nv2080ClocksChangeNotificationRec_timeStamp(c.Struct): + SIZE = 8 + nanoseconds: c.Array[ctypes.c_uint32, Literal[2]] +struct_Nv2080ClocksChangeNotificationRec_timeStamp.register_fields([('nanoseconds', c.Array[NvU32, Literal[2]], 0)]) +struct_Nv2080ClocksChangeNotificationRec.register_fields([('timeStamp', struct_Nv2080ClocksChangeNotificationRec_timeStamp, 0)]) +Nv2080ClocksChangeNotification: TypeAlias = struct_Nv2080ClocksChangeNotificationRec +@c.record +class struct_Nv2080WorkloadModulationChangeNotificationRec(c.Struct): + SIZE = 12 + timeStamp: struct_Nv2080WorkloadModulationChangeNotificationRec_timeStamp + WorkloadModulationEnabled: int +@c.record +class struct_Nv2080WorkloadModulationChangeNotificationRec_timeStamp(c.Struct): + SIZE = 8 + nanoseconds: c.Array[ctypes.c_uint32, Literal[2]] +struct_Nv2080WorkloadModulationChangeNotificationRec_timeStamp.register_fields([('nanoseconds', c.Array[NvU32, Literal[2]], 0)]) +NvBool: TypeAlias = ctypes.c_ubyte +struct_Nv2080WorkloadModulationChangeNotificationRec.register_fields([('timeStamp', struct_Nv2080WorkloadModulationChangeNotificationRec_timeStamp, 0), ('WorkloadModulationEnabled', NvBool, 8)]) +Nv2080WorkloadModulationChangeNotification: TypeAlias = struct_Nv2080WorkloadModulationChangeNotificationRec +@c.record +class Nv2080HotplugNotification(c.Struct): + SIZE = 8 + plugDisplayMask: int + unplugDisplayMask: int +Nv2080HotplugNotification.register_fields([('plugDisplayMask', NvU32, 0), ('unplugDisplayMask', NvU32, 4)]) +@c.record +class Nv2080PowerEventNotification(c.Struct): + SIZE = 8 + bSwitchToAC: int + bGPUCapabilityChanged: int + displayMaskAffected: int +Nv2080PowerEventNotification.register_fields([('bSwitchToAC', NvBool, 0), ('bGPUCapabilityChanged', NvBool, 1), ('displayMaskAffected', NvU32, 4)]) +@c.record +class struct_Nv2080DpIrqNotificationRec(c.Struct): + SIZE = 4 + displayId: int +struct_Nv2080DpIrqNotificationRec.register_fields([('displayId', NvU32, 0)]) +Nv2080DpIrqNotification: TypeAlias = struct_Nv2080DpIrqNotificationRec +@c.record +class struct_Nv2080DstateXusbPpcNotificationRec(c.Struct): + SIZE = 8 + dstateXusb: int + dstatePpc: int +struct_Nv2080DstateXusbPpcNotificationRec.register_fields([('dstateXusb', NvU32, 0), ('dstatePpc', NvU32, 4)]) +Nv2080DstateXusbPpcNotification: TypeAlias = struct_Nv2080DstateXusbPpcNotificationRec +@c.record +class struct_Nv2080XusbPpcConnectStateNotificationRec(c.Struct): + SIZE = 1 + bConnected: int +struct_Nv2080XusbPpcConnectStateNotificationRec.register_fields([('bConnected', NvBool, 0)]) +Nv2080XusbPpcConnectStateNotification: TypeAlias = struct_Nv2080XusbPpcConnectStateNotificationRec +@c.record +class struct_Nv2080ACPIEvent(c.Struct): + SIZE = 4 + event: int +struct_Nv2080ACPIEvent.register_fields([('event', NvU32, 0)]) +Nv2080ACPIEvent: TypeAlias = struct_Nv2080ACPIEvent +@c.record +class struct__NV2080_COOLER_DIAG_ZONE_NOTIFICATION_REC(c.Struct): + SIZE = 4 + currentZone: int +struct__NV2080_COOLER_DIAG_ZONE_NOTIFICATION_REC.register_fields([('currentZone', NvU32, 0)]) +NV2080_COOLER_DIAG_ZONE_NOTIFICATION_REC: TypeAlias = struct__NV2080_COOLER_DIAG_ZONE_NOTIFICATION_REC +@c.record +class struct__NV2080_THERM_DIAG_ZONE_NOTIFICATION_REC(c.Struct): + SIZE = 4 + currentZone: int +struct__NV2080_THERM_DIAG_ZONE_NOTIFICATION_REC.register_fields([('currentZone', NvU32, 0)]) +NV2080_THERM_DIAG_ZONE_NOTIFICATION_REC: TypeAlias = struct__NV2080_THERM_DIAG_ZONE_NOTIFICATION_REC +@c.record +class struct_Nv2080AudioHdcpRequestRec(c.Struct): + SIZE = 8 + displayId: int + requestedState: int +struct_Nv2080AudioHdcpRequestRec.register_fields([('displayId', NvU32, 0), ('requestedState', NvU32, 4)]) +Nv2080AudioHdcpRequest: TypeAlias = struct_Nv2080AudioHdcpRequestRec +@c.record +class struct_Nv2080GC5GpuReadyParams(c.Struct): + SIZE = 12 + event: int + sciIntr0: int + sciIntr1: int +struct_Nv2080GC5GpuReadyParams.register_fields([('event', NvU32, 0), ('sciIntr0', NvU32, 4), ('sciIntr1', NvU32, 8)]) +Nv2080GC5GpuReadyParams: TypeAlias = struct_Nv2080GC5GpuReadyParams +@c.record +class Nv2080PrivRegAccessFaultNotification(c.Struct): + SIZE = 4 + errAddr: int +Nv2080PrivRegAccessFaultNotification.register_fields([('errAddr', NvU32, 0)]) +@c.record +class struct_Nv2080DstateHdaCodecNotificationRec(c.Struct): + SIZE = 4 + dstateHdaCodec: int +struct_Nv2080DstateHdaCodecNotificationRec.register_fields([('dstateHdaCodec', NvU32, 0)]) +Nv2080DstateHdaCodecNotification: TypeAlias = struct_Nv2080DstateHdaCodecNotificationRec +@c.record +class struct_Nv2080HdmiFrlRequestNotificationRec(c.Struct): + SIZE = 4 + displayId: int +struct_Nv2080HdmiFrlRequestNotificationRec.register_fields([('displayId', NvU32, 0)]) +Nv2080HdmiFrlRequestNotification: TypeAlias = struct_Nv2080HdmiFrlRequestNotificationRec +@c.record +class struct__NV2080_PLATFORM_POWER_MODE_CHANGE_STATUS(c.Struct): + SIZE = 3 + platformPowerModeIndex: int + platformPowerModeMask: int + eventReason: int +NvU8: TypeAlias = ctypes.c_ubyte +struct__NV2080_PLATFORM_POWER_MODE_CHANGE_STATUS.register_fields([('platformPowerModeIndex', NvU8, 0), ('platformPowerModeMask', NvU8, 1), ('eventReason', NvU8, 2)]) +NV2080_PLATFORM_POWER_MODE_CHANGE_STATUS: TypeAlias = struct__NV2080_PLATFORM_POWER_MODE_CHANGE_STATUS +@c.record +class struct__NV2080_NOTIFIERS_USE_GC6_REDUCED_THRESHOLD_UPDATE(c.Struct): + SIZE = 1 + workloadType: int +struct__NV2080_NOTIFIERS_USE_GC6_REDUCED_THRESHOLD_UPDATE.register_fields([('workloadType', NvU8, 0)]) +NV2080_NOTIFIERS_USE_GC6_REDUCED_THRESHOLD_UPDATE: TypeAlias = struct__NV2080_NOTIFIERS_USE_GC6_REDUCED_THRESHOLD_UPDATE +@c.record +class Nv2080QosIntrNotification(c.Struct): + SIZE = 4 + engineType: int +Nv2080QosIntrNotification.register_fields([('engineType', NvU32, 0)]) +@c.record +class Nv2080EccDbeNotification(c.Struct): + SIZE = 8 + physAddress: int +Nv2080EccDbeNotification.register_fields([('physAddress', NvU64, 0)]) +@c.record +class Nv2080LpwrDifrPrefetchNotification(c.Struct): + SIZE = 4 + l2CacheSize: int +Nv2080LpwrDifrPrefetchNotification.register_fields([('l2CacheSize', NvU32, 0)]) +@c.record +class Nv2080NvlinkLnkChangeNotification(c.Struct): + SIZE = 8 + GpuId: int + linkId: int +Nv2080NvlinkLnkChangeNotification.register_fields([('GpuId', NvU32, 0), ('linkId', NvU32, 4)]) +@c.record +class struct_Nvc56fControl_struct(c.Struct): + SIZE = 512 + Ignored00: c.Array[ctypes.c_uint32, Literal[16]] + Put: int + Get: int + Reference: int + PutHi: int + Ignored01: c.Array[ctypes.c_uint32, Literal[2]] + TopLevelGet: int + TopLevelGetHi: int + GetHi: int + Ignored02: c.Array[ctypes.c_uint32, Literal[7]] + Ignored03: int + Ignored04: c.Array[ctypes.c_uint32, Literal[1]] + GPGet: int + GPPut: int + Ignored05: c.Array[ctypes.c_uint32, Literal[92]] +struct_Nvc56fControl_struct.register_fields([('Ignored00', c.Array[NvU32, Literal[16]], 0), ('Put', NvU32, 64), ('Get', NvU32, 68), ('Reference', NvU32, 72), ('PutHi', NvU32, 76), ('Ignored01', c.Array[NvU32, Literal[2]], 80), ('TopLevelGet', NvU32, 88), ('TopLevelGetHi', NvU32, 92), ('GetHi', NvU32, 96), ('Ignored02', c.Array[NvU32, Literal[7]], 100), ('Ignored03', NvU32, 128), ('Ignored04', c.Array[NvU32, Literal[1]], 132), ('GPGet', NvU32, 136), ('GPPut', NvU32, 140), ('Ignored05', c.Array[NvU32, Literal[92]], 144)]) +@c.record +class volatile_struct_Nvc56fControl_struct(c.Struct): + SIZE = 512 + Ignored00: c.Array[ctypes.c_uint32, Literal[16]] + Put: int + Get: int + Reference: int + PutHi: int + Ignored01: c.Array[ctypes.c_uint32, Literal[2]] + TopLevelGet: int + TopLevelGetHi: int + GetHi: int + Ignored02: c.Array[ctypes.c_uint32, Literal[7]] + Ignored03: int + Ignored04: c.Array[ctypes.c_uint32, Literal[1]] + GPGet: int + GPPut: int + Ignored05: c.Array[ctypes.c_uint32, Literal[92]] +Nvc56fControl: TypeAlias = volatile_struct_Nvc56fControl_struct +volatile_struct_Nvc56fControl_struct.register_fields([('Ignored00', c.Array[NvU32, Literal[16]], 0), ('Put', NvU32, 64), ('Get', NvU32, 68), ('Reference', NvU32, 72), ('PutHi', NvU32, 76), ('Ignored01', c.Array[NvU32, Literal[2]], 80), ('TopLevelGet', NvU32, 88), ('TopLevelGetHi', NvU32, 92), ('GetHi', NvU32, 96), ('Ignored02', c.Array[NvU32, Literal[7]], 100), ('Ignored03', NvU32, 128), ('Ignored04', c.Array[NvU32, Literal[1]], 132), ('GPGet', NvU32, 136), ('GPPut', NvU32, 140), ('Ignored05', c.Array[NvU32, Literal[92]], 144)]) +AmpereAControlGPFifo: TypeAlias = volatile_struct_Nvc56fControl_struct +@c.record +class struct_Nvc86fControl_struct(c.Struct): + SIZE = 512 + Ignored00: c.Array[ctypes.c_uint32, Literal[16]] + Put: int + Get: int + Reference: int + PutHi: int + Ignored01: c.Array[ctypes.c_uint32, Literal[2]] + TopLevelGet: int + TopLevelGetHi: int + GetHi: int + Ignored02: c.Array[ctypes.c_uint32, Literal[7]] + Ignored03: int + Ignored04: c.Array[ctypes.c_uint32, Literal[1]] + GPGet: int + GPPut: int + Ignored05: c.Array[ctypes.c_uint32, Literal[92]] +struct_Nvc86fControl_struct.register_fields([('Ignored00', c.Array[NvU32, Literal[16]], 0), ('Put', NvU32, 64), ('Get', NvU32, 68), ('Reference', NvU32, 72), ('PutHi', NvU32, 76), ('Ignored01', c.Array[NvU32, Literal[2]], 80), ('TopLevelGet', NvU32, 88), ('TopLevelGetHi', NvU32, 92), ('GetHi', NvU32, 96), ('Ignored02', c.Array[NvU32, Literal[7]], 100), ('Ignored03', NvU32, 128), ('Ignored04', c.Array[NvU32, Literal[1]], 132), ('GPGet', NvU32, 136), ('GPPut', NvU32, 140), ('Ignored05', c.Array[NvU32, Literal[92]], 144)]) +@c.record +class volatile_struct_Nvc86fControl_struct(c.Struct): + SIZE = 512 + Ignored00: c.Array[ctypes.c_uint32, Literal[16]] + Put: int + Get: int + Reference: int + PutHi: int + Ignored01: c.Array[ctypes.c_uint32, Literal[2]] + TopLevelGet: int + TopLevelGetHi: int + GetHi: int + Ignored02: c.Array[ctypes.c_uint32, Literal[7]] + Ignored03: int + Ignored04: c.Array[ctypes.c_uint32, Literal[1]] + GPGet: int + GPPut: int + Ignored05: c.Array[ctypes.c_uint32, Literal[92]] +Nvc86fControl: TypeAlias = volatile_struct_Nvc86fControl_struct +volatile_struct_Nvc86fControl_struct.register_fields([('Ignored00', c.Array[NvU32, Literal[16]], 0), ('Put', NvU32, 64), ('Get', NvU32, 68), ('Reference', NvU32, 72), ('PutHi', NvU32, 76), ('Ignored01', c.Array[NvU32, Literal[2]], 80), ('TopLevelGet', NvU32, 88), ('TopLevelGetHi', NvU32, 92), ('GetHi', NvU32, 96), ('Ignored02', c.Array[NvU32, Literal[7]], 100), ('Ignored03', NvU32, 128), ('Ignored04', c.Array[NvU32, Literal[1]], 132), ('GPGet', NvU32, 136), ('GPPut', NvU32, 140), ('Ignored05', c.Array[NvU32, Literal[92]], 144)]) +HopperAControlGPFifo: TypeAlias = volatile_struct_Nvc86fControl_struct +@c.record +class struct_Nvc96fControl_struct(c.Struct): + SIZE = 512 + Ignored00: c.Array[ctypes.c_uint32, Literal[35]] + GPPut: int + Ignored01: c.Array[ctypes.c_uint32, Literal[92]] +struct_Nvc96fControl_struct.register_fields([('Ignored00', c.Array[NvU32, Literal[35]], 0), ('GPPut', NvU32, 140), ('Ignored01', c.Array[NvU32, Literal[92]], 144)]) +@c.record +class volatile_struct_Nvc96fControl_struct(c.Struct): + SIZE = 512 + Ignored00: c.Array[ctypes.c_uint32, Literal[35]] + GPPut: int + Ignored01: c.Array[ctypes.c_uint32, Literal[92]] +Nvc96fControl: TypeAlias = volatile_struct_Nvc96fControl_struct +volatile_struct_Nvc96fControl_struct.register_fields([('Ignored00', c.Array[NvU32, Literal[35]], 0), ('GPPut', NvU32, 140), ('Ignored01', c.Array[NvU32, Literal[92]], 144)]) +BlackwellAControlGPFifo: TypeAlias = volatile_struct_Nvc96fControl_struct +@c.record +class struct_NV83DE_ALLOC_PARAMETERS(c.Struct): + SIZE = 12 + hDebuggerClient_Obsolete: int + hAppClient: int + hClass3dObject: int +struct_NV83DE_ALLOC_PARAMETERS.register_fields([('hDebuggerClient_Obsolete', NvHandle, 0), ('hAppClient', NvHandle, 4), ('hClass3dObject', NvHandle, 8)]) +NV83DE_ALLOC_PARAMETERS: TypeAlias = struct_NV83DE_ALLOC_PARAMETERS +@c.record +class struct_NVB2CC_ALLOC_PARAMETERS(c.Struct): + SIZE = 8 + hClientTarget: int + hContextTarget: int +struct_NVB2CC_ALLOC_PARAMETERS.register_fields([('hClientTarget', NvHandle, 0), ('hContextTarget', NvHandle, 4)]) +NVB2CC_ALLOC_PARAMETERS: TypeAlias = struct_NVB2CC_ALLOC_PARAMETERS +@c.record +class UVM_RESERVE_VA_PARAMS(c.Struct): + SIZE = 24 + requestedBase: int + length: int + rmStatus: int +NV_STATUS: TypeAlias = ctypes.c_uint32 +UVM_RESERVE_VA_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_RELEASE_VA_PARAMS(c.Struct): + SIZE = 24 + requestedBase: int + length: int + rmStatus: int +UVM_RELEASE_VA_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_REGION_COMMIT_PARAMS(c.Struct): + SIZE = 48 + requestedBase: int + length: int + streamId: int + gpuUuid: struct_nv_uuid + rmStatus: int +UvmStream: TypeAlias = ctypes.c_uint64 +@c.record +class struct_nv_uuid(c.Struct): + SIZE = 16 + uuid: c.Array[ctypes.c_ubyte, Literal[16]] +NvProcessorUuid: TypeAlias = struct_nv_uuid +struct_nv_uuid.register_fields([('uuid', c.Array[NvU8, Literal[16]], 0)]) +UVM_REGION_COMMIT_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('streamId', UvmStream, 16), ('gpuUuid', NvProcessorUuid, 24), ('rmStatus', NV_STATUS, 40)]) +@c.record +class UVM_REGION_DECOMMIT_PARAMS(c.Struct): + SIZE = 24 + requestedBase: int + length: int + rmStatus: int +UVM_REGION_DECOMMIT_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_REGION_SET_STREAM_PARAMS(c.Struct): + SIZE = 48 + requestedBase: int + length: int + newStreamId: int + gpuUuid: struct_nv_uuid + rmStatus: int +UVM_REGION_SET_STREAM_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('newStreamId', UvmStream, 16), ('gpuUuid', NvProcessorUuid, 24), ('rmStatus', NV_STATUS, 40)]) +@c.record +class UVM_SET_STREAM_RUNNING_PARAMS(c.Struct): + SIZE = 16 + streamId: int + rmStatus: int +UVM_SET_STREAM_RUNNING_PARAMS.register_fields([('streamId', UvmStream, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_SET_STREAM_STOPPED_PARAMS(c.Struct): + SIZE = 272 + streamIdArray: c.Array[ctypes.c_uint64, Literal[32]] + nStreams: int + rmStatus: int +UVM_SET_STREAM_STOPPED_PARAMS.register_fields([('streamIdArray', c.Array[UvmStream, Literal[32]], 0), ('nStreams', NvU64, 256), ('rmStatus', NV_STATUS, 264)]) +@c.record +class UVM_ADD_SESSION_PARAMS(c.Struct): + SIZE = 8 + pidTarget: int + rmStatus: int +UVM_ADD_SESSION_PARAMS.register_fields([('pidTarget', NvU32, 0), ('rmStatus', NV_STATUS, 4)]) +@c.record +class UVM_ENABLE_COUNTERS_PARAMS(c.Struct): + SIZE = 904 + config: c.Array[UvmCounterConfig, Literal[32]] + count: int + rmStatus: int +@c.record +class UvmCounterConfig(c.Struct): + SIZE = 28 + scope: int + name: int + gpuid: struct_nv_uuid + state: int +UvmCounterConfig.register_fields([('scope', NvU32, 0), ('name', NvU32, 4), ('gpuid', NvProcessorUuid, 8), ('state', NvU32, 24)]) +UVM_ENABLE_COUNTERS_PARAMS.register_fields([('config', c.Array[UvmCounterConfig, Literal[32]], 0), ('count', NvU32, 896), ('rmStatus', NV_STATUS, 900)]) +@c.record +class UVM_MAP_COUNTER_PARAMS(c.Struct): + SIZE = 40 + scope: int + counterName: int + gpuUuid: struct_nv_uuid + addr: ctypes.c_void_p + rmStatus: int +UVM_MAP_COUNTER_PARAMS.register_fields([('scope', NvU32, 0), ('counterName', NvU32, 4), ('gpuUuid', NvProcessorUuid, 8), ('addr', NvP64, 24), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_CREATE_EVENT_QUEUE_PARAMS(c.Struct): + SIZE = 40 + eventQueueIndex: int + queueSize: int + notificationCount: int + notificationHandle: int + timeStampType: int + rmStatus: int +UVM_CREATE_EVENT_QUEUE_PARAMS.register_fields([('eventQueueIndex', NvU32, 0), ('queueSize', NvU64, 8), ('notificationCount', NvU64, 16), ('notificationHandle', NvU64, 24), ('timeStampType', NvU32, 32), ('rmStatus', NV_STATUS, 36)]) +@c.record +class UVM_REMOVE_EVENT_QUEUE_PARAMS(c.Struct): + SIZE = 8 + eventQueueIndex: int + rmStatus: int +UVM_REMOVE_EVENT_QUEUE_PARAMS.register_fields([('eventQueueIndex', NvU32, 0), ('rmStatus', NV_STATUS, 4)]) +@c.record +class UVM_MAP_EVENT_QUEUE_PARAMS(c.Struct): + SIZE = 56 + eventQueueIndex: int + userRODataAddr: ctypes.c_void_p + userRWDataAddr: ctypes.c_void_p + readIndexAddr: ctypes.c_void_p + writeIndexAddr: ctypes.c_void_p + queueBufferAddr: ctypes.c_void_p + rmStatus: int +UVM_MAP_EVENT_QUEUE_PARAMS.register_fields([('eventQueueIndex', NvU32, 0), ('userRODataAddr', NvP64, 8), ('userRWDataAddr', NvP64, 16), ('readIndexAddr', NvP64, 24), ('writeIndexAddr', NvP64, 32), ('queueBufferAddr', NvP64, 40), ('rmStatus', NV_STATUS, 48)]) +@c.record +class UVM_EVENT_CTRL_PARAMS(c.Struct): + SIZE = 16 + eventQueueIndex: int + eventType: int + enable: int + rmStatus: int +NvS32: TypeAlias = ctypes.c_int32 +UVM_EVENT_CTRL_PARAMS.register_fields([('eventQueueIndex', NvU32, 0), ('eventType', NvS32, 4), ('enable', NvU32, 8), ('rmStatus', NV_STATUS, 12)]) +@c.record +class UVM_GET_GPU_UUID_TABLE_PARAMS(c.Struct): + SIZE = 520 + gpuUuidArray: c.Array[struct_nv_uuid, Literal[32]] + validCount: int + rmStatus: int +UVM_GET_GPU_UUID_TABLE_PARAMS.register_fields([('gpuUuidArray', c.Array[NvProcessorUuid, Literal[32]], 0), ('validCount', NvU32, 512), ('rmStatus', NV_STATUS, 516)]) +@c.record +class UVM_REGISTER_GPU_VASPACE_PARAMS(c.Struct): + SIZE = 32 + gpuUuid: struct_nv_uuid + rmCtrlFd: int + hClient: int + hVaSpace: int + rmStatus: int +UVM_REGISTER_GPU_VASPACE_PARAMS.register_fields([('gpuUuid', NvProcessorUuid, 0), ('rmCtrlFd', NvS32, 16), ('hClient', NvHandle, 20), ('hVaSpace', NvHandle, 24), ('rmStatus', NV_STATUS, 28)]) +@c.record +class UVM_UNREGISTER_GPU_VASPACE_PARAMS(c.Struct): + SIZE = 20 + gpuUuid: struct_nv_uuid + rmStatus: int +UVM_UNREGISTER_GPU_VASPACE_PARAMS.register_fields([('gpuUuid', NvProcessorUuid, 0), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_REGISTER_CHANNEL_PARAMS(c.Struct): + SIZE = 56 + gpuUuid: struct_nv_uuid + rmCtrlFd: int + hClient: int + hChannel: int + base: int + length: int + rmStatus: int +UVM_REGISTER_CHANNEL_PARAMS.register_fields([('gpuUuid', NvProcessorUuid, 0), ('rmCtrlFd', NvS32, 16), ('hClient', NvHandle, 20), ('hChannel', NvHandle, 24), ('base', NvU64, 32), ('length', NvU64, 40), ('rmStatus', NV_STATUS, 48)]) +@c.record +class UVM_UNREGISTER_CHANNEL_PARAMS(c.Struct): + SIZE = 12 + hClient: int + hChannel: int + rmStatus: int +UVM_UNREGISTER_CHANNEL_PARAMS.register_fields([('hClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_ENABLE_PEER_ACCESS_PARAMS(c.Struct): + SIZE = 36 + gpuUuidA: struct_nv_uuid + gpuUuidB: struct_nv_uuid + rmStatus: int +UVM_ENABLE_PEER_ACCESS_PARAMS.register_fields([('gpuUuidA', NvProcessorUuid, 0), ('gpuUuidB', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_DISABLE_PEER_ACCESS_PARAMS(c.Struct): + SIZE = 36 + gpuUuidA: struct_nv_uuid + gpuUuidB: struct_nv_uuid + rmStatus: int +UVM_DISABLE_PEER_ACCESS_PARAMS.register_fields([('gpuUuidA', NvProcessorUuid, 0), ('gpuUuidB', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_MAP_EXTERNAL_ALLOCATION_PARAMS(c.Struct): + SIZE = 9264 + base: int + length: int + offset: int + perGpuAttributes: c.Array[UvmGpuMappingAttributes, Literal[256]] + gpuAttributesCount: int + rmCtrlFd: int + hClient: int + hMemory: int + rmStatus: int +@c.record +class UvmGpuMappingAttributes(c.Struct): + SIZE = 36 + gpuUuid: struct_nv_uuid + gpuMappingType: int + gpuCachingType: int + gpuFormatType: int + gpuElementBits: int + gpuCompressionType: int +UvmGpuMappingAttributes.register_fields([('gpuUuid', NvProcessorUuid, 0), ('gpuMappingType', NvU32, 16), ('gpuCachingType', NvU32, 20), ('gpuFormatType', NvU32, 24), ('gpuElementBits', NvU32, 28), ('gpuCompressionType', NvU32, 32)]) +UVM_MAP_EXTERNAL_ALLOCATION_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('offset', NvU64, 16), ('perGpuAttributes', c.Array[UvmGpuMappingAttributes, Literal[256]], 24), ('gpuAttributesCount', NvU64, 9240), ('rmCtrlFd', NvS32, 9248), ('hClient', NvU32, 9252), ('hMemory', NvU32, 9256), ('rmStatus', NV_STATUS, 9260)]) +@c.record +class UVM_FREE_PARAMS(c.Struct): + SIZE = 16 + base: int + rmStatus: int +UVM_FREE_PARAMS.register_fields([('base', NvU64, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_MEM_MAP_PARAMS(c.Struct): + SIZE = 24 + regionBase: ctypes.c_void_p + regionLength: int + rmStatus: int +UVM_MEM_MAP_PARAMS.register_fields([('regionBase', NvP64, 0), ('regionLength', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_REGISTER_GPU_PARAMS(c.Struct): + SIZE = 40 + gpu_uuid: struct_nv_uuid + numaEnabled: int + numaNodeId: int + rmCtrlFd: int + hClient: int + hSmcPartRef: int + rmStatus: int +UVM_REGISTER_GPU_PARAMS.register_fields([('gpu_uuid', NvProcessorUuid, 0), ('numaEnabled', NvBool, 16), ('numaNodeId', NvS32, 20), ('rmCtrlFd', NvS32, 24), ('hClient', NvHandle, 28), ('hSmcPartRef', NvHandle, 32), ('rmStatus', NV_STATUS, 36)]) +@c.record +class UVM_UNREGISTER_GPU_PARAMS(c.Struct): + SIZE = 20 + gpu_uuid: struct_nv_uuid + rmStatus: int +UVM_UNREGISTER_GPU_PARAMS.register_fields([('gpu_uuid', NvProcessorUuid, 0), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_PAGEABLE_MEM_ACCESS_PARAMS(c.Struct): + SIZE = 8 + pageableMemAccess: int + rmStatus: int +UVM_PAGEABLE_MEM_ACCESS_PARAMS.register_fields([('pageableMemAccess', NvBool, 0), ('rmStatus', NV_STATUS, 4)]) +@c.record +class UVM_SET_PREFERRED_LOCATION_PARAMS(c.Struct): + SIZE = 40 + requestedBase: int + length: int + preferredLocation: struct_nv_uuid + preferredCpuNumaNode: int + rmStatus: int +UVM_SET_PREFERRED_LOCATION_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('preferredLocation', NvProcessorUuid, 16), ('preferredCpuNumaNode', NvS32, 32), ('rmStatus', NV_STATUS, 36)]) +@c.record +class UVM_UNSET_PREFERRED_LOCATION_PARAMS(c.Struct): + SIZE = 24 + requestedBase: int + length: int + rmStatus: int +UVM_UNSET_PREFERRED_LOCATION_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_ENABLE_READ_DUPLICATION_PARAMS(c.Struct): + SIZE = 24 + requestedBase: int + length: int + rmStatus: int +UVM_ENABLE_READ_DUPLICATION_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_DISABLE_READ_DUPLICATION_PARAMS(c.Struct): + SIZE = 24 + requestedBase: int + length: int + rmStatus: int +UVM_DISABLE_READ_DUPLICATION_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_SET_ACCESSED_BY_PARAMS(c.Struct): + SIZE = 40 + requestedBase: int + length: int + accessedByUuid: struct_nv_uuid + rmStatus: int +UVM_SET_ACCESSED_BY_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('accessedByUuid', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_UNSET_ACCESSED_BY_PARAMS(c.Struct): + SIZE = 40 + requestedBase: int + length: int + accessedByUuid: struct_nv_uuid + rmStatus: int +UVM_UNSET_ACCESSED_BY_PARAMS.register_fields([('requestedBase', NvU64, 0), ('length', NvU64, 8), ('accessedByUuid', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_MIGRATE_PARAMS(c.Struct): + SIZE = 80 + base: int + length: int + destinationUuid: struct_nv_uuid + flags: int + semaphoreAddress: int + semaphorePayload: int + cpuNumaNode: int + userSpaceStart: int + userSpaceLength: int + rmStatus: int +UVM_MIGRATE_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('destinationUuid', NvProcessorUuid, 16), ('flags', NvU32, 32), ('semaphoreAddress', NvU64, 40), ('semaphorePayload', NvU32, 48), ('cpuNumaNode', NvS32, 52), ('userSpaceStart', NvU64, 56), ('userSpaceLength', NvU64, 64), ('rmStatus', NV_STATUS, 72)]) +@c.record +class UVM_TOOLS_INIT_EVENT_TRACKER_PARAMS(c.Struct): + SIZE = 56 + queueBuffer: int + queueBufferSize: int + controlBuffer: int + processor: struct_nv_uuid + allProcessors: int + uvmFd: int + rmStatus: int +UVM_TOOLS_INIT_EVENT_TRACKER_PARAMS.register_fields([('queueBuffer', NvU64, 0), ('queueBufferSize', NvU64, 8), ('controlBuffer', NvU64, 16), ('processor', NvProcessorUuid, 24), ('allProcessors', NvU32, 40), ('uvmFd', NvU32, 44), ('rmStatus', NV_STATUS, 48)]) +@c.record +class UVM_TOOLS_SET_NOTIFICATION_THRESHOLD_PARAMS(c.Struct): + SIZE = 8 + notificationThreshold: int + rmStatus: int +UVM_TOOLS_SET_NOTIFICATION_THRESHOLD_PARAMS.register_fields([('notificationThreshold', NvU32, 0), ('rmStatus', NV_STATUS, 4)]) +@c.record +class UVM_TOOLS_EVENT_QUEUE_ENABLE_EVENTS_PARAMS(c.Struct): + SIZE = 16 + eventTypeFlags: int + rmStatus: int +UVM_TOOLS_EVENT_QUEUE_ENABLE_EVENTS_PARAMS.register_fields([('eventTypeFlags', NvU64, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_TOOLS_EVENT_QUEUE_DISABLE_EVENTS_PARAMS(c.Struct): + SIZE = 16 + eventTypeFlags: int + rmStatus: int +UVM_TOOLS_EVENT_QUEUE_DISABLE_EVENTS_PARAMS.register_fields([('eventTypeFlags', NvU64, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_TOOLS_ENABLE_COUNTERS_PARAMS(c.Struct): + SIZE = 16 + counterTypeFlags: int + rmStatus: int +UVM_TOOLS_ENABLE_COUNTERS_PARAMS.register_fields([('counterTypeFlags', NvU64, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_TOOLS_DISABLE_COUNTERS_PARAMS(c.Struct): + SIZE = 16 + counterTypeFlags: int + rmStatus: int +UVM_TOOLS_DISABLE_COUNTERS_PARAMS.register_fields([('counterTypeFlags', NvU64, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS(c.Struct): + SIZE = 40 + buffer: int + size: int + targetVa: int + bytesRead: int + rmStatus: int +UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS.register_fields([('buffer', NvU64, 0), ('size', NvU64, 8), ('targetVa', NvU64, 16), ('bytesRead', NvU64, 24), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS(c.Struct): + SIZE = 40 + buffer: int + size: int + targetVa: int + bytesWritten: int + rmStatus: int +UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS.register_fields([('buffer', NvU64, 0), ('size', NvU64, 8), ('targetVa', NvU64, 16), ('bytesWritten', NvU64, 24), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_TOOLS_GET_PROCESSOR_UUID_TABLE_PARAMS(c.Struct): + SIZE = 16 + tablePtr: int + rmStatus: int +UVM_TOOLS_GET_PROCESSOR_UUID_TABLE_PARAMS.register_fields([('tablePtr', NvU64, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS(c.Struct): + SIZE = 40 + base: int + length: int + gpuUuid: struct_nv_uuid + rmStatus: int +UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('gpuUuid', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_UNMAP_EXTERNAL_PARAMS(c.Struct): + SIZE = 40 + base: int + length: int + gpuUuid: struct_nv_uuid + rmStatus: int +UVM_UNMAP_EXTERNAL_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('gpuUuid', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_TOOLS_FLUSH_EVENTS_PARAMS(c.Struct): + SIZE = 4 + rmStatus: int +UVM_TOOLS_FLUSH_EVENTS_PARAMS.register_fields([('rmStatus', NV_STATUS, 0)]) +@c.record +class UVM_ALLOC_SEMAPHORE_POOL_PARAMS(c.Struct): + SIZE = 9248 + base: int + length: int + perGpuAttributes: c.Array[UvmGpuMappingAttributes, Literal[256]] + gpuAttributesCount: int + rmStatus: int +UVM_ALLOC_SEMAPHORE_POOL_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('perGpuAttributes', c.Array[UvmGpuMappingAttributes, Literal[256]], 16), ('gpuAttributesCount', NvU64, 9232), ('rmStatus', NV_STATUS, 9240)]) +@c.record +class UVM_CLEAN_UP_ZOMBIE_RESOURCES_PARAMS(c.Struct): + SIZE = 4 + rmStatus: int +UVM_CLEAN_UP_ZOMBIE_RESOURCES_PARAMS.register_fields([('rmStatus', NV_STATUS, 0)]) +@c.record +class UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS(c.Struct): + SIZE = 24 + gpu_uuid: struct_nv_uuid + pageableMemAccess: int + rmStatus: int +UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS.register_fields([('gpu_uuid', NvProcessorUuid, 0), ('pageableMemAccess', NvBool, 16), ('rmStatus', NV_STATUS, 20)]) +@c.record +class UVM_POPULATE_PAGEABLE_PARAMS(c.Struct): + SIZE = 24 + base: int + length: int + flags: int + rmStatus: int +UVM_POPULATE_PAGEABLE_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('flags', NvU32, 16), ('rmStatus', NV_STATUS, 20)]) +@c.record +class UVM_VALIDATE_VA_RANGE_PARAMS(c.Struct): + SIZE = 24 + base: int + length: int + rmStatus: int +UVM_VALIDATE_VA_RANGE_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_CREATE_EXTERNAL_RANGE_PARAMS(c.Struct): + SIZE = 24 + base: int + length: int + rmStatus: int +UVM_CREATE_EXTERNAL_RANGE_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('rmStatus', NV_STATUS, 16)]) +@c.record +class UVM_MAP_EXTERNAL_SPARSE_PARAMS(c.Struct): + SIZE = 40 + base: int + length: int + gpuUuid: struct_nv_uuid + rmStatus: int +UVM_MAP_EXTERNAL_SPARSE_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('gpuUuid', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_MM_INITIALIZE_PARAMS(c.Struct): + SIZE = 8 + uvmFd: int + rmStatus: int +UVM_MM_INITIALIZE_PARAMS.register_fields([('uvmFd', NvS32, 0), ('rmStatus', NV_STATUS, 4)]) +UVM_TOOLS_INIT_EVENT_TRACKER_V2_PARAMS: TypeAlias = UVM_TOOLS_INIT_EVENT_TRACKER_PARAMS +UVM_TOOLS_GET_PROCESSOR_UUID_TABLE_V2_PARAMS: TypeAlias = UVM_TOOLS_GET_PROCESSOR_UUID_TABLE_PARAMS +@c.record +class UVM_ALLOC_DEVICE_P2P_PARAMS(c.Struct): + SIZE = 56 + base: int + length: int + offset: int + gpuUuid: struct_nv_uuid + rmCtrlFd: int + hClient: int + hMemory: int + rmStatus: int +UVM_ALLOC_DEVICE_P2P_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('offset', NvU64, 16), ('gpuUuid', NvProcessorUuid, 24), ('rmCtrlFd', NvS32, 40), ('hClient', NvU32, 44), ('hMemory', NvU32, 48), ('rmStatus', NV_STATUS, 52)]) +@c.record +class UVM_CLEAR_ALL_ACCESS_COUNTERS_PARAMS(c.Struct): + SIZE = 4 + rmStatus: int +UVM_CLEAR_ALL_ACCESS_COUNTERS_PARAMS.register_fields([('rmStatus', NV_STATUS, 0)]) +@c.record +class UVM_DISCARD_PARAMS(c.Struct): + SIZE = 32 + base: int + length: int + flags: int + rmStatus: int +UVM_DISCARD_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('flags', NvU64, 16), ('rmStatus', NV_STATUS, 24)]) +@c.record +class UVM_QUERY_RESIDENCY_PARAMS(c.Struct): + SIZE = 80 + base: int + length: int + samplingStride: int + isManagedMemory: int + numSamples: int + pageAddresses: int + pageStatus: int + residency: struct_nv_uuid + resident_nid: int + rmStatus: int +UVM_QUERY_RESIDENCY_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('samplingStride', NvU64, 16), ('isManagedMemory', NvBool, 24), ('numSamples', NvU64, 32), ('pageAddresses', NvU64, 40), ('pageStatus', NvU64, 48), ('residency', NvProcessorUuid, 56), ('resident_nid', NvS32, 72), ('rmStatus', NV_STATUS, 76)]) +@c.record +class UVM_IS_DMA_BUF_IMPORT_SUPPORTED_PARAMS(c.Struct): + SIZE = 8 + dmaBufImportSupported: int + rmStatus: int +UVM_IS_DMA_BUF_IMPORT_SUPPORTED_PARAMS.register_fields([('dmaBufImportSupported', NvBool, 0), ('rmStatus', NV_STATUS, 4)]) +@c.record +class UVM_IMPORT_DMA_BUF_PARAMS(c.Struct): + SIZE = 24 + base: int + length: int + handleFd: int + rmStatus: int +UVM_IMPORT_DMA_BUF_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('handleFd', NvS32, 16), ('rmStatus', NV_STATUS, 20)]) +@c.record +class UVM_MAP_DMA_BUF_PARAMS(c.Struct): + SIZE = 40 + base: int + length: int + gpuUuid: struct_nv_uuid + rmStatus: int +UVM_MAP_DMA_BUF_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('gpuUuid', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_UNMAP_DMA_BUF_PARAMS(c.Struct): + SIZE = 40 + base: int + length: int + gpuUuid: struct_nv_uuid + rmStatus: int +UVM_UNMAP_DMA_BUF_PARAMS.register_fields([('base', NvU64, 0), ('length', NvU64, 8), ('gpuUuid', NvProcessorUuid, 16), ('rmStatus', NV_STATUS, 32)]) +@c.record +class UVM_IS_8_SUPPORTED_PARAMS(c.Struct): + SIZE = 8 + is8Supported: int + rmStatus: int +UVM_IS_8_SUPPORTED_PARAMS.register_fields([('is8Supported', NvU32, 0), ('rmStatus', NV_STATUS, 4)]) +@c.record +class UVM_INITIALIZE_PARAMS(c.Struct): + SIZE = 16 + flags: int + rmStatus: int +UVM_INITIALIZE_PARAMS.register_fields([('flags', NvU64, 0), ('rmStatus', NV_STATUS, 8)]) +@c.record +class nv_pci_info_t(c.Struct): + SIZE = 12 + domain: int + bus: int + slot: int + function: int + vendor_id: int + device_id: int +NvU16: TypeAlias = ctypes.c_uint16 +nv_pci_info_t.register_fields([('domain', NvU32, 0), ('bus', NvU8, 4), ('slot', NvU8, 5), ('function', NvU8, 6), ('vendor_id', NvU16, 8), ('device_id', NvU16, 10)]) +@c.record +class struct_nv_ioctl_xfer(c.Struct): + SIZE = 16 + cmd: int + size: int + ptr: ctypes.c_void_p +struct_nv_ioctl_xfer.register_fields([('cmd', NvU32, 0), ('size', NvU32, 4), ('ptr', NvP64, 8)]) +nv_ioctl_xfer_t: TypeAlias = struct_nv_ioctl_xfer +@c.record +class struct_nv_ioctl_card_info(c.Struct): + SIZE = 72 + valid: int + pci_info: nv_pci_info_t + gpu_id: int + interrupt_line: int + reg_address: int + reg_size: int + fb_address: int + fb_size: int + minor_number: int + dev_name: c.Array[ctypes.c_ubyte, Literal[10]] +struct_nv_ioctl_card_info.register_fields([('valid', NvBool, 0), ('pci_info', nv_pci_info_t, 4), ('gpu_id', NvU32, 16), ('interrupt_line', NvU16, 20), ('reg_address', NvU64, 24), ('reg_size', NvU64, 32), ('fb_address', NvU64, 40), ('fb_size', NvU64, 48), ('minor_number', NvU32, 56), ('dev_name', c.Array[NvU8, Literal[10]], 60)]) +nv_ioctl_card_info_t: TypeAlias = struct_nv_ioctl_card_info +@c.record +class struct_nv_ioctl_alloc_os_event(c.Struct): + SIZE = 16 + hClient: int + hDevice: int + fd: int + Status: int +struct_nv_ioctl_alloc_os_event.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('fd', NvU32, 8), ('Status', NvU32, 12)]) +nv_ioctl_alloc_os_event_t: TypeAlias = struct_nv_ioctl_alloc_os_event +@c.record +class struct_nv_ioctl_free_os_event(c.Struct): + SIZE = 16 + hClient: int + hDevice: int + fd: int + Status: int +struct_nv_ioctl_free_os_event.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('fd', NvU32, 8), ('Status', NvU32, 12)]) +nv_ioctl_free_os_event_t: TypeAlias = struct_nv_ioctl_free_os_event +@c.record +class struct_nv_ioctl_status_code(c.Struct): + SIZE = 12 + domain: int + bus: int + slot: int + status: int +struct_nv_ioctl_status_code.register_fields([('domain', NvU32, 0), ('bus', NvU8, 4), ('slot', NvU8, 5), ('status', NvU32, 8)]) +nv_ioctl_status_code_t: TypeAlias = struct_nv_ioctl_status_code +@c.record +class struct_nv_ioctl_rm_api_version(c.Struct): + SIZE = 72 + cmd: int + reply: int + versionString: c.Array[ctypes.c_char, Literal[64]] +struct_nv_ioctl_rm_api_version.register_fields([('cmd', NvU32, 0), ('reply', NvU32, 4), ('versionString', c.Array[ctypes.c_char, Literal[64]], 8)]) +nv_ioctl_rm_api_version_t: TypeAlias = struct_nv_ioctl_rm_api_version +@c.record +class struct_nv_ioctl_query_device_intr(c.Struct): + SIZE = 8 + intrStatus: int + status: int +struct_nv_ioctl_query_device_intr.register_fields([('intrStatus', NvU32, 0), ('status', NvU32, 4)]) +nv_ioctl_query_device_intr_t: TypeAlias = struct_nv_ioctl_query_device_intr +@c.record +class struct_nv_ioctl_sys_params(c.Struct): + SIZE = 8 + memblock_size: int +struct_nv_ioctl_sys_params.register_fields([('memblock_size', NvU64, 0)]) +nv_ioctl_sys_params_t: TypeAlias = struct_nv_ioctl_sys_params +@c.record +class struct_nv_ioctl_register_fd(c.Struct): + SIZE = 4 + ctl_fd: int +struct_nv_ioctl_register_fd.register_fields([('ctl_fd', ctypes.c_int32, 0)]) +nv_ioctl_register_fd_t: TypeAlias = struct_nv_ioctl_register_fd +@c.record +class struct_nv_ioctl_export_to_dma_buf_fd(c.Struct): + SIZE = 2608 + fd: int + hClient: int + totalObjects: int + numObjects: int + index: int + totalSize: int + mappingType: int + bAllowMmap: int + handles: c.Array[ctypes.c_uint32, Literal[128]] + offsets: c.Array[ctypes.c_uint64, Literal[128]] + sizes: c.Array[ctypes.c_uint64, Literal[128]] + status: int +struct_nv_ioctl_export_to_dma_buf_fd.register_fields([('fd', ctypes.c_int32, 0), ('hClient', NvHandle, 4), ('totalObjects', NvU32, 8), ('numObjects', NvU32, 12), ('index', NvU32, 16), ('totalSize', NvU64, 24), ('mappingType', NvU8, 32), ('bAllowMmap', NvBool, 33), ('handles', c.Array[NvHandle, Literal[128]], 36), ('offsets', c.Array[NvU64, Literal[128]], 552), ('sizes', c.Array[NvU64, Literal[128]], 1576), ('status', NvU32, 2600)]) +nv_ioctl_export_to_dma_buf_fd_t: TypeAlias = struct_nv_ioctl_export_to_dma_buf_fd +@c.record +class struct_nv_ioctl_wait_open_complete(c.Struct): + SIZE = 8 + rc: int + adapterStatus: int +struct_nv_ioctl_wait_open_complete.register_fields([('rc', ctypes.c_int32, 0), ('adapterStatus', NvU32, 4)]) +nv_ioctl_wait_open_complete_t: TypeAlias = struct_nv_ioctl_wait_open_complete +@c.record +class struct_offline_addresses(c.Struct): + SIZE = 520 + addresses: c.Array[ctypes.c_uint64, Literal[64]] + numEntries: int +uint64_t: TypeAlias = ctypes.c_uint64 +uint32_t: TypeAlias = ctypes.c_uint32 +struct_offline_addresses.register_fields([('addresses', c.Array[uint64_t, Literal[64]], 0), ('numEntries', uint32_t, 512)]) +nv_offline_addresses_t: TypeAlias = struct_offline_addresses +@c.record +class struct_nv_ioctl_numa_info(c.Struct): + SIZE = 560 + nid: int + status: int + memblock_size: int + numa_mem_addr: int + numa_mem_size: int + use_auto_online: int + offline_addresses: struct_offline_addresses +int32_t: TypeAlias = ctypes.c_int32 +uint8_t: TypeAlias = ctypes.c_ubyte +struct_nv_ioctl_numa_info.register_fields([('nid', int32_t, 0), ('status', int32_t, 4), ('memblock_size', uint64_t, 8), ('numa_mem_addr', uint64_t, 16), ('numa_mem_size', uint64_t, 24), ('use_auto_online', uint8_t, 32), ('offline_addresses', nv_offline_addresses_t, 40)]) +nv_ioctl_numa_info_t: TypeAlias = struct_nv_ioctl_numa_info +@c.record +class struct_nv_ioctl_set_numa_status(c.Struct): + SIZE = 4 + status: int +struct_nv_ioctl_set_numa_status.register_fields([('status', int32_t, 0)]) +nv_ioctl_set_numa_status_t: TypeAlias = struct_nv_ioctl_set_numa_status +@c.record +class nv_ioctl_nvos02_parameters_with_fd(c.Struct): + SIZE = 56 + params: NVOS02_PARAMETERS + fd: int +@c.record +class NVOS02_PARAMETERS(c.Struct): + SIZE = 48 + hRoot: int + hObjectParent: int + hObjectNew: int + hClass: int + flags: int + pMemory: ctypes.c_void_p + limit: int + status: int +NVOS02_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('hObjectNew', NvHandle, 8), ('hClass', NvV32, 12), ('flags', NvV32, 16), ('pMemory', NvP64, 24), ('limit', NvU64, 32), ('status', NvV32, 40)]) +nv_ioctl_nvos02_parameters_with_fd.register_fields([('params', NVOS02_PARAMETERS, 0), ('fd', ctypes.c_int32, 48)]) +@c.record +class nv_ioctl_nvos33_parameters_with_fd(c.Struct): + SIZE = 56 + params: NVOS33_PARAMETERS + fd: int +@c.record +class NVOS33_PARAMETERS(c.Struct): + SIZE = 48 + hClient: int + hDevice: int + hMemory: int + offset: int + length: int + pLinearAddress: ctypes.c_void_p + status: int + flags: int +NVOS33_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hMemory', NvHandle, 8), ('offset', NvU64, 16), ('length', NvU64, 24), ('pLinearAddress', NvP64, 32), ('status', NvU32, 40), ('flags', NvU32, 44)]) +nv_ioctl_nvos33_parameters_with_fd.register_fields([('params', NVOS33_PARAMETERS, 0), ('fd', ctypes.c_int32, 48)]) +@c.record +class struct_NV_MEMORY_DESC_PARAMS(c.Struct): + SIZE = 24 + base: int + size: int + addressSpace: int + cacheAttrib: int +struct_NV_MEMORY_DESC_PARAMS.register_fields([('base', NvU64, 0), ('size', NvU64, 8), ('addressSpace', NvU32, 16), ('cacheAttrib', NvU32, 20)]) +NV_MEMORY_DESC_PARAMS: TypeAlias = struct_NV_MEMORY_DESC_PARAMS +@c.record +class struct_NV_CHANNEL_ALLOC_PARAMS(c.Struct): + SIZE = 376 + hObjectError: int + hObjectBuffer: int + gpFifoOffset: int + gpFifoEntries: int + flags: int + hContextShare: int + hVASpace: int + hHandleVASpace: int + hUserdMemory: c.Array[ctypes.c_uint32, Literal[8]] + userdOffset: c.Array[ctypes.c_uint64, Literal[8]] + engineType: int + cid: int + subDeviceId: int + hObjectEccError: int + instanceMem: struct_NV_MEMORY_DESC_PARAMS + userdMem: struct_NV_MEMORY_DESC_PARAMS + ramfcMem: struct_NV_MEMORY_DESC_PARAMS + mthdbufMem: struct_NV_MEMORY_DESC_PARAMS + hPhysChannelGroup: int + internalFlags: int + errorNotifierMem: struct_NV_MEMORY_DESC_PARAMS + eccErrorNotifierMem: struct_NV_MEMORY_DESC_PARAMS + ProcessID: int + SubProcessID: int + encryptIv: c.Array[ctypes.c_uint32, Literal[3]] + decryptIv: c.Array[ctypes.c_uint32, Literal[3]] + hmacNonce: c.Array[ctypes.c_uint32, Literal[8]] + tpcConfigID: int +struct_NV_CHANNEL_ALLOC_PARAMS.register_fields([('hObjectError', NvHandle, 0), ('hObjectBuffer', NvHandle, 4), ('gpFifoOffset', NvU64, 8), ('gpFifoEntries', NvU32, 16), ('flags', NvU32, 20), ('hContextShare', NvHandle, 24), ('hVASpace', NvHandle, 28), ('hHandleVASpace', NvHandle, 32), ('hUserdMemory', c.Array[NvHandle, Literal[8]], 36), ('userdOffset', c.Array[NvU64, Literal[8]], 72), ('engineType', NvU32, 136), ('cid', NvU32, 140), ('subDeviceId', NvU32, 144), ('hObjectEccError', NvHandle, 148), ('instanceMem', NV_MEMORY_DESC_PARAMS, 152), ('userdMem', NV_MEMORY_DESC_PARAMS, 176), ('ramfcMem', NV_MEMORY_DESC_PARAMS, 200), ('mthdbufMem', NV_MEMORY_DESC_PARAMS, 224), ('hPhysChannelGroup', NvHandle, 248), ('internalFlags', NvU32, 252), ('errorNotifierMem', NV_MEMORY_DESC_PARAMS, 256), ('eccErrorNotifierMem', NV_MEMORY_DESC_PARAMS, 280), ('ProcessID', NvU32, 304), ('SubProcessID', NvU32, 308), ('encryptIv', c.Array[NvU32, Literal[3]], 312), ('decryptIv', c.Array[NvU32, Literal[3]], 324), ('hmacNonce', c.Array[NvU32, Literal[8]], 336), ('tpcConfigID', NvU32, 368)]) +NV_CHANNEL_ALLOC_PARAMS: TypeAlias = struct_NV_CHANNEL_ALLOC_PARAMS +NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS: TypeAlias = struct_NV_CHANNEL_ALLOC_PARAMS +@c.record +class NVOS00_PARAMETERS(c.Struct): + SIZE = 16 + hRoot: int + hObjectParent: int + hObjectOld: int + status: int +NVOS00_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('hObjectOld', NvHandle, 8), ('status', NvV32, 12)]) +@c.record +class NVOS05_PARAMETERS(c.Struct): + SIZE = 20 + hRoot: int + hObjectParent: int + hObjectNew: int + hClass: int + status: int +NVOS05_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('hObjectNew', NvHandle, 8), ('hClass', NvV32, 12), ('status', NvV32, 16)]) +Callback1ArgVoidReturn: TypeAlias = c.CFUNCTYPE[None, [ctypes.c_void_p]] +Callback5ArgVoidReturn: TypeAlias = c.CFUNCTYPE[None, [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32]] +@c.record +class NVOS10_EVENT_KERNEL_CALLBACK(c.Struct): + SIZE = 16 + func: c.CFUNCTYPE[None, [ctypes.c_void_p]] + arg: ctypes.c_void_p +NVOS10_EVENT_KERNEL_CALLBACK.register_fields([('func', Callback1ArgVoidReturn, 0), ('arg', ctypes.c_void_p, 8)]) +@c.record +class NVOS10_EVENT_KERNEL_CALLBACK_EX(c.Struct): + SIZE = 16 + func: c.CFUNCTYPE[None, [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32]] + arg: ctypes.c_void_p +NVOS10_EVENT_KERNEL_CALLBACK_EX.register_fields([('func', Callback5ArgVoidReturn, 0), ('arg', ctypes.c_void_p, 8)]) +@c.record +class NVOS_I2C_ACCESS_PARAMS(c.Struct): + SIZE = 32 + hClient: int + hDevice: int + paramSize: int + paramStructPtr: ctypes.c_void_p + status: int +NVOS_I2C_ACCESS_PARAMS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('paramSize', NvU32, 8), ('paramStructPtr', NvP64, 16), ('status', NvV32, 24)]) +@c.record +class NVOS21_PARAMETERS(c.Struct): + SIZE = 32 + hRoot: int + hObjectParent: int + hObjectNew: int + hClass: int + pAllocParms: ctypes.c_void_p + paramsSize: int + status: int +NVOS21_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('hObjectNew', NvHandle, 8), ('hClass', NvV32, 12), ('pAllocParms', NvP64, 16), ('paramsSize', NvU32, 24), ('status', NvV32, 28)]) +@c.record +class NVOS64_PARAMETERS(c.Struct): + SIZE = 48 + hRoot: int + hObjectParent: int + hObjectNew: int + hClass: int + pAllocParms: ctypes.c_void_p + pRightsRequested: ctypes.c_void_p + paramsSize: int + flags: int + status: int +NVOS64_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('hObjectNew', NvHandle, 8), ('hClass', NvV32, 12), ('pAllocParms', NvP64, 16), ('pRightsRequested', NvP64, 24), ('paramsSize', NvU32, 32), ('flags', NvU32, 36), ('status', NvV32, 40)]) +@c.record +class NVOS62_PARAMETERS(c.Struct): + SIZE = 24 + hRoot: int + hObjectParent: int + hObjectNew: int + hClass: int + paramSize: int + status: int +NVOS62_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('hObjectNew', NvHandle, 8), ('hClass', NvV32, 12), ('paramSize', NvU32, 16), ('status', NvV32, 20)]) +@c.record +class NVOS65_PARAMETERS(c.Struct): + SIZE = 32 + hRoot: int + hObjectParent: int + hObjectNew: int + hClass: int + paramSize: int + versionMagic: int + maskSize: int + status: int +NVOS65_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('hObjectNew', NvHandle, 8), ('hClass', NvV32, 12), ('paramSize', NvU32, 16), ('versionMagic', NvU32, 20), ('maskSize', NvU32, 24), ('status', NvV32, 28)]) +@c.record +class NVOS30_PARAMETERS(c.Struct): + SIZE = 56 + hClient: int + hDevice: int + hChannel: int + numChannels: int + phClients: ctypes.c_void_p + phDevices: ctypes.c_void_p + phChannels: ctypes.c_void_p + flags: int + timeout: int + status: int +NVOS30_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hChannel', NvHandle, 8), ('numChannels', NvV32, 12), ('phClients', NvP64, 16), ('phDevices', NvP64, 24), ('phChannels', NvP64, 32), ('flags', NvV32, 40), ('timeout', NvV32, 44), ('status', NvV32, 48)]) +BindResultFunc: TypeAlias = c.CFUNCTYPE[None, [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32]] +@c.record +class NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS(c.Struct): + SIZE = 16 + sgt: ctypes.c_void_p + gem: ctypes.c_void_p +NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS.register_fields([('sgt', NvP64, 0), ('gem', NvP64, 8)]) +@c.record +class NVOS32_PARAMETERS(c.Struct): + SIZE = 184 + hRoot: int + hObjectParent: int + function: int + hVASpace: int + ivcHeapNumber: int + status: int + total: int + free: int + data: NVOS32_PARAMETERS_data +NvS16: TypeAlias = ctypes.c_int16 +@c.record +class NVOS32_PARAMETERS_data(c.Struct): + SIZE = 144 + AllocSize: NVOS32_PARAMETERS_data_AllocSize + AllocTiledPitchHeight: NVOS32_PARAMETERS_data_AllocTiledPitchHeight + Free: NVOS32_PARAMETERS_data_Free + ReleaseCompr: NVOS32_PARAMETERS_data_ReleaseCompr + ReacquireCompr: NVOS32_PARAMETERS_data_ReacquireCompr + Info: NVOS32_PARAMETERS_data_Info + AllocSizeRange: NVOS32_PARAMETERS_data_AllocSizeRange + AllocHintAlignment: NVOS32_PARAMETERS_data_AllocHintAlignment + HwAlloc: NVOS32_PARAMETERS_data_HwAlloc + HwFree: NVOS32_PARAMETERS_data_HwFree + AllocOsDesc: NVOS32_PARAMETERS_data_AllocOsDesc +@c.record +class NVOS32_PARAMETERS_data_AllocSize(c.Struct): + SIZE = 120 + owner: int + hMemory: int + type: int + flags: int + attr: int + format: int + comprCovg: int + zcullCovg: int + partitionStride: int + width: int + height: int + size: int + alignment: int + offset: int + limit: int + address: ctypes.c_void_p + rangeBegin: int + rangeEnd: int + attr2: int + ctagOffset: int + numaNode: int +NVOS32_PARAMETERS_data_AllocSize.register_fields([('owner', NvU32, 0), ('hMemory', NvHandle, 4), ('type', NvU32, 8), ('flags', NvU32, 12), ('attr', NvU32, 16), ('format', NvU32, 20), ('comprCovg', NvU32, 24), ('zcullCovg', NvU32, 28), ('partitionStride', NvU32, 32), ('width', NvU32, 36), ('height', NvU32, 40), ('size', NvU64, 48), ('alignment', NvU64, 56), ('offset', NvU64, 64), ('limit', NvU64, 72), ('address', NvP64, 80), ('rangeBegin', NvU64, 88), ('rangeEnd', NvU64, 96), ('attr2', NvU32, 104), ('ctagOffset', NvU32, 108), ('numaNode', NvS32, 112)]) +@c.record +class NVOS32_PARAMETERS_data_AllocTiledPitchHeight(c.Struct): + SIZE = 120 + owner: int + hMemory: int + type: int + flags: int + height: int + pitch: int + attr: int + width: int + format: int + comprCovg: int + zcullCovg: int + partitionStride: int + size: int + alignment: int + offset: int + limit: int + address: ctypes.c_void_p + rangeBegin: int + rangeEnd: int + attr2: int + ctagOffset: int + numaNode: int +NVOS32_PARAMETERS_data_AllocTiledPitchHeight.register_fields([('owner', NvU32, 0), ('hMemory', NvHandle, 4), ('type', NvU32, 8), ('flags', NvU32, 12), ('height', NvU32, 16), ('pitch', NvS32, 20), ('attr', NvU32, 24), ('width', NvU32, 28), ('format', NvU32, 32), ('comprCovg', NvU32, 36), ('zcullCovg', NvU32, 40), ('partitionStride', NvU32, 44), ('size', NvU64, 48), ('alignment', NvU64, 56), ('offset', NvU64, 64), ('limit', NvU64, 72), ('address', NvP64, 80), ('rangeBegin', NvU64, 88), ('rangeEnd', NvU64, 96), ('attr2', NvU32, 104), ('ctagOffset', NvU32, 108), ('numaNode', NvS32, 112)]) +@c.record +class NVOS32_PARAMETERS_data_Free(c.Struct): + SIZE = 12 + owner: int + hMemory: int + flags: int +NVOS32_PARAMETERS_data_Free.register_fields([('owner', NvU32, 0), ('hMemory', NvHandle, 4), ('flags', NvU32, 8)]) +@c.record +class NVOS32_PARAMETERS_data_ReleaseCompr(c.Struct): + SIZE = 12 + owner: int + flags: int + hMemory: int +NVOS32_PARAMETERS_data_ReleaseCompr.register_fields([('owner', NvU32, 0), ('flags', NvU32, 4), ('hMemory', NvHandle, 8)]) +@c.record +class NVOS32_PARAMETERS_data_ReacquireCompr(c.Struct): + SIZE = 12 + owner: int + flags: int + hMemory: int +NVOS32_PARAMETERS_data_ReacquireCompr.register_fields([('owner', NvU32, 0), ('flags', NvU32, 4), ('hMemory', NvHandle, 8)]) +@c.record +class NVOS32_PARAMETERS_data_Info(c.Struct): + SIZE = 32 + attr: int + offset: int + size: int + base: int +NVOS32_PARAMETERS_data_Info.register_fields([('attr', NvU32, 0), ('offset', NvU64, 8), ('size', NvU64, 16), ('base', NvU64, 24)]) +@c.record +class NVOS32_PARAMETERS_data_AllocSizeRange(c.Struct): + SIZE = 112 + owner: int + hMemory: int + type: int + flags: int + attr: int + format: int + comprCovg: int + zcullCovg: int + partitionStride: int + size: int + alignment: int + offset: int + limit: int + rangeBegin: int + rangeEnd: int + address: ctypes.c_void_p + attr2: int + ctagOffset: int + numaNode: int +NVOS32_PARAMETERS_data_AllocSizeRange.register_fields([('owner', NvU32, 0), ('hMemory', NvHandle, 4), ('type', NvU32, 8), ('flags', NvU32, 12), ('attr', NvU32, 16), ('format', NvU32, 20), ('comprCovg', NvU32, 24), ('zcullCovg', NvU32, 28), ('partitionStride', NvU32, 32), ('size', NvU64, 40), ('alignment', NvU64, 48), ('offset', NvU64, 56), ('limit', NvU64, 64), ('rangeBegin', NvU64, 72), ('rangeEnd', NvU64, 80), ('address', NvP64, 88), ('attr2', NvU32, 96), ('ctagOffset', NvU32, 100), ('numaNode', NvS32, 104)]) +@c.record +class NVOS32_PARAMETERS_data_AllocHintAlignment(c.Struct): + SIZE = 56 + alignType: int + alignAttr: int + alignInputFlags: int + alignSize: int + alignHeight: int + alignWidth: int + alignPitch: int + alignPad: int + alignMask: int + alignKind: int + alignAdjust: int + alignAttr2: int +NVOS32_PARAMETERS_data_AllocHintAlignment.register_fields([('alignType', NvU32, 0), ('alignAttr', NvU32, 4), ('alignInputFlags', NvU32, 8), ('alignSize', NvU64, 16), ('alignHeight', NvU32, 24), ('alignWidth', NvU32, 28), ('alignPitch', NvU32, 32), ('alignPad', NvU32, 36), ('alignMask', NvU32, 40), ('alignKind', NvU32, 44), ('alignAdjust', NvU32, 48), ('alignAttr2', NvU32, 52)]) +@c.record +class NVOS32_PARAMETERS_data_HwAlloc(c.Struct): + SIZE = 144 + allocOwner: int + allochMemory: int + flags: int + allocType: int + allocAttr: int + allocInputFlags: int + allocSize: int + allocHeight: int + allocWidth: int + allocPitch: int + allocMask: int + allocComprCovg: int + allocZcullCovg: int + bindResultFunc: ctypes.c_void_p + pHandle: ctypes.c_void_p + hResourceHandle: int + retAttr: int + kind: int + osDeviceHandle: int + allocAttr2: int + retAttr2: int + allocAddr: int + comprInfo: NVOS32_PARAMETERS_data_HwAlloc_comprInfo + uncompressedKind: int +@c.record +class NVOS32_PARAMETERS_data_HwAlloc_comprInfo(c.Struct): + SIZE = 24 + compPageShift: int + compressedKind: int + compTagLineMin: int + compPageIndexLo: int + compPageIndexHi: int + compTagLineMultiplier: int +NVOS32_PARAMETERS_data_HwAlloc_comprInfo.register_fields([('compPageShift', NvU32, 0), ('compressedKind', NvU32, 4), ('compTagLineMin', NvU32, 8), ('compPageIndexLo', NvU32, 12), ('compPageIndexHi', NvU32, 16), ('compTagLineMultiplier', NvU32, 20)]) +NVOS32_PARAMETERS_data_HwAlloc.register_fields([('allocOwner', NvU32, 0), ('allochMemory', NvHandle, 4), ('flags', NvU32, 8), ('allocType', NvU32, 12), ('allocAttr', NvU32, 16), ('allocInputFlags', NvU32, 20), ('allocSize', NvU64, 24), ('allocHeight', NvU32, 32), ('allocWidth', NvU32, 36), ('allocPitch', NvU32, 40), ('allocMask', NvU32, 44), ('allocComprCovg', NvU32, 48), ('allocZcullCovg', NvU32, 52), ('bindResultFunc', NvP64, 56), ('pHandle', NvP64, 64), ('hResourceHandle', NvHandle, 72), ('retAttr', NvU32, 76), ('kind', NvU32, 80), ('osDeviceHandle', NvU64, 88), ('allocAttr2', NvU32, 96), ('retAttr2', NvU32, 100), ('allocAddr', NvU64, 104), ('comprInfo', NVOS32_PARAMETERS_data_HwAlloc_comprInfo, 112), ('uncompressedKind', NvU32, 136)]) +@c.record +class NVOS32_PARAMETERS_data_HwFree(c.Struct): + SIZE = 8 + hResourceHandle: int + flags: int +NVOS32_PARAMETERS_data_HwFree.register_fields([('hResourceHandle', NvHandle, 0), ('flags', NvU32, 4)]) +@c.record +class NVOS32_PARAMETERS_data_AllocOsDesc(c.Struct): + SIZE = 48 + hMemory: int + type: int + flags: int + attr: int + attr2: int + descriptor: ctypes.c_void_p + limit: int + descriptorType: int +NVOS32_PARAMETERS_data_AllocOsDesc.register_fields([('hMemory', NvHandle, 0), ('type', NvU32, 4), ('flags', NvU32, 8), ('attr', NvU32, 12), ('attr2', NvU32, 16), ('descriptor', NvP64, 24), ('limit', NvU64, 32), ('descriptorType', NvU32, 40)]) +NVOS32_PARAMETERS_data.register_fields([('AllocSize', NVOS32_PARAMETERS_data_AllocSize, 0), ('AllocTiledPitchHeight', NVOS32_PARAMETERS_data_AllocTiledPitchHeight, 0), ('Free', NVOS32_PARAMETERS_data_Free, 0), ('ReleaseCompr', NVOS32_PARAMETERS_data_ReleaseCompr, 0), ('ReacquireCompr', NVOS32_PARAMETERS_data_ReacquireCompr, 0), ('Info', NVOS32_PARAMETERS_data_Info, 0), ('AllocSizeRange', NVOS32_PARAMETERS_data_AllocSizeRange, 0), ('AllocHintAlignment', NVOS32_PARAMETERS_data_AllocHintAlignment, 0), ('HwAlloc', NVOS32_PARAMETERS_data_HwAlloc, 0), ('HwFree', NVOS32_PARAMETERS_data_HwFree, 0), ('AllocOsDesc', NVOS32_PARAMETERS_data_AllocOsDesc, 0)]) +NVOS32_PARAMETERS.register_fields([('hRoot', NvHandle, 0), ('hObjectParent', NvHandle, 4), ('function', NvU32, 8), ('hVASpace', NvHandle, 12), ('ivcHeapNumber', NvS16, 16), ('status', NvV32, 20), ('total', NvU64, 24), ('free', NvU64, 32), ('data', NVOS32_PARAMETERS_data, 40)]) +@c.record +class NV_CONTEXT_DMA_ALLOCATION_PARAMS(c.Struct): + SIZE = 32 + hSubDevice: int + flags: int + hMemory: int + offset: int + limit: int +NV_CONTEXT_DMA_ALLOCATION_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('flags', NvV32, 4), ('hMemory', NvHandle, 8), ('offset', NvU64, 16), ('limit', NvU64, 24)]) +@c.record +class NV_MEMORY_ALLOCATION_PARAMS(c.Struct): + SIZE = 128 + owner: int + type: int + flags: int + width: int + height: int + pitch: int + attr: int + attr2: int + format: int + comprCovg: int + zcullCovg: int + rangeLo: int + rangeHi: int + size: int + alignment: int + offset: int + limit: int + address: ctypes.c_void_p + ctagOffset: int + hVASpace: int + internalflags: int + tag: int + numaNode: int +NV_MEMORY_ALLOCATION_PARAMS.register_fields([('owner', NvU32, 0), ('type', NvU32, 4), ('flags', NvU32, 8), ('width', NvU32, 12), ('height', NvU32, 16), ('pitch', NvS32, 20), ('attr', NvU32, 24), ('attr2', NvU32, 28), ('format', NvU32, 32), ('comprCovg', NvU32, 36), ('zcullCovg', NvU32, 40), ('rangeLo', NvU64, 48), ('rangeHi', NvU64, 56), ('size', NvU64, 64), ('alignment', NvU64, 72), ('offset', NvU64, 80), ('limit', NvU64, 88), ('address', NvP64, 96), ('ctagOffset', NvU32, 104), ('hVASpace', NvHandle, 108), ('internalflags', NvU32, 112), ('tag', NvU32, 116), ('numaNode', NvS32, 120)]) +@c.record +class NV_OS_DESC_MEMORY_ALLOCATION_PARAMS(c.Struct): + SIZE = 40 + type: int + flags: int + attr: int + attr2: int + descriptor: ctypes.c_void_p + limit: int + descriptorType: int + tag: int +NV_OS_DESC_MEMORY_ALLOCATION_PARAMS.register_fields([('type', NvU32, 0), ('flags', NvU32, 4), ('attr', NvU32, 8), ('attr2', NvU32, 12), ('descriptor', NvP64, 16), ('limit', NvU64, 24), ('descriptorType', NvU32, 32), ('tag', NvU32, 36)]) +@c.record +class NV_USER_LOCAL_DESC_MEMORY_ALLOCATION_PARAMS(c.Struct): + SIZE = 32 + flags: int + physAddr: int + size: int + tag: int + bGuestAllocated: int +NV_USER_LOCAL_DESC_MEMORY_ALLOCATION_PARAMS.register_fields([('flags', NvU32, 0), ('physAddr', NvU64, 8), ('size', NvU64, 16), ('tag', NvU32, 24), ('bGuestAllocated', NvBool, 28)]) +@c.record +class NV_MEMORY_HW_RESOURCES_ALLOCATION_PARAMS(c.Struct): + SIZE = 120 + owner: int + flags: int + type: int + attr: int + attr2: int + height: int + width: int + pitch: int + alignment: int + comprCovg: int + zcullCovg: int + kind: int + bindResultFunc: ctypes.c_void_p + pHandle: ctypes.c_void_p + osDeviceHandle: int + size: int + allocAddr: int + compPageShift: int + compressedKind: int + compTagLineMin: int + compPageIndexLo: int + compPageIndexHi: int + compTagLineMultiplier: int + uncompressedKind: int + tag: int +NV_MEMORY_HW_RESOURCES_ALLOCATION_PARAMS.register_fields([('owner', NvU32, 0), ('flags', NvU32, 4), ('type', NvU32, 8), ('attr', NvU32, 12), ('attr2', NvU32, 16), ('height', NvU32, 20), ('width', NvU32, 24), ('pitch', NvU32, 28), ('alignment', NvU32, 32), ('comprCovg', NvU32, 36), ('zcullCovg', NvU32, 40), ('kind', NvU32, 44), ('bindResultFunc', NvP64, 48), ('pHandle', NvP64, 56), ('osDeviceHandle', NvU64, 64), ('size', NvU64, 72), ('allocAddr', NvU64, 80), ('compPageShift', NvU32, 88), ('compressedKind', NvU32, 92), ('compTagLineMin', NvU32, 96), ('compPageIndexLo', NvU32, 100), ('compPageIndexHi', NvU32, 104), ('compTagLineMultiplier', NvU32, 108), ('uncompressedKind', NvU32, 112), ('tag', NvU32, 116)]) +@c.record +class NVOS34_PARAMETERS(c.Struct): + SIZE = 32 + hClient: int + hDevice: int + hMemory: int + pLinearAddress: ctypes.c_void_p + status: int + flags: int +NVOS34_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hMemory', NvHandle, 8), ('pLinearAddress', NvP64, 16), ('status', NvU32, 24), ('flags', NvU32, 28)]) +@c.record +class NVOS38_PARAMETERS(c.Struct): + SIZE = 72 + hClient: int + hObject: int + AccessType: int + DevNodeLength: int + pDevNode: ctypes.c_void_p + ParmStrLength: int + pParmStr: ctypes.c_void_p + BinaryDataLength: int + pBinaryData: ctypes.c_void_p + Data: int + Entry: int + status: int +NVOS38_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hObject', NvHandle, 4), ('AccessType', NvV32, 8), ('DevNodeLength', NvV32, 12), ('pDevNode', NvP64, 16), ('ParmStrLength', NvV32, 24), ('pParmStr', NvP64, 32), ('BinaryDataLength', NvV32, 40), ('pBinaryData', NvP64, 48), ('Data', NvV32, 56), ('Entry', NvV32, 60), ('status', NvV32, 64)]) +@c.record +class NVOS39_PARAMETERS(c.Struct): + SIZE = 56 + hObjectParent: int + hSubDevice: int + hObjectNew: int + hClass: int + flags: int + selector: int + hMemory: int + offset: int + limit: int + status: int +NVOS39_PARAMETERS.register_fields([('hObjectParent', NvHandle, 0), ('hSubDevice', NvHandle, 4), ('hObjectNew', NvHandle, 8), ('hClass', NvV32, 12), ('flags', NvV32, 16), ('selector', NvU32, 20), ('hMemory', NvHandle, 24), ('offset', NvU64, 32), ('limit', NvU64, 40), ('status', NvV32, 48)]) +@c.record +class NvUnixEvent(c.Struct): + SIZE = 16 + hObject: int + NotifyIndex: int + info32: int + info16: int +NvUnixEvent.register_fields([('hObject', NvHandle, 0), ('NotifyIndex', NvV32, 4), ('info32', NvV32, 8), ('info16', NvU16, 12)]) +@c.record +class NVOS41_PARAMETERS(c.Struct): + SIZE = 16 + pEvent: ctypes.c_void_p + MoreEvents: int + status: int +NVOS41_PARAMETERS.register_fields([('pEvent', NvP64, 0), ('MoreEvents', NvV32, 8), ('status', NvV32, 12)]) +@c.record +class NVOS2C_PARAMETERS(c.Struct): + SIZE = 32 + hClient: int + hDevice: int + offset: int + bar: int + bytes: int + write: int + data: int + status: int +NVOS2C_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('offset', NvU32, 8), ('bar', NvU32, 12), ('bytes', NvU32, 16), ('write', NvU32, 20), ('data', NvU32, 24), ('status', NvU32, 28)]) +@c.record +class NVOS46_PARAMETERS(c.Struct): + SIZE = 64 + hClient: int + hDevice: int + hDma: int + hMemory: int + offset: int + length: int + flags: int + flags2: int + kindOverride: int + dmaOffset: int + status: int +NVOS46_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hDma', NvHandle, 8), ('hMemory', NvHandle, 12), ('offset', NvU64, 16), ('length', NvU64, 24), ('flags', NvV32, 32), ('flags2', NvV32, 36), ('kindOverride', NvV32, 40), ('dmaOffset', NvU64, 48), ('status', NvV32, 56)]) +NV_MAP_MEMORY_DMA_PARAMETERS: TypeAlias = NVOS46_PARAMETERS +@c.record +class NVOS47_PARAMETERS(c.Struct): + SIZE = 48 + hClient: int + hDevice: int + hDma: int + hMemory: int + flags: int + dmaOffset: int + size: int + status: int +NVOS47_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hDma', NvHandle, 8), ('hMemory', NvHandle, 12), ('flags', NvV32, 16), ('dmaOffset', NvU64, 24), ('size', NvU64, 32), ('status', NvV32, 40)]) +NV_UNMAP_MEMORY_DMA_PARAMETERS: TypeAlias = NVOS47_PARAMETERS +@c.record +class NVOS49_PARAMETERS(c.Struct): + SIZE = 16 + hClient: int + hChannel: int + hCtxDma: int + status: int +NVOS49_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('hCtxDma', NvHandle, 8), ('status', NvV32, 12)]) +@c.record +class NVOS54_PARAMETERS(c.Struct): + SIZE = 32 + hClient: int + hObject: int + cmd: int + flags: int + params: ctypes.c_void_p + paramsSize: int + status: int +NVOS54_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hObject', NvHandle, 4), ('cmd', NvV32, 8), ('flags', NvU32, 12), ('params', NvP64, 16), ('paramsSize', NvU32, 24), ('status', NvV32, 28)]) +@c.record +class NVOS63_PARAMETERS(c.Struct): + SIZE = 20 + hClient: int + hObject: int + cmd: int + paramsSize: int + status: int +NVOS63_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hObject', NvHandle, 4), ('cmd', NvV32, 8), ('paramsSize', NvU32, 12), ('status', NvV32, 16)]) +@c.record +class NVOS55_PARAMETERS(c.Struct): + SIZE = 28 + hClient: int + hParent: int + hObject: int + hClientSrc: int + hObjectSrc: int + flags: int + status: int +NVOS55_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hParent', NvHandle, 4), ('hObject', NvHandle, 8), ('hClientSrc', NvHandle, 12), ('hObjectSrc', NvHandle, 16), ('flags', NvU32, 20), ('status', NvU32, 24)]) +@c.record +class NVOS56_PARAMETERS(c.Struct): + SIZE = 40 + hClient: int + hDevice: int + hMemory: int + pOldCpuAddress: ctypes.c_void_p + pNewCpuAddress: ctypes.c_void_p + status: int +NVOS56_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hMemory', NvHandle, 8), ('pOldCpuAddress', NvP64, 16), ('pNewCpuAddress', NvP64, 24), ('status', NvV32, 32)]) +@c.record +class NVOS57_PARAMETERS(c.Struct): + SIZE = 24 + hClient: int + hObject: int + sharePolicy: struct_RS_SHARE_POLICY + status: int +@c.record +class struct_RS_SHARE_POLICY(c.Struct): + SIZE = 12 + target: int + accessMask: struct_RS_ACCESS_MASK + type: int + action: int +RS_SHARE_POLICY: TypeAlias = struct_RS_SHARE_POLICY +@c.record +class struct_RS_ACCESS_MASK(c.Struct): + SIZE = 4 + limbs: c.Array[ctypes.c_uint32, Literal[1]] +RS_ACCESS_MASK: TypeAlias = struct_RS_ACCESS_MASK +RsAccessLimb: TypeAlias = ctypes.c_uint32 +struct_RS_ACCESS_MASK.register_fields([('limbs', c.Array[RsAccessLimb, Literal[1]], 0)]) +struct_RS_SHARE_POLICY.register_fields([('target', NvU32, 0), ('accessMask', RS_ACCESS_MASK, 4), ('type', NvU16, 8), ('action', NvU8, 10)]) +NVOS57_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hObject', NvHandle, 4), ('sharePolicy', RS_SHARE_POLICY, 8), ('status', NvU32, 20)]) +NVPOWERSTATE_STAGE: TypeAlias = ctypes.c_uint32 +@c.record +class NVPOWERSTATE_FAILURE_ENGINE_TRANSITION(c.Struct): + SIZE = 24 + classId: int + cumulativeTimeus: int + engineMaxTimeus: int + engineMaxTimeClassId: int +NVPOWERSTATE_FAILURE_ENGINE_TRANSITION.register_fields([('classId', NvU32, 0), ('cumulativeTimeus', NvU64, 8), ('engineMaxTimeus', NvU32, 16), ('engineMaxTimeClassId', NvU32, 20)]) +@c.record +class NVPOWERSTATE_FAILURE_WAIT_FOR_GFW_BOOT_OK(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_WAIT_FOR_GFW_BOOT_OK.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_INIT_LIBOS_LOGGING_STRUCTURES(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_INIT_LIBOS_LOGGING_STRUCTURES.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_GSP_PREPARE_FOR_BOOTSTRAP(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_GSP_PREPARE_FOR_BOOTSTRAP.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_GSP_BOOTSTRAP(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_GSP_BOOTSTRAP.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_BOOT_GSP_RM_PROXY(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_BOOT_GSP_RM_PROXY.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_VBIOS_HANDLE_SECURE_BOOT(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_VBIOS_HANDLE_SECURE_BOOT.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_RESTORE_PCIE_CONFIG_REGISTERS(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_RESTORE_PCIE_CONFIG_REGISTERS.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_GCX_BOOT_TIMER_CB_SCHEDULE(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_GCX_BOOT_TIMER_CB_SCHEDULE.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_POWER_MANAGEMENT_RESUME_PRE_LOAD_PHYSICAL_UNATTACHED(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_POWER_MANAGEMENT_RESUME_PRE_LOAD_PHYSICAL_UNATTACHED.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_PMS_EXPECTED_CHECKPOINT_DONE(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_PMS_EXPECTED_CHECKPOINT_DONE.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_POLL_FOR_NVLINK_READY(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_POLL_FOR_NVLINK_READY.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_CE_STATE_PRE_LOAD(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_CE_STATE_PRE_LOAD.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_LOAD_PROXY_UCODE_EARLY_INIT(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_LOAD_PROXY_UCODE_EARLY_INIT.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_RESTORE_NON_WPR_REGION(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_RESTORE_NON_WPR_REGION.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_PMU_20_OS_BOOTSTRAP(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_PMU_20_OS_BOOTSTRAP.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_ENGINE(c.Struct): + SIZE = 24 + engineTransition: NVPOWERSTATE_FAILURE_ENGINE_TRANSITION +NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_ENGINE.register_fields([('engineTransition', NVPOWERSTATE_FAILURE_ENGINE_TRANSITION, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_UNKNOWN(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_UNKNOWN.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_LOAD_ENGINE(c.Struct): + SIZE = 24 + engineTransition: NVPOWERSTATE_FAILURE_ENGINE_TRANSITION +NVPOWERSTATE_FAILURE_STATE_LOAD_ENGINE.register_fields([('engineTransition', NVPOWERSTATE_FAILURE_ENGINE_TRANSITION, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_LOAD_UNKNOWN(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_STATE_LOAD_UNKNOWN.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_POST_LOAD_ENGINE(c.Struct): + SIZE = 24 + engineTransition: NVPOWERSTATE_FAILURE_ENGINE_TRANSITION +NVPOWERSTATE_FAILURE_STATE_POST_LOAD_ENGINE.register_fields([('engineTransition', NVPOWERSTATE_FAILURE_ENGINE_TRANSITION, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_POST_LOAD_UNKNOWN(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_STATE_POST_LOAD_UNKNOWN.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_LOAD_PHYSICAL(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_STATE_LOAD_PHYSICAL.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_GSP_PREPARE_SUSPEND_RESUME_DATA(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_GSP_PREPARE_SUSPEND_RESUME_DATA.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_MC_POINTER_NULL(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_MC_POINTER_NULL.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_SAVE_PCIE_CONFIG_REGISTERS(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_SAVE_PCIE_CONFIG_REGISTERS.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_ENGINE(c.Struct): + SIZE = 24 + engineTransition: NVPOWERSTATE_FAILURE_ENGINE_TRANSITION +NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_ENGINE.register_fields([('engineTransition', NVPOWERSTATE_FAILURE_ENGINE_TRANSITION, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_UNKNOWN(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_UNKNOWN.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_UNLOAD_ENGINE(c.Struct): + SIZE = 24 + engineTransition: NVPOWERSTATE_FAILURE_ENGINE_TRANSITION +NVPOWERSTATE_FAILURE_STATE_UNLOAD_ENGINE.register_fields([('engineTransition', NVPOWERSTATE_FAILURE_ENGINE_TRANSITION, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_UNLOAD_UNKNOWN(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_STATE_UNLOAD_UNKNOWN.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_ENGINE(c.Struct): + SIZE = 24 + engineTransition: NVPOWERSTATE_FAILURE_ENGINE_TRANSITION +NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_ENGINE.register_fields([('engineTransition', NVPOWERSTATE_FAILURE_ENGINE_TRANSITION, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_UNKNOWN(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_UNKNOWN.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_GSP_UNLOAD_RM(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_GSP_UNLOAD_RM.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_MONITOR_STATE_0(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_MONITOR_STATE_0.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_MONITOR_STATE_1(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_MONITOR_STATE_1.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_MONITOR_STATE_HIBERNATE(c.Struct): + SIZE = 1 + rsvd: int +NVPOWERSTATE_FAILURE_MONITOR_STATE_HIBERNATE.register_fields([('rsvd', NvU8, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_SET_POWER_STATE_UNKNOWN(c.Struct): + SIZE = 4 + state: int +NVPOWERSTATE_FAILURE_SET_POWER_STATE_UNKNOWN.register_fields([('state', NvU32, 0)]) +@c.record +class NVPOWERSTATE_FAILURE_DATA(c.Struct): + SIZE = 24 + waitForGfwBootOk: NVPOWERSTATE_FAILURE_WAIT_FOR_GFW_BOOT_OK + initLibosLoggingStructures: NVPOWERSTATE_FAILURE_INIT_LIBOS_LOGGING_STRUCTURES + gspPrepareForBootstrap: NVPOWERSTATE_FAILURE_GSP_PREPARE_FOR_BOOTSTRAP + gspBootstrap: NVPOWERSTATE_FAILURE_GSP_BOOTSTRAP + bootGspRmProxy: NVPOWERSTATE_FAILURE_BOOT_GSP_RM_PROXY + vbiosHandleSecureBoot: NVPOWERSTATE_FAILURE_VBIOS_HANDLE_SECURE_BOOT + restorePcieConfigRegisters: NVPOWERSTATE_FAILURE_RESTORE_PCIE_CONFIG_REGISTERS + gcxBootTimerCbSchedule: NVPOWERSTATE_FAILURE_GCX_BOOT_TIMER_CB_SCHEDULE + powerManagementResumePreLoadPhysicalUnattached: NVPOWERSTATE_FAILURE_POWER_MANAGEMENT_RESUME_PRE_LOAD_PHYSICAL_UNATTACHED + pmsExpectedCheckpointDone: NVPOWERSTATE_FAILURE_PMS_EXPECTED_CHECKPOINT_DONE + pollForNvlinkReady: NVPOWERSTATE_FAILURE_POLL_FOR_NVLINK_READY + ceStatePreLoad: NVPOWERSTATE_FAILURE_CE_STATE_PRE_LOAD + loadProxyUcodeEarlyInit: NVPOWERSTATE_FAILURE_LOAD_PROXY_UCODE_EARLY_INIT + restoreNonWprRegion: NVPOWERSTATE_FAILURE_RESTORE_NON_WPR_REGION + pmu20OsBootstrap: NVPOWERSTATE_FAILURE_PMU_20_OS_BOOTSTRAP + statePreLoadEngine: NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_ENGINE + statePreLoadUnknown: NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_UNKNOWN + stateLoadEngine: NVPOWERSTATE_FAILURE_STATE_LOAD_ENGINE + stateLoadUnknown: NVPOWERSTATE_FAILURE_STATE_LOAD_UNKNOWN + statePostLoadEngine: NVPOWERSTATE_FAILURE_STATE_POST_LOAD_ENGINE + statePostLoadUnknown: NVPOWERSTATE_FAILURE_STATE_POST_LOAD_UNKNOWN + stateLoadPhysical: NVPOWERSTATE_FAILURE_STATE_LOAD_PHYSICAL + gspPrepareSuspendResumeData: NVPOWERSTATE_FAILURE_GSP_PREPARE_SUSPEND_RESUME_DATA + mcPointerNull: NVPOWERSTATE_FAILURE_MC_POINTER_NULL + savePcieConfigRegisters: NVPOWERSTATE_FAILURE_SAVE_PCIE_CONFIG_REGISTERS + statePreUnloadEngine: NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_ENGINE + statePreUnloadUnknown: NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_UNKNOWN + stateUnloadEngine: NVPOWERSTATE_FAILURE_STATE_UNLOAD_ENGINE + stateUnloadUnknown: NVPOWERSTATE_FAILURE_STATE_UNLOAD_UNKNOWN + statePostUnloadEngine: NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_ENGINE + statePostUnloadUnknown: NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_UNKNOWN + gspUnloadRm: NVPOWERSTATE_FAILURE_GSP_UNLOAD_RM + monitorState0: NVPOWERSTATE_FAILURE_MONITOR_STATE_0 + monitorState1: NVPOWERSTATE_FAILURE_MONITOR_STATE_1 + monitorStateHibernate: NVPOWERSTATE_FAILURE_MONITOR_STATE_HIBERNATE + setPowerStateUnknown: NVPOWERSTATE_FAILURE_SET_POWER_STATE_UNKNOWN +NVPOWERSTATE_FAILURE_DATA.register_fields([('waitForGfwBootOk', NVPOWERSTATE_FAILURE_WAIT_FOR_GFW_BOOT_OK, 0), ('initLibosLoggingStructures', NVPOWERSTATE_FAILURE_INIT_LIBOS_LOGGING_STRUCTURES, 0), ('gspPrepareForBootstrap', NVPOWERSTATE_FAILURE_GSP_PREPARE_FOR_BOOTSTRAP, 0), ('gspBootstrap', NVPOWERSTATE_FAILURE_GSP_BOOTSTRAP, 0), ('bootGspRmProxy', NVPOWERSTATE_FAILURE_BOOT_GSP_RM_PROXY, 0), ('vbiosHandleSecureBoot', NVPOWERSTATE_FAILURE_VBIOS_HANDLE_SECURE_BOOT, 0), ('restorePcieConfigRegisters', NVPOWERSTATE_FAILURE_RESTORE_PCIE_CONFIG_REGISTERS, 0), ('gcxBootTimerCbSchedule', NVPOWERSTATE_FAILURE_GCX_BOOT_TIMER_CB_SCHEDULE, 0), ('powerManagementResumePreLoadPhysicalUnattached', NVPOWERSTATE_FAILURE_POWER_MANAGEMENT_RESUME_PRE_LOAD_PHYSICAL_UNATTACHED, 0), ('pmsExpectedCheckpointDone', NVPOWERSTATE_FAILURE_PMS_EXPECTED_CHECKPOINT_DONE, 0), ('pollForNvlinkReady', NVPOWERSTATE_FAILURE_POLL_FOR_NVLINK_READY, 0), ('ceStatePreLoad', NVPOWERSTATE_FAILURE_CE_STATE_PRE_LOAD, 0), ('loadProxyUcodeEarlyInit', NVPOWERSTATE_FAILURE_LOAD_PROXY_UCODE_EARLY_INIT, 0), ('restoreNonWprRegion', NVPOWERSTATE_FAILURE_RESTORE_NON_WPR_REGION, 0), ('pmu20OsBootstrap', NVPOWERSTATE_FAILURE_PMU_20_OS_BOOTSTRAP, 0), ('statePreLoadEngine', NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_ENGINE, 0), ('statePreLoadUnknown', NVPOWERSTATE_FAILURE_STATE_PRE_LOAD_UNKNOWN, 0), ('stateLoadEngine', NVPOWERSTATE_FAILURE_STATE_LOAD_ENGINE, 0), ('stateLoadUnknown', NVPOWERSTATE_FAILURE_STATE_LOAD_UNKNOWN, 0), ('statePostLoadEngine', NVPOWERSTATE_FAILURE_STATE_POST_LOAD_ENGINE, 0), ('statePostLoadUnknown', NVPOWERSTATE_FAILURE_STATE_POST_LOAD_UNKNOWN, 0), ('stateLoadPhysical', NVPOWERSTATE_FAILURE_STATE_LOAD_PHYSICAL, 0), ('gspPrepareSuspendResumeData', NVPOWERSTATE_FAILURE_GSP_PREPARE_SUSPEND_RESUME_DATA, 0), ('mcPointerNull', NVPOWERSTATE_FAILURE_MC_POINTER_NULL, 0), ('savePcieConfigRegisters', NVPOWERSTATE_FAILURE_SAVE_PCIE_CONFIG_REGISTERS, 0), ('statePreUnloadEngine', NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_ENGINE, 0), ('statePreUnloadUnknown', NVPOWERSTATE_FAILURE_STATE_PRE_UNLOAD_UNKNOWN, 0), ('stateUnloadEngine', NVPOWERSTATE_FAILURE_STATE_UNLOAD_ENGINE, 0), ('stateUnloadUnknown', NVPOWERSTATE_FAILURE_STATE_UNLOAD_UNKNOWN, 0), ('statePostUnloadEngine', NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_ENGINE, 0), ('statePostUnloadUnknown', NVPOWERSTATE_FAILURE_STATE_POST_UNLOAD_UNKNOWN, 0), ('gspUnloadRm', NVPOWERSTATE_FAILURE_GSP_UNLOAD_RM, 0), ('monitorState0', NVPOWERSTATE_FAILURE_MONITOR_STATE_0, 0), ('monitorState1', NVPOWERSTATE_FAILURE_MONITOR_STATE_1, 0), ('monitorStateHibernate', NVPOWERSTATE_FAILURE_MONITOR_STATE_HIBERNATE, 0), ('setPowerStateUnknown', NVPOWERSTATE_FAILURE_SET_POWER_STATE_UNKNOWN, 0)]) +@c.record +class NVPOWERSTATE_FAILURE(c.Struct): + SIZE = 32 + stage: int + status: int + data: NVPOWERSTATE_FAILURE_DATA +NVPOWERSTATE_FAILURE.register_fields([('stage', NVPOWERSTATE_STAGE, 0), ('status', NV_STATUS, 4), ('data', NVPOWERSTATE_FAILURE_DATA, 8)]) +@c.record +class NVPOWERSTATE_PARAMETERS(c.Struct): + SIZE = 64 + deviceReference: int + head: int + state: int + forceMonitorState: int + bForcePerfBiosLevel: int + bIsD3HotTransition: int + bForcePowerStateFail: int + errorStatus: int + fastBootPowerState: int + bGC8Transition: int + bGC8InputRailCutOff: int + failure: NVPOWERSTATE_FAILURE +NVPOWERSTATE_PARAMETERS.register_fields([('deviceReference', NvU32, 0), ('head', NvU32, 4), ('state', NvU32, 8), ('forceMonitorState', NvU8, 12), ('bForcePerfBiosLevel', NvU8, 13), ('bIsD3HotTransition', NvU8, 14), ('bForcePowerStateFail', NvU8, 15), ('errorStatus', NvU32, 16), ('fastBootPowerState', NvU32, 20), ('bGC8Transition', NvU8, 24), ('bGC8InputRailCutOff', NvU8, 25), ('failure', NVPOWERSTATE_FAILURE, 32)]) +PNVPOWERSTATE_PARAMETERS: TypeAlias = c.POINTER[NVPOWERSTATE_PARAMETERS] +@c.record +class NV_GR_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 16 + version: int + flags: int + size: int + caps: int +NV_GR_ALLOCATION_PARAMETERS.register_fields([('version', NvU32, 0), ('flags', NvU32, 4), ('size', NvU32, 8), ('caps', NvU32, 12)]) +ChannelPBSize: dict[int, str] = {(PB_SIZE_4KB:=0): 'PB_SIZE_4KB', (PB_SIZE_8KB:=1): 'PB_SIZE_8KB', (PB_SIZE_16KB:=2): 'PB_SIZE_16KB', (PB_SIZE_32KB:=3): 'PB_SIZE_32KB', (PB_SIZE_64KB:=4): 'PB_SIZE_64KB'} +@c.record +class NV50VAIO_CHANNELDMA_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 40 + channelInstance: int + hObjectBuffer: int + hObjectNotify: int + offset: int + pControl: ctypes.c_void_p + flags: int + channelPBSize: int + subDeviceId: int +NV50VAIO_CHANNELDMA_ALLOCATION_PARAMETERS.register_fields([('channelInstance', NvV32, 0), ('hObjectBuffer', NvHandle, 4), ('hObjectNotify', NvHandle, 8), ('offset', NvU32, 12), ('pControl', NvP64, 16), ('flags', NvU32, 24), ('channelPBSize', ctypes.c_uint32, 28), ('subDeviceId', NvU32, 32)]) +@c.record +class NV50VAIO_CHANNELPIO_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 16 + channelInstance: int + hObjectNotify: int + pControl: ctypes.c_void_p +NV50VAIO_CHANNELPIO_ALLOCATION_PARAMETERS.register_fields([('channelInstance', NvV32, 0), ('hObjectNotify', NvHandle, 4), ('pControl', NvP64, 8)]) +@c.record +class NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 28 + hObjectError: int + hObjectEccError: int + hVASpace: int + engineType: int + bIsCallingContextVgpuPlugin: int + reserved: int + internalFlags: int +NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS.register_fields([('hObjectError', NvHandle, 0), ('hObjectEccError', NvHandle, 4), ('hVASpace', NvHandle, 8), ('engineType', NvU32, 12), ('bIsCallingContextVgpuPlugin', NvBool, 16), ('reserved', NvU32, 20), ('internalFlags', NvU32, 24)]) +@c.record +class NV_SWRUNLIST_ALLOCATION_PARAMS(c.Struct): + SIZE = 12 + engineId: int + maxTSGs: int + qosIntrEnableMask: int +NV_SWRUNLIST_ALLOCATION_PARAMS.register_fields([('engineId', NvU32, 0), ('maxTSGs', NvU32, 4), ('qosIntrEnableMask', NvU32, 8)]) +@c.record +class NV_ME_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 8 + size: int + caps: int +NV_ME_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('caps', NvU32, 4)]) +@c.record +class NV_NVDEC_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 12 + size: int + prohibitMultipleInstances: int + engineInstance: int +NV_NVDEC_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('prohibitMultipleInstances', NvU32, 4), ('engineInstance', NvU32, 8)]) +@c.record +class NV_VP_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 80 + size: int + caps: int + flags: int + altUcode: int + rawUcode: ctypes.c_void_p + rawUcodeSize: int + numSubClasses: int + numSubSets: int + subClasses: ctypes.c_void_p + prohibitMultipleInstances: int + pControl: ctypes.c_void_p + hMemoryCmdBuffer: int + offset: int +NV_VP_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('caps', NvU32, 4), ('flags', NvU32, 8), ('altUcode', NvU32, 12), ('rawUcode', NvP64, 16), ('rawUcodeSize', NvU32, 24), ('numSubClasses', NvU32, 28), ('numSubSets', NvU32, 32), ('subClasses', NvP64, 40), ('prohibitMultipleInstances', NvU32, 48), ('pControl', NvP64, 56), ('hMemoryCmdBuffer', NvHandle, 64), ('offset', NvU64, 72)]) +@c.record +class NV_PPP_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 8 + size: int + prohibitMultipleInstances: int +NV_PPP_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('prohibitMultipleInstances', NvU32, 4)]) +@c.record +class NV_NVENC_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 12 + size: int + prohibitMultipleInstances: int + engineInstance: int +NV_NVENC_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('prohibitMultipleInstances', NvU32, 4), ('engineInstance', NvU32, 8)]) +@c.record +class NV_SEC2_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 8 + size: int + prohibitMultipleInstances: int +NV_SEC2_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('prohibitMultipleInstances', NvU32, 4)]) +@c.record +class NV_NVJPG_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 12 + size: int + prohibitMultipleInstances: int + engineInstance: int +NV_NVJPG_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('prohibitMultipleInstances', NvU32, 4), ('engineInstance', NvU32, 8)]) +@c.record +class NV_OFA_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 12 + size: int + prohibitMultipleInstances: int + engineInstance: int +NV_OFA_ALLOCATION_PARAMETERS.register_fields([('size', NvU32, 0), ('prohibitMultipleInstances', NvU32, 4), ('engineInstance', NvU32, 8)]) +@c.record +class NVOS61_PARAMETERS(c.Struct): + SIZE = 56 + hClient: int + hDevice: int + hVblank: int + pProc: c.CFUNCTYPE[None, [ctypes.c_void_p, ctypes.c_void_p]] + LogicalHead: int + pParm1: ctypes.c_void_p + pParm2: ctypes.c_void_p + bAdd: int + status: int +OSVBLANKCALLBACKPROC: TypeAlias = c.CFUNCTYPE[None, [ctypes.c_void_p, ctypes.c_void_p]] +NVOS61_PARAMETERS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hVblank', NvHandle, 8), ('pProc', OSVBLANKCALLBACKPROC, 16), ('LogicalHead', NvV32, 24), ('pParm1', ctypes.c_void_p, 32), ('pParm2', ctypes.c_void_p, 40), ('bAdd', NvU32, 48), ('status', NvV32, 52)]) +@c.record +class NV_VASPACE_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 56 + index: int + flags: int + vaSize: int + vaStartInternal: int + vaLimitInternal: int + bigPageSize: int + vaBase: int + pasid: int +NV_VASPACE_ALLOCATION_PARAMETERS.register_fields([('index', NvU32, 0), ('flags', NvV32, 4), ('vaSize', NvU64, 8), ('vaStartInternal', NvU64, 16), ('vaLimitInternal', NvU64, 24), ('bigPageSize', NvU32, 32), ('vaBase', NvU64, 40), ('pasid', NvU32, 48)]) +@c.record +class NV_CTXSHARE_ALLOCATION_PARAMETERS(c.Struct): + SIZE = 12 + hVASpace: int + flags: int + subctxId: int +NV_CTXSHARE_ALLOCATION_PARAMETERS.register_fields([('hVASpace', NvHandle, 0), ('flags', NvU32, 4), ('subctxId', NvU32, 8)]) +@c.record +class NV_TIMEOUT_CONTROL_PARAMETERS(c.Struct): + SIZE = 12 + cmd: int + timeoutInMs: int + deviceInstance: int +NV_TIMEOUT_CONTROL_PARAMETERS.register_fields([('cmd', NvU32, 0), ('timeoutInMs', NvU32, 4), ('deviceInstance', NvU32, 8)]) +NV_VIDMEM_ACCESS_BIT_ALLOCATION_PARAMS_ADDR_SPACE: dict[int, str] = {(NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_DEFAULT:=0): 'NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_DEFAULT', (NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_COH:=1): 'NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_COH', (NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_NCOH:=2): 'NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_NCOH', (NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_VID:=3): 'NV_VIDMEM_ACCESS_BIT_BUFFER_ADDR_SPACE_VID'} +@c.record +class NV_VIDMEM_ACCESS_BIT_ALLOCATION_PARAMS(c.Struct): + SIZE = 528 + bDirtyTracking: int + granularity: int + accessBitMask: c.Array[ctypes.c_uint64, Literal[64]] + noOfEntries: int + addrSpace: int +NV_VIDMEM_ACCESS_BIT_ALLOCATION_PARAMS.register_fields([('bDirtyTracking', NvBool, 0), ('granularity', NvU32, 4), ('accessBitMask', c.Array[NvU64, Literal[64]], 8), ('noOfEntries', NvU32, 520), ('addrSpace', ctypes.c_uint32, 524)]) +@c.record +class NV_HOPPER_USERMODE_A_PARAMS(c.Struct): + SIZE = 2 + bBar1Mapping: int + bPriv: int +NV_HOPPER_USERMODE_A_PARAMS.register_fields([('bBar1Mapping', NvBool, 0), ('bPriv', NvBool, 1)]) +@c.record +class struct_NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS(c.Struct): + SIZE = 4 + workSubmitToken: int +struct_NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS.register_fields([('workSubmitToken', NvU32, 0)]) +NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS: TypeAlias = struct_NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS +@c.record +class struct_NVC36F_CTRL_GPFIFO_UPDATE_FAULT_METHOD_BUFFER_PARAMS(c.Struct): + SIZE = 16 + bar2Addr: c.Array[ctypes.c_uint64, Literal[2]] +struct_NVC36F_CTRL_GPFIFO_UPDATE_FAULT_METHOD_BUFFER_PARAMS.register_fields([('bar2Addr', c.Array[NvU64, Literal[2]], 0)]) +NVC36F_CTRL_GPFIFO_UPDATE_FAULT_METHOD_BUFFER_PARAMS: TypeAlias = struct_NVC36F_CTRL_GPFIFO_UPDATE_FAULT_METHOD_BUFFER_PARAMS +@c.record +class struct_NVC36F_CTRL_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX_PARAMS(c.Struct): + SIZE = 4 + index: int +struct_NVC36F_CTRL_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX_PARAMS.register_fields([('index', NvU32, 0)]) +NVC36F_CTRL_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX_PARAMS: TypeAlias = struct_NVC36F_CTRL_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX_PARAMS +NVC36F_CTRL_INTERNAL_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS: TypeAlias = struct_NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_CAPABILITIES_PARAMS(c.Struct): + SIZE = 6 + cpuCapability: int + gpusCapability: int + environment: int + ccFeature: int + devToolsMode: int + multiGpuMode: int +struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_CAPABILITIES_PARAMS.register_fields([('cpuCapability', NvU8, 0), ('gpusCapability', NvU8, 1), ('environment', NvU8, 2), ('ccFeature', NvU8, 3), ('devToolsMode', NvU8, 4), ('multiGpuMode', NvU8, 5)]) +NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_CAPABILITIES_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_CAPABILITIES_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_GPUS_STATE_PARAMS(c.Struct): + SIZE = 1 + bAcceptClientRequest: int +struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_GPUS_STATE_PARAMS.register_fields([('bAcceptClientRequest', NvBool, 0)]) +NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_GPUS_STATE_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_GPUS_STATE_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_SET_GPUS_STATE_PARAMS(c.Struct): + SIZE = 1 + bAcceptClientRequest: int +struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_SET_GPUS_STATE_PARAMS.register_fields([('bAcceptClientRequest', NvBool, 0)]) +NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_SET_GPUS_STATE_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_SET_GPUS_STATE_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_VIDMEM_SIZE_PARAMS(c.Struct): + SIZE = 24 + hSubDevice: int + protectedMemSizeInKb: int + unprotectedMemSizeInKb: int +struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_VIDMEM_SIZE_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('protectedMemSizeInKb', NvU64, 8), ('unprotectedMemSizeInKb', NvU64, 16)]) +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_VIDMEM_SIZE_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_VIDMEM_SIZE_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_SET_VIDMEM_SIZE_PARAMS(c.Struct): + SIZE = 24 + hSubDevice: int + protectedMemSizeInKb: int + unprotectedMemSizeInKb: int +struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_SET_VIDMEM_SIZE_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('protectedMemSizeInKb', NvU64, 8), ('unprotectedMemSizeInKb', NvU64, 16)]) +NV_CONF_COMPUTE_CTRL_CMD_GPU_SET_VIDMEM_SIZE_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_SET_VIDMEM_SIZE_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_GET_NUM_SUPPORTED_CC_SECURE_CHANNELS_PARAMS(c.Struct): + SIZE = 12 + hSubDevice: int + numSupportedSec2CCSecureChannels: int + numSupportedCeCCSecureChannels: int +struct_NV_CONF_COMPUTE_CTRL_CMD_GET_NUM_SUPPORTED_CC_SECURE_CHANNELS_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('numSupportedSec2CCSecureChannels', NvU32, 4), ('numSupportedCeCCSecureChannels', NvU32, 8)]) +NV_CONF_COMPUTE_CTRL_CMD_GET_NUM_SUPPORTED_CC_SECURE_CHANNELS_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_GET_NUM_SUPPORTED_CC_SECURE_CHANNELS_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_CERTIFICATE_PARAMS(c.Struct): + SIZE = 9228 + hSubDevice: int + certChain: c.Array[ctypes.c_ubyte, Literal[4096]] + certChainSize: int + attestationCertChain: c.Array[ctypes.c_ubyte, Literal[5120]] + attestationCertChainSize: int +struct_NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_CERTIFICATE_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('certChain', c.Array[NvU8, Literal[4096]], 4), ('certChainSize', NvU32, 4100), ('attestationCertChain', c.Array[NvU8, Literal[5120]], 4104), ('attestationCertChainSize', NvU32, 9224)]) +NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_CERTIFICATE_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_CERTIFICATE_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_ATTESTATION_REPORT_PARAMS(c.Struct): + SIZE = 12336 + hSubDevice: int + nonce: c.Array[ctypes.c_ubyte, Literal[32]] + attestationReport: c.Array[ctypes.c_ubyte, Literal[8192]] + attestationReportSize: int + isCecAttestationReportPresent: int + cecAttestationReport: c.Array[ctypes.c_ubyte, Literal[4096]] + cecAttestationReportSize: int +struct_NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_ATTESTATION_REPORT_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('nonce', c.Array[NvU8, Literal[32]], 4), ('attestationReport', c.Array[NvU8, Literal[8192]], 36), ('attestationReportSize', NvU32, 8228), ('isCecAttestationReportPresent', NvBool, 8232), ('cecAttestationReport', c.Array[NvU8, Literal[4096]], 8233), ('cecAttestationReportSize', NvU32, 12332)]) +NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_ATTESTATION_REPORT_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_ATTESTATION_REPORT_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_NUM_SECURE_CHANNELS_PARAMS(c.Struct): + SIZE = 12 + hSubDevice: int + maxSec2Channels: int + maxCeChannels: int +struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_NUM_SECURE_CHANNELS_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('maxSec2Channels', NvU32, 4), ('maxCeChannels', NvU32, 8)]) +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_NUM_SECURE_CHANNELS_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_NUM_SECURE_CHANNELS_PARAMS +@c.record +class struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_KEY_ROTATION_STATE_PARAMS(c.Struct): + SIZE = 8 + hSubDevice: int + keyRotationState: int +struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_KEY_ROTATION_STATE_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('keyRotationState', NvU32, 4)]) +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_KEY_ROTATION_STATE_PARAMS: TypeAlias = struct_NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_KEY_ROTATION_STATE_PARAMS +@c.record +class struct_NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS(c.Struct): + SIZE = 3 + bEnable: int + bSkipSubmit: int + bSkipEnable: int +NVA06C_CTRL_GPFIFO_SCHEDULE_PARAMS: TypeAlias = struct_NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS +struct_NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS.register_fields([('bEnable', NvBool, 0), ('bSkipSubmit', NvBool, 1), ('bSkipEnable', NvBool, 2)]) +@c.record +class struct_NVA06F_CTRL_BIND_PARAMS(c.Struct): + SIZE = 4 + engineType: int +NVA06C_CTRL_BIND_PARAMS: TypeAlias = struct_NVA06F_CTRL_BIND_PARAMS +struct_NVA06F_CTRL_BIND_PARAMS.register_fields([('engineType', NvU32, 0)]) +@c.record +class struct_NVA06C_CTRL_TIMESLICE_PARAMS(c.Struct): + SIZE = 8 + timesliceUs: int +struct_NVA06C_CTRL_TIMESLICE_PARAMS.register_fields([('timesliceUs', NvU64, 0)]) +NVA06C_CTRL_TIMESLICE_PARAMS: TypeAlias = struct_NVA06C_CTRL_TIMESLICE_PARAMS +NVA06C_CTRL_SET_TIMESLICE_PARAMS: TypeAlias = struct_NVA06C_CTRL_TIMESLICE_PARAMS +NVA06C_CTRL_GET_TIMESLICE_PARAMS: TypeAlias = struct_NVA06C_CTRL_TIMESLICE_PARAMS +@c.record +class struct_NVA06C_CTRL_PREEMPT_PARAMS(c.Struct): + SIZE = 8 + bWait: int + bManualTimeout: int + timeoutUs: int +struct_NVA06C_CTRL_PREEMPT_PARAMS.register_fields([('bWait', NvBool, 0), ('bManualTimeout', NvBool, 1), ('timeoutUs', NvU32, 4)]) +NVA06C_CTRL_PREEMPT_PARAMS: TypeAlias = struct_NVA06C_CTRL_PREEMPT_PARAMS +@c.record +class struct_NVA06C_CTRL_GET_INFO_PARAMS(c.Struct): + SIZE = 4 + tsgID: int +struct_NVA06C_CTRL_GET_INFO_PARAMS.register_fields([('tsgID', NvU32, 0)]) +NVA06C_CTRL_GET_INFO_PARAMS: TypeAlias = struct_NVA06C_CTRL_GET_INFO_PARAMS +@c.record +class struct_NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS(c.Struct): + SIZE = 4 + tsgInterleaveLevel: int +struct_NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS.register_fields([('tsgInterleaveLevel', NvU32, 0)]) +NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS: TypeAlias = struct_NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS +NVA06C_CTRL_SET_INTERLEAVE_LEVEL_PARAMS: TypeAlias = struct_NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS +NVA06C_CTRL_GET_INTERLEAVE_LEVEL_PARAMS: TypeAlias = struct_NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS +enum_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE: dict[int, str] = {(NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE_NONE:=0): 'NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE_NONE', (NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE_64B:=1): 'NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE_64B', (NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE_128B:=2): 'NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE_128B'} +NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD(c.Struct): + SIZE = 4 + size: int +struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD.register_fields([('size', NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_SIZE, 0)]) +NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD: TypeAlias = struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD +@c.record +class struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS(c.Struct): + SIZE = 8 + l1: struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD + t1: struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD +struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS.register_fields([('l1', NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD, 0), ('t1', NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_FIELD, 4)]) +NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS: TypeAlias = struct_NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS +@c.record +class struct_NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS(c.Struct): + SIZE = 88 + methodBufferMemdesc: c.Array[struct_NV2080_CTRL_INTERNAL_MEMDESC_INFO, Literal[2]] + bar2Addr: c.Array[ctypes.c_uint64, Literal[2]] + numValidEntries: int +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMDESC_INFO(c.Struct): + SIZE = 32 + base: int + size: int + alignment: int + addressSpace: int + cpuCacheAttrib: int +NV2080_CTRL_INTERNAL_MEMDESC_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMDESC_INFO +struct_NV2080_CTRL_INTERNAL_MEMDESC_INFO.register_fields([('base', NvU64, 0), ('size', NvU64, 8), ('alignment', NvU64, 16), ('addressSpace', NvU32, 24), ('cpuCacheAttrib', NvU32, 28)]) +struct_NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS.register_fields([('methodBufferMemdesc', c.Array[NV2080_CTRL_INTERNAL_MEMDESC_INFO, Literal[2]], 0), ('bar2Addr', c.Array[NvU64, Literal[2]], 64), ('numValidEntries', NvU32, 80)]) +NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS: TypeAlias = struct_NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS +@c.record +class struct_NVA06C_CTRL_MAKE_REALTIME_PARAMS(c.Struct): + SIZE = 1 + bRealtime: int +struct_NVA06C_CTRL_MAKE_REALTIME_PARAMS.register_fields([('bRealtime', NvBool, 0)]) +NVA06C_CTRL_MAKE_REALTIME_PARAMS: TypeAlias = struct_NVA06C_CTRL_MAKE_REALTIME_PARAMS +NVA06C_CTRL_INTERNAL_GPFIFO_SCHEDULE_PARAMS: TypeAlias = struct_NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS +NVA06C_CTRL_INTERNAL_SET_TIMESLICE_PARAMS: TypeAlias = struct_NVA06C_CTRL_TIMESLICE_PARAMS +@c.record +class struct_NV90F1_CTRL_VASPACE_GET_GMMU_FORMAT_PARAMS(c.Struct): + SIZE = 16 + hSubDevice: int + subDeviceId: int + pFmt: ctypes.c_void_p +struct_NV90F1_CTRL_VASPACE_GET_GMMU_FORMAT_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('subDeviceId', NvU32, 4), ('pFmt', NvP64, 8)]) +NV90F1_CTRL_VASPACE_GET_GMMU_FORMAT_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_GET_GMMU_FORMAT_PARAMS +@c.record +class struct_NV_CTRL_VASPACE_PAGE_LEVEL(c.Struct): + SIZE = 112 + pFmt: c.POINTER[struct_MMU_FMT_LEVEL] + levelFmt: struct_MMU_FMT_LEVEL + sublevelFmt: c.Array[struct_MMU_FMT_LEVEL, Literal[2]] + physAddress: int + aperture: int + size: int + entryIndex: int +@c.record +class struct_MMU_FMT_LEVEL(c.Struct): + SIZE = 24 + virtAddrBitLo: int + virtAddrBitHi: int + entrySize: int + bPageTable: int + numSubLevels: int + pageLevelIdTag: int + subLevels: c.POINTER[struct_MMU_FMT_LEVEL] +struct_MMU_FMT_LEVEL.register_fields([('virtAddrBitLo', NvU8, 0), ('virtAddrBitHi', NvU8, 1), ('entrySize', NvU8, 2), ('bPageTable', NvBool, 3), ('numSubLevels', NvU8, 4), ('pageLevelIdTag', NvU32, 8), ('subLevels', c.POINTER[struct_MMU_FMT_LEVEL], 16)]) +MMU_FMT_LEVEL: TypeAlias = struct_MMU_FMT_LEVEL +struct_NV_CTRL_VASPACE_PAGE_LEVEL.register_fields([('pFmt', c.POINTER[struct_MMU_FMT_LEVEL], 0), ('levelFmt', MMU_FMT_LEVEL, 8), ('sublevelFmt', c.Array[MMU_FMT_LEVEL, Literal[2]], 32), ('physAddress', NvU64, 80), ('aperture', NvU32, 88), ('size', NvU64, 96), ('entryIndex', NvU32, 104)]) +NV_CTRL_VASPACE_PAGE_LEVEL: TypeAlias = struct_NV_CTRL_VASPACE_PAGE_LEVEL +@c.record +class struct_NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS(c.Struct): + SIZE = 712 + hSubDevice: int + subDeviceId: int + virtAddress: int + pageSize: int + flags: int + numLevels: int + levels: c.Array[struct_NV_CTRL_VASPACE_PAGE_LEVEL, Literal[6]] +struct_NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('subDeviceId', NvU32, 4), ('virtAddress', NvU64, 8), ('pageSize', NvU64, 16), ('flags', NvU64, 24), ('numLevels', NvU32, 32), ('levels', c.Array[NV_CTRL_VASPACE_PAGE_LEVEL, Literal[6]], 40)]) +NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS +@c.record +class struct_NV90F1_CTRL_VASPACE_RESERVE_ENTRIES_PARAMS(c.Struct): + SIZE = 32 + hSubDevice: int + subDeviceId: int + pageSize: int + virtAddrLo: int + virtAddrHi: int +struct_NV90F1_CTRL_VASPACE_RESERVE_ENTRIES_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('subDeviceId', NvU32, 4), ('pageSize', NvU64, 8), ('virtAddrLo', NvU64, 16), ('virtAddrHi', NvU64, 24)]) +NV90F1_CTRL_VASPACE_RESERVE_ENTRIES_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_RESERVE_ENTRIES_PARAMS +@c.record +class struct_NV90F1_CTRL_VASPACE_RELEASE_ENTRIES_PARAMS(c.Struct): + SIZE = 32 + hSubDevice: int + subDeviceId: int + pageSize: int + virtAddrLo: int + virtAddrHi: int +struct_NV90F1_CTRL_VASPACE_RELEASE_ENTRIES_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('subDeviceId', NvU32, 4), ('pageSize', NvU64, 8), ('virtAddrLo', NvU64, 16), ('virtAddrHi', NvU64, 24)]) +NV90F1_CTRL_VASPACE_RELEASE_ENTRIES_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_RELEASE_ENTRIES_PARAMS +NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_VERIF_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS +@c.record +class struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS(c.Struct): + SIZE = 184 + hSubDevice: int + subDeviceId: int + pageSize: int + virtAddrLo: int + virtAddrHi: int + numLevelsToCopy: int + levels: c.Array[struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS_level, Literal[6]] +@c.record +class struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS_level(c.Struct): + SIZE = 24 + physAddress: int + size: int + aperture: int + pageShift: int +struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS_level.register_fields([('physAddress', NvU64, 0), ('size', NvU64, 8), ('aperture', NvU32, 16), ('pageShift', NvU8, 20)]) +struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('subDeviceId', NvU32, 4), ('pageSize', NvU64, 8), ('virtAddrLo', NvU64, 16), ('virtAddrHi', NvU64, 24), ('numLevelsToCopy', NvU32, 32), ('levels', c.Array[struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS_level, Literal[6]], 40)]) +NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS +@c.record +class struct_NV90F1_CTRL_VASPACE_GET_HOST_RM_MANAGED_SIZE_PARAMS(c.Struct): + SIZE = 16 + hSubDevice: int + subDeviceId: int + requiredVaRange: int +struct_NV90F1_CTRL_VASPACE_GET_HOST_RM_MANAGED_SIZE_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('subDeviceId', NvU32, 4), ('requiredVaRange', NvU64, 8)]) +NV90F1_CTRL_VASPACE_GET_HOST_RM_MANAGED_SIZE_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_GET_HOST_RM_MANAGED_SIZE_PARAMS +@c.record +class struct_NV90F1_CTRL_VASPACE_GET_VAS_HEAP_INFO_PARAMS(c.Struct): + SIZE = 56 + hSubDevice: int + subDeviceId: int + bytesFree: int + bytesTotal: int + largestFreeOffset: int + largestFreeSize: int + usableBytesFree: int + numFreeBlocks: int +struct_NV90F1_CTRL_VASPACE_GET_VAS_HEAP_INFO_PARAMS.register_fields([('hSubDevice', NvHandle, 0), ('subDeviceId', NvU32, 4), ('bytesFree', NvU64, 8), ('bytesTotal', NvU64, 16), ('largestFreeOffset', NvU64, 24), ('largestFreeSize', NvU64, 32), ('usableBytesFree', NvU64, 40), ('numFreeBlocks', NvU32, 48)]) +NV90F1_CTRL_VASPACE_GET_VAS_HEAP_INFO_PARAMS: TypeAlias = struct_NV90F1_CTRL_VASPACE_GET_VAS_HEAP_INFO_PARAMS +NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS: TypeAlias = struct_NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS +NVA06F_CTRL_BIND_PARAMS: TypeAlias = struct_NVA06F_CTRL_BIND_PARAMS +@c.record +class struct_NVA06F_CTRL_SET_ERROR_NOTIFIER_PARAMS(c.Struct): + SIZE = 1 + bNotifyEachChannelInTSG: int +struct_NVA06F_CTRL_SET_ERROR_NOTIFIER_PARAMS.register_fields([('bNotifyEachChannelInTSG', NvBool, 0)]) +NVA06F_CTRL_SET_ERROR_NOTIFIER_PARAMS: TypeAlias = struct_NVA06F_CTRL_SET_ERROR_NOTIFIER_PARAMS +@c.record +class struct_NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS(c.Struct): + SIZE = 4 + channelInterleaveLevel: int +struct_NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS.register_fields([('channelInterleaveLevel', NvU32, 0)]) +NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS: TypeAlias = struct_NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS +NVA06F_CTRL_SET_INTERLEAVE_LEVEL_PARAMS: TypeAlias = struct_NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS +NVA06F_CTRL_GET_INTERLEAVE_LEVEL_PARAMS: TypeAlias = struct_NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS +@c.record +class struct_NVA06F_CTRL_RESTART_RUNLIST_PARAMS(c.Struct): + SIZE = 2 + bForceRestart: int + bBypassWait: int +struct_NVA06F_CTRL_RESTART_RUNLIST_PARAMS.register_fields([('bForceRestart', NvBool, 0), ('bBypassWait', NvBool, 1)]) +NVA06F_CTRL_RESTART_RUNLIST_PARAMS: TypeAlias = struct_NVA06F_CTRL_RESTART_RUNLIST_PARAMS +@c.record +class struct_NVA06F_CTRL_STOP_CHANNEL_PARAMS(c.Struct): + SIZE = 1 + bImmediate: int +struct_NVA06F_CTRL_STOP_CHANNEL_PARAMS.register_fields([('bImmediate', NvBool, 0)]) +NVA06F_CTRL_STOP_CHANNEL_PARAMS: TypeAlias = struct_NVA06F_CTRL_STOP_CHANNEL_PARAMS +@c.record +class struct_NVA06F_CTRL_GET_CONTEXT_ID_PARAMS(c.Struct): + SIZE = 4 + contextId: int +struct_NVA06F_CTRL_GET_CONTEXT_ID_PARAMS.register_fields([('contextId', NvU32, 0)]) +NVA06F_CTRL_GET_CONTEXT_ID_PARAMS: TypeAlias = struct_NVA06F_CTRL_GET_CONTEXT_ID_PARAMS +@c.record +class struct_NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS(c.Struct): + SIZE = 12 + hObject: int + mapFlags: int + addrSpaceType: int +struct_NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS.register_fields([('hObject', NvHandle, 0), ('mapFlags', NvU32, 4), ('addrSpaceType', NvU32, 8)]) +NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS: TypeAlias = struct_NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS +@c.record +class struct_NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS(c.Struct): + SIZE = 16 + hObject: int + index: int + data: struct_NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS_data +@c.record +class struct_NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS_data(c.Struct): + SIZE = 8 + hResult: int + iResult: int +struct_NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS_data.register_fields([('hResult', NvHandle, 0), ('iResult', NvU64, 0)]) +struct_NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS.register_fields([('hObject', NvHandle, 0), ('index', NvU32, 4), ('data', struct_NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS_data, 8)]) +NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_CLIENT_GET_ACCESS_RIGHTS_PARAMS(c.Struct): + SIZE = 12 + hObject: int + hClient: int + maskResult: struct_RS_ACCESS_MASK +struct_NV0000_CTRL_CLIENT_GET_ACCESS_RIGHTS_PARAMS.register_fields([('hObject', NvHandle, 0), ('hClient', NvHandle, 4), ('maskResult', RS_ACCESS_MASK, 8)]) +NV0000_CTRL_CLIENT_GET_ACCESS_RIGHTS_PARAMS: TypeAlias = struct_NV0000_CTRL_CLIENT_GET_ACCESS_RIGHTS_PARAMS +@c.record +class struct_NV0000_CTRL_CLIENT_SET_INHERITED_SHARE_POLICY_PARAMS(c.Struct): + SIZE = 12 + sharePolicy: struct_RS_SHARE_POLICY +struct_NV0000_CTRL_CLIENT_SET_INHERITED_SHARE_POLICY_PARAMS.register_fields([('sharePolicy', RS_SHARE_POLICY, 0)]) +NV0000_CTRL_CLIENT_SET_INHERITED_SHARE_POLICY_PARAMS: TypeAlias = struct_NV0000_CTRL_CLIENT_SET_INHERITED_SHARE_POLICY_PARAMS +@c.record +class struct_NV0000_CTRL_CMD_CLIENT_GET_CHILD_HANDLE_PARAMS(c.Struct): + SIZE = 12 + hParent: int + classId: int + hObject: int +struct_NV0000_CTRL_CMD_CLIENT_GET_CHILD_HANDLE_PARAMS.register_fields([('hParent', NvHandle, 0), ('classId', NvU32, 4), ('hObject', NvHandle, 8)]) +NV0000_CTRL_CMD_CLIENT_GET_CHILD_HANDLE_PARAMS: TypeAlias = struct_NV0000_CTRL_CMD_CLIENT_GET_CHILD_HANDLE_PARAMS +@c.record +class struct_NV0000_CTRL_CLIENT_SHARE_OBJECT_PARAMS(c.Struct): + SIZE = 16 + hObject: int + sharePolicy: struct_RS_SHARE_POLICY +struct_NV0000_CTRL_CLIENT_SHARE_OBJECT_PARAMS.register_fields([('hObject', NvHandle, 0), ('sharePolicy', RS_SHARE_POLICY, 4)]) +NV0000_CTRL_CLIENT_SHARE_OBJECT_PARAMS: TypeAlias = struct_NV0000_CTRL_CLIENT_SHARE_OBJECT_PARAMS +@c.record +class struct_NV0000_CTRL_CLIENT_OBJECTS_ARE_DUPLICATES_PARAMS(c.Struct): + SIZE = 12 + hObject1: int + hObject2: int + bDuplicates: int +struct_NV0000_CTRL_CLIENT_OBJECTS_ARE_DUPLICATES_PARAMS.register_fields([('hObject1', NvHandle, 0), ('hObject2', NvHandle, 4), ('bDuplicates', NvBool, 8)]) +NV0000_CTRL_CLIENT_OBJECTS_ARE_DUPLICATES_PARAMS: TypeAlias = struct_NV0000_CTRL_CLIENT_OBJECTS_ARE_DUPLICATES_PARAMS +@c.record +class struct_NV0000_CTRL_CLIENT_SUBSCRIBE_TO_IMEX_CHANNEL_PARAMS(c.Struct): + SIZE = 16 + devDescriptor: int + channel: int +struct_NV0000_CTRL_CLIENT_SUBSCRIBE_TO_IMEX_CHANNEL_PARAMS.register_fields([('devDescriptor', NvU64, 0), ('channel', NvU32, 8)]) +NV0000_CTRL_CLIENT_SUBSCRIBE_TO_IMEX_CHANNEL_PARAMS: TypeAlias = struct_NV0000_CTRL_CLIENT_SUBSCRIBE_TO_IMEX_CHANNEL_PARAMS +@c.record +class struct_NV0000_CTRL_DIAG_GET_LOCK_METER_STATE_PARAMS(c.Struct): + SIZE = 16 + state: int + count: int + missedCount: int + bCircularBuffer: int +struct_NV0000_CTRL_DIAG_GET_LOCK_METER_STATE_PARAMS.register_fields([('state', NvU32, 0), ('count', NvU32, 4), ('missedCount', NvU32, 8), ('bCircularBuffer', NvBool, 12)]) +NV0000_CTRL_DIAG_GET_LOCK_METER_STATE_PARAMS: TypeAlias = struct_NV0000_CTRL_DIAG_GET_LOCK_METER_STATE_PARAMS +@c.record +class struct_NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_PARAMS(c.Struct): + SIZE = 8 + state: int + bCircularBuffer: int +struct_NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_PARAMS.register_fields([('state', NvU32, 0), ('bCircularBuffer', NvBool, 4)]) +NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_PARAMS: TypeAlias = struct_NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_PARAMS +@c.record +class struct_NV0000_CTRL_DIAG_LOCK_METER_ENTRY(c.Struct): + SIZE = 56 + counter: int + line: int + filename: c.Array[ctypes.c_ubyte, Literal[12]] + tag: int + cpuNum: int + irql: int + threadId: int + data0: int + data1: int + data2: int +struct_NV0000_CTRL_DIAG_LOCK_METER_ENTRY.register_fields([('counter', NvU64, 0), ('line', NvU32, 8), ('filename', c.Array[NvU8, Literal[12]], 12), ('tag', NvU16, 24), ('cpuNum', NvU8, 26), ('irql', NvU8, 27), ('threadId', NvU64, 32), ('data0', NvU32, 40), ('data1', NvU32, 44), ('data2', NvU32, 48)]) +NV0000_CTRL_DIAG_LOCK_METER_ENTRY: TypeAlias = struct_NV0000_CTRL_DIAG_LOCK_METER_ENTRY +@c.record +class struct_NV0000_CTRL_DIAG_GET_LOCK_METER_ENTRIES_PARAMS(c.Struct): + SIZE = 3592 + entryCount: int + entries: c.Array[struct_NV0000_CTRL_DIAG_LOCK_METER_ENTRY, Literal[64]] +struct_NV0000_CTRL_DIAG_GET_LOCK_METER_ENTRIES_PARAMS.register_fields([('entryCount', NvU32, 0), ('entries', c.Array[NV0000_CTRL_DIAG_LOCK_METER_ENTRY, Literal[64]], 8)]) +NV0000_CTRL_DIAG_GET_LOCK_METER_ENTRIES_PARAMS: TypeAlias = struct_NV0000_CTRL_DIAG_GET_LOCK_METER_ENTRIES_PARAMS +@c.record +class struct_RPC_METER_ENTRY(c.Struct): + SIZE = 32 + startTimeInNs: int + endTimeInNs: int + rpcDataTag: int + rpcExtraData: int +struct_RPC_METER_ENTRY.register_fields([('startTimeInNs', NvU64, 0), ('endTimeInNs', NvU64, 8), ('rpcDataTag', NvU64, 16), ('rpcExtraData', NvU64, 24)]) +RPC_METER_ENTRY: TypeAlias = struct_RPC_METER_ENTRY +@c.record +class struct_NV0000_CTRL_DIAG_PROFILE_RPC_PARAMS(c.Struct): + SIZE = 4 + rpcProfileCmd: int +struct_NV0000_CTRL_DIAG_PROFILE_RPC_PARAMS.register_fields([('rpcProfileCmd', NvU32, 0)]) +NV0000_CTRL_DIAG_PROFILE_RPC_PARAMS: TypeAlias = struct_NV0000_CTRL_DIAG_PROFILE_RPC_PARAMS +@c.record +class struct_NV0000_CTRL_DIAG_DUMP_RPC_PARAMS(c.Struct): + SIZE = 3224 + firstEntryOffset: int + outputEntryCount: int + remainingEntryCount: int + elapsedTimeInNs: int + rpcProfilerBuffer: c.Array[struct_RPC_METER_ENTRY, Literal[100]] +struct_NV0000_CTRL_DIAG_DUMP_RPC_PARAMS.register_fields([('firstEntryOffset', NvU32, 0), ('outputEntryCount', NvU32, 4), ('remainingEntryCount', NvU32, 8), ('elapsedTimeInNs', NvU64, 16), ('rpcProfilerBuffer', c.Array[RPC_METER_ENTRY, Literal[100]], 24)]) +NV0000_CTRL_DIAG_DUMP_RPC_PARAMS: TypeAlias = struct_NV0000_CTRL_DIAG_DUMP_RPC_PARAMS +@c.record +class struct_NV0000_CTRL_EVENT_SET_NOTIFICATION_PARAMS(c.Struct): + SIZE = 8 + event: int + action: int +struct_NV0000_CTRL_EVENT_SET_NOTIFICATION_PARAMS.register_fields([('event', NvU32, 0), ('action', NvU32, 4)]) +NV0000_CTRL_EVENT_SET_NOTIFICATION_PARAMS: TypeAlias = struct_NV0000_CTRL_EVENT_SET_NOTIFICATION_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_EVENT_DATA_DISPLAY_CHANGE(c.Struct): + SIZE = 4 + deviceMask: int +struct_NV0000_CTRL_SYSTEM_EVENT_DATA_DISPLAY_CHANGE.register_fields([('deviceMask', NvU32, 0)]) +NV0000_CTRL_SYSTEM_EVENT_DATA_DISPLAY_CHANGE: TypeAlias = struct_NV0000_CTRL_SYSTEM_EVENT_DATA_DISPLAY_CHANGE +@c.record +class struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND(c.Struct): + SIZE = 4 + gpuId: int +struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND.register_fields([('gpuId', NvU32, 0)]) +NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND: TypeAlias = struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND +@c.record +class struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND(c.Struct): + SIZE = 4 + gpuId: int +struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND.register_fields([('gpuId', NvU32, 0)]) +NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND: TypeAlias = struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND +@c.record +class struct_NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND(c.Struct): + SIZE = 8 + gpuId: int + bBind: int +struct_NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND.register_fields([('gpuId', NvU32, 0), ('bBind', NvBool, 4)]) +NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND: TypeAlias = struct_NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND +@c.record +class struct_NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS(c.Struct): + SIZE = 12 + event: int + data: struct_NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS_data +@c.record +class struct_NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS_data(c.Struct): + SIZE = 8 + display: struct_NV0000_CTRL_SYSTEM_EVENT_DATA_DISPLAY_CHANGE + vgpuUnbind: struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND + vgpuBind: struct_NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND + gpuBindUnbind: struct_NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND +struct_NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS_data.register_fields([('display', NV0000_CTRL_SYSTEM_EVENT_DATA_DISPLAY_CHANGE, 0), ('vgpuUnbind', NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_UNBIND, 0), ('vgpuBind', NV0000_CTRL_SYSTEM_EVENT_DATA_VGPU_BIND, 0), ('gpuBindUnbind', NV0000_CTRL_SYSTEM_EVENT_DATA_GPU_BIND_UNBIND, 0)]) +struct_NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS.register_fields([('event', NvU32, 0), ('data', struct_NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS_data, 4)]) +NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS: TypeAlias = struct_NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_LINK_MASK(c.Struct): + SIZE = 16 + lenMasks: int + masks: c.Array[ctypes.c_uint64, Literal[1]] +NV0000_CTRL_NVLINK_LINK_MASK: TypeAlias = struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_NVLINK_LINK_MASK.register_fields([('lenMasks', NvU8, 0), ('masks', c.Array[NvU64, Literal[1]], 8)]) +@c.record +class struct_NV0000_CTRL_GPU_GET_ATTACHED_IDS_PARAMS(c.Struct): + SIZE = 128 + gpuIds: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV0000_CTRL_GPU_GET_ATTACHED_IDS_PARAMS.register_fields([('gpuIds', c.Array[NvU32, Literal[32]], 0)]) +NV0000_CTRL_GPU_GET_ATTACHED_IDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_ATTACHED_IDS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_ID_INFO_PARAMS(c.Struct): + SIZE = 40 + gpuId: int + gpuFlags: int + deviceInstance: int + subDeviceInstance: int + szName: ctypes.c_void_p + sliStatus: int + boardId: int + gpuInstance: int + numaId: int +struct_NV0000_CTRL_GPU_GET_ID_INFO_PARAMS.register_fields([('gpuId', NvU32, 0), ('gpuFlags', NvU32, 4), ('deviceInstance', NvU32, 8), ('subDeviceInstance', NvU32, 12), ('szName', NvP64, 16), ('sliStatus', NvU32, 24), ('boardId', NvU32, 28), ('gpuInstance', NvU32, 32), ('numaId', NvS32, 36)]) +NV0000_CTRL_GPU_GET_ID_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_ID_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS(c.Struct): + SIZE = 32 + gpuId: int + gpuFlags: int + deviceInstance: int + subDeviceInstance: int + sliStatus: int + boardId: int + gpuInstance: int + numaId: int +struct_NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS.register_fields([('gpuId', NvU32, 0), ('gpuFlags', NvU32, 4), ('deviceInstance', NvU32, 8), ('subDeviceInstance', NvU32, 12), ('sliStatus', NvU32, 16), ('boardId', NvU32, 20), ('gpuInstance', NvU32, 24), ('numaId', NvS32, 28)]) +NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_INIT_STATUS_PARAMS(c.Struct): + SIZE = 8 + gpuId: int + status: int +struct_NV0000_CTRL_GPU_GET_INIT_STATUS_PARAMS.register_fields([('gpuId', NvU32, 0), ('status', NvU32, 4)]) +NV0000_CTRL_GPU_GET_INIT_STATUS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_INIT_STATUS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_DEVICE_IDS_PARAMS(c.Struct): + SIZE = 4 + deviceIds: int +struct_NV0000_CTRL_GPU_GET_DEVICE_IDS_PARAMS.register_fields([('deviceIds', NvU32, 0)]) +NV0000_CTRL_GPU_GET_DEVICE_IDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_DEVICE_IDS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS(c.Struct): + SIZE = 384 + gpuIds: c.Array[ctypes.c_uint32, Literal[32]] + excludedGpuIds: c.Array[ctypes.c_uint32, Literal[32]] + gpuFlags: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS.register_fields([('gpuIds', c.Array[NvU32, Literal[32]], 0), ('excludedGpuIds', c.Array[NvU32, Literal[32]], 128), ('gpuFlags', c.Array[NvU32, Literal[32]], 256)]) +NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_PCI_INFO_PARAMS(c.Struct): + SIZE = 12 + gpuId: int + domain: int + bus: int + slot: int +struct_NV0000_CTRL_GPU_GET_PCI_INFO_PARAMS.register_fields([('gpuId', NvU32, 0), ('domain', NvU32, 4), ('bus', NvU16, 8), ('slot', NvU16, 10)]) +NV0000_CTRL_GPU_GET_PCI_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_PCI_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_ATTACH_IDS_PARAMS(c.Struct): + SIZE = 132 + gpuIds: c.Array[ctypes.c_uint32, Literal[32]] + failedId: int +struct_NV0000_CTRL_GPU_ATTACH_IDS_PARAMS.register_fields([('gpuIds', c.Array[NvU32, Literal[32]], 0), ('failedId', NvU32, 128)]) +NV0000_CTRL_GPU_ATTACH_IDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_ATTACH_IDS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_DETACH_IDS_PARAMS(c.Struct): + SIZE = 128 + gpuIds: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV0000_CTRL_GPU_DETACH_IDS_PARAMS.register_fields([('gpuIds', c.Array[NvU32, Literal[32]], 0)]) +NV0000_CTRL_GPU_DETACH_IDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_DETACH_IDS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_VIDEO_LINKS(c.Struct): + SIZE = 36 + gpuId: int + connectedGpuIds: c.Array[ctypes.c_uint32, Literal[8]] +struct_NV0000_CTRL_GPU_VIDEO_LINKS.register_fields([('gpuId', NvU32, 0), ('connectedGpuIds', c.Array[NvU32, Literal[8]], 4)]) +NV0000_CTRL_GPU_VIDEO_LINKS: TypeAlias = struct_NV0000_CTRL_GPU_VIDEO_LINKS +@c.record +class struct_NV0000_CTRL_GPU_GET_VIDEO_LINKS_PARAMS(c.Struct): + SIZE = 1152 + links: c.Array[struct_NV0000_CTRL_GPU_VIDEO_LINKS, Literal[32]] +struct_NV0000_CTRL_GPU_GET_VIDEO_LINKS_PARAMS.register_fields([('links', c.Array[NV0000_CTRL_GPU_VIDEO_LINKS, Literal[32]], 0)]) +NV0000_CTRL_GPU_GET_VIDEO_LINKS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_VIDEO_LINKS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_UUID_INFO_PARAMS(c.Struct): + SIZE = 272 + gpuUuid: c.Array[ctypes.c_ubyte, Literal[256]] + flags: int + gpuId: int + deviceInstance: int + subdeviceInstance: int +struct_NV0000_CTRL_GPU_GET_UUID_INFO_PARAMS.register_fields([('gpuUuid', c.Array[NvU8, Literal[256]], 0), ('flags', NvU32, 256), ('gpuId', NvU32, 260), ('deviceInstance', NvU32, 264), ('subdeviceInstance', NvU32, 268)]) +NV0000_CTRL_GPU_GET_UUID_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_UUID_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS(c.Struct): + SIZE = 268 + gpuId: int + flags: int + gpuUuid: c.Array[ctypes.c_ubyte, Literal[256]] + uuidStrLen: int +struct_NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS.register_fields([('gpuId', NvU32, 0), ('flags', NvU32, 4), ('gpuUuid', c.Array[NvU8, Literal[256]], 8), ('uuidStrLen', NvU32, 264)]) +NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_MODIFY_DRAIN_STATE_PARAMS(c.Struct): + SIZE = 12 + gpuId: int + newState: int + flags: int +struct_NV0000_CTRL_GPU_MODIFY_DRAIN_STATE_PARAMS.register_fields([('gpuId', NvU32, 0), ('newState', NvU32, 4), ('flags', NvU32, 8)]) +NV0000_CTRL_GPU_MODIFY_DRAIN_STATE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_MODIFY_DRAIN_STATE_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_QUERY_DRAIN_STATE_PARAMS(c.Struct): + SIZE = 12 + gpuId: int + drainState: int + flags: int +struct_NV0000_CTRL_GPU_QUERY_DRAIN_STATE_PARAMS.register_fields([('gpuId', NvU32, 0), ('drainState', NvU32, 4), ('flags', NvU32, 8)]) +NV0000_CTRL_GPU_QUERY_DRAIN_STATE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_QUERY_DRAIN_STATE_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_DISCOVER_PARAMS(c.Struct): + SIZE = 8 + domain: int + bus: int + slot: int + function: int +struct_NV0000_CTRL_GPU_DISCOVER_PARAMS.register_fields([('domain', NvU32, 0), ('bus', NvU8, 4), ('slot', NvU8, 5), ('function', NvU8, 6)]) +NV0000_CTRL_GPU_DISCOVER_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_DISCOVER_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_MEMOP_ENABLE_PARAMS(c.Struct): + SIZE = 4 + enableMask: int +struct_NV0000_CTRL_GPU_GET_MEMOP_ENABLE_PARAMS.register_fields([('enableMask', NvU32, 0)]) +NV0000_CTRL_GPU_GET_MEMOP_ENABLE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_MEMOP_ENABLE_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_IDLE_CHANNELS_PARAMS(c.Struct): + SIZE = 48 + hDevice: int + hChannel: int + numChannels: int + phClients: ctypes.c_void_p + phDevices: ctypes.c_void_p + phChannels: ctypes.c_void_p + flags: int + timeout: int +struct_NV0000_CTRL_GPU_IDLE_CHANNELS_PARAMS.register_fields([('hDevice', NvHandle, 0), ('hChannel', NvHandle, 4), ('numChannels', NvV32, 8), ('phClients', NvP64, 16), ('phDevices', NvP64, 24), ('phChannels', NvP64, 32), ('flags', NvV32, 40), ('timeout', NvV32, 44)]) +NV0000_CTRL_GPU_IDLE_CHANNELS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_IDLE_CHANNELS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_PUSH_UCODE_IMAGE_PARAMS(c.Struct): + SIZE = 24 + image: int + totalSize: int + pData: ctypes.c_void_p +struct_NV0000_CTRL_GPU_PUSH_UCODE_IMAGE_PARAMS.register_fields([('image', NvU8, 0), ('totalSize', NvU64, 8), ('pData', NvP64, 16)]) +NV0000_CTRL_GPU_PUSH_UCODE_IMAGE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_PUSH_UCODE_IMAGE_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_SET_NVLINK_BW_MODE_PARAMS(c.Struct): + SIZE = 2 + mode: int +struct_NV0000_CTRL_GPU_SET_NVLINK_BW_MODE_PARAMS.register_fields([('mode', NvU16, 0)]) +NV0000_CTRL_GPU_SET_NVLINK_BW_MODE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_SET_NVLINK_BW_MODE_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_GET_NVLINK_BW_MODE_PARAMS(c.Struct): + SIZE = 2 + mode: int + bwModeScope: int +struct_NV0000_CTRL_GPU_GET_NVLINK_BW_MODE_PARAMS.register_fields([('mode', NvU8, 0), ('bwModeScope', NvU8, 1)]) +NV0000_CTRL_GPU_GET_NVLINK_BW_MODE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_NVLINK_BW_MODE_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_ACTIVE_DEVICE(c.Struct): + SIZE = 12 + gpuId: int + gpuInstanceId: int + computeInstanceId: int +struct_NV0000_CTRL_GPU_ACTIVE_DEVICE.register_fields([('gpuId', NvU32, 0), ('gpuInstanceId', NvU32, 4), ('computeInstanceId', NvU32, 8)]) +NV0000_CTRL_GPU_ACTIVE_DEVICE: TypeAlias = struct_NV0000_CTRL_GPU_ACTIVE_DEVICE +@c.record +class struct_NV0000_CTRL_GPU_GET_ACTIVE_DEVICE_IDS_PARAMS(c.Struct): + SIZE = 3076 + numDevices: int + devices: c.Array[struct_NV0000_CTRL_GPU_ACTIVE_DEVICE, Literal[256]] +struct_NV0000_CTRL_GPU_GET_ACTIVE_DEVICE_IDS_PARAMS.register_fields([('numDevices', NvU32, 0), ('devices', c.Array[NV0000_CTRL_GPU_ACTIVE_DEVICE, Literal[256]], 4)]) +NV0000_CTRL_GPU_GET_ACTIVE_DEVICE_IDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_GET_ACTIVE_DEVICE_IDS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_ASYNC_ATTACH_ID_PARAMS(c.Struct): + SIZE = 4 + gpuId: int +struct_NV0000_CTRL_GPU_ASYNC_ATTACH_ID_PARAMS.register_fields([('gpuId', NvU32, 0)]) +NV0000_CTRL_GPU_ASYNC_ATTACH_ID_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_ASYNC_ATTACH_ID_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_WAIT_ATTACH_ID_PARAMS(c.Struct): + SIZE = 4 + gpuId: int +struct_NV0000_CTRL_GPU_WAIT_ATTACH_ID_PARAMS.register_fields([('gpuId', NvU32, 0)]) +NV0000_CTRL_GPU_WAIT_ATTACH_ID_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_WAIT_ATTACH_ID_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_PCI_READ_WRITE_PARAMS(c.Struct): + SIZE = 16 + bWrite: int + pciDomain: int + pciBus: int + pciDevice: int + pciFunction: int + pciOffset: int + dataSize: int + data: int +struct_NV0000_CTRL_GPU_PCI_READ_WRITE_PARAMS.register_fields([('bWrite', NvBool, 0), ('pciDomain', NvU16, 2), ('pciBus', NvU8, 4), ('pciDevice', NvU8, 5), ('pciFunction', NvU8, 6), ('pciOffset', NvU8, 7), ('dataSize', NvU32, 8), ('data', NvU32, 12)]) +NV0000_CTRL_GPU_PCI_READ_WRITE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_PCI_READ_WRITE_PARAMS +@c.record +class struct_NV0000_CTRL_GPUACCT_SET_ACCOUNTING_STATE_PARAMS(c.Struct): + SIZE = 12 + gpuId: int + pid: int + newState: int +struct_NV0000_CTRL_GPUACCT_SET_ACCOUNTING_STATE_PARAMS.register_fields([('gpuId', NvU32, 0), ('pid', NvU32, 4), ('newState', NvU32, 8)]) +NV0000_CTRL_GPUACCT_SET_ACCOUNTING_STATE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPUACCT_SET_ACCOUNTING_STATE_PARAMS +@c.record +class struct_NV0000_CTRL_GPUACCT_GET_ACCOUNTING_STATE_PARAMS(c.Struct): + SIZE = 12 + gpuId: int + pid: int + state: int +struct_NV0000_CTRL_GPUACCT_GET_ACCOUNTING_STATE_PARAMS.register_fields([('gpuId', NvU32, 0), ('pid', NvU32, 4), ('state', NvU32, 8)]) +NV0000_CTRL_GPUACCT_GET_ACCOUNTING_STATE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPUACCT_GET_ACCOUNTING_STATE_PARAMS +@c.record +class struct_NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_PARAMS(c.Struct): + SIZE = 48 + gpuId: int + pid: int + subPid: int + gpuUtil: int + fbUtil: int + maxFbUsage: int + startTime: int + endTime: int +struct_NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_PARAMS.register_fields([('gpuId', NvU32, 0), ('pid', NvU32, 4), ('subPid', NvU32, 8), ('gpuUtil', NvU32, 12), ('fbUtil', NvU32, 16), ('maxFbUsage', NvU64, 24), ('startTime', NvU64, 32), ('endTime', NvU64, 40)]) +NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_GPUACCT_GET_ACCOUNTING_PIDS_PARAMS(c.Struct): + SIZE = 16012 + gpuId: int + pid: int + pidTbl: c.Array[ctypes.c_uint32, Literal[4000]] + pidCount: int +struct_NV0000_CTRL_GPUACCT_GET_ACCOUNTING_PIDS_PARAMS.register_fields([('gpuId', NvU32, 0), ('pid', NvU32, 4), ('pidTbl', c.Array[NvU32, Literal[4000]], 8), ('pidCount', NvU32, 16008)]) +NV0000_CTRL_GPUACCT_GET_ACCOUNTING_PIDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GPUACCT_GET_ACCOUNTING_PIDS_PARAMS +@c.record +class struct_NV0000_CTRL_GPUACCT_CLEAR_ACCOUNTING_DATA_PARAMS(c.Struct): + SIZE = 8 + gpuId: int + pid: int +struct_NV0000_CTRL_GPUACCT_CLEAR_ACCOUNTING_DATA_PARAMS.register_fields([('gpuId', NvU32, 0), ('pid', NvU32, 4)]) +NV0000_CTRL_GPUACCT_CLEAR_ACCOUNTING_DATA_PARAMS: TypeAlias = struct_NV0000_CTRL_GPUACCT_CLEAR_ACCOUNTING_DATA_PARAMS +@c.record +class struct_NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_V2_PARAMS(c.Struct): + SIZE = 64 + gpuId: int + pid: int + subPid: int + gpuUtil: int + fbUtil: int + sampleCount: int + maxFbUsage: int + startTime: int + endTime: int + sumGpuUtil: int + sumFbUtil: int +struct_NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_V2_PARAMS.register_fields([('gpuId', NvU32, 0), ('pid', NvU32, 4), ('subPid', NvU32, 8), ('gpuUtil', NvU32, 12), ('fbUtil', NvU32, 16), ('sampleCount', NvU32, 20), ('maxFbUsage', NvU64, 24), ('startTime', NvU64, 32), ('endTime', NvU64, 40), ('sumGpuUtil', NvU64, 48), ('sumFbUtil', NvU64, 56)]) +NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_V2_PARAMS: TypeAlias = struct_NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_V2_PARAMS +@c.record +class struct_NV0000_CTRL_GSYNC_GET_ATTACHED_IDS_PARAMS(c.Struct): + SIZE = 16 + gsyncIds: c.Array[ctypes.c_uint32, Literal[4]] +struct_NV0000_CTRL_GSYNC_GET_ATTACHED_IDS_PARAMS.register_fields([('gsyncIds', c.Array[NvU32, Literal[4]], 0)]) +NV0000_CTRL_GSYNC_GET_ATTACHED_IDS_PARAMS: TypeAlias = struct_NV0000_CTRL_GSYNC_GET_ATTACHED_IDS_PARAMS +@c.record +class struct_NV0000_CTRL_GSYNC_GET_ID_INFO_PARAMS(c.Struct): + SIZE = 12 + gsyncId: int + gsyncFlags: int + gsyncInstance: int +struct_NV0000_CTRL_GSYNC_GET_ID_INFO_PARAMS.register_fields([('gsyncId', NvU32, 0), ('gsyncFlags', NvU32, 4), ('gsyncInstance', NvU32, 8)]) +NV0000_CTRL_GSYNC_GET_ID_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_GSYNC_GET_ID_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_NVD_GET_DUMP_SIZE_PARAMS(c.Struct): + SIZE = 8 + component: int + size: int +struct_NV0000_CTRL_NVD_GET_DUMP_SIZE_PARAMS.register_fields([('component', NvU32, 0), ('size', NvU32, 4)]) +NV0000_CTRL_NVD_GET_DUMP_SIZE_PARAMS: TypeAlias = struct_NV0000_CTRL_NVD_GET_DUMP_SIZE_PARAMS +@c.record +class struct_NV0000_CTRL_NVD_GET_DUMP_PARAMS(c.Struct): + SIZE = 16 + pBuffer: ctypes.c_void_p + component: int + size: int +struct_NV0000_CTRL_NVD_GET_DUMP_PARAMS.register_fields([('pBuffer', NvP64, 0), ('component', NvU32, 8), ('size', NvU32, 12)]) +NV0000_CTRL_NVD_GET_DUMP_PARAMS: TypeAlias = struct_NV0000_CTRL_NVD_GET_DUMP_PARAMS +@c.record +class struct_NV0000_CTRL_NVD_GET_TIMESTAMP_PARAMS(c.Struct): + SIZE = 16 + timestamp: int + cpuClkId: int +struct_NV0000_CTRL_NVD_GET_TIMESTAMP_PARAMS.register_fields([('timestamp', NvU64, 0), ('cpuClkId', NvU8, 8)]) +NV0000_CTRL_NVD_GET_TIMESTAMP_PARAMS: TypeAlias = struct_NV0000_CTRL_NVD_GET_TIMESTAMP_PARAMS +@c.record +class struct_NV0000_CTRL_NVD_GET_NVLOG_INFO_PARAMS(c.Struct): + SIZE = 15404 + component: int + version: int + runtimeSizes: c.Array[ctypes.c_ubyte, Literal[16]] + printFlags: int + signature: c.Array[ctypes.c_uint32, Literal[4]] + bufferTags: c.Array[ctypes.c_uint32, Literal[3840]] +struct_NV0000_CTRL_NVD_GET_NVLOG_INFO_PARAMS.register_fields([('component', NvU32, 0), ('version', NvU32, 4), ('runtimeSizes', c.Array[NvU8, Literal[16]], 8), ('printFlags', NvU32, 24), ('signature', c.Array[NvU32, Literal[4]], 28), ('bufferTags', c.Array[NvU32, Literal[3840]], 44)]) +NV0000_CTRL_NVD_GET_NVLOG_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_NVD_GET_NVLOG_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_NVD_GET_NVLOG_BUFFER_INFO_PARAMS(c.Struct): + SIZE = 28 + component: int + buffer: int + tag: int + size: int + flags: int + pos: int + overflow: int +struct_NV0000_CTRL_NVD_GET_NVLOG_BUFFER_INFO_PARAMS.register_fields([('component', NvU32, 0), ('buffer', NvU32, 4), ('tag', NvU32, 8), ('size', NvU32, 12), ('flags', NvU32, 16), ('pos', NvU32, 20), ('overflow', NvU32, 24)]) +NV0000_CTRL_NVD_GET_NVLOG_BUFFER_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_NVD_GET_NVLOG_BUFFER_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_NVD_GET_NVLOG_PARAMS(c.Struct): + SIZE = 4016 + component: int + buffer: int + blockNum: int + size: int + data: c.Array[ctypes.c_ubyte, Literal[4000]] +struct_NV0000_CTRL_NVD_GET_NVLOG_PARAMS.register_fields([('component', NvU32, 0), ('buffer', NvU32, 4), ('blockNum', NvU32, 8), ('size', NvU32, 12), ('data', c.Array[NvU8, Literal[4000]], 16)]) +NV0000_CTRL_NVD_GET_NVLOG_PARAMS: TypeAlias = struct_NV0000_CTRL_NVD_GET_NVLOG_PARAMS +@c.record +class struct_NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_ENTRY(c.Struct): + SIZE = 12 + tag: int + value: int + attribute: int +struct_NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_ENTRY.register_fields([('tag', NvU32, 0), ('value', NvU32, 4), ('attribute', NvU32, 8)]) +NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_ENTRY: TypeAlias = struct_NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_ENTRY +@c.record +class struct_NV0000_CTRL_CMD_NVD_GET_RCERR_RPT_PARAMS(c.Struct): + SIZE = 2436 + reqIdx: int + rptIdx: int + GPUTag: int + rptTime: int + startIdx: int + endIdx: int + rptType: int + flags: int + rptCount: int + owner: int + processId: int + report: c.Array[struct_NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_ENTRY, Literal[200]] +struct_NV0000_CTRL_CMD_NVD_GET_RCERR_RPT_PARAMS.register_fields([('reqIdx', NvU16, 0), ('rptIdx', NvU16, 2), ('GPUTag', NvU32, 4), ('rptTime', NvU32, 8), ('startIdx', NvU16, 12), ('endIdx', NvU16, 14), ('rptType', NvU16, 16), ('flags', NvU32, 20), ('rptCount', NvU16, 24), ('owner', NvU32, 28), ('processId', NvU32, 32), ('report', c.Array[NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_ENTRY, Literal[200]], 36)]) +NV0000_CTRL_CMD_NVD_GET_RCERR_RPT_PARAMS: TypeAlias = struct_NV0000_CTRL_CMD_NVD_GET_RCERR_RPT_PARAMS +@c.record +class struct_NV0000_CTRL_NVD_GET_DPC_ISR_TS_PARAMS(c.Struct): + SIZE = 16 + tsBufferSize: int + pTSBuffer: ctypes.c_void_p +struct_NV0000_CTRL_NVD_GET_DPC_ISR_TS_PARAMS.register_fields([('tsBufferSize', NvU32, 0), ('pTSBuffer', NvP64, 8)]) +NV0000_CTRL_NVD_GET_DPC_ISR_TS_PARAMS: TypeAlias = struct_NV0000_CTRL_NVD_GET_DPC_ISR_TS_PARAMS +@c.record +class struct_NV0000_CTRL_SET_SUB_PROCESS_ID_PARAMS(c.Struct): + SIZE = 104 + subProcessID: int + subProcessName: c.Array[ctypes.c_char, Literal[100]] +struct_NV0000_CTRL_SET_SUB_PROCESS_ID_PARAMS.register_fields([('subProcessID', NvU32, 0), ('subProcessName', c.Array[ctypes.c_char, Literal[100]], 4)]) +NV0000_CTRL_SET_SUB_PROCESS_ID_PARAMS: TypeAlias = struct_NV0000_CTRL_SET_SUB_PROCESS_ID_PARAMS +@c.record +class struct_NV0000_CTRL_DISABLE_SUB_PROCESS_USERD_ISOLATION_PARAMS(c.Struct): + SIZE = 1 + bIsSubProcessDisabled: int +struct_NV0000_CTRL_DISABLE_SUB_PROCESS_USERD_ISOLATION_PARAMS.register_fields([('bIsSubProcessDisabled', NvBool, 0)]) +NV0000_CTRL_DISABLE_SUB_PROCESS_USERD_ISOLATION_PARAMS: TypeAlias = struct_NV0000_CTRL_DISABLE_SUB_PROCESS_USERD_ISOLATION_PARAMS +@c.record +class struct_NV0000_SYNC_GPU_BOOST_INFO_PARAMS(c.Struct): + SIZE = 1 + bEnabled: int +struct_NV0000_SYNC_GPU_BOOST_INFO_PARAMS.register_fields([('bEnabled', NvBool, 0)]) +NV0000_SYNC_GPU_BOOST_INFO_PARAMS: TypeAlias = struct_NV0000_SYNC_GPU_BOOST_INFO_PARAMS +@c.record +class struct_NV0000_SYNC_GPU_BOOST_GROUP_CONFIG(c.Struct): + SIZE = 140 + gpuCount: int + gpuIds: c.Array[ctypes.c_uint32, Literal[32]] + boostGroupId: int + bBridgeless: int +struct_NV0000_SYNC_GPU_BOOST_GROUP_CONFIG.register_fields([('gpuCount', NvU32, 0), ('gpuIds', c.Array[NvU32, Literal[32]], 4), ('boostGroupId', NvU32, 132), ('bBridgeless', NvBool, 136)]) +NV0000_SYNC_GPU_BOOST_GROUP_CONFIG: TypeAlias = struct_NV0000_SYNC_GPU_BOOST_GROUP_CONFIG +@c.record +class struct_NV0000_SYNC_GPU_BOOST_GROUP_CREATE_PARAMS(c.Struct): + SIZE = 140 + boostConfig: struct_NV0000_SYNC_GPU_BOOST_GROUP_CONFIG +struct_NV0000_SYNC_GPU_BOOST_GROUP_CREATE_PARAMS.register_fields([('boostConfig', NV0000_SYNC_GPU_BOOST_GROUP_CONFIG, 0)]) +NV0000_SYNC_GPU_BOOST_GROUP_CREATE_PARAMS: TypeAlias = struct_NV0000_SYNC_GPU_BOOST_GROUP_CREATE_PARAMS +@c.record +class struct_NV0000_SYNC_GPU_BOOST_GROUP_DESTROY_PARAMS(c.Struct): + SIZE = 4 + boostGroupId: int +struct_NV0000_SYNC_GPU_BOOST_GROUP_DESTROY_PARAMS.register_fields([('boostGroupId', NvU32, 0)]) +NV0000_SYNC_GPU_BOOST_GROUP_DESTROY_PARAMS: TypeAlias = struct_NV0000_SYNC_GPU_BOOST_GROUP_DESTROY_PARAMS +@c.record +class struct_NV0000_SYNC_GPU_BOOST_GROUP_INFO_PARAMS(c.Struct): + SIZE = 2244 + groupCount: int + pBoostGroups: c.Array[struct_NV0000_SYNC_GPU_BOOST_GROUP_CONFIG, Literal[16]] +struct_NV0000_SYNC_GPU_BOOST_GROUP_INFO_PARAMS.register_fields([('groupCount', NvU32, 0), ('pBoostGroups', c.Array[NV0000_SYNC_GPU_BOOST_GROUP_CONFIG, Literal[16]], 4)]) +NV0000_SYNC_GPU_BOOST_GROUP_INFO_PARAMS: TypeAlias = struct_NV0000_SYNC_GPU_BOOST_GROUP_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_FEATURES_PARAMS(c.Struct): + SIZE = 4 + featuresMask: int +struct_NV0000_CTRL_SYSTEM_GET_FEATURES_PARAMS.register_fields([('featuresMask', NvU32, 0)]) +NV0000_CTRL_SYSTEM_GET_FEATURES_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_FEATURES_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS(c.Struct): + SIZE = 40 + sizeOfStrings: int + pDriverVersionBuffer: ctypes.c_void_p + pVersionBuffer: ctypes.c_void_p + pTitleBuffer: ctypes.c_void_p + changelistNumber: int + officialChangelistNumber: int +struct_NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS.register_fields([('sizeOfStrings', NvU32, 0), ('pDriverVersionBuffer', NvP64, 8), ('pVersionBuffer', NvP64, 16), ('pTitleBuffer', NvP64, 24), ('changelistNumber', NvU32, 32), ('officialChangelistNumber', NvU32, 36)]) +NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS +enum_NV0000_CTRL_SYSTEM_SH_SOC_TYPE: dict[int, str] = {(NV0000_CTRL_SYSTEM_SH_SOC_TYPE_NA:=0): 'NV0000_CTRL_SYSTEM_SH_SOC_TYPE_NA', (NV0000_CTRL_SYSTEM_SH_SOC_TYPE_NV_GRACE:=1): 'NV0000_CTRL_SYSTEM_SH_SOC_TYPE_NV_GRACE'} +NV0000_CTRL_SYSTEM_SH_SOC_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_CPU_INFO_PARAMS(c.Struct): + SIZE = 108 + type: int + capabilities: int + clock: int + L1DataCacheSize: int + L2DataCacheSize: int + dataCacheLineSize: int + numLogicalCpus: int + numPhysicalCpus: int + name: c.Array[ctypes.c_ubyte, Literal[52]] + family: int + model: int + stepping: int + coresOnDie: int + bCCEnabled: int + selfHostedSocType: int +struct_NV0000_CTRL_SYSTEM_GET_CPU_INFO_PARAMS.register_fields([('type', NvU32, 0), ('capabilities', NvU32, 4), ('clock', NvU32, 8), ('L1DataCacheSize', NvU32, 12), ('L2DataCacheSize', NvU32, 16), ('dataCacheLineSize', NvU32, 20), ('numLogicalCpus', NvU32, 24), ('numPhysicalCpus', NvU32, 28), ('name', c.Array[NvU8, Literal[52]], 32), ('family', NvU32, 84), ('model', NvU32, 88), ('stepping', NvU8, 92), ('coresOnDie', NvU32, 96), ('bCCEnabled', NvBool, 100), ('selfHostedSocType', NV0000_CTRL_SYSTEM_SH_SOC_TYPE, 104)]) +NV0000_CTRL_SYSTEM_GET_CPU_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_CPU_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_CHIPSET_INFO_PARAMS(c.Struct): + SIZE = 156 + vendorId: int + deviceId: int + subSysVendorId: int + subSysDeviceId: int + HBvendorId: int + HBdeviceId: int + HBsubSysVendorId: int + HBsubSysDeviceId: int + sliBondId: int + vendorNameString: c.Array[ctypes.c_ubyte, Literal[32]] + subSysVendorNameString: c.Array[ctypes.c_ubyte, Literal[32]] + chipsetNameString: c.Array[ctypes.c_ubyte, Literal[32]] + sliBondNameString: c.Array[ctypes.c_ubyte, Literal[32]] + flags: int + chipsetId: int +struct_NV0000_CTRL_SYSTEM_GET_CHIPSET_INFO_PARAMS.register_fields([('vendorId', NvU16, 0), ('deviceId', NvU16, 2), ('subSysVendorId', NvU16, 4), ('subSysDeviceId', NvU16, 6), ('HBvendorId', NvU16, 8), ('HBdeviceId', NvU16, 10), ('HBsubSysVendorId', NvU16, 12), ('HBsubSysDeviceId', NvU16, 14), ('sliBondId', NvU32, 16), ('vendorNameString', c.Array[NvU8, Literal[32]], 20), ('subSysVendorNameString', c.Array[NvU8, Literal[32]], 52), ('chipsetNameString', c.Array[NvU8, Literal[32]], 84), ('sliBondNameString', c.Array[NvU8, Literal[32]], 116), ('flags', NvU32, 148), ('chipsetId', NvU32, 152)]) +NV0000_CTRL_SYSTEM_GET_CHIPSET_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_CHIPSET_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_VRR_COOKIE_PRESENT_PARAMS(c.Struct): + SIZE = 1 + bIsPresent: int +struct_NV0000_CTRL_SYSTEM_GET_VRR_COOKIE_PRESENT_PARAMS.register_fields([('bIsPresent', NvBool, 0)]) +NV0000_CTRL_SYSTEM_GET_VRR_COOKIE_PRESENT_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_VRR_COOKIE_PRESENT_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_LOCK_TIMES_PARAMS(c.Struct): + SIZE = 40 + waitApiLock: int + holdRoApiLock: int + holdRwApiLock: int + waitGpuLock: int + holdGpuLock: int +struct_NV0000_CTRL_SYSTEM_GET_LOCK_TIMES_PARAMS.register_fields([('waitApiLock', NvU64, 0), ('holdRoApiLock', NvU64, 8), ('holdRwApiLock', NvU64, 16), ('waitGpuLock', NvU64, 24), ('holdGpuLock', NvU64, 32)]) +NV0000_CTRL_SYSTEM_GET_LOCK_TIMES_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_LOCK_TIMES_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_CLASSLIST_PARAMS(c.Struct): + SIZE = 132 + numClasses: int + classes: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV0000_CTRL_SYSTEM_GET_CLASSLIST_PARAMS.register_fields([('numClasses', NvU32, 0), ('classes', c.Array[NvU32, Literal[32]], 4)]) +NV0000_CTRL_SYSTEM_GET_CLASSLIST_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_CLASSLIST_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_NOTIFY_EVENT_PARAMS(c.Struct): + SIZE = 12 + eventType: int + eventData: int + bEventDataForced: int +struct_NV0000_CTRL_SYSTEM_NOTIFY_EVENT_PARAMS.register_fields([('eventType', NvU32, 0), ('eventData', NvU32, 4), ('bEventDataForced', NvBool, 8)]) +NV0000_CTRL_SYSTEM_NOTIFY_EVENT_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_NOTIFY_EVENT_PARAMS +@c.record +class struct_NV0000_CTRL_CMD_SYSTEM_GET_PLATFORM_TYPE_PARAMS(c.Struct): + SIZE = 4 + systemType: int +struct_NV0000_CTRL_CMD_SYSTEM_GET_PLATFORM_TYPE_PARAMS.register_fields([('systemType', NvU32, 0)]) +NV0000_CTRL_CMD_SYSTEM_GET_PLATFORM_TYPE_PARAMS: TypeAlias = struct_NV0000_CTRL_CMD_SYSTEM_GET_PLATFORM_TYPE_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_DEBUG_RMMSG_CTRL_PARAMS(c.Struct): + SIZE = 520 + cmd: int + count: int + data: c.Array[ctypes.c_ubyte, Literal[512]] +struct_NV0000_CTRL_SYSTEM_DEBUG_RMMSG_CTRL_PARAMS.register_fields([('cmd', NvU32, 0), ('count', NvU32, 4), ('data', c.Array[NvU8, Literal[512]], 8)]) +NV0000_CTRL_SYSTEM_DEBUG_RMMSG_CTRL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_DEBUG_RMMSG_CTRL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS(c.Struct): + SIZE = 8 + command: int + locale: int + data: int +struct_NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS.register_fields([('command', NvU16, 0), ('locale', NvU16, 2), ('data', NvU32, 4)]) +NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS(c.Struct): + SIZE = 136 + cmdCount: int + succeeded: int + cmdData: c.Array[struct_NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS_cmdData, Literal[16]] +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS_cmdData(c.Struct): + SIZE = 8 + command: int + locale: int + data: int +struct_NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS_cmdData.register_fields([('command', NvU16, 0), ('locale', NvU16, 2), ('data', NvU32, 4)]) +struct_NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS.register_fields([('cmdCount', NvU32, 0), ('succeeded', NvU32, 4), ('cmdData', c.Array[struct_NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS_cmdData, Literal[16]], 8)]) +NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS(c.Struct): + SIZE = 176 + gpuIds: c.Array[ctypes.c_uint32, Literal[32]] + gpuCount: int + p2pCaps: int + p2pOptimalReadCEs: int + p2pOptimalWriteCEs: int + p2pCapsStatus: c.Array[ctypes.c_ubyte, Literal[9]] + busPeerIds: ctypes.c_void_p + busEgmPeerIds: ctypes.c_void_p +struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS.register_fields([('gpuIds', c.Array[NvU32, Literal[32]], 0), ('gpuCount', NvU32, 128), ('p2pCaps', NvU32, 132), ('p2pOptimalReadCEs', NvU32, 136), ('p2pOptimalWriteCEs', NvU32, 140), ('p2pCapsStatus', c.Array[NvU8, Literal[9]], 144), ('busPeerIds', NvP64, 160), ('busEgmPeerIds', NvP64, 168)]) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_V2_PARAMS(c.Struct): + SIZE = 8348 + gpuIds: c.Array[ctypes.c_uint32, Literal[32]] + gpuCount: int + p2pCaps: int + p2pOptimalReadCEs: int + p2pOptimalWriteCEs: int + p2pCapsStatus: c.Array[ctypes.c_ubyte, Literal[9]] + busPeerIds: c.Array[ctypes.c_uint32, Literal[1024]] + busEgmPeerIds: c.Array[ctypes.c_uint32, Literal[1024]] +struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_V2_PARAMS.register_fields([('gpuIds', c.Array[NvU32, Literal[32]], 0), ('gpuCount', NvU32, 128), ('p2pCaps', NvU32, 132), ('p2pOptimalReadCEs', NvU32, 136), ('p2pOptimalWriteCEs', NvU32, 140), ('p2pCapsStatus', c.Array[NvU8, Literal[9]], 144), ('busPeerIds', c.Array[NvU32, Literal[1024]], 156), ('busEgmPeerIds', c.Array[NvU32, Literal[1024]], 4252)]) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_V2_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_V2_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_MATRIX_PARAMS(c.Struct): + SIZE = 1352 + grpACount: int + grpBCount: int + gpuIdGrpA: c.Array[ctypes.c_uint32, Literal[8]] + gpuIdGrpB: c.Array[ctypes.c_uint32, Literal[8]] + p2pCaps: c.Array[c.Array[ctypes.c_uint32, Literal[8]], Literal[8]] + a2bOptimalReadCes: c.Array[c.Array[ctypes.c_uint32, Literal[8]], Literal[8]] + a2bOptimalWriteCes: c.Array[c.Array[ctypes.c_uint32, Literal[8]], Literal[8]] + b2aOptimalReadCes: c.Array[c.Array[ctypes.c_uint32, Literal[8]], Literal[8]] + b2aOptimalWriteCes: c.Array[c.Array[ctypes.c_uint32, Literal[8]], Literal[8]] +struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_MATRIX_PARAMS.register_fields([('grpACount', NvU32, 0), ('grpBCount', NvU32, 4), ('gpuIdGrpA', c.Array[NvU32, Literal[8]], 8), ('gpuIdGrpB', c.Array[NvU32, Literal[8]], 40), ('p2pCaps', c.Array[c.Array[NvU32, Literal[8]], Literal[8]], 72), ('a2bOptimalReadCes', c.Array[c.Array[NvU32, Literal[8]], Literal[8]], 328), ('a2bOptimalWriteCes', c.Array[c.Array[NvU32, Literal[8]], Literal[8]], 584), ('b2aOptimalReadCes', c.Array[c.Array[NvU32, Literal[8]], Literal[8]], 840), ('b2aOptimalWriteCes', c.Array[c.Array[NvU32, Literal[8]], Literal[8]], 1096)]) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_MATRIX_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_P2P_CAPS_MATRIX_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_CTRL_PARAMS(c.Struct): + SIZE = 28 + cmd: int + input: c.Array[ctypes.c_int32, Literal[2]] + result: c.Array[ctypes.c_int32, Literal[4]] +struct_NV0000_CTRL_SYSTEM_GPS_CTRL_PARAMS.register_fields([('cmd', NvU32, 0), ('input', c.Array[NvS32, Literal[2]], 4), ('result', c.Array[NvS32, Literal[4]], 12)]) +NV0000_CTRL_SYSTEM_GPS_CTRL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_CTRL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSOR_COUNTERS_PARAMS(c.Struct): + SIZE = 148 + objHndl: int + blockId: int + nextExpectedSampleTimems: int + countersReq: int + countersReturned: int + counterBlock: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSOR_COUNTERS_PARAMS.register_fields([('objHndl', NvU32, 0), ('blockId', NvU32, 4), ('nextExpectedSampleTimems', NvU32, 8), ('countersReq', NvU32, 12), ('countersReturned', NvU32, 16), ('counterBlock', c.Array[NvU32, Literal[32]], 20)]) +NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSOR_COUNTERS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSOR_COUNTERS_PARAMS +NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSORS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSOR_COUNTERS_PARAMS +NV0000_CTRL_SYSTEM_GPS_GET_EXTENDED_PERF_SENSORS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSOR_COUNTERS_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_CALL_ACPI_PARAMS(c.Struct): + SIZE = 1164 + cmd: int + input: int + resultSz: int + result: c.Array[ctypes.c_uint32, Literal[288]] +struct_NV0000_CTRL_SYSTEM_GPS_CALL_ACPI_PARAMS.register_fields([('cmd', NvU32, 0), ('input', NvU32, 4), ('resultSz', NvU32, 8), ('result', c.Array[NvU32, Literal[288]], 12)]) +NV0000_CTRL_SYSTEM_GPS_CALL_ACPI_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_CALL_ACPI_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_PARAMS(c.Struct): + SIZE = 40 + method: int + inData: ctypes.c_void_p + inDataSize: int + outStatus: int + outData: ctypes.c_void_p + outDataSize: int +struct_NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_PARAMS.register_fields([('method', NvU32, 0), ('inData', NvP64, 8), ('inDataSize', NvU16, 16), ('outStatus', NvU32, 20), ('outData', NvP64, 24), ('outDataSize', NvU16, 32)]) +NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_ENABLE_ETW_EVENTS_PARAMS(c.Struct): + SIZE = 4 + moduleMask: int +struct_NV0000_CTRL_SYSTEM_ENABLE_ETW_EVENTS_PARAMS.register_fields([('moduleMask', NvU32, 0)]) +NV0000_CTRL_SYSTEM_ENABLE_ETW_EVENTS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_ENABLE_ETW_EVENTS_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE(c.Struct): + SIZE = 8 + frameTime: int + renderTime: int + targetTime: int + sleepTime: int + sampleNumber: int +struct_NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE.register_fields([('frameTime', NvU16, 0), ('renderTime', NvU16, 2), ('targetTime', NvU16, 4), ('sleepTime', NvU8, 6), ('sampleNumber', NvU8, 7)]) +NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_GET_FRM_DATA_PARAMS(c.Struct): + SIZE = 514 + samples: c.Array[struct_NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE, Literal[64]] + nextSampleNumber: int +struct_NV0000_CTRL_SYSTEM_GPS_GET_FRM_DATA_PARAMS.register_fields([('samples', c.Array[NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE, Literal[64]], 0), ('nextSampleNumber', NvU8, 512)]) +NV0000_CTRL_SYSTEM_GPS_GET_FRM_DATA_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_GET_FRM_DATA_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GPS_SET_FRM_DATA_PARAMS(c.Struct): + SIZE = 8 + sampleData: struct_NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE +struct_NV0000_CTRL_SYSTEM_GPS_SET_FRM_DATA_PARAMS.register_fields([('sampleData', NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE, 0)]) +NV0000_CTRL_SYSTEM_GPS_SET_FRM_DATA_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GPS_SET_FRM_DATA_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_VGX_SYSTEM_INFO_PARAMS(c.Struct): + SIZE = 1288 + szHostDriverVersionBuffer: c.Array[ctypes.c_char, Literal[256]] + szHostVersionBuffer: c.Array[ctypes.c_char, Literal[256]] + szHostTitleBuffer: c.Array[ctypes.c_char, Literal[256]] + szPluginTitleBuffer: c.Array[ctypes.c_char, Literal[256]] + szHostUnameBuffer: c.Array[ctypes.c_char, Literal[256]] + iHostChangelistNumber: int + iPluginChangelistNumber: int +struct_NV0000_CTRL_SYSTEM_GET_VGX_SYSTEM_INFO_PARAMS.register_fields([('szHostDriverVersionBuffer', c.Array[ctypes.c_char, Literal[256]], 0), ('szHostVersionBuffer', c.Array[ctypes.c_char, Literal[256]], 256), ('szHostTitleBuffer', c.Array[ctypes.c_char, Literal[256]], 512), ('szPluginTitleBuffer', c.Array[ctypes.c_char, Literal[256]], 768), ('szHostUnameBuffer', c.Array[ctypes.c_char, Literal[256]], 1024), ('iHostChangelistNumber', NvU32, 1280), ('iPluginChangelistNumber', NvU32, 1284)]) +NV0000_CTRL_SYSTEM_GET_VGX_SYSTEM_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_VGX_SYSTEM_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_GPUS_POWER_STATUS_PARAMS(c.Struct): + SIZE = 65 + gpuCount: int + gpuBus: c.Array[ctypes.c_ubyte, Literal[32]] + gpuExternalPowerStatus: c.Array[ctypes.c_ubyte, Literal[32]] +struct_NV0000_CTRL_SYSTEM_GET_GPUS_POWER_STATUS_PARAMS.register_fields([('gpuCount', NvU8, 0), ('gpuBus', c.Array[NvU8, Literal[32]], 1), ('gpuExternalPowerStatus', c.Array[NvU8, Literal[32]], 33)]) +NV0000_CTRL_SYSTEM_GET_GPUS_POWER_STATUS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_GPUS_POWER_STATUS_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_PARAMS(c.Struct): + SIZE = 1 + privStatusFlags: int +struct_NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_PARAMS.register_fields([('privStatusFlags', NvU8, 0)]) +NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_PARAMS +enum_NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS: dict[int, str] = {(NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_SKIP:=1): 'NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_SKIP', (NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_UNINITIALIZED:=2): 'NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_UNINITIALIZED', (NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_IN_PROGRESS:=3): 'NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_IN_PROGRESS', (NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_INITIALIZED:=4): 'NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS_INITIALIZED'} +NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_FABRIC_STATUS_PARAMS(c.Struct): + SIZE = 4 + fabricStatus: int +struct_NV0000_CTRL_SYSTEM_GET_FABRIC_STATUS_PARAMS.register_fields([('fabricStatus', NV0000_CTRL_GET_SYSTEM_FABRIC_STATUS, 0)]) +NV0000_CTRL_SYSTEM_GET_FABRIC_STATUS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_FABRIC_STATUS_PARAMS +@c.record +class struct_NV0000_CTRL_VGPU_GET_VGPU_VERSION_PARAMS(c.Struct): + SIZE = 16 + host_min_supported_version: int + host_max_supported_version: int + user_min_supported_version: int + user_max_supported_version: int +struct_NV0000_CTRL_VGPU_GET_VGPU_VERSION_PARAMS.register_fields([('host_min_supported_version', NvU32, 0), ('host_max_supported_version', NvU32, 4), ('user_min_supported_version', NvU32, 8), ('user_max_supported_version', NvU32, 12)]) +NV0000_CTRL_VGPU_GET_VGPU_VERSION_PARAMS: TypeAlias = struct_NV0000_CTRL_VGPU_GET_VGPU_VERSION_PARAMS +@c.record +class struct_NV0000_CTRL_VGPU_SET_VGPU_VERSION_PARAMS(c.Struct): + SIZE = 8 + min_version: int + max_version: int +struct_NV0000_CTRL_VGPU_SET_VGPU_VERSION_PARAMS.register_fields([('min_version', NvU32, 0), ('max_version', NvU32, 4)]) +NV0000_CTRL_VGPU_SET_VGPU_VERSION_PARAMS: TypeAlias = struct_NV0000_CTRL_VGPU_SET_VGPU_VERSION_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_RM_INSTANCE_ID_PARAMS(c.Struct): + SIZE = 8 + rm_instance_id: int +struct_NV0000_CTRL_SYSTEM_GET_RM_INSTANCE_ID_PARAMS.register_fields([('rm_instance_id', NvU64, 0)]) +NV0000_CTRL_SYSTEM_GET_RM_INSTANCE_ID_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_RM_INSTANCE_ID_PARAMS +@c.record +class struct_NV0000_CTRL_CMD_SYSTEM_GET_SYSTEM_POWER_LIMIT(c.Struct): + SIZE = 24 + batteryStateOfChargePercent: int + batteryCurrentLimitmA: int + restOfSytemReservedPowermW: int + minCpuTdpmW: int + maxCpuTdpmW: int + shortTimescaleBatteryCurrentLimitmA: int +struct_NV0000_CTRL_CMD_SYSTEM_GET_SYSTEM_POWER_LIMIT.register_fields([('batteryStateOfChargePercent', NvU8, 0), ('batteryCurrentLimitmA', NvU32, 4), ('restOfSytemReservedPowermW', NvU32, 8), ('minCpuTdpmW', NvU32, 12), ('maxCpuTdpmW', NvU32, 16), ('shortTimescaleBatteryCurrentLimitmA', NvU32, 20)]) +NV0000_CTRL_CMD_SYSTEM_GET_SYSTEM_POWER_LIMIT: TypeAlias = struct_NV0000_CTRL_CMD_SYSTEM_GET_SYSTEM_POWER_LIMIT +enum_QBOOST_CPU_TDP_CONTROL_TYPE: dict[int, str] = {(QBOOST_CPU_TDP_CONTROL_TYPE_DC_ONLY:=0): 'QBOOST_CPU_TDP_CONTROL_TYPE_DC_ONLY', (QBOOST_CPU_TDP_CONTROL_TYPE_DC_AC:=1): 'QBOOST_CPU_TDP_CONTROL_TYPE_DC_AC'} +QBOOST_CPU_TDP_CONTROL_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS(c.Struct): + SIZE = 876 + supportedFuncs: int + gpuId: int + tpp: int + ratedTgp: int + subFunc: int + ctgpOffsetmW: int + targetTppOffsetmW: int + maxOutputOffsetmW: int + minOutputOffsetmW: int + ctgpBattOffsetmW: int + targetTppBattOffsetmW: int + maxOutputBattOffsetmW: int + minOutputBattOffsetmW: int + dcRosReserveOverridemW: int + dcTspLongTimescaleLimitOverridemA: int + dcTspShortTimescaleLimitOverridemA: int + bEnableForAC: int + bEnableForDC: int + version: int + samplingPeriodmS: int + samplingMulti: int + filterType: int + filterParam: struct_NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS_filterParam + filterReserved: int + bIsBoostController: int + incRatio: int + decRatio: int + bSupportBatt: int + cpuType: int + gpuType: int + sysPwrIndex: int + sysPwrGetInfo: c.Array[struct_NV0000_CTRL_CMD_SYSTEM_GET_SYSTEM_POWER_LIMIT, Literal[32]] + bIsTspSupported: int + sysPwrLimitsTableVersion: int + type: int + cpuTdpmw: int + cpuTdpControlType: int +@c.record +class struct_NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS_filterParam(c.Struct): + SIZE = 1 + weight: int + windowSize: int +struct_NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS_filterParam.register_fields([('weight', NvU8, 0), ('windowSize', NvU8, 0)]) +struct_NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS.register_fields([('supportedFuncs', NvU32, 0), ('gpuId', NvU32, 4), ('tpp', NvU32, 8), ('ratedTgp', NvU32, 12), ('subFunc', NvU32, 16), ('ctgpOffsetmW', NvS32, 20), ('targetTppOffsetmW', NvS32, 24), ('maxOutputOffsetmW', NvS32, 28), ('minOutputOffsetmW', NvS32, 32), ('ctgpBattOffsetmW', NvS32, 36), ('targetTppBattOffsetmW', NvS32, 40), ('maxOutputBattOffsetmW', NvS32, 44), ('minOutputBattOffsetmW', NvS32, 48), ('dcRosReserveOverridemW', NvU32, 52), ('dcTspLongTimescaleLimitOverridemA', NvU32, 56), ('dcTspShortTimescaleLimitOverridemA', NvU32, 60), ('bEnableForAC', NvBool, 64), ('bEnableForDC', NvBool, 65), ('version', NvU8, 66), ('samplingPeriodmS', NvU16, 68), ('samplingMulti', NvU16, 70), ('filterType', NvU8, 72), ('filterParam', struct_NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS_filterParam, 73), ('filterReserved', NvU16, 74), ('bIsBoostController', NvBool, 76), ('incRatio', NvU16, 78), ('decRatio', NvU16, 80), ('bSupportBatt', NvBool, 82), ('cpuType', NvU8, 83), ('gpuType', NvU8, 84), ('sysPwrIndex', NvU32, 88), ('sysPwrGetInfo', c.Array[NV0000_CTRL_CMD_SYSTEM_GET_SYSTEM_POWER_LIMIT, Literal[32]], 92), ('bIsTspSupported', NvBool, 860), ('sysPwrLimitsTableVersion', NvU8, 861), ('type', NvU32, 864), ('cpuTdpmw', NvU32, 868), ('cpuTdpControlType', QBOOST_CPU_TDP_CONTROL_TYPE, 872)]) +NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_CMD_SYSTEM_SYNC_EXTERNAL_FABRIC_MGMT_PARAMS(c.Struct): + SIZE = 1 + bExternalFabricMgmt: int +struct_NV0000_CTRL_CMD_SYSTEM_SYNC_EXTERNAL_FABRIC_MGMT_PARAMS.register_fields([('bExternalFabricMgmt', NvBool, 0)]) +NV0000_CTRL_CMD_SYSTEM_SYNC_EXTERNAL_FABRIC_MGMT_PARAMS: TypeAlias = struct_NV0000_CTRL_CMD_SYSTEM_SYNC_EXTERNAL_FABRIC_MGMT_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_CLIENT_DATABASE_INFO_PARAMS(c.Struct): + SIZE = 16 + clientCount: int + resourceCount: int +struct_NV0000_CTRL_SYSTEM_GET_CLIENT_DATABASE_INFO_PARAMS.register_fields([('clientCount', NvU32, 0), ('resourceCount', NvU64, 8)]) +NV0000_CTRL_SYSTEM_GET_CLIENT_DATABASE_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_CLIENT_DATABASE_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_PARAMS(c.Struct): + SIZE = 1032 + driverVersionBuffer: c.Array[ctypes.c_char, Literal[256]] + versionBuffer: c.Array[ctypes.c_char, Literal[256]] + driverBranch: c.Array[ctypes.c_char, Literal[256]] + titleBuffer: c.Array[ctypes.c_char, Literal[256]] + changelistNumber: int + officialChangelistNumber: int +struct_NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_PARAMS.register_fields([('driverVersionBuffer', c.Array[ctypes.c_char, Literal[256]], 0), ('versionBuffer', c.Array[ctypes.c_char, Literal[256]], 256), ('driverBranch', c.Array[ctypes.c_char, Literal[256]], 512), ('titleBuffer', c.Array[ctypes.c_char, Literal[256]], 768), ('changelistNumber', NvU32, 1024), ('officialChangelistNumber', NvU32, 1028)]) +NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_PARAMS(c.Struct): + SIZE = 8 + cmd: int + mode: int +struct_NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_PARAMS.register_fields([('cmd', NvU32, 0), ('mode', NvU32, 4)]) +NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CONTROL_PARAMS(c.Struct): + SIZE = 8 + command: int + locale: int + data: int +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CONTROL_PARAMS.register_fields([('command', NvU16, 0), ('locale', NvU16, 2), ('data', NvU32, 4)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CONTROL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CONTROL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS(c.Struct): + SIZE = 136 + cmdCount: int + succeeded: int + cmdData: c.Array[struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS_cmdData, Literal[16]] +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS_cmdData(c.Struct): + SIZE = 8 + command: int + locale: int + data: int +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS_cmdData.register_fields([('command', NvU16, 0), ('locale', NvU16, 2), ('data', NvU32, 4)]) +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS.register_fields([('cmdCount', NvU32, 0), ('succeeded', NvU32, 4), ('cmdData', c.Array[struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS_cmdData, Literal[16]], 8)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CTRL_PARAMS(c.Struct): + SIZE = 28 + cmd: int + input: c.Array[ctypes.c_int32, Literal[2]] + result: c.Array[ctypes.c_int32, Literal[4]] +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CTRL_PARAMS.register_fields([('cmd', NvU32, 0), ('input', c.Array[NvS32, Literal[2]], 4), ('result', c.Array[NvS32, Literal[4]], 12)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CTRL_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CTRL_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSOR_COUNTERS_PARAMS(c.Struct): + SIZE = 148 + objHndl: int + blockId: int + nextExpectedSampleTimems: int + countersReq: int + countersReturned: int + counterBlock: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSOR_COUNTERS_PARAMS.register_fields([('objHndl', NvU32, 0), ('blockId', NvU32, 4), ('nextExpectedSampleTimems', NvU32, 8), ('countersReq', NvU32, 12), ('countersReturned', NvU32, 16), ('counterBlock', c.Array[NvU32, Literal[32]], 20)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSOR_COUNTERS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSOR_COUNTERS_PARAMS +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSORS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSOR_COUNTERS_PARAMS +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_EXTENDED_PERF_SENSORS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSOR_COUNTERS_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CALL_ACPI_PARAMS(c.Struct): + SIZE = 1164 + cmd: int + input: int + resultSz: int + result: c.Array[ctypes.c_uint32, Literal[288]] +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CALL_ACPI_PARAMS.register_fields([('cmd', NvU32, 0), ('input', NvU32, 4), ('resultSz', NvU32, 8), ('result', c.Array[NvU32, Literal[288]], 12)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CALL_ACPI_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CALL_ACPI_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE(c.Struct): + SIZE = 8 + frameTime: int + renderTime: int + targetTime: int + sleepTime: int + sampleNumber: int +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE.register_fields([('frameTime', NvU16, 0), ('renderTime', NvU16, 2), ('targetTime', NvU16, 4), ('sleepTime', NvU8, 6), ('sampleNumber', NvU8, 7)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_FRM_DATA_PARAMS(c.Struct): + SIZE = 514 + samples: c.Array[struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE, Literal[64]] + nextSampleNumber: int +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_FRM_DATA_PARAMS.register_fields([('samples', c.Array[NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE, Literal[64]], 0), ('nextSampleNumber', NvU8, 512)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_FRM_DATA_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_FRM_DATA_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_SET_FRM_DATA_PARAMS(c.Struct): + SIZE = 8 + sampleData: struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE +struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_SET_FRM_DATA_PARAMS.register_fields([('sampleData', NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE, 0)]) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_SET_FRM_DATA_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_SET_FRM_DATA_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_READ_CPER_PARAMS(c.Struct): + SIZE = 1064 + cperTypeMask: int + uuid: c.Array[ctypes.c_ubyte, Literal[16]] + cperCursor: int + buffer: c.Array[ctypes.c_ubyte, Literal[1024]] + bufferSize: int +struct_NV0000_CTRL_SYSTEM_READ_CPER_PARAMS.register_fields([('cperTypeMask', NvU64, 0), ('uuid', c.Array[NvU8, Literal[16]], 8), ('cperCursor', NvU64, 24), ('buffer', c.Array[NvU8, Literal[1024]], 32), ('bufferSize', NvU32, 1056)]) +NV0000_CTRL_SYSTEM_READ_CPER_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_READ_CPER_PARAMS +@c.record +class struct_NV0000_CTRL_SYSTEM_SOC_PERF_LIMIT(c.Struct): + SIZE = 12 + minLevelOrFreqKHz: int + maxLevelOrFreqKHz: int + clientUsageId: int + bwResource: int +struct_NV0000_CTRL_SYSTEM_SOC_PERF_LIMIT.register_fields([('minLevelOrFreqKHz', NvU32, 0), ('maxLevelOrFreqKHz', NvU32, 4), ('clientUsageId', NvU8, 8), ('bwResource', NvU8, 9)]) +NV0000_CTRL_SYSTEM_SOC_PERF_LIMIT: TypeAlias = struct_NV0000_CTRL_SYSTEM_SOC_PERF_LIMIT +@c.record +class struct_NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMITS_PARAMS(c.Struct): + SIZE = 200 + numLimits: int + limits: c.Array[struct_NV0000_CTRL_SYSTEM_SOC_PERF_LIMIT, Literal[16]] + bWaitForCompletion: int +struct_NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMITS_PARAMS.register_fields([('numLimits', NvU32, 0), ('limits', c.Array[NV0000_CTRL_SYSTEM_SOC_PERF_LIMIT, Literal[16]], 4), ('bWaitForCompletion', NvBool, 196)]) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMITS_PARAMS: TypeAlias = struct_NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMITS_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS(c.Struct): + SIZE = 40 + offset: int + length: int + cacheOps: int + hDevice: int + hObject: int + internalOnly: int +struct_NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS.register_fields([('offset', NvU64, 0), ('length', NvU64, 8), ('cacheOps', NvU32, 16), ('hDevice', NvHandle, 20), ('hObject', NvHandle, 24), ('internalOnly', NvU64, 32)]) +NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_GET_CONTROL_FILE_DESCRIPTOR_PARAMS(c.Struct): + SIZE = 4 + fd: int +struct_NV0000_CTRL_OS_UNIX_GET_CONTROL_FILE_DESCRIPTOR_PARAMS.register_fields([('fd', NvS32, 0)]) +NV0000_CTRL_OS_UNIX_GET_CONTROL_FILE_DESCRIPTOR_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_GET_CONTROL_FILE_DESCRIPTOR_PARAMS +enum_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE: dict[int, str] = {(NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_NONE:=0): 'NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_NONE', (NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM:=1): 'NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM'} +NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT(c.Struct): + SIZE = 16 + type: int + data: struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data +@c.record +class struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data(c.Struct): + SIZE = 12 + rmObject: struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data_rmObject +@c.record +class struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data_rmObject(c.Struct): + SIZE = 12 + hDevice: int + hParent: int + hObject: int +struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data_rmObject.register_fields([('hDevice', NvHandle, 0), ('hParent', NvHandle, 4), ('hObject', NvHandle, 8)]) +struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data.register_fields([('rmObject', struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data_rmObject, 0)]) +struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT.register_fields([('type', NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE, 0), ('data', struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_data, 4)]) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECT: TypeAlias = struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT +@c.record +class struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS(c.Struct): + SIZE = 24 + object: struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT + fd: int + flags: int +struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS.register_fields([('object', NV0000_CTRL_OS_UNIX_EXPORT_OBJECT, 0), ('fd', NvS32, 16), ('flags', NvU32, 20)]) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS(c.Struct): + SIZE = 20 + fd: int + object: struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECT +struct_NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS.register_fields([('fd', NvS32, 0), ('object', NV0000_CTRL_OS_UNIX_EXPORT_OBJECT, 4)]) +NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS +@c.record +class struct_NV0000_CTRL_OS_GET_GPU_INFO_PARAMS(c.Struct): + SIZE = 8 + gpuId: int + minorNum: int +struct_NV0000_CTRL_OS_GET_GPU_INFO_PARAMS.register_fields([('gpuId', NvU32, 0), ('minorNum', NvU32, 4)]) +NV0000_CTRL_OS_GET_GPU_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_GET_GPU_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS(c.Struct): + SIZE = 80 + fd: int + deviceInstance: int + gpuInstanceId: int + maxObjects: int + metadata: c.Array[ctypes.c_ubyte, Literal[64]] +struct_NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS.register_fields([('fd', NvS32, 0), ('deviceInstance', NvU32, 4), ('gpuInstanceId', NvU32, 8), ('maxObjects', NvU16, 12), ('metadata', c.Array[NvU8, Literal[64]], 14)]) +NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS(c.Struct): + SIZE = 76 + hDevice: int + maxObjects: int + metadata: c.Array[ctypes.c_ubyte, Literal[64]] + fd: int +struct_NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS.register_fields([('hDevice', NvHandle, 0), ('maxObjects', NvU16, 4), ('metadata', c.Array[NvU8, Literal[64]], 6), ('fd', NvS32, 72)]) +NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS(c.Struct): + SIZE = 2128 + fd: int + hDevice: int + maxObjects: int + metadata: c.Array[ctypes.c_ubyte, Literal[64]] + objects: c.Array[ctypes.c_uint32, Literal[512]] + numObjects: int + index: int +struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS.register_fields([('fd', NvS32, 0), ('hDevice', NvHandle, 4), ('maxObjects', NvU16, 8), ('metadata', c.Array[NvU8, Literal[64]], 10), ('objects', c.Array[NvHandle, Literal[512]], 76), ('numObjects', NvU16, 2124), ('index', NvU16, 2126)]) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS(c.Struct): + SIZE = 652 + fd: int + hParent: int + objects: c.Array[ctypes.c_uint32, Literal[128]] + objectTypes: c.Array[ctypes.c_ubyte, Literal[128]] + numObjects: int + index: int +struct_NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS.register_fields([('fd', NvS32, 0), ('hParent', NvHandle, 4), ('objects', c.Array[NvHandle, Literal[128]], 8), ('objectTypes', c.Array[NvU8, Literal[128]], 520), ('numObjects', NvU16, 648), ('index', NvU16, 650)]) +NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_MEMACCT_SET_LIMITS_PARAMS(c.Struct): + SIZE = 24 + cgroupFd: int + gpuId: int + softLimit: int + hardLimit: int +struct_NV0000_CTRL_OS_UNIX_MEMACCT_SET_LIMITS_PARAMS.register_fields([('cgroupFd', NvS32, 0), ('gpuId', NvU32, 4), ('softLimit', NvU64, 8), ('hardLimit', NvU64, 16)]) +NV0000_CTRL_OS_UNIX_MEMACCT_SET_LIMITS_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_MEMACCT_SET_LIMITS_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_MEMACCT_LIMITS(c.Struct): + SIZE = 32 + gpuId: int + soft: int + hard: int + current: int +struct_NV0000_CTRL_OS_UNIX_MEMACCT_LIMITS.register_fields([('gpuId', NvU32, 0), ('soft', NvU64, 8), ('hard', NvU64, 16), ('current', NvU64, 24)]) +NV0000_CTRL_OS_UNIX_MEMACCT_LIMITS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_MEMACCT_LIMITS +@c.record +class struct_NV0000_CTRL_OS_UNIX_MEMACCT_GET_LIMITS_PARAMS(c.Struct): + SIZE = 1032 + cgroupFd: int + count: int + limits: c.Array[struct_NV0000_CTRL_OS_UNIX_MEMACCT_LIMITS, Literal[32]] +struct_NV0000_CTRL_OS_UNIX_MEMACCT_GET_LIMITS_PARAMS.register_fields([('cgroupFd', NvS32, 0), ('count', NvU32, 4), ('limits', c.Array[NV0000_CTRL_OS_UNIX_MEMACCT_LIMITS, Literal[32]], 8)]) +NV0000_CTRL_OS_UNIX_MEMACCT_GET_LIMITS_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_MEMACCT_GET_LIMITS_PARAMS +@c.record +class struct_NV0000_CTRL_OS_UNIX_MEMACCT_GET_IMPL_PARAMS(c.Struct): + SIZE = 4 + impl: int +struct_NV0000_CTRL_OS_UNIX_MEMACCT_GET_IMPL_PARAMS.register_fields([('impl', NvU32, 0)]) +NV0000_CTRL_OS_UNIX_MEMACCT_GET_IMPL_PARAMS: TypeAlias = struct_NV0000_CTRL_OS_UNIX_MEMACCT_GET_IMPL_PARAMS +@c.record +class struct_NV0000_CTRL_VGPU_CREATE_DEVICE_PARAMS(c.Struct): + SIZE = 40 + vgpuName: c.Array[ctypes.c_ubyte, Literal[16]] + gpuPciId: int + gpuPciBdf: int + vgpuTypeId: int + vgpuId: int + gpuInstanceId: int + placementId: int +struct_NV0000_CTRL_VGPU_CREATE_DEVICE_PARAMS.register_fields([('vgpuName', c.Array[NvU8, Literal[16]], 0), ('gpuPciId', NvU32, 16), ('gpuPciBdf', NvU32, 20), ('vgpuTypeId', NvU32, 24), ('vgpuId', NvU16, 28), ('gpuInstanceId', NvU32, 32), ('placementId', NvU32, 36)]) +NV0000_CTRL_VGPU_CREATE_DEVICE_PARAMS: TypeAlias = struct_NV0000_CTRL_VGPU_CREATE_DEVICE_PARAMS +@c.record +class struct_NV0000_CTRL_VGPU_GET_INSTANCES_PARAMS(c.Struct): + SIZE = 1036 + gpuPciId: int + gpuPciBdf: int + numVgpuTypes: int + vgpuTypeIds: c.Array[ctypes.c_uint32, Literal[128]] + availableInstances: c.Array[ctypes.c_uint32, Literal[128]] +struct_NV0000_CTRL_VGPU_GET_INSTANCES_PARAMS.register_fields([('gpuPciId', NvU32, 0), ('gpuPciBdf', NvU32, 4), ('numVgpuTypes', NvU32, 8), ('vgpuTypeIds', c.Array[NvU32, Literal[128]], 12), ('availableInstances', c.Array[NvU32, Literal[128]], 524)]) +NV0000_CTRL_VGPU_GET_INSTANCES_PARAMS: TypeAlias = struct_NV0000_CTRL_VGPU_GET_INSTANCES_PARAMS +@c.record +class struct_NV0000_CTRL_VGPU_DELETE_DEVICE_PARAMS(c.Struct): + SIZE = 18 + vgpuName: c.Array[ctypes.c_ubyte, Literal[16]] + vgpuId: int +struct_NV0000_CTRL_VGPU_DELETE_DEVICE_PARAMS.register_fields([('vgpuName', c.Array[NvU8, Literal[16]], 0), ('vgpuId', NvU16, 16)]) +NV0000_CTRL_VGPU_DELETE_DEVICE_PARAMS: TypeAlias = struct_NV0000_CTRL_VGPU_DELETE_DEVICE_PARAMS +@c.record +class struct_NV0000_CTRL_VGPU_VFIO_NOTIFY_RM_STATUS_PARAMS(c.Struct): + SIZE = 8 + returnStatus: int + gpuId: int +struct_NV0000_CTRL_VGPU_VFIO_NOTIFY_RM_STATUS_PARAMS.register_fields([('returnStatus', NvU32, 0), ('gpuId', NvU32, 4)]) +NV0000_CTRL_VGPU_VFIO_NOTIFY_RM_STATUS_PARAMS: TypeAlias = struct_NV0000_CTRL_VGPU_VFIO_NOTIFY_RM_STATUS_PARAMS +@c.record +class struct_NV0000_CTRL_GPU_UPDATE_SYSFS_NODE_PARAMS(c.Struct): + SIZE = 24 + vgpuName: c.Array[ctypes.c_ubyte, Literal[16]] + mode: int + sysfs_val: int +struct_NV0000_CTRL_GPU_UPDATE_SYSFS_NODE_PARAMS.register_fields([('vgpuName', c.Array[NvU8, Literal[16]], 0), ('mode', NvU32, 16), ('sysfs_val', NvU32, 20)]) +NV0000_CTRL_GPU_UPDATE_SYSFS_NODE_PARAMS: TypeAlias = struct_NV0000_CTRL_GPU_UPDATE_SYSFS_NODE_PARAMS +@c.record +class struct_NV0080_CTRL_BIF_RESET_PARAMS(c.Struct): + SIZE = 4 + flags: int +struct_NV0080_CTRL_BIF_RESET_PARAMS.register_fields([('flags', NvU32, 0)]) +NV0080_CTRL_BIF_RESET_PARAMS: TypeAlias = struct_NV0080_CTRL_BIF_RESET_PARAMS +@c.record +class struct_NV0080_CTRL_BIF_ASPM_CYA_UPDATE_PARAMS(c.Struct): + SIZE = 2 + bL0sEnable: int + bL1Enable: int +struct_NV0080_CTRL_BIF_ASPM_CYA_UPDATE_PARAMS.register_fields([('bL0sEnable', NvBool, 0), ('bL1Enable', NvBool, 1)]) +NV0080_CTRL_BIF_ASPM_CYA_UPDATE_PARAMS: TypeAlias = struct_NV0080_CTRL_BIF_ASPM_CYA_UPDATE_PARAMS +@c.record +class struct_NV0080_CTRL_CMD_BIF_GET_PCIE_POWER_CONTROL_MASK_PARAMS(c.Struct): + SIZE = 12 + pciePowerControlMask: int + pciePowerControlIdentifiedKeyOrder: int + pciePowerControlIdentifiedKeyLocation: int +struct_NV0080_CTRL_CMD_BIF_GET_PCIE_POWER_CONTROL_MASK_PARAMS.register_fields([('pciePowerControlMask', NvU32, 0), ('pciePowerControlIdentifiedKeyOrder', NvU32, 4), ('pciePowerControlIdentifiedKeyLocation', NvU32, 8)]) +NV0080_CTRL_CMD_BIF_GET_PCIE_POWER_CONTROL_MASK_PARAMS: TypeAlias = struct_NV0080_CTRL_CMD_BIF_GET_PCIE_POWER_CONTROL_MASK_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK(c.Struct): + SIZE = 32 + pageSize: int + pteEntrySize: int + comptagLine: int + kind: int + pteFlags: int +struct_NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK.register_fields([('pageSize', NvU64, 0), ('pteEntrySize', NvU64, 8), ('comptagLine', NvU32, 16), ('kind', NvU32, 20), ('pteFlags', NvU32, 24)]) +NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK: TypeAlias = struct_NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK +@c.record +class struct_NV0080_CTRL_DMA_GET_PTE_INFO_PARAMS(c.Struct): + SIZE = 184 + gpuAddr: int + subDeviceId: int + skipVASpaceInit: int + pteBlocks: c.Array[struct_NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK, Literal[5]] + hVASpace: int +struct_NV0080_CTRL_DMA_GET_PTE_INFO_PARAMS.register_fields([('gpuAddr', NvU64, 0), ('subDeviceId', NvU32, 8), ('skipVASpaceInit', NvU8, 12), ('pteBlocks', c.Array[NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK, Literal[5]], 16), ('hVASpace', NvHandle, 176)]) +NV0080_CTRL_DMA_GET_PTE_INFO_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_GET_PTE_INFO_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_SET_PTE_INFO_PARAMS(c.Struct): + SIZE = 184 + gpuAddr: int + subDeviceId: int + pteBlocks: c.Array[struct_NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK, Literal[5]] + hVASpace: int +struct_NV0080_CTRL_DMA_SET_PTE_INFO_PARAMS.register_fields([('gpuAddr', NvU64, 0), ('subDeviceId', NvU32, 8), ('pteBlocks', c.Array[NV0080_CTRL_DMA_PTE_INFO_PTE_BLOCK, Literal[5]], 16), ('hVASpace', NvHandle, 176)]) +NV0080_CTRL_DMA_SET_PTE_INFO_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_SET_PTE_INFO_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS(c.Struct): + SIZE = 104 + pageCount: int + hwResource: struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_hwResource + comprInfo: struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_comprInfo + offset: int + gpuAddr: int + pageArray: ctypes.c_void_p + pteMem: ctypes.c_void_p + pteMemPfn: int + pageSize: int + startPageIndex: int + flags: int + hSrcVASpace: int + hTgtVASpace: int + peerId: int +@c.record +class struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_hwResource(c.Struct): + SIZE = 16 + hClient: int + hDevice: int + hMemory: int + subDeviceId: int +struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_hwResource.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('hMemory', NvHandle, 8), ('subDeviceId', NvU32, 12)]) +@c.record +class struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_comprInfo(c.Struct): + SIZE = 12 + fbKind: int + sysKind: int + compTagStartOffset: int +struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_comprInfo.register_fields([('fbKind', NvU32, 0), ('sysKind', NvU32, 4), ('compTagStartOffset', NvU32, 8)]) +struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS.register_fields([('pageCount', NvU32, 0), ('hwResource', struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_hwResource, 4), ('comprInfo', struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_comprInfo, 20), ('offset', NvU64, 32), ('gpuAddr', NvU64, 40), ('pageArray', NvP64, 48), ('pteMem', NvP64, 56), ('pteMemPfn', NvU32, 64), ('pageSize', NvU32, 68), ('startPageIndex', NvU32, 72), ('flags', NvU64, 80), ('hSrcVASpace', NvHandle, 88), ('hTgtVASpace', NvHandle, 92), ('peerId', NvU32, 96)]) +NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_FLUSH_PARAMS(c.Struct): + SIZE = 4 + targetUnit: int +struct_NV0080_CTRL_DMA_FLUSH_PARAMS.register_fields([('targetUnit', NvU32, 0)]) +NV0080_CTRL_DMA_FLUSH_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_FLUSH_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT(c.Struct): + SIZE = 8 + pageTableSize: int + pageTableCoverage: int +struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT.register_fields([('pageTableSize', NvU32, 0), ('pageTableCoverage', NvU32, 4)]) +NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT: TypeAlias = struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT +@c.record +class struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS(c.Struct): + SIZE = 192 + vaBitCount: int + pdeCoverageBitCount: int + num4KPageTableFormats: int + bigPageSize: int + compressionPageSize: int + dualPageTableSupported: int + idealVRAMPageSize: int + pageTableBigFormat: struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT + pageTable4KFormat: c.Array[struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT, Literal[16]] + hVASpace: int + vaRangeLo: int + vaSpaceId: int + supportedPageSizeMask: int +struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS.register_fields([('vaBitCount', NvU32, 0), ('pdeCoverageBitCount', NvU32, 4), ('num4KPageTableFormats', NvU32, 8), ('bigPageSize', NvU32, 12), ('compressionPageSize', NvU32, 16), ('dualPageTableSupported', NvU32, 20), ('idealVRAMPageSize', NvU32, 24), ('pageTableBigFormat', NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT, 28), ('pageTable4KFormat', c.Array[NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PAGE_TABLE_FORMAT, Literal[16]], 36), ('hVASpace', NvHandle, 164), ('vaRangeLo', NvU64, 168), ('vaSpaceId', NvU32, 176), ('supportedPageSizeMask', NvU64, 184)]) +NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_PDE_INFO_PTE_BLOCK(c.Struct): + SIZE = 32 + ptePhysAddr: int + pteCacheAttrib: int + pteEntrySize: int + pageSize: int + pteAddrSpace: int + pdeVASpaceSize: int + pdeFlags: int +struct_NV0080_CTRL_DMA_PDE_INFO_PTE_BLOCK.register_fields([('ptePhysAddr', NvU64, 0), ('pteCacheAttrib', NvU32, 8), ('pteEntrySize', NvU32, 12), ('pageSize', NvU32, 16), ('pteAddrSpace', NvU32, 20), ('pdeVASpaceSize', NvU32, 24), ('pdeFlags', NvU32, 28)]) +NV0080_CTRL_DMA_PDE_INFO_PTE_BLOCK: TypeAlias = struct_NV0080_CTRL_DMA_PDE_INFO_PTE_BLOCK +@c.record +class struct_NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS(c.Struct): + SIZE = 208 + gpuAddr: int + pdeVirtAddr: int + pdeEntrySize: int + pdeAddrSpace: int + pdeSize: int + subDeviceId: int + pteBlocks: c.Array[struct_NV0080_CTRL_DMA_PDE_INFO_PTE_BLOCK, Literal[5]] + pdbAddr: int + hVASpace: int +struct_NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS.register_fields([('gpuAddr', NvU64, 0), ('pdeVirtAddr', NvU64, 8), ('pdeEntrySize', NvU32, 16), ('pdeAddrSpace', NvU32, 20), ('pdeSize', NvU32, 24), ('subDeviceId', NvU32, 28), ('pteBlocks', c.Array[NV0080_CTRL_DMA_PDE_INFO_PTE_BLOCK, Literal[5]], 32), ('pdbAddr', NvU64, 192), ('hVASpace', NvHandle, 200)]) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_INVALIDATE_TLB_PARAMS(c.Struct): + SIZE = 8 + hVASpace: int + flags: int +struct_NV0080_CTRL_DMA_INVALIDATE_TLB_PARAMS.register_fields([('hVASpace', NvHandle, 0), ('flags', NvU32, 4)]) +NV0080_CTRL_DMA_INVALIDATE_TLB_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_INVALIDATE_TLB_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_GET_CAPS_PARAMS(c.Struct): + SIZE = 12 + capsTblSize: int + capsTbl: c.Array[ctypes.c_ubyte, Literal[8]] +struct_NV0080_CTRL_DMA_GET_CAPS_PARAMS.register_fields([('capsTblSize', NvU32, 0), ('capsTbl', c.Array[NvU8, Literal[8]], 4)]) +NV0080_CTRL_DMA_GET_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_GET_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_SET_VA_SPACE_SIZE_PARAMS(c.Struct): + SIZE = 16 + vaSpaceSize: int + hVASpace: int +struct_NV0080_CTRL_DMA_SET_VA_SPACE_SIZE_PARAMS.register_fields([('vaSpaceSize', NvU64, 0), ('hVASpace', NvHandle, 8)]) +NV0080_CTRL_DMA_SET_VA_SPACE_SIZE_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_SET_VA_SPACE_SIZE_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_UPDATE_PDE_2_PAGE_TABLE_PARAMS(c.Struct): + SIZE = 16 + physAddr: int + numEntries: int + aperture: int +struct_NV0080_CTRL_DMA_UPDATE_PDE_2_PAGE_TABLE_PARAMS.register_fields([('physAddr', NvU64, 0), ('numEntries', NvU32, 8), ('aperture', NvU32, 12)]) +NV0080_CTRL_DMA_UPDATE_PDE_2_PAGE_TABLE_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_UPDATE_PDE_2_PAGE_TABLE_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_UPDATE_PDE_2_PARAMS(c.Struct): + SIZE = 64 + pdeIndex: int + flags: int + ptParams: c.Array[struct_NV0080_CTRL_DMA_UPDATE_PDE_2_PAGE_TABLE_PARAMS, Literal[2]] + hVASpace: int + pPdeBuffer: ctypes.c_void_p + subDeviceId: int +struct_NV0080_CTRL_DMA_UPDATE_PDE_2_PARAMS.register_fields([('pdeIndex', NvU32, 0), ('flags', NvU32, 4), ('ptParams', c.Array[NV0080_CTRL_DMA_UPDATE_PDE_2_PAGE_TABLE_PARAMS, Literal[2]], 8), ('hVASpace', NvHandle, 40), ('pPdeBuffer', NvP64, 48), ('subDeviceId', NvU32, 56)]) +NV0080_CTRL_DMA_UPDATE_PDE_2_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_UPDATE_PDE_2_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_SET_DEFAULT_VASPACE_PARAMS(c.Struct): + SIZE = 4 + hVASpace: int +struct_NV0080_CTRL_DMA_SET_DEFAULT_VASPACE_PARAMS.register_fields([('hVASpace', NvHandle, 0)]) +NV0080_CTRL_DMA_SET_DEFAULT_VASPACE_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_SET_DEFAULT_VASPACE_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS(c.Struct): + SIZE = 32 + physAddress: int + numEntries: int + flags: int + hVASpace: int + chId: int + subDeviceId: int + pasid: int +struct_NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS.register_fields([('physAddress', NvU64, 0), ('numEntries', NvU32, 8), ('flags', NvU32, 12), ('hVASpace', NvHandle, 16), ('chId', NvU32, 20), ('subDeviceId', NvU32, 24), ('pasid', NvU32, 28)]) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS +@c.record +class struct_NV0080_CTRL_DMA_UNSET_PAGE_DIRECTORY_PARAMS(c.Struct): + SIZE = 8 + hVASpace: int + subDeviceId: int +struct_NV0080_CTRL_DMA_UNSET_PAGE_DIRECTORY_PARAMS.register_fields([('hVASpace', NvHandle, 0), ('subDeviceId', NvU32, 4)]) +NV0080_CTRL_DMA_UNSET_PAGE_DIRECTORY_PARAMS: TypeAlias = struct_NV0080_CTRL_DMA_UNSET_PAGE_DIRECTORY_PARAMS +@c.record +class struct_NV0080_CTRL_FB_GET_CAPS_PARAMS(c.Struct): + SIZE = 16 + capsTblSize: int + capsTbl: ctypes.c_void_p +struct_NV0080_CTRL_FB_GET_CAPS_PARAMS.register_fields([('capsTblSize', NvU32, 0), ('capsTbl', NvP64, 8)]) +NV0080_CTRL_FB_GET_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_FB_GET_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS(c.Struct): + SIZE = 88 + Size: int + Address: int + AddressSpace: int + MaxCompbitLine: int + comptagsPerCacheLine: int + cacheLineSize: int + cacheLineSizePerSlice: int + cacheLineFetchAlignment: int + backingStoreBase: int + gobsPerComptagPerSlice: int + backingStoreCbcBase: int + comptaglineAllocationPolicy: int + privRegionStartOffset: int + cbcCoveragePerSlice: int + cbcRegionSize: int +struct_NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS.register_fields([('Size', NvU64, 0), ('Address', NvU64, 8), ('AddressSpace', NvU32, 16), ('MaxCompbitLine', NvU32, 20), ('comptagsPerCacheLine', NvU32, 24), ('cacheLineSize', NvU32, 28), ('cacheLineSizePerSlice', NvU32, 32), ('cacheLineFetchAlignment', NvU32, 36), ('backingStoreBase', NvU64, 40), ('gobsPerComptagPerSlice', NvU32, 48), ('backingStoreCbcBase', NvU32, 52), ('comptaglineAllocationPolicy', NvU32, 56), ('privRegionStartOffset', NvU64, 64), ('cbcCoveragePerSlice', NvU32, 72), ('cbcRegionSize', NvU64, 80)]) +NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS: TypeAlias = struct_NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS +@c.record +class struct_NV0080_CTRL_FB_GET_CAPS_V2_PARAMS(c.Struct): + SIZE = 3 + capsTbl: c.Array[ctypes.c_ubyte, Literal[3]] +struct_NV0080_CTRL_FB_GET_CAPS_V2_PARAMS.register_fields([('capsTbl', c.Array[NvU8, Literal[3]], 0)]) +NV0080_CTRL_FB_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_FB_GET_CAPS_V2_PARAMS +@c.record +class struct_NV0080_CTRL_FB_SET_DEFAULT_VIDMEM_PHYSICALITY_PARAMS(c.Struct): + SIZE = 4 + value: int +struct_NV0080_CTRL_FB_SET_DEFAULT_VIDMEM_PHYSICALITY_PARAMS.register_fields([('value', NvU32, 0)]) +NV0080_CTRL_FB_SET_DEFAULT_VIDMEM_PHYSICALITY_PARAMS: TypeAlias = struct_NV0080_CTRL_FB_SET_DEFAULT_VIDMEM_PHYSICALITY_PARAMS +enum_NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY: dict[int, str] = {(NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_DEFAULT:=0): 'NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_DEFAULT', (NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_NONCONTIGUOUS:=1): 'NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_NONCONTIGUOUS', (NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_CONTIGUOUS:=2): 'NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_CONTIGUOUS', (NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_ALLOW_NONCONTIGUOUS:=3): 'NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY_ALLOW_NONCONTIGUOUS'} +NV0080_CTRL_FB_DEFAULT_VIDMEM_PHYSICALITY: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV0080_CTRL_FIFO_GET_CAPS_PARAMS(c.Struct): + SIZE = 16 + capsTblSize: int + capsTbl: ctypes.c_void_p +struct_NV0080_CTRL_FIFO_GET_CAPS_PARAMS.register_fields([('capsTblSize', NvU32, 0), ('capsTbl', NvP64, 8)]) +NV0080_CTRL_FIFO_GET_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_GET_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS(c.Struct): + SIZE = 12 + engineId: int + alignment: int + size: int +struct_NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS.register_fields([('engineId', NvU32, 0), ('alignment', NvU32, 4), ('size', NvU32, 8)]) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS(c.Struct): + SIZE = 24 + numChannels: int + pChannelHandleList: ctypes.c_void_p + pChannelList: ctypes.c_void_p +struct_NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS.register_fields([('numChannels', NvU32, 0), ('pChannelHandleList', NvP64, 8), ('pChannelList', NvP64, 16)]) +NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_GET_LATENCY_BUFFER_SIZE_PARAMS(c.Struct): + SIZE = 12 + engineID: int + gpEntries: int + pbEntries: int +struct_NV0080_CTRL_FIFO_GET_LATENCY_BUFFER_SIZE_PARAMS.register_fields([('engineID', NvU32, 0), ('gpEntries', NvU32, 4), ('pbEntries', NvU32, 8)]) +NV0080_CTRL_FIFO_GET_LATENCY_BUFFER_SIZE_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_GET_LATENCY_BUFFER_SIZE_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_PARAMS(c.Struct): + SIZE = 16 + hChannel: int + property: int + value: int +struct_NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_PARAMS.register_fields([('hChannel', NvHandle, 0), ('property', NvU32, 4), ('value', NvU64, 8)]) +NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_STOP_RUNLIST_PARAMS(c.Struct): + SIZE = 4 + engineID: int +struct_NV0080_CTRL_FIFO_STOP_RUNLIST_PARAMS.register_fields([('engineID', NvU32, 0)]) +NV0080_CTRL_FIFO_STOP_RUNLIST_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_STOP_RUNLIST_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_START_RUNLIST_PARAMS(c.Struct): + SIZE = 4 + engineID: int +struct_NV0080_CTRL_FIFO_START_RUNLIST_PARAMS.register_fields([('engineID', NvU32, 0)]) +NV0080_CTRL_FIFO_START_RUNLIST_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_START_RUNLIST_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_GET_CAPS_V2_PARAMS(c.Struct): + SIZE = 2 + capsTbl: c.Array[ctypes.c_ubyte, Literal[2]] +struct_NV0080_CTRL_FIFO_GET_CAPS_V2_PARAMS.register_fields([('capsTbl', c.Array[NvU8, Literal[2]], 0)]) +NV0080_CTRL_FIFO_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_GET_CAPS_V2_PARAMS +@c.record +class struct_NV0080_CTRL_FIFO_IDLE_CHANNELS_PARAMS(c.Struct): + SIZE = 16396 + numChannels: int + hChannels: c.Array[ctypes.c_uint32, Literal[4096]] + flags: int + timeout: int +struct_NV0080_CTRL_FIFO_IDLE_CHANNELS_PARAMS.register_fields([('numChannels', NvU32, 0), ('hChannels', c.Array[NvHandle, Literal[4096]], 4), ('flags', NvU32, 16388), ('timeout', NvU32, 16392)]) +NV0080_CTRL_FIFO_IDLE_CHANNELS_PARAMS: TypeAlias = struct_NV0080_CTRL_FIFO_IDLE_CHANNELS_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_CLASSLIST_PARAMS(c.Struct): + SIZE = 16 + numClasses: int + classList: ctypes.c_void_p +struct_NV0080_CTRL_GPU_GET_CLASSLIST_PARAMS.register_fields([('numClasses', NvU32, 0), ('classList', NvP64, 8)]) +NV0080_CTRL_GPU_GET_CLASSLIST_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_CLASSLIST_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_NUM_SUBDEVICES_PARAMS(c.Struct): + SIZE = 4 + numSubDevices: int +struct_NV0080_CTRL_GPU_GET_NUM_SUBDEVICES_PARAMS.register_fields([('numSubDevices', NvU32, 0)]) +NV0080_CTRL_GPU_GET_NUM_SUBDEVICES_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_NUM_SUBDEVICES_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_VIDLINK_ORDER_PARAMS(c.Struct): + SIZE = 36 + ConnectionCount: int + Order: c.Array[ctypes.c_uint32, Literal[8]] +struct_NV0080_CTRL_GPU_GET_VIDLINK_ORDER_PARAMS.register_fields([('ConnectionCount', NvU32, 0), ('Order', c.Array[NvU32, Literal[8]], 4)]) +NV0080_CTRL_GPU_GET_VIDLINK_ORDER_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_VIDLINK_ORDER_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_SET_DISPLAY_OWNER_PARAMS(c.Struct): + SIZE = 4 + subDeviceInstance: int +struct_NV0080_CTRL_GPU_SET_DISPLAY_OWNER_PARAMS.register_fields([('subDeviceInstance', NvU32, 0)]) +NV0080_CTRL_GPU_SET_DISPLAY_OWNER_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_SET_DISPLAY_OWNER_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_DISPLAY_OWNER_PARAMS(c.Struct): + SIZE = 4 + subDeviceInstance: int +struct_NV0080_CTRL_GPU_GET_DISPLAY_OWNER_PARAMS.register_fields([('subDeviceInstance', NvU32, 0)]) +NV0080_CTRL_GPU_GET_DISPLAY_OWNER_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_DISPLAY_OWNER_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_SET_VIDLINK_PARAMS(c.Struct): + SIZE = 4 + enable: int +struct_NV0080_CTRL_GPU_SET_VIDLINK_PARAMS.register_fields([('enable', NvU32, 0)]) +NV0080_CTRL_GPU_SET_VIDLINK_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_SET_VIDLINK_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_MODIFY_SW_STATE_PERSISTENCE_PARAMS(c.Struct): + SIZE = 4 + newState: int +struct_NV0080_CTRL_GPU_MODIFY_SW_STATE_PERSISTENCE_PARAMS.register_fields([('newState', NvU32, 0)]) +NV0080_CTRL_GPU_MODIFY_SW_STATE_PERSISTENCE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_MODIFY_SW_STATE_PERSISTENCE_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_QUERY_SW_STATE_PERSISTENCE_PARAMS(c.Struct): + SIZE = 4 + swStatePersistence: int +struct_NV0080_CTRL_GPU_QUERY_SW_STATE_PERSISTENCE_PARAMS.register_fields([('swStatePersistence', NvU32, 0)]) +NV0080_CTRL_GPU_QUERY_SW_STATE_PERSISTENCE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_QUERY_SW_STATE_PERSISTENCE_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_VIRTUALIZATION_MODE_PARAMS(c.Struct): + SIZE = 8 + virtualizationMode: int + isGridBuild: int +struct_NV0080_CTRL_GPU_GET_VIRTUALIZATION_MODE_PARAMS.register_fields([('virtualizationMode', NvU32, 0), ('isGridBuild', NvBool, 4)]) +NV0080_CTRL_GPU_GET_VIRTUALIZATION_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_VIRTUALIZATION_MODE_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS(c.Struct): + SIZE = 12 + defaultSetting: int + currentSetting: int + pendingSetting: int +struct_NV0080_CTRL_GPU_GET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS.register_fields([('defaultSetting', NvU32, 0), ('currentSetting', NvU32, 4), ('pendingSetting', NvU32, 8)]) +NV0080_CTRL_GPU_GET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_SET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS(c.Struct): + SIZE = 4 + setting: int +struct_NV0080_CTRL_GPU_SET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS.register_fields([('setting', NvU32, 0)]) +NV0080_CTRL_GPU_SET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_SET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_VGX_CAPS_PARAMS(c.Struct): + SIZE = 1 + isVgx: int +struct_NV0080_CTRL_GPU_GET_VGX_CAPS_PARAMS.register_fields([('isVgx', NvBool, 0)]) +NV0080_CTRL_GPU_GET_VGX_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_VGX_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS(c.Struct): + SIZE = 80 + totalVFs: int + firstVfOffset: int + vfFeatureMask: int + FirstVFBar0Address: int + FirstVFBar1Address: int + FirstVFBar2Address: int + bar0Size: int + bar1Size: int + bar2Size: int + b64bitBar0: int + b64bitBar1: int + b64bitBar2: int + bSriovEnabled: int + bSriovHeavyEnabled: int + bEmulateVFBar0TlbInvalidationRegister: int + bClientRmAllocatedCtxBuffer: int + bNonPowerOf2ChannelCountSupported: int + bVfResizableBAR1Supported: int +struct_NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS.register_fields([('totalVFs', NvU32, 0), ('firstVfOffset', NvU32, 4), ('vfFeatureMask', NvU32, 8), ('FirstVFBar0Address', NvU64, 16), ('FirstVFBar1Address', NvU64, 24), ('FirstVFBar2Address', NvU64, 32), ('bar0Size', NvU64, 40), ('bar1Size', NvU64, 48), ('bar2Size', NvU64, 56), ('b64bitBar0', NvBool, 64), ('b64bitBar1', NvBool, 65), ('b64bitBar2', NvBool, 66), ('bSriovEnabled', NvBool, 67), ('bSriovHeavyEnabled', NvBool, 68), ('bEmulateVFBar0TlbInvalidationRegister', NvBool, 69), ('bClientRmAllocatedCtxBuffer', NvBool, 70), ('bNonPowerOf2ChannelCountSupported', NvBool, 71), ('bVfResizableBAR1Supported', NvBool, 72)]) +NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_CLASSLIST_V2_PARAMS(c.Struct): + SIZE = 804 + numClasses: int + classList: c.Array[ctypes.c_uint32, Literal[200]] +struct_NV0080_CTRL_GPU_GET_CLASSLIST_V2_PARAMS.register_fields([('numClasses', NvU32, 0), ('classList', c.Array[NvU32, Literal[200]], 4)]) +NV0080_CTRL_GPU_GET_CLASSLIST_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_CLASSLIST_V2_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_FIND_SUBDEVICE_HANDLE_PARAM(c.Struct): + SIZE = 8 + subDeviceInst: int + hSubDevice: int +struct_NV0080_CTRL_GPU_FIND_SUBDEVICE_HANDLE_PARAM.register_fields([('subDeviceInst', NvU32, 0), ('hSubDevice', NvHandle, 4)]) +NV0080_CTRL_GPU_FIND_SUBDEVICE_HANDLE_PARAM: TypeAlias = struct_NV0080_CTRL_GPU_FIND_SUBDEVICE_HANDLE_PARAM +@c.record +class struct_NV0080_CTRL_GPU_GET_BRAND_CAPS_PARAMS(c.Struct): + SIZE = 4 + brands: int +struct_NV0080_CTRL_GPU_GET_BRAND_CAPS_PARAMS.register_fields([('brands', NvU32, 0)]) +NV0080_CTRL_GPU_GET_BRAND_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_BRAND_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_SET_VGPU_VF_BAR1_SIZE_PARAMS(c.Struct): + SIZE = 8 + vfBar1SizeMB: int + numVfs: int +struct_NV0080_CTRL_GPU_SET_VGPU_VF_BAR1_SIZE_PARAMS.register_fields([('vfBar1SizeMB', NvU32, 0), ('numVfs', NvU32, 4)]) +NV0080_CTRL_GPU_SET_VGPU_VF_BAR1_SIZE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_SET_VGPU_VF_BAR1_SIZE_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_SET_VGPU_HETEROGENEOUS_MODE_PARAMS(c.Struct): + SIZE = 8 + bHeterogeneousMode: int + gpuInstanceId: int +struct_NV0080_CTRL_GPU_SET_VGPU_HETEROGENEOUS_MODE_PARAMS.register_fields([('bHeterogeneousMode', NvBool, 0), ('gpuInstanceId', NvU32, 4)]) +NV0080_CTRL_GPU_SET_VGPU_HETEROGENEOUS_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_SET_VGPU_HETEROGENEOUS_MODE_PARAMS +@c.record +class struct_NV0080_CTRL_GPU_GET_VGPU_HETEROGENEOUS_MODE_PARAMS(c.Struct): + SIZE = 8 + bHeterogeneousMode: int + gpuInstanceId: int +struct_NV0080_CTRL_GPU_GET_VGPU_HETEROGENEOUS_MODE_PARAMS.register_fields([('bHeterogeneousMode', NvBool, 0), ('gpuInstanceId', NvU32, 4)]) +NV0080_CTRL_GPU_GET_VGPU_HETEROGENEOUS_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GPU_GET_VGPU_HETEROGENEOUS_MODE_PARAMS +@c.record +class struct_NV0080_CTRL_GR_ROUTE_INFO(c.Struct): + SIZE = 16 + flags: int + route: int +struct_NV0080_CTRL_GR_ROUTE_INFO.register_fields([('flags', NvU32, 0), ('route', NvU64, 8)]) +NV0080_CTRL_GR_ROUTE_INFO: TypeAlias = struct_NV0080_CTRL_GR_ROUTE_INFO +@c.record +class struct_NV0080_CTRL_GR_GET_CAPS_PARAMS(c.Struct): + SIZE = 16 + capsTblSize: int + capsTbl: ctypes.c_void_p +struct_NV0080_CTRL_GR_GET_CAPS_PARAMS.register_fields([('capsTblSize', NvU32, 0), ('capsTbl', NvP64, 8)]) +NV0080_CTRL_GR_GET_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_GET_CAPS_PARAMS +@c.record +class struct_NVXXXX_CTRL_XXX_INFO(c.Struct): + SIZE = 8 + index: int + data: int +NV0080_CTRL_GR_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +struct_NVXXXX_CTRL_XXX_INFO.register_fields([('index', NvU32, 0), ('data', NvU32, 4)]) +@c.record +class struct_NV0080_CTRL_GR_GET_INFO_PARAMS(c.Struct): + SIZE = 16 + grInfoListSize: int + grInfoList: ctypes.c_void_p +struct_NV0080_CTRL_GR_GET_INFO_PARAMS.register_fields([('grInfoListSize', NvU32, 0), ('grInfoList', NvP64, 8)]) +NV0080_CTRL_GR_GET_INFO_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_GET_INFO_PARAMS +enum_NV0080_CTRL_GR_TPC_PARTITION_MODE: dict[int, str] = {(NV0080_CTRL_GR_TPC_PARTITION_MODE_NONE:=0): 'NV0080_CTRL_GR_TPC_PARTITION_MODE_NONE', (NV0080_CTRL_GR_TPC_PARTITION_MODE_STATIC:=1): 'NV0080_CTRL_GR_TPC_PARTITION_MODE_STATIC', (NV0080_CTRL_GR_TPC_PARTITION_MODE_DYNAMIC:=2): 'NV0080_CTRL_GR_TPC_PARTITION_MODE_DYNAMIC'} +NV0080_CTRL_GR_TPC_PARTITION_MODE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS(c.Struct): + SIZE = 32 + hChannelGroup: int + mode: int + bEnableAllTpcs: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS.register_fields([('hChannelGroup', NvHandle, 0), ('mode', NV0080_CTRL_GR_TPC_PARTITION_MODE, 4), ('bEnableAllTpcs', NvBool, 8), ('grRouteInfo', NV0080_CTRL_GR_ROUTE_INFO, 16)]) +NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS +NV0080_CTRL_GR_GET_TPC_PARTITION_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS +NV0080_CTRL_GR_SET_TPC_PARTITION_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS +@c.record +class struct_NV0080_CTRL_GR_GET_CAPS_V2_PARAMS(c.Struct): + SIZE = 48 + capsTbl: c.Array[ctypes.c_ubyte, Literal[23]] + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + bCapsPopulated: int +struct_NV0080_CTRL_GR_GET_CAPS_V2_PARAMS.register_fields([('capsTbl', c.Array[NvU8, Literal[23]], 0), ('grRouteInfo', NV0080_CTRL_GR_ROUTE_INFO, 24), ('bCapsPopulated', NvBool, 40)]) +NV0080_CTRL_GR_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_GET_CAPS_V2_PARAMS +@c.record +class struct_NV0080_CTRL_GR_GET_INFO_V2_PARAMS(c.Struct): + SIZE = 528 + grInfoListSize: int + grInfoList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[63]] + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV0080_CTRL_GR_GET_INFO_V2_PARAMS.register_fields([('grInfoListSize', NvU32, 0), ('grInfoList', c.Array[NV0080_CTRL_GR_INFO, Literal[63]], 4), ('grRouteInfo', NV0080_CTRL_GR_ROUTE_INFO, 512)]) +NV0080_CTRL_GR_GET_INFO_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_GET_INFO_V2_PARAMS +@c.record +class struct_NV0080_CTRL_HOST_GET_CAPS_PARAMS(c.Struct): + SIZE = 16 + capsTblSize: int + capsTbl: ctypes.c_void_p +struct_NV0080_CTRL_HOST_GET_CAPS_PARAMS.register_fields([('capsTblSize', NvU32, 0), ('capsTbl', NvP64, 8)]) +NV0080_CTRL_HOST_GET_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_HOST_GET_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_HOST_GET_CAPS_V2_PARAMS(c.Struct): + SIZE = 3 + capsTbl: c.Array[ctypes.c_ubyte, Literal[3]] +struct_NV0080_CTRL_HOST_GET_CAPS_V2_PARAMS.register_fields([('capsTbl', c.Array[NvU8, Literal[3]], 0)]) +NV0080_CTRL_HOST_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_HOST_GET_CAPS_V2_PARAMS +@c.record +class struct_NV0080_CTRL_CMD_INTERNAL_GR_GET_TPC_PARTITION_MODE_FINN_PARAMS(c.Struct): + SIZE = 32 + params: struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS +struct_NV0080_CTRL_CMD_INTERNAL_GR_GET_TPC_PARTITION_MODE_FINN_PARAMS.register_fields([('params', NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS, 0)]) +NV0080_CTRL_CMD_INTERNAL_GR_GET_TPC_PARTITION_MODE_FINN_PARAMS: TypeAlias = struct_NV0080_CTRL_CMD_INTERNAL_GR_GET_TPC_PARTITION_MODE_FINN_PARAMS +@c.record +class struct_NV0080_CTRL_CMD_INTERNAL_GR_SET_TPC_PARTITION_MODE_FINN_PARAMS(c.Struct): + SIZE = 32 + params: struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS +struct_NV0080_CTRL_CMD_INTERNAL_GR_SET_TPC_PARTITION_MODE_FINN_PARAMS.register_fields([('params', NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS, 0)]) +NV0080_CTRL_CMD_INTERNAL_GR_SET_TPC_PARTITION_MODE_FINN_PARAMS: TypeAlias = struct_NV0080_CTRL_CMD_INTERNAL_GR_SET_TPC_PARTITION_MODE_FINN_PARAMS +@c.record +class struct_NV0080_CTRL_INTERNAL_FIFO_RC_AND_PERMANENTLY_DISABLE_CHANNELS_PARAMS(c.Struct): + SIZE = 804 + numClients: int + clientHandles: c.Array[ctypes.c_uint32, Literal[200]] +struct_NV0080_CTRL_INTERNAL_FIFO_RC_AND_PERMANENTLY_DISABLE_CHANNELS_PARAMS.register_fields([('numClients', NvU32, 0), ('clientHandles', c.Array[NvHandle, Literal[200]], 4)]) +NV0080_CTRL_INTERNAL_FIFO_RC_AND_PERMANENTLY_DISABLE_CHANNELS_PARAMS: TypeAlias = struct_NV0080_CTRL_INTERNAL_FIFO_RC_AND_PERMANENTLY_DISABLE_CHANNELS_PARAMS +@c.record +class struct_NV0080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS(c.Struct): + SIZE = 1 + bZbcSurfacesExist: int +struct_NV0080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS.register_fields([('bZbcSurfacesExist', NvBool, 0)]) +NV0080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS: TypeAlias = struct_NV0080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS +@c.record +class struct_NV0080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS(c.Struct): + SIZE = 1 + bTeardown: int +struct_NV0080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS.register_fields([('bTeardown', NvBool, 0)]) +NV0080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS: TypeAlias = struct_NV0080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS +NV0080_CTRL_INTERNAL_KGR_INIT_BUG4208224_WAR_PARAMS: TypeAlias = struct_NV0080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS +@c.record +class struct_NV0080_CTRL_NVDEC_GET_CAPS_PARAMS(c.Struct): + SIZE = 24 + capsTblSize: int + capsTbl: ctypes.c_void_p + instanceId: int +struct_NV0080_CTRL_NVDEC_GET_CAPS_PARAMS.register_fields([('capsTblSize', NvU32, 0), ('capsTbl', NvP64, 8), ('instanceId', NvU32, 16)]) +NV0080_CTRL_NVDEC_GET_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_NVDEC_GET_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_NVDEC_GET_CAPS_PARAMS_V2(c.Struct): + SIZE = 12 + capsTbl: c.Array[ctypes.c_ubyte, Literal[8]] + instanceId: int +struct_NV0080_CTRL_NVDEC_GET_CAPS_PARAMS_V2.register_fields([('capsTbl', c.Array[NvU8, Literal[8]], 0), ('instanceId', NvU32, 8)]) +NV0080_CTRL_NVDEC_GET_CAPS_PARAMS_V2: TypeAlias = struct_NV0080_CTRL_NVDEC_GET_CAPS_PARAMS_V2 +@c.record +class struct_NV0080_CTRL_NVENC_GET_CAPS_PARAMS(c.Struct): + SIZE = 16 + capsTblSize: int + capsTbl: ctypes.c_void_p +struct_NV0080_CTRL_NVENC_GET_CAPS_PARAMS.register_fields([('capsTblSize', NvU32, 0), ('capsTbl', NvP64, 8)]) +NV0080_CTRL_NVENC_GET_CAPS_PARAMS: TypeAlias = struct_NV0080_CTRL_NVENC_GET_CAPS_PARAMS +@c.record +class struct_NV0080_CTRL_NVENC_GET_CAPS_V2_PARAMS(c.Struct): + SIZE = 12 + capsTbl: c.Array[ctypes.c_ubyte, Literal[6]] + instanceId: int +struct_NV0080_CTRL_NVENC_GET_CAPS_V2_PARAMS.register_fields([('capsTbl', c.Array[NvU8, Literal[6]], 0), ('instanceId', NvU32, 8)]) +NV0080_CTRL_NVENC_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_NVENC_GET_CAPS_V2_PARAMS +@c.record +class struct_NV0080_CTRL_NVJPG_GET_CAPS_V2_PARAMS(c.Struct): + SIZE = 16 + capsTbl: c.Array[ctypes.c_ubyte, Literal[9]] + instanceId: int +struct_NV0080_CTRL_NVJPG_GET_CAPS_V2_PARAMS.register_fields([('capsTbl', c.Array[NvU8, Literal[9]], 0), ('instanceId', NvU32, 12)]) +NV0080_CTRL_NVJPG_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_NVJPG_GET_CAPS_V2_PARAMS +@c.record +class struct_NV0080_CTRL_PERF_SLI_GPU_BOOST_SYNC_CONTROL_PARAMS(c.Struct): + SIZE = 1 + bActivate: int +struct_NV0080_CTRL_PERF_SLI_GPU_BOOST_SYNC_CONTROL_PARAMS.register_fields([('bActivate', NvBool, 0)]) +NV0080_CTRL_PERF_SLI_GPU_BOOST_SYNC_CONTROL_PARAMS: TypeAlias = struct_NV0080_CTRL_PERF_SLI_GPU_BOOST_SYNC_CONTROL_PARAMS +@c.record +class struct_NV0080_CTRL_PERF_CUDA_LIMIT_CONTROL_PARAMS(c.Struct): + SIZE = 1 + bCudaLimit: int +struct_NV0080_CTRL_PERF_CUDA_LIMIT_CONTROL_PARAMS.register_fields([('bCudaLimit', NvBool, 0)]) +NV0080_CTRL_PERF_CUDA_LIMIT_CONTROL_PARAMS: TypeAlias = struct_NV0080_CTRL_PERF_CUDA_LIMIT_CONTROL_PARAMS +@c.record +class struct_NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS(c.Struct): + SIZE = 4 + cmd: int +struct_NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS.register_fields([('cmd', NvU32, 0)]) +NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS: TypeAlias = struct_NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS +@c.record +class struct_NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS(c.Struct): + SIZE = 32 + subDeviceInstance: int + width: int + height: int + depth: int + pitch: int + baseAddress: int + size: int +struct_NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS.register_fields([('subDeviceInstance', NvU32, 0), ('width', NvU16, 4), ('height', NvU16, 6), ('depth', NvU16, 8), ('pitch', NvU16, 10), ('baseAddress', NvU64, 16), ('size', NvU64, 24)]) +NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS: TypeAlias = struct_NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS +enum_NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG: dict[int, str] = {(NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_INVALID:=0): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_INVALID', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_DEVICE_CTRL_STATUS:=1): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_DEVICE_CTRL_STATUS', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_LINK_CTRL_STATUS:=2): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_LINK_CTRL_STATUS', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_DEVICE_CTRL_STATUS_2:=3): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_DEVICE_CTRL_STATUS_2', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_LINK_CAP:=4): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_LINK_CAP', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_L1_PM_SUBSTATES_CTRL1:=5): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_L1_PM_SUBSTATES_CTRL1', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_STATUS:=6): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_STATUS', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_STATUS:=7): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_STATUS', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_STATUS:=8): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_STATUS', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_MASK:=9): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_MASK', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_MASK:=10): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_MASK', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_MASK:=11): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_MASK', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_SEVERITY:=12): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_CHECK_FAILED_SEVERITY', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_SEVERITY:=13): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_MISROUTED_TLP_SEVERITY', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_SEVERITY:=14): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_PCRC_CHECK_FAILED_SEVERITY', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TLP_ENCRYPTION_STATUS:=15): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TLP_ENCRYPTION_STATUS', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_LINK_STREAM_ENABLED:=16): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_LINK_STREAM_ENABLED', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_SELECTIVE_STREAM_ENABLED:=17): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_SELECTIVE_STREAM_ENABLED', (NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_FLOW_THROUGH_STREAM_ENABLED:=18): 'NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG_FN0_IDE_TYPE_FLOW_THROUGH_STREAM_ENABLED'} +NV2080_CTRL_BIF_PARAM_ID_CONFIG_REG: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_BIF_GET_DATA_PARAMS(c.Struct): + SIZE = 268 + featureId: int + paramId: c.Array[ctypes.c_uint32, Literal[32]] + paramVal: c.Array[ctypes.c_uint32, Literal[32]] + flags: int + paramSize: int +struct_NV2080_CTRL_BIF_GET_DATA_PARAMS.register_fields([('featureId', NvU32, 0), ('paramId', c.Array[NvU32, Literal[32]], 4), ('paramVal', c.Array[NvU32, Literal[32]], 132), ('flags', NvU32, 260), ('paramSize', NvU32, 264)]) +NV2080_CTRL_BIF_GET_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_BIF_GET_DATA_PARAMS +enum_NV2080_CTRL_BIF_FEATURE_ID: dict[int, str] = {(NV2080_CTRL_BIF_FEATURE_ID_INVALID:=0): 'NV2080_CTRL_BIF_FEATURE_ID_INVALID', (NV2080_CTRL_BIF_FEATURE_ID_CONFIG_REG:=1): 'NV2080_CTRL_BIF_FEATURE_ID_CONFIG_REG'} +NV2080_CTRL_BIF_FEATURE_ID: TypeAlias = ctypes.c_uint32 +NV2080_CTRL_BIOS_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_BIOS_GET_INFO_PARAMS(c.Struct): + SIZE = 16 + biosInfoListSize: int + biosInfoList: ctypes.c_void_p +struct_NV2080_CTRL_BIOS_GET_INFO_PARAMS.register_fields([('biosInfoListSize', NvU32, 0), ('biosInfoList', NvP64, 8)]) +NV2080_CTRL_BIOS_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_BIOS_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS(c.Struct): + SIZE = 124 + biosInfoListSize: int + biosInfoList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[15]] +struct_NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS.register_fields([('biosInfoListSize', NvU32, 0), ('biosInfoList', c.Array[NV2080_CTRL_BIOS_INFO, Literal[15]], 4)]) +NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_BIOS_NBSI_REG_STRING(c.Struct): + SIZE = 520 + size: int + type: int + value: struct_NV2080_CTRL_BIOS_NBSI_REG_STRING_value +@c.record +class struct_NV2080_CTRL_BIOS_NBSI_REG_STRING_value(c.Struct): + SIZE = 512 + ascii: c.Array[ctypes.c_ubyte, Literal[256]] + unicode: c.Array[ctypes.c_uint16, Literal[256]] + hash: int +struct_NV2080_CTRL_BIOS_NBSI_REG_STRING_value.register_fields([('ascii', c.Array[NvU8, Literal[256]], 0), ('unicode', c.Array[NvU16, Literal[256]], 0), ('hash', NvU16, 0)]) +struct_NV2080_CTRL_BIOS_NBSI_REG_STRING.register_fields([('size', NvU32, 0), ('type', NvU32, 4), ('value', struct_NV2080_CTRL_BIOS_NBSI_REG_STRING_value, 8)]) +NV2080_CTRL_BIOS_NBSI_REG_STRING: TypeAlias = struct_NV2080_CTRL_BIOS_NBSI_REG_STRING +@c.record +class struct_NV2080_CTRL_BIOS_GET_NBSI_PARAMS(c.Struct): + SIZE = 1064 + module: int + path: struct_NV2080_CTRL_BIOS_NBSI_REG_STRING + valueName: struct_NV2080_CTRL_BIOS_NBSI_REG_STRING + retBuf: ctypes.c_void_p + retSize: int + errorCode: int +struct_NV2080_CTRL_BIOS_GET_NBSI_PARAMS.register_fields([('module', NvU32, 0), ('path', NV2080_CTRL_BIOS_NBSI_REG_STRING, 4), ('valueName', NV2080_CTRL_BIOS_NBSI_REG_STRING, 524), ('retBuf', NvP64, 1048), ('retSize', NvU32, 1056), ('errorCode', NvU32, 1060)]) +NV2080_CTRL_BIOS_GET_NBSI_PARAMS: TypeAlias = struct_NV2080_CTRL_BIOS_GET_NBSI_PARAMS +@c.record +class struct_NV2080_CTRL_BIOS_GET_NBSI_V2_PARAMS(c.Struct): + SIZE = 1308 + module: int + path: struct_NV2080_CTRL_BIOS_NBSI_REG_STRING + valueName: struct_NV2080_CTRL_BIOS_NBSI_REG_STRING + retBuf: c.Array[ctypes.c_ubyte, Literal[256]] + retSize: int + errorCode: int +struct_NV2080_CTRL_BIOS_GET_NBSI_V2_PARAMS.register_fields([('module', NvU32, 0), ('path', NV2080_CTRL_BIOS_NBSI_REG_STRING, 4), ('valueName', NV2080_CTRL_BIOS_NBSI_REG_STRING, 524), ('retBuf', c.Array[NvU8, Literal[256]], 1044), ('retSize', NvU32, 1300), ('errorCode', NvU32, 1304)]) +NV2080_CTRL_BIOS_GET_NBSI_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_BIOS_GET_NBSI_V2_PARAMS +@c.record +class struct_NV2080_CTRL_BIOS_GET_NBSI_OBJ_PARAMS(c.Struct): + SIZE = 40 + globType: int + globIndex: int + globSource: int + retBufOffset: int + retBuf: ctypes.c_void_p + retSize: int + totalObjSize: int + errorCode: int +struct_NV2080_CTRL_BIOS_GET_NBSI_OBJ_PARAMS.register_fields([('globType', NvU16, 0), ('globIndex', NvU8, 2), ('globSource', NvU16, 4), ('retBufOffset', NvU32, 8), ('retBuf', NvP64, 16), ('retSize', NvU32, 24), ('totalObjSize', NvU32, 28), ('errorCode', NvU32, 32)]) +NV2080_CTRL_BIOS_GET_NBSI_OBJ_PARAMS: TypeAlias = struct_NV2080_CTRL_BIOS_GET_NBSI_OBJ_PARAMS +@c.record +class struct_NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS(c.Struct): + SIZE = 48 + BoardID: int + chipSKU: c.Array[ctypes.c_char, Literal[9]] + chipSKUMod: c.Array[ctypes.c_char, Literal[5]] + skuConfigVersion: int + project: c.Array[ctypes.c_char, Literal[5]] + projectSKU: c.Array[ctypes.c_char, Literal[5]] + CDP: c.Array[ctypes.c_char, Literal[6]] + projectSKUMod: c.Array[ctypes.c_char, Literal[2]] + businessCycle: int +struct_NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS.register_fields([('BoardID', NvU32, 0), ('chipSKU', c.Array[ctypes.c_char, Literal[9]], 4), ('chipSKUMod', c.Array[ctypes.c_char, Literal[5]], 13), ('skuConfigVersion', NvU32, 20), ('project', c.Array[ctypes.c_char, Literal[5]], 24), ('projectSKU', c.Array[ctypes.c_char, Literal[5]], 29), ('CDP', c.Array[ctypes.c_char, Literal[6]], 34), ('projectSKUMod', c.Array[ctypes.c_char, Literal[2]], 40), ('businessCycle', NvU32, 44)]) +NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BIOS_GET_POST_TIME_PARAMS(c.Struct): + SIZE = 8 + vbiosPostTime: int +struct_NV2080_CTRL_CMD_BIOS_GET_POST_TIME_PARAMS.register_fields([('vbiosPostTime', NvU64, 0)]) +NV2080_CTRL_CMD_BIOS_GET_POST_TIME_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BIOS_GET_POST_TIME_PARAMS +@c.record +class struct_NV2080_CTRL_BIOS_GET_UEFI_SUPPORT_PARAMS(c.Struct): + SIZE = 8 + version: int + flags: int +struct_NV2080_CTRL_BIOS_GET_UEFI_SUPPORT_PARAMS.register_fields([('version', NvU32, 0), ('flags', NvU32, 4)]) +NV2080_CTRL_BIOS_GET_UEFI_SUPPORT_PARAMS: TypeAlias = struct_NV2080_CTRL_BIOS_GET_UEFI_SUPPORT_PARAMS +NvBoardObjIdx: TypeAlias = ctypes.c_uint16 +NvBoardObjMaskIdx: TypeAlias = ctypes.c_uint16 +NV2080_CTRL_BOARDOBJGRP_MASK_PRIMITIVE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_BOARDOBJGRP_MASK(c.Struct): + SIZE = 4 + pData: c.Array[ctypes.c_uint32, Literal[1]] +struct_NV2080_CTRL_BOARDOBJGRP_MASK.register_fields([('pData', c.Array[NvU32, Literal[1]], 0)]) +NV2080_CTRL_BOARDOBJGRP_MASK: TypeAlias = struct_NV2080_CTRL_BOARDOBJGRP_MASK +PNV2080_CTRL_BOARDOBJGRP_MASK: TypeAlias = c.POINTER[struct_NV2080_CTRL_BOARDOBJGRP_MASK] +@c.record +class struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32(c.Struct): + SIZE = 4 + super: struct_NV2080_CTRL_BOARDOBJGRP_MASK +struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32.register_fields([('super', NV2080_CTRL_BOARDOBJGRP_MASK, 0)]) +NV2080_CTRL_BOARDOBJGRP_MASK_E32: TypeAlias = struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32 +PNV2080_CTRL_BOARDOBJGRP_MASK_E32: TypeAlias = c.POINTER[struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32] +@c.record +class struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255(c.Struct): + SIZE = 32 + super: struct_NV2080_CTRL_BOARDOBJGRP_MASK + pDataE255: c.Array[ctypes.c_uint32, Literal[7]] +struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255.register_fields([('super', NV2080_CTRL_BOARDOBJGRP_MASK, 0), ('pDataE255', c.Array[NvU32, Literal[7]], 4)]) +NV2080_CTRL_BOARDOBJGRP_MASK_E255: TypeAlias = struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255 +PNV2080_CTRL_BOARDOBJGRP_MASK_E255: TypeAlias = c.POINTER[struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255] +@c.record +class struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512(c.Struct): + SIZE = 64 + super: struct_NV2080_CTRL_BOARDOBJGRP_MASK + pDataE512: c.Array[ctypes.c_uint32, Literal[15]] +struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512.register_fields([('super', NV2080_CTRL_BOARDOBJGRP_MASK, 0), ('pDataE512', c.Array[NvU32, Literal[15]], 4)]) +NV2080_CTRL_BOARDOBJGRP_MASK_E512: TypeAlias = struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512 +PNV2080_CTRL_BOARDOBJGRP_MASK_E512: TypeAlias = c.POINTER[struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512] +@c.record +class struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024(c.Struct): + SIZE = 128 + super: struct_NV2080_CTRL_BOARDOBJGRP_MASK + pDataE1024: c.Array[ctypes.c_uint32, Literal[31]] +struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024.register_fields([('super', NV2080_CTRL_BOARDOBJGRP_MASK, 0), ('pDataE1024', c.Array[NvU32, Literal[31]], 4)]) +NV2080_CTRL_BOARDOBJGRP_MASK_E1024: TypeAlias = struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024 +PNV2080_CTRL_BOARDOBJGRP_MASK_E1024: TypeAlias = c.POINTER[struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024] +@c.record +class struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048(c.Struct): + SIZE = 256 + super: struct_NV2080_CTRL_BOARDOBJGRP_MASK + pDataE2048: c.Array[ctypes.c_uint32, Literal[63]] +struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048.register_fields([('super', NV2080_CTRL_BOARDOBJGRP_MASK, 0), ('pDataE2048', c.Array[NvU32, Literal[63]], 4)]) +NV2080_CTRL_BOARDOBJGRP_MASK_E2048: TypeAlias = struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048 +PNV2080_CTRL_BOARDOBJGRP_MASK_E2048: TypeAlias = c.POINTER[struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048] +@c.record +class struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192(c.Struct): + SIZE = 1024 + super: struct_NV2080_CTRL_BOARDOBJGRP_MASK + pDataE8192: c.Array[ctypes.c_uint32, Literal[255]] +struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192.register_fields([('super', NV2080_CTRL_BOARDOBJGRP_MASK, 0), ('pDataE8192', c.Array[NvU32, Literal[255]], 4)]) +NV2080_CTRL_BOARDOBJGRP_MASK_E8192: TypeAlias = struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192 +PNV2080_CTRL_BOARDOBJGRP_MASK_E8192: TypeAlias = c.POINTER[struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192] +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO(c.Struct): + SIZE = 1 + type: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO.register_fields([('type', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS(c.Struct): + SIZE = 1 + type: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS.register_fields([('type', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL(c.Struct): + SIZE = 1 + type: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL.register_fields([('type', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS(c.Struct): + SIZE = 1 + type: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS.register_fields([('type', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJ_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO(c.Struct): + SIZE = 1 + classType: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO.register_fields([('classType', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_SUPER_ALIAS(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_SUPER_ALIAS.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_SUPER_ALIAS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_SUPER_ALIAS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS(c.Struct): + SIZE = 1 + classType: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS.register_fields([('classType', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_SUPER_ALIAS(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_SUPER_ALIAS.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_SUPER_ALIAS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_SUPER_ALIAS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL(c.Struct): + SIZE = 1 + classType: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL.register_fields([('classType', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_SUPER_ALIAS(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_SUPER_ALIAS.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_SUPER_ALIAS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_SUPER_ALIAS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS(c.Struct): + SIZE = 2 + classType: int + bInitialized: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS.register_fields([('classType', NvU8, 0), ('bInitialized', NvBool, 1)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_SUPER_ALIAS(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_SUPER_ALIAS.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_SUPER_ALIAS: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_SUPER_ALIAS +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E32(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E32.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E32, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E32: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E32 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E32(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E32.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E32, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E32: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E32 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E32(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E32.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E32, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E32: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E32 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E32(c.Struct): + SIZE = 8 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E32 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E32.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E32, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E32: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E32 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E255(c.Struct): + SIZE = 36 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E255.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E255, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E255: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E255 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E255(c.Struct): + SIZE = 36 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E255.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E255, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E255: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E255 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E255(c.Struct): + SIZE = 36 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E255.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E255, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E255: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E255 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E255(c.Struct): + SIZE = 36 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E255.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E255, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E255: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E255 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E512(c.Struct): + SIZE = 68 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E512.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E512, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E512: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E512 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E512(c.Struct): + SIZE = 68 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E512.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E512, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E512: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E512 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E512(c.Struct): + SIZE = 68 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E512.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E512, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E512: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E512 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E512(c.Struct): + SIZE = 68 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E512 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E512.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E512, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E512: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E512 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E1024(c.Struct): + SIZE = 132 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E1024.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E1024, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E1024: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E1024 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E1024(c.Struct): + SIZE = 132 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E1024.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E1024, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E1024: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E1024 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E1024(c.Struct): + SIZE = 132 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E1024.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E1024, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E1024: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E1024 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E1024(c.Struct): + SIZE = 132 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E1024 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E1024.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E1024, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E1024: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E1024 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E2048(c.Struct): + SIZE = 260 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E2048.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E2048, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E2048: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E2048 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E2048(c.Struct): + SIZE = 260 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E2048.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E2048, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E2048: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E2048 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E2048(c.Struct): + SIZE = 260 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E2048.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E2048, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E2048: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E2048 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E2048(c.Struct): + SIZE = 260 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E2048 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E2048.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E2048, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E2048: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E2048 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E8192(c.Struct): + SIZE = 1028 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E8192.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E8192, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E8192: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_E8192 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E8192(c.Struct): + SIZE = 1028 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E8192.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E8192, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E8192: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_E8192 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E8192(c.Struct): + SIZE = 1028 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E8192.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E8192, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E8192: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_E8192 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E8192(c.Struct): + SIZE = 1028 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS + objMask: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E8192 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E8192.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS, 0), ('objMask', NV2080_CTRL_BOARDOBJGRP_MASK_E8192, 4)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E8192: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_E8192 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_INFO_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_STATUS_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_CONTROL_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_INTERFACE(c.Struct): + SIZE = 1 + rsvd: int +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_INTERFACE.register_fields([('rsvd', NvU8, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_INTERFACE: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_INTERFACE +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_INTERFACE +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_INTERFACE, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_20(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_20.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_20: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_20 +@c.record +class struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X(c.Struct): + SIZE = 1 + super: struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_20 +struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X.register_fields([('super', NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_20, 0)]) +NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X: TypeAlias = struct_NV2080_CTRL_BOARDOBJ_BOARDOBJGRP_CLIENT_READINGS_IFACE_MODEL_PMU_INIT_1X +@c.record +class struct_NV2080_CTRL_BUS_GET_PCI_INFO_PARAMS(c.Struct): + SIZE = 16 + pciDeviceId: int + pciSubSystemId: int + pciRevisionId: int + pciExtDeviceId: int +struct_NV2080_CTRL_BUS_GET_PCI_INFO_PARAMS.register_fields([('pciDeviceId', NvU32, 0), ('pciSubSystemId', NvU32, 4), ('pciRevisionId', NvU32, 8), ('pciExtDeviceId', NvU32, 12)]) +NV2080_CTRL_BUS_GET_PCI_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_PCI_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_GET_PCI_INFO_V2_PARAMS(c.Struct): + SIZE = 28 + pciDeviceId: int + pciSubSystemId: int + pciRevisionId: int + pciExtDeviceId: int + pciProgrammingInterface: int + pciSubClass: int + pciBaseClass: int +struct_NV2080_CTRL_BUS_GET_PCI_INFO_V2_PARAMS.register_fields([('pciDeviceId', NvU32, 0), ('pciSubSystemId', NvU32, 4), ('pciRevisionId', NvU32, 8), ('pciExtDeviceId', NvU32, 12), ('pciProgrammingInterface', NvU32, 16), ('pciSubClass', NvU32, 20), ('pciBaseClass', NvU32, 24)]) +NV2080_CTRL_BUS_GET_PCI_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_PCI_INFO_V2_PARAMS +NV2080_CTRL_BUS_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_BUS_GET_INFO_PARAMS(c.Struct): + SIZE = 16 + busInfoListSize: int + busInfoList: ctypes.c_void_p +struct_NV2080_CTRL_BUS_GET_INFO_PARAMS.register_fields([('busInfoListSize', NvU32, 0), ('busInfoList', NvP64, 8)]) +NV2080_CTRL_BUS_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_GET_INFO_V2_PARAMS(c.Struct): + SIZE = 420 + busInfoListSize: int + busInfoList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[52]] +struct_NV2080_CTRL_BUS_GET_INFO_V2_PARAMS.register_fields([('busInfoListSize', NvU32, 0), ('busInfoList', c.Array[NV2080_CTRL_BUS_INFO, Literal[52]], 4)]) +NV2080_CTRL_BUS_GET_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_PCI_BAR_INFO(c.Struct): + SIZE = 24 + flags: int + barSize: int + barSizeBytes: int + barOffset: int +struct_NV2080_CTRL_BUS_PCI_BAR_INFO.register_fields([('flags', NvU32, 0), ('barSize', NvU32, 4), ('barSizeBytes', NvU64, 8), ('barOffset', NvU64, 16)]) +NV2080_CTRL_BUS_PCI_BAR_INFO: TypeAlias = struct_NV2080_CTRL_BUS_PCI_BAR_INFO +@c.record +class struct_NV2080_CTRL_BUS_GET_PCI_BAR_INFO_PARAMS(c.Struct): + SIZE = 200 + pciBarCount: int + pciBarInfo: c.Array[struct_NV2080_CTRL_BUS_PCI_BAR_INFO, Literal[8]] +struct_NV2080_CTRL_BUS_GET_PCI_BAR_INFO_PARAMS.register_fields([('pciBarCount', NvU32, 0), ('pciBarInfo', c.Array[NV2080_CTRL_BUS_PCI_BAR_INFO, Literal[8]], 8)]) +NV2080_CTRL_BUS_GET_PCI_BAR_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_PCI_BAR_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_PARAMS(c.Struct): + SIZE = 8 + pcieLinkWidth: int + failingReason: int +struct_NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_PARAMS.register_fields([('pcieLinkWidth', NvU32, 0), ('failingReason', NvU32, 4)]) +NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_SET_PCIE_SPEED_PARAMS(c.Struct): + SIZE = 4 + busSpeed: int +struct_NV2080_CTRL_BUS_SET_PCIE_SPEED_PARAMS.register_fields([('busSpeed', NvU32, 0)]) +NV2080_CTRL_BUS_SET_PCIE_SPEED_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_SET_PCIE_SPEED_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_MAP_BAR2_PARAMS(c.Struct): + SIZE = 4 + hMemory: int +struct_NV2080_CTRL_BUS_MAP_BAR2_PARAMS.register_fields([('hMemory', NvHandle, 0)]) +NV2080_CTRL_BUS_MAP_BAR2_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_MAP_BAR2_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_UNMAP_BAR2_PARAMS(c.Struct): + SIZE = 4 + hMemory: int +struct_NV2080_CTRL_BUS_UNMAP_BAR2_PARAMS.register_fields([('hMemory', NvHandle, 0)]) +NV2080_CTRL_BUS_UNMAP_BAR2_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_UNMAP_BAR2_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_VERIFY_BAR2_PARAMS(c.Struct): + SIZE = 12 + hMemory: int + offset: int + size: int +struct_NV2080_CTRL_BUS_VERIFY_BAR2_PARAMS.register_fields([('hMemory', NvHandle, 0), ('offset', NvU32, 4), ('size', NvU32, 8)]) +NV2080_CTRL_BUS_VERIFY_BAR2_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_VERIFY_BAR2_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_SERVICE_GPU_MULTIFUNC_STATE_PARAMS(c.Struct): + SIZE = 8 + command: int + deviceState: int +struct_NV2080_CTRL_BUS_SERVICE_GPU_MULTIFUNC_STATE_PARAMS.register_fields([('command', NvU8, 0), ('deviceState', NvU32, 4)]) +NV2080_CTRL_BUS_SERVICE_GPU_MULTIFUNC_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_SERVICE_GPU_MULTIFUNC_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_GET_PEX_COUNTERS_PARAMS(c.Struct): + SIZE = 76 + pexCounterMask: int + pexTotalCorrectableErrors: int + pexCorrectableErrors: int + pexTotalNonFatalErrors: int + pexTotalFatalErrors: int + pexTotalUnsupportedReqs: int + pexCounters: c.Array[ctypes.c_uint16, Literal[31]] +struct_NV2080_CTRL_BUS_GET_PEX_COUNTERS_PARAMS.register_fields([('pexCounterMask', NvU32, 0), ('pexTotalCorrectableErrors', NvU32, 4), ('pexCorrectableErrors', NvU16, 8), ('pexTotalNonFatalErrors', NvU8, 10), ('pexTotalFatalErrors', NvU8, 11), ('pexTotalUnsupportedReqs', NvU8, 12), ('pexCounters', c.Array[NvU16, Literal[31]], 14)]) +NV2080_CTRL_BUS_GET_PEX_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_PEX_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_CLEAR_PEX_COUNTERS_PARAMS(c.Struct): + SIZE = 4 + pexCounterMask: int +struct_NV2080_CTRL_BUS_CLEAR_PEX_COUNTERS_PARAMS.register_fields([('pexCounterMask', NvU32, 0)]) +NV2080_CTRL_BUS_CLEAR_PEX_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_CLEAR_PEX_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_FREEZE_PEX_COUNTERS_PARAMS(c.Struct): + SIZE = 8 + pexCounterMask: int + bFreezeRmCounter: int +struct_NV2080_CTRL_BUS_FREEZE_PEX_COUNTERS_PARAMS.register_fields([('pexCounterMask', NvU32, 0), ('bFreezeRmCounter', NvBool, 4)]) +NV2080_CTRL_BUS_FREEZE_PEX_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_FREEZE_PEX_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_PEX_LANE_COUNTERS_PARAMS(c.Struct): + SIZE = 18 + pexLaneErrorStatus: int + pexLaneCounter: c.Array[ctypes.c_ubyte, Literal[16]] +struct_NV2080_CTRL_CMD_BUS_GET_PEX_LANE_COUNTERS_PARAMS.register_fields([('pexLaneErrorStatus', NvU16, 0), ('pexLaneCounter', c.Array[NvU8, Literal[16]], 2)]) +NV2080_CTRL_CMD_BUS_GET_PEX_LANE_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_PEX_LANE_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_PCIE_LTR_LATENCY_PARAMS(c.Struct): + SIZE = 12 + bPexLtrRegkeyOverride: int + bPexRootPortLtrSupported: int + bPexGpuLtrSupported: int + pexLtrSnoopLatencyValue: int + pexLtrSnoopLatencyScale: int + pexLtrNoSnoopLatencyValue: int + pexLtrNoSnoopLatencyScale: int +struct_NV2080_CTRL_CMD_BUS_GET_PCIE_LTR_LATENCY_PARAMS.register_fields([('bPexLtrRegkeyOverride', NvBool, 0), ('bPexRootPortLtrSupported', NvBool, 1), ('bPexGpuLtrSupported', NvBool, 2), ('pexLtrSnoopLatencyValue', NvU16, 4), ('pexLtrSnoopLatencyScale', NvU8, 6), ('pexLtrNoSnoopLatencyValue', NvU16, 8), ('pexLtrNoSnoopLatencyScale', NvU8, 10)]) +NV2080_CTRL_CMD_BUS_GET_PCIE_LTR_LATENCY_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_PCIE_LTR_LATENCY_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_SET_PCIE_LTR_LATENCY_PARAMS(c.Struct): + SIZE = 8 + pexLtrSnoopLatencyValue: int + pexLtrSnoopLatencyScale: int + pexLtrNoSnoopLatencyValue: int + pexLtrNoSnoopLatencyScale: int +struct_NV2080_CTRL_CMD_BUS_SET_PCIE_LTR_LATENCY_PARAMS.register_fields([('pexLtrSnoopLatencyValue', NvU16, 0), ('pexLtrSnoopLatencyScale', NvU8, 2), ('pexLtrNoSnoopLatencyValue', NvU16, 4), ('pexLtrNoSnoopLatencyScale', NvU8, 6)]) +NV2080_CTRL_CMD_BUS_SET_PCIE_LTR_LATENCY_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_SET_PCIE_LTR_LATENCY_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_GET_PEX_UTIL_COUNTERS_PARAMS(c.Struct): + SIZE = 32 + pexCounterMask: int + pexCounters: c.Array[ctypes.c_uint32, Literal[7]] +struct_NV2080_CTRL_BUS_GET_PEX_UTIL_COUNTERS_PARAMS.register_fields([('pexCounterMask', NvU32, 0), ('pexCounters', c.Array[NvU32, Literal[7]], 4)]) +NV2080_CTRL_BUS_GET_PEX_UTIL_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_PEX_UTIL_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_CLEAR_PEX_UTIL_COUNTERS_PARAMS(c.Struct): + SIZE = 4 + pexCounterMask: int +struct_NV2080_CTRL_BUS_CLEAR_PEX_UTIL_COUNTERS_PARAMS.register_fields([('pexCounterMask', NvU32, 0)]) +NV2080_CTRL_BUS_CLEAR_PEX_UTIL_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_CLEAR_PEX_UTIL_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_CONTROL_PUBLIC_ASPM_BITS_PARAMS(c.Struct): + SIZE = 1 + bEnable: int +struct_NV2080_CTRL_CMD_BUS_CONTROL_PUBLIC_ASPM_BITS_PARAMS.register_fields([('bEnable', NvBool, 0)]) +NV2080_CTRL_CMD_BUS_CONTROL_PUBLIC_ASPM_BITS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_CONTROL_PUBLIC_ASPM_BITS_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_GET_NVLINK_PEER_ID_MASK_PARAMS(c.Struct): + SIZE = 128 + nvlinkPeerIdMask: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV2080_CTRL_BUS_GET_NVLINK_PEER_ID_MASK_PARAMS.register_fields([('nvlinkPeerIdMask', c.Array[NvU32, Literal[32]], 0)]) +NV2080_CTRL_BUS_GET_NVLINK_PEER_ID_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_NVLINK_PEER_ID_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_SET_EOM_PARAMETERS_PARAMS(c.Struct): + SIZE = 3 + eomMode: int + eomNblks: int + eomNerrs: int +struct_NV2080_CTRL_CMD_BUS_SET_EOM_PARAMETERS_PARAMS.register_fields([('eomMode', NvU8, 0), ('eomNblks', NvU8, 1), ('eomNerrs', NvU8, 2)]) +NV2080_CTRL_CMD_BUS_SET_EOM_PARAMETERS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_SET_EOM_PARAMETERS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_UPHY_DLN_CFG_SPACE_PARAMS(c.Struct): + SIZE = 12 + regAddress: int + laneSelectMask: int + regValue: int +struct_NV2080_CTRL_CMD_BUS_GET_UPHY_DLN_CFG_SPACE_PARAMS.register_fields([('regAddress', NvU32, 0), ('laneSelectMask', NvU32, 4), ('regValue', NvU16, 8)]) +NV2080_CTRL_CMD_BUS_GET_UPHY_DLN_CFG_SPACE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_UPHY_DLN_CFG_SPACE_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_GET_EOM_STATUS_PARAMS(c.Struct): + SIZE = 76 + eomMode: int + eomNblks: int + eomNerrs: int + eomBerEyeSel: int + eomPamEyeSel: int + laneMask: int + eomStatus: c.Array[ctypes.c_uint16, Literal[32]] +struct_NV2080_CTRL_BUS_GET_EOM_STATUS_PARAMS.register_fields([('eomMode', NvU8, 0), ('eomNblks', NvU8, 1), ('eomNerrs', NvU8, 2), ('eomBerEyeSel', NvU8, 3), ('eomPamEyeSel', NvU8, 4), ('laneMask', NvU32, 8), ('eomStatus', c.Array[NvU16, Literal[32]], 12)]) +NV2080_CTRL_BUS_GET_EOM_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_EOM_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_PARAMS(c.Struct): + SIZE = 12 + capType: int + dbdf: int + atomicsCaps: int +struct_NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_PARAMS.register_fields([('capType', NvU32, 0), ('dbdf', NvU32, 4), ('atomicsCaps', NvU32, 8)]) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_PCIE_GPU_ATOMIC_OP_INFO(c.Struct): + SIZE = 8 + bSupported: int + attributes: int +struct_NV2080_CTRL_BUS_PCIE_GPU_ATOMIC_OP_INFO.register_fields([('bSupported', NvBool, 0), ('attributes', NvU32, 4)]) +NV2080_CTRL_BUS_PCIE_GPU_ATOMIC_OP_INFO: TypeAlias = struct_NV2080_CTRL_BUS_PCIE_GPU_ATOMIC_OP_INFO +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS_PARAMS(c.Struct): + SIZE = 112 + capType: int + dbdf: int + atomicOp: c.Array[struct_NV2080_CTRL_BUS_PCIE_GPU_ATOMIC_OP_INFO, Literal[13]] +struct_NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS_PARAMS.register_fields([('capType', NvU32, 0), ('dbdf', NvU32, 4), ('atomicOp', c.Array[NV2080_CTRL_BUS_PCIE_GPU_ATOMIC_OP_INFO, Literal[13]], 8)]) +NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_C2C_INFO_PARAMS(c.Struct): + SIZE = 28 + bIsLinkUp: int + bLinkInHS: int + bEncryptionEnabled: int + nrLinks: int + maxNrLinks: int + linkMask: int + perLinkBwMBps: int + perLinkLaneWidth: int + remoteType: int +struct_NV2080_CTRL_CMD_BUS_GET_C2C_INFO_PARAMS.register_fields([('bIsLinkUp', NvBool, 0), ('bLinkInHS', NvBool, 1), ('bEncryptionEnabled', NvBool, 2), ('nrLinks', NvU32, 4), ('maxNrLinks', NvU32, 8), ('linkMask', NvU32, 12), ('perLinkBwMBps', NvU32, 16), ('perLinkLaneWidth', NvU32, 20), ('remoteType', NvU32, 24)]) +NV2080_CTRL_CMD_BUS_GET_C2C_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_C2C_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_SYSMEM_ACCESS_PARAMS(c.Struct): + SIZE = 1 + bDisable: int +struct_NV2080_CTRL_BUS_SYSMEM_ACCESS_PARAMS.register_fields([('bDisable', NvBool, 0)]) +NV2080_CTRL_BUS_SYSMEM_ACCESS_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_SYSMEM_ACCESS_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS(c.Struct): + SIZE = 168 + errCnts: c.Array[struct_NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS_errCnt, Literal[14]] +@c.record +class struct_NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS_errCnt(c.Struct): + SIZE = 12 + nrCrcErrIntr: int + nrReplayErrIntr: int + nrReplayB2bErrIntr: int +struct_NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS_errCnt.register_fields([('nrCrcErrIntr', NvU32, 0), ('nrReplayErrIntr', NvU32, 4), ('nrReplayB2bErrIntr', NvU32, 8)]) +struct_NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS.register_fields([('errCnts', c.Array[struct_NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS_errCnt, Literal[14]], 0)]) +NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_SET_P2P_MAPPING_PARAMS(c.Struct): + SIZE = 32 + connectionType: int + peerId: int + bEgmPeer: int + bSpaAccessOnly: int + bUseUuid: int + remoteGpuId: int + remoteGpuUuid: c.Array[ctypes.c_ubyte, Literal[16]] +struct_NV2080_CTRL_BUS_SET_P2P_MAPPING_PARAMS.register_fields([('connectionType', NvU32, 0), ('peerId', NvU32, 4), ('bEgmPeer', NvBool, 8), ('bSpaAccessOnly', NvBool, 9), ('bUseUuid', NvBool, 10), ('remoteGpuId', NvU32, 12), ('remoteGpuUuid', c.Array[NvU8, Literal[16]], 16)]) +NV2080_CTRL_BUS_SET_P2P_MAPPING_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_SET_P2P_MAPPING_PARAMS +@c.record +class struct_NV2080_CTRL_BUS_UNSET_P2P_MAPPING_PARAMS(c.Struct): + SIZE = 32 + connectionType: int + peerId: int + bUseUuid: int + remoteGpuId: int + remoteGpuUuid: c.Array[ctypes.c_ubyte, Literal[16]] +struct_NV2080_CTRL_BUS_UNSET_P2P_MAPPING_PARAMS.register_fields([('connectionType', NvU32, 0), ('peerId', NvU32, 4), ('bUseUuid', NvBool, 8), ('remoteGpuId', NvU32, 12), ('remoteGpuUuid', c.Array[NvU8, Literal[16]], 16)]) +NV2080_CTRL_BUS_UNSET_P2P_MAPPING_PARAMS: TypeAlias = struct_NV2080_CTRL_BUS_UNSET_P2P_MAPPING_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_PARAMS(c.Struct): + SIZE = 4 + atomicsCaps: int +struct_NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_PARAMS.register_fields([('atomicsCaps', NvU32, 0)]) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_C2C_LPWR_STATS_PARAMS(c.Struct): + SIZE = 88 + c2cStateSupportMask: int + bCl3Support: int + cl3EntryCount: int + cl3ExitCount: int + cl3ResidentTimeUs: int + cl3AvgEntryLatencyUs: int + cl3AvgExitLatencyUs: int + cl3PstateSupportMask: int + cl3DisallowReasonMask: int + bCl4Support: int + cl4EntryCount: int + cl4ExitCount: int + cl4ResidentTimeUs: int + cl4AvgEntryLatencyUs: int + cl4AvgExitLatencyUs: int + cl4PstateSupportMask: int + cl4DisallowReasonMask: int + c2cLpwrStateAllowedMask: int + localPowerState: int + remotePowerState: int + cl3IdleThresholdUs: int + cl4IdleThresholdUs: int +struct_NV2080_CTRL_CMD_BUS_GET_C2C_LPWR_STATS_PARAMS.register_fields([('c2cStateSupportMask', NvU32, 0), ('bCl3Support', NvBool, 4), ('cl3EntryCount', NvU32, 8), ('cl3ExitCount', NvU32, 12), ('cl3ResidentTimeUs', NvU32, 16), ('cl3AvgEntryLatencyUs', NvU32, 20), ('cl3AvgExitLatencyUs', NvU32, 24), ('cl3PstateSupportMask', NvU32, 28), ('cl3DisallowReasonMask', NvU32, 32), ('bCl4Support', NvBool, 36), ('cl4EntryCount', NvU32, 40), ('cl4ExitCount', NvU32, 44), ('cl4ResidentTimeUs', NvU32, 48), ('cl4AvgEntryLatencyUs', NvU32, 52), ('cl4AvgExitLatencyUs', NvU32, 56), ('cl4PstateSupportMask', NvU32, 60), ('cl4DisallowReasonMask', NvU32, 64), ('c2cLpwrStateAllowedMask', NvU32, 68), ('localPowerState', NvU32, 72), ('remotePowerState', NvU32, 76), ('cl3IdleThresholdUs', NvU32, 80), ('cl4IdleThresholdUs', NvU32, 84)]) +NV2080_CTRL_CMD_BUS_GET_C2C_LPWR_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_C2C_LPWR_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_STATE_VOTE_PARAMS(c.Struct): + SIZE = 8 + c2cLpwrStateId: int + bAllowed: int +struct_NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_STATE_VOTE_PARAMS.register_fields([('c2cLpwrStateId', NvU32, 0), ('bAllowed', NvBool, 4)]) +NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_STATE_VOTE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_STATE_VOTE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_IDLE_THRESHOLD_PARAMS(c.Struct): + SIZE = 8 + c2cLpwrStateId: int + idleThresholdUs: int +struct_NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_IDLE_THRESHOLD_PARAMS.register_fields([('c2cLpwrStateId', NvU32, 0), ('idleThresholdUs', NvU32, 4)]) +NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_IDLE_THRESHOLD_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_IDLE_THRESHOLD_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_BUS_GET_C2C_PACKET_COUNTERS_PARAMS(c.Struct): + SIZE = 56 + packetCounters: c.Array[ctypes.c_uint32, Literal[14]] +struct_NV2080_CTRL_CMD_BUS_GET_C2C_PACKET_COUNTERS_PARAMS.register_fields([('packetCounters', c.Array[NvU32, Literal[14]], 0)]) +NV2080_CTRL_CMD_BUS_GET_C2C_PACKET_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_BUS_GET_C2C_PACKET_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_CAPS_PARAMS(c.Struct): + SIZE = 16 + ceEngineType: int + capsTblSize: int + capsTbl: ctypes.c_void_p +struct_NV2080_CTRL_CE_GET_CAPS_PARAMS.register_fields([('ceEngineType', NvU32, 0), ('capsTblSize', NvU32, 4), ('capsTbl', NvP64, 8)]) +NV2080_CTRL_CE_GET_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_CAPS_V2_PARAMS(c.Struct): + SIZE = 8 + ceEngineType: int + capsTbl: c.Array[ctypes.c_ubyte, Literal[2]] +struct_NV2080_CTRL_CE_GET_CAPS_V2_PARAMS.register_fields([('ceEngineType', NvU32, 0), ('capsTbl', c.Array[NvU8, Literal[2]], 4)]) +NV2080_CTRL_CE_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_CAPS_V2_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_CE_PCE_MASK_PARAMS(c.Struct): + SIZE = 8 + ceEngineType: int + pceMask: int +struct_NV2080_CTRL_CE_GET_CE_PCE_MASK_PARAMS.register_fields([('ceEngineType', NvU32, 0), ('pceMask', NvU32, 4)]) +NV2080_CTRL_CE_GET_CE_PCE_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_CE_PCE_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_CE_SET_PCE_LCE_CONFIG_PARAMS(c.Struct): + SIZE = 276 + ceEngineType: int + pceLceMap: c.Array[ctypes.c_uint32, Literal[64]] + grceSharedLceMap: c.Array[ctypes.c_uint32, Literal[4]] +struct_NV2080_CTRL_CE_SET_PCE_LCE_CONFIG_PARAMS.register_fields([('ceEngineType', NvU32, 0), ('pceLceMap', c.Array[NvU32, Literal[64]], 4), ('grceSharedLceMap', c.Array[NvU32, Literal[4]], 260)]) +NV2080_CTRL_CE_SET_PCE_LCE_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_SET_PCE_LCE_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_PARAMS(c.Struct): + SIZE = 280 + pceLceMap: c.Array[ctypes.c_uint32, Literal[64]] + grceConfig: c.Array[ctypes.c_uint32, Literal[4]] + exposeCeMask: int + bUpdateNvlinkPceLce: int +struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_PARAMS.register_fields([('pceLceMap', c.Array[NvU32, Literal[64]], 0), ('grceConfig', c.Array[NvU32, Literal[4]], 256), ('exposeCeMask', NvU32, 272), ('bUpdateNvlinkPceLce', NvBool, 276)]) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_PARAMS +@c.record +class struct_NV2080_CTRL_CE_UPDATE_CLASS_DB_PARAMS(c.Struct): + SIZE = 4 + stubbedCeMask: int +struct_NV2080_CTRL_CE_UPDATE_CLASS_DB_PARAMS.register_fields([('stubbedCeMask', NvU32, 0)]) +NV2080_CTRL_CE_UPDATE_CLASS_DB_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_UPDATE_CLASS_DB_PARAMS +NV2080_CTRL_CE_GET_PHYSICAL_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_CAPS_V2_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS(c.Struct): + SIZE = 4 + size: int +struct_NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS.register_fields([('size', NvU32, 0)]) +NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_HUB_PCE_MASK_PARAMS(c.Struct): + SIZE = 132 + hshubPceMasks: c.Array[ctypes.c_uint32, Literal[32]] + fbhubPceMask: int +struct_NV2080_CTRL_CE_GET_HUB_PCE_MASK_PARAMS.register_fields([('hshubPceMasks', c.Array[NvU32, Literal[32]], 0), ('fbhubPceMask', NvU32, 128)]) +NV2080_CTRL_CE_GET_HUB_PCE_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_HUB_PCE_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_ALL_CAPS_PARAMS(c.Struct): + SIZE = 136 + capsTbl: c.Array[c.Array[ctypes.c_ubyte, Literal[2]], Literal[64]] + present: int +struct_NV2080_CTRL_CE_GET_ALL_CAPS_PARAMS.register_fields([('capsTbl', c.Array[c.Array[NvU8, Literal[2]], Literal[64]], 0), ('present', NvU64, 128)]) +NV2080_CTRL_CE_GET_ALL_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_ALL_CAPS_PARAMS +NV2080_CTRL_CE_GET_ALL_PHYSICAL_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_ALL_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_LCE_SHIM_INFO_PARAMS(c.Struct): + SIZE = 12 + ceEngineType: int + shimInstance: int + shimLocalLceIdx: int +struct_NV2080_CTRL_CE_GET_LCE_SHIM_INFO_PARAMS.register_fields([('ceEngineType', NvU32, 0), ('shimInstance', NvU32, 4), ('shimLocalLceIdx', NvU32, 8)]) +NV2080_CTRL_CE_GET_LCE_SHIM_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_LCE_SHIM_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_V2_PARAMS(c.Struct): + SIZE = 284 + pceLceMap: c.Array[ctypes.c_uint32, Literal[64]] + grceConfig: c.Array[ctypes.c_uint32, Literal[4]] + exposeCeMask: int + bUpdateNvlinkPceLce: int + shimInstance: int +struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_V2_PARAMS.register_fields([('pceLceMap', c.Array[NvU32, Literal[64]], 0), ('grceConfig', c.Array[NvU32, Literal[4]], 256), ('exposeCeMask', NvU32, 272), ('bUpdateNvlinkPceLce', NvBool, 276), ('shimInstance', NvU32, 280)]) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_V2_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_HUB_PCE_MASK_V2_PARAMS(c.Struct): + SIZE = 136 + connectingHubPceMasks: c.Array[ctypes.c_uint32, Literal[32]] + fbhubPceMask: int + shimInstance: int +struct_NV2080_CTRL_CE_GET_HUB_PCE_MASK_V2_PARAMS.register_fields([('connectingHubPceMasks', c.Array[NvU32, Literal[32]], 0), ('fbhubPceMask', NvU32, 128), ('shimInstance', NvU32, 132)]) +NV2080_CTRL_CE_GET_HUB_PCE_MASK_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_HUB_PCE_MASK_V2_PARAMS +enum_NV2080_CTRL_CE_LCE_TYPE: dict[int, str] = {(NV2080_CTRL_CE_LCE_TYPE_PCIE:=1): 'NV2080_CTRL_CE_LCE_TYPE_PCIE', (NV2080_CTRL_CE_LCE_TYPE_DECOMP:=2): 'NV2080_CTRL_CE_LCE_TYPE_DECOMP', (NV2080_CTRL_CE_LCE_TYPE_SCRUB:=3): 'NV2080_CTRL_CE_LCE_TYPE_SCRUB', (NV2080_CTRL_CE_LCE_TYPE_NVLINK_PEER:=4): 'NV2080_CTRL_CE_LCE_TYPE_NVLINK_PEER', (NV2080_CTRL_CE_LCE_TYPE_C2C:=5): 'NV2080_CTRL_CE_LCE_TYPE_C2C', (NV2080_CTRL_CE_LCE_TYPE_PCIE_RD:=6): 'NV2080_CTRL_CE_LCE_TYPE_PCIE_RD', (NV2080_CTRL_CE_LCE_TYPE_PCIE_WR:=7): 'NV2080_CTRL_CE_LCE_TYPE_PCIE_WR', (NV2080_CTRL_CE_LCE_TYPE_C2C_H2D:=8): 'NV2080_CTRL_CE_LCE_TYPE_C2C_H2D', (NV2080_CTRL_CE_LCE_TYPE_C2C_D2H:=9): 'NV2080_CTRL_CE_LCE_TYPE_C2C_D2H', (NV2080_CTRL_CE_LCE_TYPE_CC_WORK_SUBMIT:=10): 'NV2080_CTRL_CE_LCE_TYPE_CC_WORK_SUBMIT'} +NV2080_CTRL_CE_LCE_TYPE: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_CE_SHIM_PREFERENCE: dict[int, str] = {(NV2080_CTRL_CE_SHIM_PREFERENCE_SAME_SHIM:=1): 'NV2080_CTRL_CE_SHIM_PREFERENCE_SAME_SHIM', (NV2080_CTRL_CE_SHIM_PREFERENCE_DIFFERENT_SHIM:=2): 'NV2080_CTRL_CE_SHIM_PREFERENCE_DIFFERENT_SHIM'} +NV2080_CTRL_CE_SHIM_PREFERENCE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_TYPE_PARAMS(c.Struct): + SIZE = 32 + lceType: int + metadataForLceType: int + numPces: int + numLces: int + supportedPceMask: int + supportedLceMask: int + pcePerHshub: int + multipleLceShimPreference: int +struct_NV2080_CTRL_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_TYPE_PARAMS.register_fields([('lceType', NV2080_CTRL_CE_LCE_TYPE, 0), ('metadataForLceType', NvU32, 4), ('numPces', NvU32, 8), ('numLces', NvU32, 12), ('supportedPceMask', NvU32, 16), ('supportedLceMask', NvU32, 20), ('pcePerHshub', NvU32, 24), ('multipleLceShimPreference', NV2080_CTRL_CE_SHIM_PREFERENCE, 28)]) +NV2080_CTRL_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_TYPE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_TYPE_PARAMS +@c.record +class struct_NV2080_CTRL_CE_GET_DECOMP_LCE_MASK_PARAMS(c.Struct): + SIZE = 16 + decompLceMask: int + shimInstance: int +struct_NV2080_CTRL_CE_GET_DECOMP_LCE_MASK_PARAMS.register_fields([('decompLceMask', NvU64, 0), ('shimInstance', NvU32, 8)]) +NV2080_CTRL_CE_GET_DECOMP_LCE_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_GET_DECOMP_LCE_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_CE_IS_DECOMP_LCE_ENABLED_PARAMS(c.Struct): + SIZE = 8 + lceIndex: int + bDecompEnabled: int +struct_NV2080_CTRL_CE_IS_DECOMP_LCE_ENABLED_PARAMS.register_fields([('lceIndex', NvU32, 0), ('bDecompEnabled', NvBool, 4)]) +NV2080_CTRL_CE_IS_DECOMP_LCE_ENABLED_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_IS_DECOMP_LCE_ENABLED_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_MIG_GPU_INSTANCE_PARAMS(c.Struct): + SIZE = 24 + lceMask: int + numLces: int + pceAvailableMask: int + numLcesToMap: int + lceAvailableMask: int + numMinPcesPerLce: int +struct_NV2080_CTRL_CMD_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_MIG_GPU_INSTANCE_PARAMS.register_fields([('lceMask', NvU32, 0), ('numLces', NvU32, 4), ('pceAvailableMask', NvU32, 8), ('numLcesToMap', NvU32, 12), ('lceAvailableMask', NvU32, 16), ('numMinPcesPerLce', NvU32, 20)]) +NV2080_CTRL_CMD_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_MIG_GPU_INSTANCE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_MIG_GPU_INSTANCE_PARAMS +@c.record +class struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MIG_MAPPINGS_PARAMS(c.Struct): + SIZE = 264 + pceLceMap: c.Array[ctypes.c_uint32, Literal[64]] + lceAvailableMask: int + shimInstance: int +struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MIG_MAPPINGS_PARAMS.register_fields([('pceLceMap', c.Array[NvU32, Literal[64]], 0), ('lceAvailableMask', NvU32, 256), ('shimInstance', NvU32, 260)]) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MIG_MAPPINGS_PARAMS: TypeAlias = struct_NV2080_CTRL_CE_UPDATE_PCE_LCE_MIG_MAPPINGS_PARAMS +@c.record +class struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE(c.Struct): + SIZE = 24 + super: struct_NV2080_CTRL_PMUMON_SAMPLE + gpcClkFreqKHz: int + dramClkFreqKHz: int + nvdClkFreqKHz: int + activeGpcClkFreqKHz: int +@c.record +class struct_NV2080_CTRL_PMUMON_SAMPLE(c.Struct): + SIZE = 8 + timestamp: int +NV2080_CTRL_PMUMON_SAMPLE: TypeAlias = struct_NV2080_CTRL_PMUMON_SAMPLE +struct_NV2080_CTRL_PMUMON_SAMPLE.register_fields([('timestamp', NvU64, 0)]) +struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE.register_fields([('super', NV2080_CTRL_PMUMON_SAMPLE, 0), ('gpcClkFreqKHz', NvU32, 8), ('dramClkFreqKHz', NvU32, 12), ('nvdClkFreqKHz', NvU32, 16), ('activeGpcClkFreqKHz', NvU32, 20)]) +NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE: TypeAlias = struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE +PNV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE: TypeAlias = c.POINTER[struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE] +@c.record +class struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES_PARAMS(c.Struct): + SIZE = 6024 + super: struct_NV2080_CTRL_PMUMON_GET_SAMPLES_SUPER + samples: c.Array[struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE, Literal[250]] +@c.record +class struct_NV2080_CTRL_PMUMON_GET_SAMPLES_SUPER(c.Struct): + SIZE = 24 + tailIndex: int + sequenceId: int + headIndex: int + numSamples: int + resetCount: int + retrievalMode: int +NV2080_CTRL_PMUMON_GET_SAMPLES_SUPER: TypeAlias = struct_NV2080_CTRL_PMUMON_GET_SAMPLES_SUPER +enum_NV2080_CTRL_PMUMON_RETRIEVAL_MODE: dict[int, str] = {(NV2080_CTRL_PMUMON_RETRIEVAL_MODE_ALL_SAMPLES:=0): 'NV2080_CTRL_PMUMON_RETRIEVAL_MODE_ALL_SAMPLES', (NV2080_CTRL_PMUMON_RETRIEVAL_MODE_LATEST_SAMPLE_ONLY:=1): 'NV2080_CTRL_PMUMON_RETRIEVAL_MODE_LATEST_SAMPLE_ONLY'} +NV2080_CTRL_PMUMON_RETRIEVAL_MODE: TypeAlias = ctypes.c_uint32 +struct_NV2080_CTRL_PMUMON_GET_SAMPLES_SUPER.register_fields([('tailIndex', NvU32, 0), ('sequenceId', NvU32, 4), ('headIndex', NvU32, 8), ('numSamples', NvU32, 12), ('resetCount', NvU32, 16), ('retrievalMode', NV2080_CTRL_PMUMON_RETRIEVAL_MODE, 20)]) +struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES_PARAMS.register_fields([('super', NV2080_CTRL_PMUMON_GET_SAMPLES_SUPER, 0), ('samples', c.Array[NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE, Literal[250]], 24)]) +NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES_PARAMS: TypeAlias = struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES_PARAMS +PNV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES_PARAMS: TypeAlias = c.POINTER[struct_NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES_PARAMS] +@c.record +class struct_NV2080_CTRL_DMA_INVALIDATE_TLB_PARAMS(c.Struct): + SIZE = 16 + hClient: int + hDevice: int + engine: int + hVASpace: int +struct_NV2080_CTRL_DMA_INVALIDATE_TLB_PARAMS.register_fields([('hClient', NvHandle, 0), ('hDevice', NvHandle, 4), ('engine', NvU32, 8), ('hVASpace', NvHandle, 12)]) +NV2080_CTRL_DMA_INVALIDATE_TLB_PARAMS: TypeAlias = struct_NV2080_CTRL_DMA_INVALIDATE_TLB_PARAMS +NV2080_CTRL_DMA_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_DMA_GET_INFO_PARAMS(c.Struct): + SIZE = 2052 + dmaInfoTblSize: int + dmaInfoTbl: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[256]] +struct_NV2080_CTRL_DMA_GET_INFO_PARAMS.register_fields([('dmaInfoTblSize', NvU32, 0), ('dmaInfoTbl', c.Array[NV2080_CTRL_DMA_INFO, Literal[256]], 4)]) +NV2080_CTRL_DMA_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_DMA_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_DMA_UPDATE_COMPTAG_INFO_TILE_INFO(c.Struct): + SIZE = 12 + srcAddr: int + dstAddr: int + relComptagIndex: int +struct_NV2080_CTRL_DMA_UPDATE_COMPTAG_INFO_TILE_INFO.register_fields([('srcAddr', NvU32, 0), ('dstAddr', NvU32, 4), ('relComptagIndex', NvU16, 8)]) +NV2080_CTRL_DMA_UPDATE_COMPTAG_INFO_TILE_INFO: TypeAlias = struct_NV2080_CTRL_DMA_UPDATE_COMPTAG_INFO_TILE_INFO +@c.record +class struct_NV2080_CTRL_DMABUF_MEM_HANDLE_INFO(c.Struct): + SIZE = 24 + hMemory: int + offset: int + size: int +struct_NV2080_CTRL_DMABUF_MEM_HANDLE_INFO.register_fields([('hMemory', NvHandle, 0), ('offset', NvU64, 8), ('size', NvU64, 16)]) +NV2080_CTRL_DMABUF_MEM_HANDLE_INFO: TypeAlias = struct_NV2080_CTRL_DMABUF_MEM_HANDLE_INFO +@c.record +class struct_NV2080_CTRL_DMABUF_EXPORT_MEM_PARAMS(c.Struct): + SIZE = 3104 + fd: int + totalObjects: int + numObjects: int + index: int + totalSize: int + mappingType: int + bAllowMmap: int + handles: c.Array[struct_NV2080_CTRL_DMABUF_MEM_HANDLE_INFO, Literal[128]] +struct_NV2080_CTRL_DMABUF_EXPORT_MEM_PARAMS.register_fields([('fd', NvS32, 0), ('totalObjects', NvU32, 4), ('numObjects', NvU32, 8), ('index', NvU32, 12), ('totalSize', NvU64, 16), ('mappingType', NvU8, 24), ('bAllowMmap', NvBool, 25), ('handles', c.Array[NV2080_CTRL_DMABUF_MEM_HANDLE_INFO, Literal[128]], 32)]) +NV2080_CTRL_DMABUF_EXPORT_MEM_PARAMS: TypeAlias = struct_NV2080_CTRL_DMABUF_EXPORT_MEM_PARAMS +@c.record +class struct_NV2080_CTRL_ECC_GET_CLIENT_EXPOSED_COUNTERS_PARAMS(c.Struct): + SIZE = 96 + sramParityUncorrectedUnique: int + sramSecDedUncorrectedUnique: int + sramCorrectedUnique: int + dramUncorrectedTotal: int + dramCorrectedTotal: int + lastClearedTimestamp: int + sramBucketL2: int + sramBucketSM: int + sramBucketPcie: int + sramBucketFirmware: int + sramBucketOther: int + sramErrorThresholdExceeded: int +struct_NV2080_CTRL_ECC_GET_CLIENT_EXPOSED_COUNTERS_PARAMS.register_fields([('sramParityUncorrectedUnique', NvU64, 0), ('sramSecDedUncorrectedUnique', NvU64, 8), ('sramCorrectedUnique', NvU64, 16), ('dramUncorrectedTotal', NvU64, 24), ('dramCorrectedTotal', NvU64, 32), ('lastClearedTimestamp', NvU32, 40), ('sramBucketL2', NvU64, 48), ('sramBucketSM', NvU64, 56), ('sramBucketPcie', NvU64, 64), ('sramBucketFirmware', NvU64, 72), ('sramBucketOther', NvU64, 80), ('sramErrorThresholdExceeded', NvBool, 88)]) +NV2080_CTRL_ECC_GET_CLIENT_EXPOSED_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_ECC_GET_CLIENT_EXPOSED_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_ECC_GET_VOLATILE_COUNTS_PARAMS(c.Struct): + SIZE = 40 + sramCorUni: int + sramUncParityUni: int + sramUncSecDedUni: int + dramCorTot: int + dramUncTot: int +struct_NV2080_CTRL_ECC_GET_VOLATILE_COUNTS_PARAMS.register_fields([('sramCorUni', NvU64, 0), ('sramUncParityUni', NvU64, 8), ('sramUncSecDedUni', NvU64, 16), ('dramCorTot', NvU64, 24), ('dramUncTot', NvU64, 32)]) +NV2080_CTRL_ECC_GET_VOLATILE_COUNTS_PARAMS: TypeAlias = struct_NV2080_CTRL_ECC_GET_VOLATILE_COUNTS_PARAMS +@c.record +class struct_eccLocation(c.Struct): + SIZE = 12 + location: int + sublocation: int + extlocation: int +struct_eccLocation.register_fields([('location', NvU32, 0), ('sublocation', NvU32, 4), ('extlocation', NvU32, 8)]) +eccLocation: TypeAlias = struct_eccLocation +@c.record +class struct_NV2080_CTRL_ECC_SRAM_UNIQUE_UNCORR_COUNTS_ENTRY(c.Struct): + SIZE = 28 + unit: int + location: struct_eccLocation + address: int + bIsParity: int + count: int +struct_NV2080_CTRL_ECC_SRAM_UNIQUE_UNCORR_COUNTS_ENTRY.register_fields([('unit', NvU32, 0), ('location', eccLocation, 4), ('address', NvU32, 16), ('bIsParity', NvBool, 20), ('count', NvU32, 24)]) +NV2080_CTRL_ECC_SRAM_UNIQUE_UNCORR_COUNTS_ENTRY: TypeAlias = struct_NV2080_CTRL_ECC_SRAM_UNIQUE_UNCORR_COUNTS_ENTRY +@c.record +class struct_NV2080_CTRL_ECC_GET_SRAM_UNIQUE_UNCORR_COUNTS_PARAMS(c.Struct): + SIZE = 16804 + entryCount: int + entries: c.Array[struct_NV2080_CTRL_ECC_SRAM_UNIQUE_UNCORR_COUNTS_ENTRY, Literal[600]] +struct_NV2080_CTRL_ECC_GET_SRAM_UNIQUE_UNCORR_COUNTS_PARAMS.register_fields([('entryCount', NvU32, 0), ('entries', c.Array[NV2080_CTRL_ECC_SRAM_UNIQUE_UNCORR_COUNTS_ENTRY, Literal[600]], 4)]) +NV2080_CTRL_ECC_GET_SRAM_UNIQUE_UNCORR_COUNTS_PARAMS: TypeAlias = struct_NV2080_CTRL_ECC_GET_SRAM_UNIQUE_UNCORR_COUNTS_PARAMS +@c.record +class struct_NV2080_CTRL_ECC_INJECT_ERROR_PARAMS(c.Struct): + SIZE = 32 + unit: int + errorType: int + location: struct_eccLocation + flags: int + address: int +struct_NV2080_CTRL_ECC_INJECT_ERROR_PARAMS.register_fields([('unit', NvU32, 0), ('errorType', NvU8, 4), ('location', eccLocation, 8), ('flags', NvU32, 20), ('address', NvU64, 24)]) +NV2080_CTRL_ECC_INJECT_ERROR_PARAMS: TypeAlias = struct_NV2080_CTRL_ECC_INJECT_ERROR_PARAMS +@c.record +class struct_NV2080_CTRL_ECC_GET_REPAIR_STATUS_PARAMS(c.Struct): + SIZE = 2 + bTpcRepairPending: int + bChannelRepairPending: int +struct_NV2080_CTRL_ECC_GET_REPAIR_STATUS_PARAMS.register_fields([('bTpcRepairPending', NvBool, 0), ('bChannelRepairPending', NvBool, 1)]) +NV2080_CTRL_ECC_GET_REPAIR_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_ECC_GET_REPAIR_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_ECC_INJECTION_SUPPORTED_PARAMS(c.Struct): + SIZE = 20 + unit: int + location: struct_eccLocation + bCorrectableSupported: int + bUncorrectableSupported: int +struct_NV2080_CTRL_ECC_INJECTION_SUPPORTED_PARAMS.register_fields([('unit', NvU32, 0), ('location', eccLocation, 4), ('bCorrectableSupported', NvBool, 16), ('bUncorrectableSupported', NvBool, 17)]) +NV2080_CTRL_ECC_INJECTION_SUPPORTED_PARAMS: TypeAlias = struct_NV2080_CTRL_ECC_INJECTION_SUPPORTED_PARAMS +@c.record +class struct_NV2080_CTRL_ECC_GET_UNREPAIRABLE_MEMORY_FLAG_PARAMS(c.Struct): + SIZE = 1 + bUnrepairableMemory: int +struct_NV2080_CTRL_ECC_GET_UNREPAIRABLE_MEMORY_FLAG_PARAMS.register_fields([('bUnrepairableMemory', NvBool, 0)]) +NV2080_CTRL_ECC_GET_UNREPAIRABLE_MEMORY_FLAG_PARAMS: TypeAlias = struct_NV2080_CTRL_ECC_GET_UNREPAIRABLE_MEMORY_FLAG_PARAMS +@c.record +class struct_NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS(c.Struct): + SIZE = 20 + event: int + action: int + bNotifyState: int + info32: int + info16: int +struct_NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS.register_fields([('event', NvU32, 0), ('action', NvU32, 4), ('bNotifyState', NvBool, 8), ('info32', NvU32, 12), ('info16', NvU16, 16)]) +NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS +enum_NV2080_EVENT_HDACODEC_DSTATE: dict[int, str] = {(NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D0:=0): 'NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D0', (NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D1:=1): 'NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D1', (NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D2:=2): 'NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D2', (NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D3_HOT:=3): 'NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D3_HOT', (NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D3_COLD:=4): 'NV2080_EVENT_HDACODEC_DEVICE_DSTATE_D3_COLD', (NV2080_EVENT_HDACODEC_DEVICE_DSTATE_DSTATE_MAX:=5): 'NV2080_EVENT_HDACODEC_DEVICE_DSTATE_DSTATE_MAX'} +NV2080_EVENT_HDACODEC_DSTATE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_EVENT_SET_MEMORY_NOTIFIES_PARAMS(c.Struct): + SIZE = 4 + hMemory: int +struct_NV2080_CTRL_EVENT_SET_MEMORY_NOTIFIES_PARAMS.register_fields([('hMemory', NvHandle, 0)]) +NV2080_CTRL_EVENT_SET_MEMORY_NOTIFIES_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_SET_MEMORY_NOTIFIES_PARAMS +@c.record +class struct_NV2080_CTRL_EVENT_SET_SEMAPHORE_MEMORY_PARAMS(c.Struct): + SIZE = 8 + hSemMemory: int + semOffset: int +struct_NV2080_CTRL_EVENT_SET_SEMAPHORE_MEMORY_PARAMS.register_fields([('hSemMemory', NvHandle, 0), ('semOffset', NvU32, 4)]) +NV2080_CTRL_EVENT_SET_SEMAPHORE_MEMORY_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_SET_SEMAPHORE_MEMORY_PARAMS +@c.record +class struct_NV2080_CTRL_EVENT_SET_GUEST_MSI_PARAMS(c.Struct): + SIZE = 48 + guestMSIAddr: int + guestMSIData: int + hSemMemory: int + isReset: int + vgpuUuid: c.Array[ctypes.c_ubyte, Literal[16]] + domainId: int +struct_NV2080_CTRL_EVENT_SET_GUEST_MSI_PARAMS.register_fields([('guestMSIAddr', NvU64, 0), ('guestMSIData', NvU32, 8), ('hSemMemory', NvHandle, 12), ('isReset', NvBool, 16), ('vgpuUuid', c.Array[NvU8, Literal[16]], 17), ('domainId', NvU64, 40)]) +NV2080_CTRL_EVENT_SET_GUEST_MSI_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_SET_GUEST_MSI_PARAMS +@c.record +class struct_NV2080_CTRL_EVENT_SET_SEMA_MEM_VALIDATION_PARAMS(c.Struct): + SIZE = 8 + hSemMemory: int + isSemaMemValidationEnabled: int +struct_NV2080_CTRL_EVENT_SET_SEMA_MEM_VALIDATION_PARAMS.register_fields([('hSemMemory', NvHandle, 0), ('isSemaMemValidationEnabled', NvBool, 4)]) +NV2080_CTRL_EVENT_SET_SEMA_MEM_VALIDATION_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_SET_SEMA_MEM_VALIDATION_PARAMS +@c.record +class struct_NV2080_CTRL_EVENT_SET_TRIGGER_FIFO_PARAMS(c.Struct): + SIZE = 4 + hEvent: int +struct_NV2080_CTRL_EVENT_SET_TRIGGER_FIFO_PARAMS.register_fields([('hEvent', NvHandle, 0)]) +NV2080_CTRL_EVENT_SET_TRIGGER_FIFO_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_SET_TRIGGER_FIFO_PARAMS +enum_NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD: dict[int, str] = {(NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD_FULL:=0): 'NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD_FULL', (NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD_SIMPLE:=1): 'NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD_SIMPLE', (NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD_CUSTOM:=2): 'NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD_CUSTOM'} +NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_PARAMS(c.Struct): + SIZE = 20 + hEventBuffer: int + recordSize: int + levelOfDetail: int + eventFilter: int + bAllUsers: int +struct_NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_PARAMS.register_fields([('hEventBuffer', NvHandle, 0), ('recordSize', NvU32, 4), ('levelOfDetail', NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_LOD, 8), ('eventFilter', NvU32, 12), ('bAllUsers', NvBool, 16)]) +NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_PARAMS +@c.record +class struct_NV2080_CTRL_EVENT_RATS_GSP_TRACE_BIND_EVTBUF_PARAMS(c.Struct): + SIZE = 32 + hEventBuffer: int + tracepointMask: int + gspLoggingBufferSize: int + gspLoggingBufferWatermark: int + targetTask: int +struct_NV2080_CTRL_EVENT_RATS_GSP_TRACE_BIND_EVTBUF_PARAMS.register_fields([('hEventBuffer', NvHandle, 0), ('tracepointMask', NvU64, 8), ('gspLoggingBufferSize', NvU32, 16), ('gspLoggingBufferWatermark', NvU32, 20), ('targetTask', NvU16, 24)]) +NV2080_CTRL_EVENT_RATS_GSP_TRACE_BIND_EVTBUF_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_RATS_GSP_TRACE_BIND_EVTBUF_PARAMS +@c.record +class struct_NV2080_CTRL_EVENT_NOCAT_BIND_EVTBUF_PARAMS(c.Struct): + SIZE = 4 + hEventBuffer: int +struct_NV2080_CTRL_EVENT_NOCAT_BIND_EVTBUF_PARAMS.register_fields([('hEventBuffer', NvHandle, 0)]) +NV2080_CTRL_EVENT_NOCAT_BIND_EVTBUF_PARAMS: TypeAlias = struct_NV2080_CTRL_EVENT_NOCAT_BIND_EVTBUF_PARAMS +NV2080_CTRL_FB_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_FB_GET_INFO_PARAMS(c.Struct): + SIZE = 16 + fbInfoListSize: int + fbInfoList: ctypes.c_void_p +struct_NV2080_CTRL_FB_GET_INFO_PARAMS.register_fields([('fbInfoListSize', NvU32, 0), ('fbInfoList', NvP64, 8)]) +NV2080_CTRL_FB_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_INFO_V2_PARAMS(c.Struct): + SIZE = 1028 + fbInfoListSize: int + fbInfoList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[128]] +struct_NV2080_CTRL_FB_GET_INFO_V2_PARAMS.register_fields([('fbInfoListSize', NvU32, 0), ('fbInfoList', c.Array[NV2080_CTRL_FB_INFO, Literal[128]], 4)]) +NV2080_CTRL_FB_GET_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_BAR1_OFFSET_PARAMS(c.Struct): + SIZE = 16 + cpuVirtAddress: ctypes.c_void_p + gpuVirtAddress: int +struct_NV2080_CTRL_FB_GET_BAR1_OFFSET_PARAMS.register_fields([('cpuVirtAddress', NvP64, 0), ('gpuVirtAddress', NvU64, 8)]) +NV2080_CTRL_FB_GET_BAR1_OFFSET_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_BAR1_OFFSET_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_CALIBRATION_LOCK_FAILED_PARAMS(c.Struct): + SIZE = 24 + flags: int + driveStrengthRiseCount: int + driveStrengthFallCount: int + driveStrengthTermCount: int + slewStrengthRiseCount: int + slewStrengthFallCount: int +struct_NV2080_CTRL_FB_GET_CALIBRATION_LOCK_FAILED_PARAMS.register_fields([('flags', NvU32, 0), ('driveStrengthRiseCount', NvU32, 4), ('driveStrengthFallCount', NvU32, 8), ('driveStrengthTermCount', NvU32, 12), ('slewStrengthRiseCount', NvU32, 16), ('slewStrengthFallCount', NvU32, 20)]) +NV2080_CTRL_FB_GET_CALIBRATION_LOCK_FAILED_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_CALIBRATION_LOCK_FAILED_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_PARAMS(c.Struct): + SIZE = 4 + flags: int +struct_NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_PARAMS.register_fields([('flags', NvU32, 0)]) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FLUSH_GPU_CACHE_PARAMS(c.Struct): + SIZE = 4024 + addressArray: c.Array[ctypes.c_uint64, Literal[500]] + addressArraySize: int + addressAlign: int + memBlockSizeBytes: int + flags: int +struct_NV2080_CTRL_FB_FLUSH_GPU_CACHE_PARAMS.register_fields([('addressArray', c.Array[NvU64, Literal[500]], 0), ('addressArraySize', NvU32, 4000), ('addressAlign', NvU32, 4004), ('memBlockSizeBytes', NvU64, 4008), ('flags', NvU32, 4016)]) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FLUSH_GPU_CACHE_PARAMS +@c.record +class struct_NV2080_CTRL_FB_IS_KIND_PARAMS(c.Struct): + SIZE = 12 + operation: int + kind: int + result: int +struct_NV2080_CTRL_FB_IS_KIND_PARAMS.register_fields([('operation', NvU32, 0), ('kind', NvU32, 4), ('result', NvBool, 8)]) +NV2080_CTRL_FB_IS_KIND_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_IS_KIND_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_GPU_CACHE_INFO_PARAMS(c.Struct): + SIZE = 16 + powerState: int + writeMode: int + bypassMode: int + rcmState: int +struct_NV2080_CTRL_FB_GET_GPU_CACHE_INFO_PARAMS.register_fields([('powerState', NvU32, 0), ('writeMode', NvU32, 4), ('bypassMode', NvU32, 8), ('rcmState', NvU32, 12)]) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_GPU_CACHE_INFO_PARAMS +enum_NV2080_FB_REGION_TAG: dict[int, str] = {(NV2080_FB_REGION_TAG_NONE:=0): 'NV2080_FB_REGION_TAG_NONE', (NV2080_FB_REGION_TAG_GSP_CARVEOUT:=1): 'NV2080_FB_REGION_TAG_GSP_CARVEOUT', (NV2080_FB_REGION_TAG_CPU_RM_RESERVED:=2): 'NV2080_FB_REGION_TAG_CPU_RM_RESERVED', (NV2080_FB_REGION_TAG_CPU_RM_RESERVED_HEAP:=3): 'NV2080_FB_REGION_TAG_CPU_RM_RESERVED_HEAP', (NV2080_FB_REGION_TAG_GSP_RM_RESERVED:=4): 'NV2080_FB_REGION_TAG_GSP_RM_RESERVED', (NV2080_FB_REGION_TAG_GSP_RM_RESERVED_HEAP:=5): 'NV2080_FB_REGION_TAG_GSP_RM_RESERVED_HEAP'} +NV2080_FB_REGION_TAG: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO(c.Struct): + SIZE = 56 + base: int + limit: int + reserved: int + performance: int + supportCompressed: int + supportISO: int + bProtected: int + blackList: c.Array[ctypes.c_ubyte, Literal[18]] + regionTag: int +struct_NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO.register_fields([('base', NvU64, 0), ('limit', NvU64, 8), ('reserved', NvU64, 16), ('performance', NvU32, 24), ('supportCompressed', NvBool, 28), ('supportISO', NvBool, 29), ('bProtected', NvBool, 30), ('blackList', c.Array[NvBool, Literal[18]], 31), ('regionTag', NV2080_FB_REGION_TAG, 52)]) +NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO: TypeAlias = struct_NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO +@c.record +class struct_NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS(c.Struct): + SIZE = 904 + numFBRegions: int + fbRegion: c.Array[struct_NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO, Literal[16]] +struct_NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS.register_fields([('numFBRegions', NvU32, 0), ('fbRegion', c.Array[NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO, Literal[16]], 8)]) +NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO(c.Struct): + SIZE = 32 + pageAddressWithEccOn: int + pageAddressWithEccOff: int + rbcAddress: int + source: int + status: int + timestamp: int +struct_NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO.register_fields([('pageAddressWithEccOn', NvU64, 0), ('pageAddressWithEccOff', NvU64, 8), ('rbcAddress', NvU32, 16), ('source', NvU32, 20), ('status', NvU32, 24), ('timestamp', NvU32, 28)]) +NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO: TypeAlias = struct_NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO +@c.record +class struct_NV2080_CTRL_FB_OFFLINE_PAGES_PARAMS(c.Struct): + SIZE = 2064 + offlined: c.Array[struct_NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO, Literal[64]] + pageSize: int + validEntries: int + numPagesAdded: int +struct_NV2080_CTRL_FB_OFFLINE_PAGES_PARAMS.register_fields([('offlined', c.Array[NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO, Literal[64]], 0), ('pageSize', NvU32, 2048), ('validEntries', NvU32, 2052), ('numPagesAdded', NvU32, 2056)]) +NV2080_CTRL_FB_OFFLINE_PAGES_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_OFFLINE_PAGES_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_OFFLINED_PAGES_PARAMS(c.Struct): + SIZE = 2056 + offlined: c.Array[struct_NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO, Literal[64]] + validEntries: int + bRetirementPending: int + retirementPending: int +struct_NV2080_CTRL_FB_GET_OFFLINED_PAGES_PARAMS.register_fields([('offlined', c.Array[NV2080_CTRL_FB_OFFLINED_ADDRESS_INFO, Literal[64]], 0), ('validEntries', NvU32, 2048), ('bRetirementPending', NvBool, 2052), ('retirementPending', NvU8, 2053)]) +NV2080_CTRL_FB_GET_OFFLINED_PAGES_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_OFFLINED_PAGES_PARAMS +enum_NV2080_CTRL_CMD_FB_ACR_QUERY_TYPE: dict[int, str] = {(NV2080_CTRL_CMD_FB_ACR_QUERY_GET_CLIENT_REGION_STATUS:=0): 'NV2080_CTRL_CMD_FB_ACR_QUERY_GET_CLIENT_REGION_STATUS', (NV2080_CTRL_CMD_FB_ACR_QUERY_GET_REGION_PROPERTY:=1): 'NV2080_CTRL_CMD_FB_ACR_QUERY_GET_REGION_PROPERTY', (NV2080_CTRL_CMD_FB_ACR_QUERY_GET_FALCON_STATUS:=2): 'NV2080_CTRL_CMD_FB_ACR_QUERY_GET_FALCON_STATUS'} +NV2080_CTRL_CMD_FB_ACR_QUERY_TYPE: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_CMD_FB_ACR_QUERY_ERROR_CODE: dict[int, str] = {(NV2080_CTRL_CMD_FB_ACR_QUERY_ERROR_NONE:=0): 'NV2080_CTRL_CMD_FB_ACR_QUERY_ERROR_NONE', (NV2080_CTRL_CMD_FB_ACR_QUERY_ERROR_INVALID_CLIENT_REQUEST:=1): 'NV2080_CTRL_CMD_FB_ACR_QUERY_ERROR_INVALID_CLIENT_REQUEST'} +NV2080_CTRL_CMD_FB_ACR_QUERY_ERROR_CODE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_ACR_REQUEST_PARAMS(c.Struct): + SIZE = 16 + clientId: int + reqReadMask: int + reqWriteMask: int + regionSize: int +struct_ACR_REQUEST_PARAMS.register_fields([('clientId', NvU32, 0), ('reqReadMask', NvU32, 4), ('reqWriteMask', NvU32, 8), ('regionSize', NvU32, 12)]) +ACR_REQUEST_PARAMS: TypeAlias = struct_ACR_REQUEST_PARAMS +@c.record +class struct_ACR_REGION_ID_PROP(c.Struct): + SIZE = 32 + regionId: int + readMask: int + writeMask: int + regionSize: int + clientMask: int + physicalAddress: int +struct_ACR_REGION_ID_PROP.register_fields([('regionId', NvU32, 0), ('readMask', NvU32, 4), ('writeMask', NvU32, 8), ('regionSize', NvU32, 12), ('clientMask', NvU32, 16), ('physicalAddress', NvU64, 24)]) +ACR_REGION_ID_PROP: TypeAlias = struct_ACR_REGION_ID_PROP +@c.record +class struct_ACR_STATUS_PARAMS(c.Struct): + SIZE = 16 + allocStatus: int + regionId: int + physicalAddress: int +struct_ACR_STATUS_PARAMS.register_fields([('allocStatus', NvU32, 0), ('regionId', NvU32, 4), ('physicalAddress', NvU64, 8)]) +ACR_STATUS_PARAMS: TypeAlias = struct_ACR_STATUS_PARAMS +@c.record +class struct_ACR_REGION_HANDLE(c.Struct): + SIZE = 20 + hClient: int + hParent: int + hMemory: int + hClass: int + hDevice: int +struct_ACR_REGION_HANDLE.register_fields([('hClient', NvHandle, 0), ('hParent', NvHandle, 4), ('hMemory', NvHandle, 8), ('hClass', NvU32, 12), ('hDevice', NvHandle, 16)]) +ACR_REGION_HANDLE: TypeAlias = struct_ACR_REGION_HANDLE +@c.record +class struct_ACR_FALCON_LS_STATUS(c.Struct): + SIZE = 4 + falconId: int + bIsInLs: int +struct_ACR_FALCON_LS_STATUS.register_fields([('falconId', NvU16, 0), ('bIsInLs', NvBool, 2)]) +ACR_FALCON_LS_STATUS: TypeAlias = struct_ACR_FALCON_LS_STATUS +@c.record +class struct_NV2080_CTRL_CMD_FB_QUERY_ACR_REGION_PARAMS(c.Struct): + SIZE = 96 + queryType: int + errorCode: int + acrRegionIdProp: struct_ACR_REGION_ID_PROP + clientReq: struct_ACR_REQUEST_PARAMS + clientReqStatus: struct_ACR_STATUS_PARAMS + handle: struct_ACR_REGION_HANDLE + falconStatus: struct_ACR_FALCON_LS_STATUS +struct_NV2080_CTRL_CMD_FB_QUERY_ACR_REGION_PARAMS.register_fields([('queryType', NV2080_CTRL_CMD_FB_ACR_QUERY_TYPE, 0), ('errorCode', NV2080_CTRL_CMD_FB_ACR_QUERY_ERROR_CODE, 4), ('acrRegionIdProp', ACR_REGION_ID_PROP, 8), ('clientReq', ACR_REQUEST_PARAMS, 40), ('clientReqStatus', ACR_STATUS_PARAMS, 56), ('handle', ACR_REGION_HANDLE, 72), ('falconStatus', ACR_FALCON_LS_STATUS, 92)]) +NV2080_CTRL_CMD_FB_QUERY_ACR_REGION_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_QUERY_ACR_REGION_PARAMS +@c.record +class struct_NV2080_CTRL_FB_CLEAR_OFFLINED_PAGES_PARAMS(c.Struct): + SIZE = 4 + sourceMask: int +struct_NV2080_CTRL_FB_CLEAR_OFFLINED_PAGES_PARAMS.register_fields([('sourceMask', NvU32, 0)]) +NV2080_CTRL_FB_CLEAR_OFFLINED_PAGES_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_CLEAR_OFFLINED_PAGES_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_INFO_PARAMS(c.Struct): + SIZE = 16 + pCompBitCopyObj: ctypes.c_void_p + pSwizzleParams: ctypes.c_void_p +struct_NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_INFO_PARAMS.register_fields([('pCompBitCopyObj', NvP64, 0), ('pSwizzleParams', NvP64, 8)]) +NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_LTC_INFO_FOR_FBP_PARAMS(c.Struct): + SIZE = 20 + fbpIndex: int + ltcMask: int + ltcCount: int + ltsMask: int + ltsCount: int +struct_NV2080_CTRL_FB_GET_LTC_INFO_FOR_FBP_PARAMS.register_fields([('fbpIndex', NvU8, 0), ('ltcMask', NvU32, 4), ('ltcCount', NvU32, 8), ('ltsMask', NvU32, 12), ('ltsCount', NvU32, 16)]) +NV2080_CTRL_FB_GET_LTC_INFO_FOR_FBP_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_LTC_INFO_FOR_FBP_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_CONTEXT_PARAMS(c.Struct): + SIZE = 72 + CBCBaseAddress: int + backingStorePA: int + backingStoreVA: c.POINTER[ctypes.c_ubyte] + backingStoreChunkPA: int + backingStoreChunkVA: c.POINTER[ctypes.c_ubyte] + backingStoreChunkSize: int + cacheWriteBitMap: c.POINTER[ctypes.c_ubyte] + backingStoreChunkOverfetch: int + PageSizeSrc: int + PageSizeDest: int +struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_CONTEXT_PARAMS.register_fields([('CBCBaseAddress', NvU32, 0), ('backingStorePA', NvU64, 8), ('backingStoreVA', c.POINTER[NvU8], 16), ('backingStoreChunkPA', NvU64, 24), ('backingStoreChunkVA', c.POINTER[NvU8], 32), ('backingStoreChunkSize', NvU32, 40), ('cacheWriteBitMap', c.POINTER[NvU8], 48), ('backingStoreChunkOverfetch', NvBool, 56), ('PageSizeSrc', NvU32, 60), ('PageSizeDest', NvU32, 64)]) +NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_CONTEXT_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_CONTEXT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_GET_COMPBITS_PARAMS(c.Struct): + SIZE = 40 + fcbits: c.POINTER[ctypes.c_uint32] + compbits: c.POINTER[ctypes.c_uint32] + dataPhysicalStart: int + surfaceOffset: int + comptagLine: int + upper64KBCompbitSel: int +struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_GET_COMPBITS_PARAMS.register_fields([('fcbits', c.POINTER[NvU32], 0), ('compbits', c.POINTER[NvU32], 8), ('dataPhysicalStart', NvU64, 16), ('surfaceOffset', NvU64, 24), ('comptagLine', NvU32, 32), ('upper64KBCompbitSel', NvBool, 36)]) +NV2080_CTRL_CMD_FB_COMPBITCOPY_GET_COMPBITS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_GET_COMPBITS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_PUT_COMPBITS_PARAMS(c.Struct): + SIZE = 40 + fcbits: int + compbits: int + writeFc: int + dataPhysicalStart: int + surfaceOffset: int + comptagLine: int + upper64KBCompbitSel: int +struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_PUT_COMPBITS_PARAMS.register_fields([('fcbits', NvU32, 0), ('compbits', NvU32, 4), ('writeFc', NvBool, 8), ('dataPhysicalStart', NvU64, 16), ('surfaceOffset', NvU64, 24), ('comptagLine', NvU32, 32), ('upper64KBCompbitSel', NvBool, 36)]) +NV2080_CTRL_CMD_FB_COMPBITCOPY_PUT_COMPBITS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_PUT_COMPBITS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_READ_COMPBITS64KB_PARAMS(c.Struct): + SIZE = 32 + SrcDataPhysicalStart: int + SrcComptagLine: int + page64KB: int + compbitBuffer: c.POINTER[ctypes.c_uint32] + upper64KBCompbitSel: int +struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_READ_COMPBITS64KB_PARAMS.register_fields([('SrcDataPhysicalStart', NvU64, 0), ('SrcComptagLine', NvU32, 8), ('page64KB', NvU32, 12), ('compbitBuffer', c.POINTER[NvU32], 16), ('upper64KBCompbitSel', NvBool, 24)]) +NV2080_CTRL_CMD_FB_COMPBITCOPY_READ_COMPBITS64KB_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_READ_COMPBITS64KB_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_WRITE_COMPBITS64KB_PARAMS(c.Struct): + SIZE = 32 + DstDataPhysicalStart: int + DstComptagLine: int + page64KB: int + compbitBuffer: c.POINTER[ctypes.c_uint32] + upper64KBCompbitSel: int +struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_WRITE_COMPBITS64KB_PARAMS.register_fields([('DstDataPhysicalStart', NvU64, 0), ('DstComptagLine', NvU32, 8), ('page64KB', NvU32, 12), ('compbitBuffer', c.POINTER[NvU32], 16), ('upper64KBCompbitSel', NvBool, 24)]) +NV2080_CTRL_CMD_FB_COMPBITCOPY_WRITE_COMPBITS64KB_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_WRITE_COMPBITS64KB_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_FORCE_BAR1_PARAMS(c.Struct): + SIZE = 1 + bForceBar1: int +struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_FORCE_BAR1_PARAMS.register_fields([('bForceBar1', NvBool, 0)]) +NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_FORCE_BAR1_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_FORCE_BAR1_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_GET_AMAP_CONF_PARAMS(c.Struct): + SIZE = 16 + pAmapConfParams: ctypes.c_void_p + pCbcSwizzleParams: ctypes.c_void_p +struct_NV2080_CTRL_CMD_FB_GET_AMAP_CONF_PARAMS.register_fields([('pAmapConfParams', NvP64, 0), ('pCbcSwizzleParams', NvP64, 8)]) +NV2080_CTRL_CMD_FB_GET_AMAP_CONF_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_GET_AMAP_CONF_PARAMS +enum_CTRL_CMD_FB_CBC_OP: dict[int, str] = {(CTRL_CMD_FB_CBC_OP_CLEAN:=0): 'CTRL_CMD_FB_CBC_OP_CLEAN', (CTRL_CMD_FB_CBC_OP_INVALIDATE:=1): 'CTRL_CMD_FB_CBC_OP_INVALIDATE'} +CTRL_CMD_FB_CBC_OP: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_CMD_FB_CBC_OP_PARAMS(c.Struct): + SIZE = 4 + fbCBCOp: int +struct_NV2080_CTRL_CMD_FB_CBC_OP_PARAMS.register_fields([('fbCBCOp', CTRL_CMD_FB_CBC_OP, 0)]) +NV2080_CTRL_CMD_FB_CBC_OP_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_CBC_OP_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_CTAGS_FOR_CBC_EVICTION_PARAMS(c.Struct): + SIZE = 512 + pCompTags: c.Array[ctypes.c_uint32, Literal[127]] + numCompTags: int +struct_NV2080_CTRL_FB_GET_CTAGS_FOR_CBC_EVICTION_PARAMS.register_fields([('pCompTags', c.Array[NvU32, Literal[127]], 0), ('numCompTags', NvU32, 508)]) +NV2080_CTRL_FB_GET_CTAGS_FOR_CBC_EVICTION_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_CTAGS_FOR_CBC_EVICTION_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_ALLOC_COMP_RESOURCE_PARAMS(c.Struct): + SIZE = 28 + attr: int + attr2: int + size: int + ctagOffset: int + hwResId: int + retCompTagLineMin: int + retCompTagLineMax: int +struct_NV2080_CTRL_CMD_FB_ALLOC_COMP_RESOURCE_PARAMS.register_fields([('attr', NvU32, 0), ('attr2', NvU32, 4), ('size', NvU32, 8), ('ctagOffset', NvU32, 12), ('hwResId', NvU32, 16), ('retCompTagLineMin', NvU32, 20), ('retCompTagLineMax', NvU32, 24)]) +NV2080_CTRL_CMD_FB_ALLOC_COMP_RESOURCE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_ALLOC_COMP_RESOURCE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_FREE_TILE_PARAMS(c.Struct): + SIZE = 4 + hwResId: int +struct_NV2080_CTRL_CMD_FB_FREE_TILE_PARAMS.register_fields([('hwResId', NvU32, 0)]) +NV2080_CTRL_CMD_FB_FREE_TILE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_FREE_TILE_PARAMS +enum_NV2080_CTRL_CMD_FB_VPR_REQUEST_TYPE: dict[int, str] = {(NV2080_CTRL_CMD_FB_SET_VPR:=0): 'NV2080_CTRL_CMD_FB_SET_VPR'} +NV2080_CTRL_CMD_FB_VPR_REQUEST_TYPE: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_CMD_FB_VPR_ERROR_CODE: dict[int, str] = {(NV2080_CTRL_CMD_FB_VPR_ERROR_GENERIC:=0): 'NV2080_CTRL_CMD_FB_VPR_ERROR_GENERIC', (NV2080_CTRL_CMD_FB_VPR_ERROR_INVALID_CLIENT_REQUEST:=1): 'NV2080_CTRL_CMD_FB_VPR_ERROR_INVALID_CLIENT_REQUEST'} +NV2080_CTRL_CMD_FB_VPR_ERROR_CODE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_VPR_REQUEST_PARAMS(c.Struct): + SIZE = 8 + startAddr: int + size: int +struct_VPR_REQUEST_PARAMS.register_fields([('startAddr', NvU32, 0), ('size', NvU32, 4)]) +VPR_REQUEST_PARAMS: TypeAlias = struct_VPR_REQUEST_PARAMS +@c.record +class struct_VPR_STATUS_PARAMS(c.Struct): + SIZE = 4 + status: int +struct_VPR_STATUS_PARAMS.register_fields([('status', NvU32, 0)]) +VPR_STATUS_PARAMS: TypeAlias = struct_VPR_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_SETUP_VPR_REGION_PARAMS(c.Struct): + SIZE = 16 + requestType: int + requestParams: struct_VPR_REQUEST_PARAMS + statusParams: struct_VPR_STATUS_PARAMS +struct_NV2080_CTRL_CMD_FB_SETUP_VPR_REGION_PARAMS.register_fields([('requestType', NV2080_CTRL_CMD_FB_VPR_REQUEST_TYPE, 0), ('requestParams', VPR_REQUEST_PARAMS, 4), ('statusParams', VPR_STATUS_PARAMS, 12)]) +NV2080_CTRL_CMD_FB_SETUP_VPR_REGION_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_SETUP_VPR_REGION_PARAMS +PNV2080_CTRL_CMD_FB_SETUP_VPR_REGION_PARAMS: TypeAlias = c.POINTER[struct_NV2080_CTRL_CMD_FB_SETUP_VPR_REGION_PARAMS] +@c.record +class struct_NV2080_CTRL_FB_GET_CLI_MANAGED_OFFLINED_PAGES_PARAMS(c.Struct): + SIZE = 264 + offlinedPages: c.Array[ctypes.c_uint32, Literal[64]] + pageSize: int + validEntries: int +struct_NV2080_CTRL_FB_GET_CLI_MANAGED_OFFLINED_PAGES_PARAMS.register_fields([('offlinedPages', c.Array[NvU32, Literal[64]], 0), ('pageSize', NvU32, 256), ('validEntries', NvU32, 260)]) +NV2080_CTRL_FB_GET_CLI_MANAGED_OFFLINED_PAGES_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_CLI_MANAGED_OFFLINED_PAGES_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_CONSTRUCT_INFO_PARAMS(c.Struct): + SIZE = 56 + defaultPageSize: int + comptagsPerCacheLine: int + unpackedComptagLinesPerCacheLine: int + compCacheLineSizePerLTC: int + unpackedCompCacheLineSizePerLTC: int + slicesPerLTC: int + numActiveLTCs: int + familyName: int + chipName: int + bitsPerRAMEntry: int + ramBankWidth: int + bitsPerComptagLine: int + ramEntriesPerCompCacheLine: int + comptagLineSize: int +struct_NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_CONSTRUCT_INFO_PARAMS.register_fields([('defaultPageSize', NvU32, 0), ('comptagsPerCacheLine', NvU32, 4), ('unpackedComptagLinesPerCacheLine', NvU32, 8), ('compCacheLineSizePerLTC', NvU32, 12), ('unpackedCompCacheLineSizePerLTC', NvU32, 16), ('slicesPerLTC', NvU32, 20), ('numActiveLTCs', NvU32, 24), ('familyName', NvU32, 28), ('chipName', NvU32, 32), ('bitsPerRAMEntry', NvU32, 36), ('ramBankWidth', NvU32, 40), ('bitsPerComptagLine', NvU32, 44), ('ramEntriesPerCompCacheLine', NvU32, 48), ('comptagLineSize', NvU32, 52)]) +NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_CONSTRUCT_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_CONSTRUCT_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_SET_RRD_PARAMS(c.Struct): + SIZE = 4 + rrd: int +struct_NV2080_CTRL_FB_SET_RRD_PARAMS.register_fields([('rrd', NvU32, 0)]) +NV2080_CTRL_FB_SET_RRD_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_SET_RRD_PARAMS +@c.record +class struct_NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_PARAMS(c.Struct): + SIZE = 1 + limit: int +struct_NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_PARAMS.register_fields([('limit', NvU8, 0)]) +NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_PARAMS +NV2080_CTRL_FB_SET_READ_LIMIT_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_PARAMS +NV2080_CTRL_FB_SET_WRITE_LIMIT_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_PARAMS +@c.record +class struct_NV2080_CTRL_FB_PATCH_PBR_FOR_MINING_PARAMS(c.Struct): + SIZE = 1 + bEnable: int +struct_NV2080_CTRL_FB_PATCH_PBR_FOR_MINING_PARAMS.register_fields([('bEnable', NvBool, 0)]) +NV2080_CTRL_FB_PATCH_PBR_FOR_MINING_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_PATCH_PBR_FOR_MINING_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_MEM_ALIGNMENT_PARAMS(c.Struct): + SIZE = 56 + alignType: int + alignAttr: int + alignInputFlags: int + alignHead: int + alignSize: int + alignHeight: int + alignWidth: int + alignPitch: int + alignPad: int + alignMask: int + alignKind: int + alignAdjust: int + alignAttr2: int +struct_NV2080_CTRL_FB_GET_MEM_ALIGNMENT_PARAMS.register_fields([('alignType', NvU32, 0), ('alignAttr', NvU32, 4), ('alignInputFlags', NvU32, 8), ('alignHead', NvU32, 12), ('alignSize', NvU64, 16), ('alignHeight', NvU32, 24), ('alignWidth', NvU32, 28), ('alignPitch', NvU32, 32), ('alignPad', NvU32, 36), ('alignMask', NvU32, 40), ('alignKind', NvU32, 44), ('alignAdjust', NvU32, 48), ('alignAttr2', NvU32, 52)]) +NV2080_CTRL_FB_GET_MEM_ALIGNMENT_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_MEM_ALIGNMENT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_GET_CBC_BASE_ADDR_PARAMS(c.Struct): + SIZE = 32 + cbcBaseAddress: int + compCacheLineSize: int + backingStoreStartPA: int + backingStoreAllocPA: int + backingStoreChunkOverfetch: int +struct_NV2080_CTRL_CMD_FB_GET_CBC_BASE_ADDR_PARAMS.register_fields([('cbcBaseAddress', NvU32, 0), ('compCacheLineSize', NvU32, 4), ('backingStoreStartPA', NvU64, 8), ('backingStoreAllocPA', NvU64, 16), ('backingStoreChunkOverfetch', NvU32, 24)]) +NV2080_CTRL_CMD_FB_GET_CBC_BASE_ADDR_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_GET_CBC_BASE_ADDR_PARAMS +@c.record +class struct_NV2080_CTRL_FB_ROW_REMAP_ENTRY(c.Struct): + SIZE = 12 + remapRegVal: int + timestamp: int + fbpa: int + sublocation: int + source: int + flags: int +struct_NV2080_CTRL_FB_ROW_REMAP_ENTRY.register_fields([('remapRegVal', NvU32, 0), ('timestamp', NvU32, 4), ('fbpa', NvU8, 8), ('sublocation', NvU8, 9), ('source', NvU8, 10), ('flags', NvU8, 11)]) +NV2080_CTRL_FB_ROW_REMAP_ENTRY: TypeAlias = struct_NV2080_CTRL_FB_ROW_REMAP_ENTRY +@c.record +class struct_NV2080_CTRL_FB_GET_REMAPPED_ROWS_PARAMS(c.Struct): + SIZE = 6152 + entryCount: int + flags: int + entries: c.Array[struct_NV2080_CTRL_FB_ROW_REMAP_ENTRY, Literal[512]] +struct_NV2080_CTRL_FB_GET_REMAPPED_ROWS_PARAMS.register_fields([('entryCount', NvU32, 0), ('flags', NvU8, 4), ('entries', c.Array[NV2080_CTRL_FB_ROW_REMAP_ENTRY, Literal[512]], 8)]) +NV2080_CTRL_FB_GET_REMAPPED_ROWS_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_REMAPPED_ROWS_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_INVALID_QUERY_PARAMS(c.Struct): + SIZE = 24 + data: c.Array[ctypes.c_ubyte, Literal[24]] +struct_NV2080_CTRL_FB_FS_INFO_INVALID_QUERY_PARAMS.register_fields([('data', c.Array[NvU8, Literal[24]], 0)]) +NV2080_CTRL_FB_FS_INFO_INVALID_QUERY_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_INVALID_QUERY_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_FBP_MASK_PARAMS(c.Struct): + SIZE = 16 + swizzId: int + fbpEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_FBP_MASK_PARAMS.register_fields([('swizzId', NvU32, 0), ('fbpEnMask', NvU64, 8)]) +NV2080_CTRL_FB_FS_INFO_FBP_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_FBP_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_LTC_MASK_PARAMS(c.Struct): + SIZE = 8 + fbpIndex: int + ltcEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_LTC_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('ltcEnMask', NvU32, 4)]) +NV2080_CTRL_FB_FS_INFO_LTC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_LTC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_LTS_MASK_PARAMS(c.Struct): + SIZE = 8 + fbpIndex: int + ltsEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_LTS_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('ltsEnMask', NvU32, 4)]) +NV2080_CTRL_FB_FS_INFO_LTS_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_LTS_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_FBPA_MASK_PARAMS(c.Struct): + SIZE = 8 + fbpIndex: int + fbpaEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_FBPA_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('fbpaEnMask', NvU32, 4)]) +NV2080_CTRL_FB_FS_INFO_FBPA_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_FBPA_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_FBPA_SUBP_MASK_PARAMS(c.Struct): + SIZE = 8 + fbpIndex: int + fbpaSubpEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_FBPA_SUBP_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('fbpaSubpEnMask', NvU32, 4)]) +NV2080_CTRL_FB_FS_INFO_FBPA_SUBP_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_FBPA_SUBP_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_FBP_LOGICAL_MAP_PARAMS(c.Struct): + SIZE = 8 + fbpIndex: int + fbpLogicalIndex: int +struct_NV2080_CTRL_FB_FS_INFO_FBP_LOGICAL_MAP_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('fbpLogicalIndex', NvU32, 4)]) +NV2080_CTRL_FB_FS_INFO_FBP_LOGICAL_MAP_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_FBP_LOGICAL_MAP_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_ROP_MASK_PARAMS(c.Struct): + SIZE = 8 + fbpIndex: int + ropEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_ROP_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('ropEnMask', NvU32, 4)]) +NV2080_CTRL_FB_FS_INFO_ROP_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_ROP_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_SYS_MASK_PARAMS(c.Struct): + SIZE = 16 + swizzId: int + sysEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_SYS_MASK_PARAMS.register_fields([('swizzId', NvU32, 0), ('sysEnMask', NvU64, 8)]) +NV2080_CTRL_FB_FS_INFO_SYS_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_SYS_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTC_MASK_PARAMS(c.Struct): + SIZE = 12 + fbpIndex: int + swizzId: int + ltcEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTC_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('swizzId', NvU32, 4), ('ltcEnMask', NvU32, 8)]) +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTS_MASK_PARAMS(c.Struct): + SIZE = 12 + fbpIndex: int + swizzId: int + ltsEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTS_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('swizzId', NvU32, 4), ('ltsEnMask', NvU32, 8)]) +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTS_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTS_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_MASK_PARAMS(c.Struct): + SIZE = 12 + fbpIndex: int + swizzId: int + fbpaEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('swizzId', NvU32, 4), ('fbpaEnMask', NvU32, 8)]) +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_ROP_MASK_PARAMS(c.Struct): + SIZE = 12 + fbpIndex: int + swizzId: int + ropEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_ROP_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('swizzId', NvU32, 4), ('ropEnMask', NvU32, 8)]) +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_ROP_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_ROP_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_SUBP_MASK_PARAMS(c.Struct): + SIZE = 16 + fbpIndex: int + swizzId: int + fbpaSubpEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_SUBP_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('swizzId', NvU32, 4), ('fbpaSubpEnMask', NvU64, 8)]) +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_SUBP_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_SUBP_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTC_MASK_PARAMS(c.Struct): + SIZE = 8 + sysIdx: int + sysl2LtcEnMask: int +struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTC_MASK_PARAMS.register_fields([('sysIdx', NvU32, 0), ('sysl2LtcEnMask', NvU32, 4)]) +NV2080_CTRL_SYSL2_FS_INFO_SYSLTC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTS_MASK_PARAMS(c.Struct): + SIZE = 16 + sysIdx: int + sysl2LtsEnMask: int +struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTS_MASK_PARAMS.register_fields([('sysIdx', NvU32, 0), ('sysl2LtsEnMask', NvU64, 8)]) +NV2080_CTRL_SYSL2_FS_INFO_SYSLTS_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTS_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_PAC_MASK_PARAMS(c.Struct): + SIZE = 8 + fbpIndex: int + pacEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_PAC_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('pacEnMask', NvU32, 4)]) +NV2080_CTRL_FB_FS_INFO_PAC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_PAC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_LOGICAL_LTC_MASK_PARAMS(c.Struct): + SIZE = 16 + fbpIndex: int + logicalLtcEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_LOGICAL_LTC_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('logicalLtcEnMask', NvU64, 8)]) +NV2080_CTRL_FB_FS_INFO_LOGICAL_LTC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_LOGICAL_LTC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LOGICAL_LTC_MASK_PARAMS(c.Struct): + SIZE = 16 + fbpIndex: int + swizzId: int + logicalLtcEnMask: int +struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LOGICAL_LTC_MASK_PARAMS.register_fields([('fbpIndex', NvU32, 0), ('swizzId', NvU32, 4), ('logicalLtcEnMask', NvU64, 8)]) +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LOGICAL_LTC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LOGICAL_LTC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_QUERY(c.Struct): + SIZE = 32 + queryType: int + reserved: c.Array[ctypes.c_ubyte, Literal[2]] + status: int + queryParams: struct_NV2080_CTRL_FB_FS_INFO_QUERY_queryParams +@c.record +class struct_NV2080_CTRL_FB_FS_INFO_QUERY_queryParams(c.Struct): + SIZE = 24 + inv: struct_NV2080_CTRL_FB_FS_INFO_INVALID_QUERY_PARAMS + fbp: struct_NV2080_CTRL_FB_FS_INFO_FBP_MASK_PARAMS + ltc: struct_NV2080_CTRL_FB_FS_INFO_LTC_MASK_PARAMS + lts: struct_NV2080_CTRL_FB_FS_INFO_LTS_MASK_PARAMS + fbpa: struct_NV2080_CTRL_FB_FS_INFO_FBPA_MASK_PARAMS + rop: struct_NV2080_CTRL_FB_FS_INFO_ROP_MASK_PARAMS + dmLtc: struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTC_MASK_PARAMS + dmLts: struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTS_MASK_PARAMS + dmFbpa: struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_MASK_PARAMS + dmRop: struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_ROP_MASK_PARAMS + dmFbpaSubp: struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_SUBP_MASK_PARAMS + fbpaSubp: struct_NV2080_CTRL_FB_FS_INFO_FBPA_SUBP_MASK_PARAMS + fbpLogicalMap: struct_NV2080_CTRL_FB_FS_INFO_FBP_LOGICAL_MAP_PARAMS + sysl2Ltc: struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTC_MASK_PARAMS + pac: struct_NV2080_CTRL_FB_FS_INFO_PAC_MASK_PARAMS + logicalLtc: struct_NV2080_CTRL_FB_FS_INFO_LOGICAL_LTC_MASK_PARAMS + dmLogicalLtc: struct_NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LOGICAL_LTC_MASK_PARAMS + sysl2Lts: struct_NV2080_CTRL_SYSL2_FS_INFO_SYSLTS_MASK_PARAMS + sys: struct_NV2080_CTRL_FB_FS_INFO_SYS_MASK_PARAMS +struct_NV2080_CTRL_FB_FS_INFO_QUERY_queryParams.register_fields([('inv', NV2080_CTRL_FB_FS_INFO_INVALID_QUERY_PARAMS, 0), ('fbp', NV2080_CTRL_FB_FS_INFO_FBP_MASK_PARAMS, 0), ('ltc', NV2080_CTRL_FB_FS_INFO_LTC_MASK_PARAMS, 0), ('lts', NV2080_CTRL_FB_FS_INFO_LTS_MASK_PARAMS, 0), ('fbpa', NV2080_CTRL_FB_FS_INFO_FBPA_MASK_PARAMS, 0), ('rop', NV2080_CTRL_FB_FS_INFO_ROP_MASK_PARAMS, 0), ('dmLtc', NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTC_MASK_PARAMS, 0), ('dmLts', NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTS_MASK_PARAMS, 0), ('dmFbpa', NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_MASK_PARAMS, 0), ('dmRop', NV2080_CTRL_FB_FS_INFO_PROFILER_MON_ROP_MASK_PARAMS, 0), ('dmFbpaSubp', NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_SUBP_MASK_PARAMS, 0), ('fbpaSubp', NV2080_CTRL_FB_FS_INFO_FBPA_SUBP_MASK_PARAMS, 0), ('fbpLogicalMap', NV2080_CTRL_FB_FS_INFO_FBP_LOGICAL_MAP_PARAMS, 0), ('sysl2Ltc', NV2080_CTRL_SYSL2_FS_INFO_SYSLTC_MASK_PARAMS, 0), ('pac', NV2080_CTRL_FB_FS_INFO_PAC_MASK_PARAMS, 0), ('logicalLtc', NV2080_CTRL_FB_FS_INFO_LOGICAL_LTC_MASK_PARAMS, 0), ('dmLogicalLtc', NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LOGICAL_LTC_MASK_PARAMS, 0), ('sysl2Lts', NV2080_CTRL_SYSL2_FS_INFO_SYSLTS_MASK_PARAMS, 0), ('sys', NV2080_CTRL_FB_FS_INFO_SYS_MASK_PARAMS, 0)]) +struct_NV2080_CTRL_FB_FS_INFO_QUERY.register_fields([('queryType', NvU16, 0), ('reserved', c.Array[NvU8, Literal[2]], 2), ('status', NvU32, 4), ('queryParams', struct_NV2080_CTRL_FB_FS_INFO_QUERY_queryParams, 8)]) +NV2080_CTRL_FB_FS_INFO_QUERY: TypeAlias = struct_NV2080_CTRL_FB_FS_INFO_QUERY +@c.record +class struct_NV2080_CTRL_FB_GET_FS_INFO_PARAMS(c.Struct): + SIZE = 3848 + numQueries: int + reserved: c.Array[ctypes.c_ubyte, Literal[6]] + queries: c.Array[struct_NV2080_CTRL_FB_FS_INFO_QUERY, Literal[120]] +struct_NV2080_CTRL_FB_GET_FS_INFO_PARAMS.register_fields([('numQueries', NvU16, 0), ('reserved', c.Array[NvU8, Literal[6]], 2), ('queries', c.Array[NV2080_CTRL_FB_FS_INFO_QUERY, Literal[120]], 8)]) +NV2080_CTRL_FB_GET_FS_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_FS_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_ROW_REMAPPER_HISTOGRAM_PARAMS(c.Struct): + SIZE = 20 + histogram: c.Array[ctypes.c_uint32, Literal[5]] +struct_NV2080_CTRL_FB_GET_ROW_REMAPPER_HISTOGRAM_PARAMS.register_fields([('histogram', c.Array[NvU32, Literal[5]], 0)]) +NV2080_CTRL_FB_GET_ROW_REMAPPER_HISTOGRAM_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_ROW_REMAPPER_HISTOGRAM_PARAMS +@c.record +class struct_NV2080_CTRL_FB_DYNAMIC_OFFLINED_ADDRESS_INFO(c.Struct): + SIZE = 16 + pageNumber: int + source: int +struct_NV2080_CTRL_FB_DYNAMIC_OFFLINED_ADDRESS_INFO.register_fields([('pageNumber', NvU64, 0), ('source', NvU8, 8)]) +NV2080_CTRL_FB_DYNAMIC_OFFLINED_ADDRESS_INFO: TypeAlias = struct_NV2080_CTRL_FB_DYNAMIC_OFFLINED_ADDRESS_INFO +@c.record +class struct_NV2080_CTRL_FB_GET_DYNAMIC_OFFLINED_PAGES_PARAMS(c.Struct): + SIZE = 1040 + offlined: c.Array[struct_NV2080_CTRL_FB_DYNAMIC_OFFLINED_ADDRESS_INFO, Literal[64]] + validEntries: int + baseIndex: int + bMore: int +struct_NV2080_CTRL_FB_GET_DYNAMIC_OFFLINED_PAGES_PARAMS.register_fields([('offlined', c.Array[NV2080_CTRL_FB_DYNAMIC_OFFLINED_ADDRESS_INFO, Literal[64]], 0), ('validEntries', NvU32, 1024), ('baseIndex', NvU32, 1028), ('bMore', NvBool, 1032)]) +NV2080_CTRL_FB_GET_DYNAMIC_OFFLINED_PAGES_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_DYNAMIC_OFFLINED_PAGES_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_ALLOCATION_INFO(c.Struct): + SIZE = 24 + client: int + flags: int + beginAddr: int + size: int +struct_NV2080_CTRL_CMD_FB_ALLOCATION_INFO.register_fields([('client', NvU32, 0), ('flags', NvU32, 4), ('beginAddr', NvU64, 8), ('size', NvU64, 16)]) +NV2080_CTRL_CMD_FB_ALLOCATION_INFO: TypeAlias = struct_NV2080_CTRL_CMD_FB_ALLOCATION_INFO +@c.record +class struct_NV2080_CTRL_CMD_FB_CLIENT_INFO(c.Struct): + SIZE = 112 + handle: int + pid: int + subProcessID: int + subProcessName: c.Array[ctypes.c_char, Literal[100]] +struct_NV2080_CTRL_CMD_FB_CLIENT_INFO.register_fields([('handle', NvHandle, 0), ('pid', NvU32, 4), ('subProcessID', NvU32, 8), ('subProcessName', c.Array[ctypes.c_char, Literal[100]], 12)]) +NV2080_CTRL_CMD_FB_CLIENT_INFO: TypeAlias = struct_NV2080_CTRL_CMD_FB_CLIENT_INFO +@c.record +class struct_NV2080_CTRL_CMD_FB_GET_CLIENT_ALLOCATION_INFO_PARAMS(c.Struct): + SIZE = 32 + allocCount: int + pAllocInfo: ctypes.c_void_p + clientCount: int + pClientInfo: ctypes.c_void_p +struct_NV2080_CTRL_CMD_FB_GET_CLIENT_ALLOCATION_INFO_PARAMS.register_fields([('allocCount', NvU64, 0), ('pAllocInfo', NvP64, 8), ('clientCount', NvU64, 16), ('pClientInfo', NvP64, 24)]) +NV2080_CTRL_CMD_FB_GET_CLIENT_ALLOCATION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_GET_CLIENT_ALLOCATION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS(c.Struct): + SIZE = 1 + bOnline: int +struct_NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS.register_fields([('bOnline', NvBool, 0)]) +NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS(c.Struct): + SIZE = 544 + numaNodeId: int + numaMemAddr: int + numaMemSize: int + numaOfflineAddressesCount: int + numaOfflineAddresses: c.Array[ctypes.c_uint64, Literal[64]] +struct_NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS.register_fields([('numaNodeId', NvS32, 0), ('numaMemAddr', NvU64, 8), ('numaMemSize', NvU64, 16), ('numaOfflineAddressesCount', NvU32, 24), ('numaOfflineAddresses', c.Array[NvU64, Literal[64]], 32)]) +NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_SEMAPHORE_SURFACE_LAYOUT_PARAMS(c.Struct): + SIZE = 32 + maxSubmittedSemaphoreValueOffset: int + monitoredFenceThresholdOffset: int + size: int + caps: int +struct_NV2080_CTRL_FB_GET_SEMAPHORE_SURFACE_LAYOUT_PARAMS.register_fields([('maxSubmittedSemaphoreValueOffset', NvU64, 0), ('monitoredFenceThresholdOffset', NvU64, 8), ('size', NvU64, 16), ('caps', NvU32, 24)]) +NV2080_CTRL_FB_GET_SEMAPHORE_SURFACE_LAYOUT_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_SEMAPHORE_SURFACE_LAYOUT_PARAMS +@c.record +class struct_NV2080_CTRL_GMMU_COMMIT_TLB_INVALIDATE_PARAMS(c.Struct): + SIZE = 8 + gfid: int + invalidateAll: int +struct_NV2080_CTRL_GMMU_COMMIT_TLB_INVALIDATE_PARAMS.register_fields([('gfid', NvU32, 0), ('invalidateAll', NvBool, 4)]) +NV2080_CTRL_GMMU_COMMIT_TLB_INVALIDATE_PARAMS: TypeAlias = struct_NV2080_CTRL_GMMU_COMMIT_TLB_INVALIDATE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FB_STATS_OWNER_INFO(c.Struct): + SIZE = 24 + allocSize: int + numBlocks: int + rsvdSize: int +struct_NV2080_CTRL_CMD_FB_STATS_OWNER_INFO.register_fields([('allocSize', NvU64, 0), ('numBlocks', NvU32, 8), ('rsvdSize', NvU64, 16)]) +NV2080_CTRL_CMD_FB_STATS_OWNER_INFO: TypeAlias = struct_NV2080_CTRL_CMD_FB_STATS_OWNER_INFO +@c.record +class struct_NV2080_CTRL_CMD_RM_FB_STATS_ENTRY(c.Struct): + SIZE = 4848 + reservedHeap: int + reservedHeapFree: int + reserved: int + fbBlockInfo: c.Array[struct_NV2080_CTRL_CMD_FB_STATS_OWNER_INFO, Literal[201]] +struct_NV2080_CTRL_CMD_RM_FB_STATS_ENTRY.register_fields([('reservedHeap', NvU64, 0), ('reservedHeapFree', NvU64, 8), ('reserved', NvU64, 16), ('fbBlockInfo', c.Array[NV2080_CTRL_CMD_FB_STATS_OWNER_INFO, Literal[201]], 24)]) +NV2080_CTRL_CMD_RM_FB_STATS_ENTRY: TypeAlias = struct_NV2080_CTRL_CMD_RM_FB_STATS_ENTRY +@c.record +class struct_NV2080_CTRL_CMD_FB_STATS_GET_PARAMS(c.Struct): + SIZE = 9728 + total: int + os: int + gspCarveout: int + miscReserved: int + cpuFbInfo: struct_NV2080_CTRL_CMD_RM_FB_STATS_ENTRY + gspFbInfo: struct_NV2080_CTRL_CMD_RM_FB_STATS_ENTRY +struct_NV2080_CTRL_CMD_FB_STATS_GET_PARAMS.register_fields([('total', NvU64, 0), ('os', NvU64, 8), ('gspCarveout', NvU64, 16), ('miscReserved', NvU64, 24), ('cpuFbInfo', NV2080_CTRL_CMD_RM_FB_STATS_ENTRY, 32), ('gspFbInfo', NV2080_CTRL_CMD_RM_FB_STATS_ENTRY, 4880)]) +NV2080_CTRL_CMD_FB_STATS_GET_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FB_STATS_GET_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_STATIC_BAR1_INFO_PARAMS(c.Struct): + SIZE = 24 + bStaticBar1Enabled: int + bStaticBar1WriteCombined: int + staticBar1StartOffset: int + staticBar1Size: int +struct_NV2080_CTRL_FB_GET_STATIC_BAR1_INFO_PARAMS.register_fields([('bStaticBar1Enabled', NvBool, 0), ('bStaticBar1WriteCombined', NvBool, 1), ('staticBar1StartOffset', NvU64, 8), ('staticBar1Size', NvU64, 16)]) +NV2080_CTRL_FB_GET_STATIC_BAR1_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_STATIC_BAR1_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION_PARAMS(c.Struct): + SIZE = 4 + currentConfiguration: int +struct_NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION_PARAMS.register_fields([('currentConfiguration', NvU32, 0)]) +NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION_PARAMS +@c.record +class struct_NV2080_CTRL_FB_SET_DRAM_ENCRYPTION_CONFIGURATION_PARAMS(c.Struct): + SIZE = 4 + newConfiguration: int +struct_NV2080_CTRL_FB_SET_DRAM_ENCRYPTION_CONFIGURATION_PARAMS.register_fields([('newConfiguration', NvU32, 0)]) +NV2080_CTRL_FB_SET_DRAM_ENCRYPTION_CONFIGURATION_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_SET_DRAM_ENCRYPTION_CONFIGURATION_PARAMS +@c.record +class struct_NV2080_CTRL_FB_DRAM_ENCRYPTION_INFOROM_SUPPORT_PARAMS(c.Struct): + SIZE = 4 + isSupported: int +struct_NV2080_CTRL_FB_DRAM_ENCRYPTION_INFOROM_SUPPORT_PARAMS.register_fields([('isSupported', NvU32, 0)]) +NV2080_CTRL_FB_DRAM_ENCRYPTION_INFOROM_SUPPORT_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_DRAM_ENCRYPTION_INFOROM_SUPPORT_PARAMS +@c.record +class struct_NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_STATUS_PARAMS(c.Struct): + SIZE = 4 + currentStatus: int +struct_NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_STATUS_PARAMS.register_fields([('currentStatus', NvU32, 0)]) +NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_MEMORY_BOOT_TRAINING_FLAGS_PARAMS(c.Struct): + SIZE = 6 + flagCollectSchmooData: int + flagWrTrHybridVrefEn: int + flagWrTrHybridNonVrefEn: int + flagRdTrHybridVrefEn: int + flagRdTrHybridNonVrefEn: int + skipBootTraining: int +struct_NV2080_CTRL_FB_GET_MEMORY_BOOT_TRAINING_FLAGS_PARAMS.register_fields([('flagCollectSchmooData', NvBool, 0), ('flagWrTrHybridVrefEn', NvBool, 1), ('flagWrTrHybridNonVrefEn', NvBool, 2), ('flagRdTrHybridVrefEn', NvBool, 3), ('flagRdTrHybridNonVrefEn', NvBool, 4), ('skipBootTraining', NvBool, 5)]) +NV2080_CTRL_FB_GET_MEMORY_BOOT_TRAINING_FLAGS_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_MEMORY_BOOT_TRAINING_FLAGS_PARAMS +enum_NV2080_CTRL_FB_GET_CARVEOUT_REGION_CARVEOUT_TYPE: dict[int, str] = {(NV2080_CTRL_FB_GET_CARVEOUT_REGION_CARVEOUT_TYPE_DISPLAY_FRM:=0): 'NV2080_CTRL_FB_GET_CARVEOUT_REGION_CARVEOUT_TYPE_DISPLAY_FRM', (NV2080_CTRL_FB_GET_CARVEOUT_REGION_CARVEOUT_TYPE_UEFI:=1): 'NV2080_CTRL_FB_GET_CARVEOUT_REGION_CARVEOUT_TYPE_UEFI'} +NV2080_CTRL_FB_GET_CARVEOUT_REGION_CARVEOUT_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO(c.Struct): + SIZE = 24 + base: int + size: int + carveoutType: int +struct_NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO.register_fields([('base', NvU64, 0), ('size', NvU64, 8), ('carveoutType', NV2080_CTRL_FB_GET_CARVEOUT_REGION_CARVEOUT_TYPE, 16)]) +NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO: TypeAlias = struct_NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO +@c.record +class struct_NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO_PARAMS(c.Struct): + SIZE = 200 + numCarveoutRegions: int + carveoutRegion: c.Array[struct_NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO, Literal[8]] +struct_NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO_PARAMS.register_fields([('numCarveoutRegions', NvU32, 0), ('carveoutRegion', c.Array[NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO, Literal[8]], 8)]) +NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_BANK_REMAP_HISTOGRAM(c.Struct): + SIZE = 8 + maxSpareGroupCount: int + noSpareGroupCount: int +struct_NV2080_CTRL_FB_BANK_REMAP_HISTOGRAM.register_fields([('maxSpareGroupCount', NvU32, 0), ('noSpareGroupCount', NvU32, 4)]) +NV2080_CTRL_FB_BANK_REMAP_HISTOGRAM: TypeAlias = struct_NV2080_CTRL_FB_BANK_REMAP_HISTOGRAM +@c.record +class struct_NV2080_CTRL_FB_BANK_REMAP_ENTRY(c.Struct): + SIZE = 16 + remapRegVal: int + timestamp: int + fbpa: int + channel: int + pseudoChannel: int + group: int + source: int + flags: int +struct_NV2080_CTRL_FB_BANK_REMAP_ENTRY.register_fields([('remapRegVal', NvU32, 0), ('timestamp', NvU32, 4), ('fbpa', NvU8, 8), ('channel', NvU8, 9), ('pseudoChannel', NvU8, 10), ('group', NvU8, 11), ('source', NvU8, 12), ('flags', NvU8, 13)]) +NV2080_CTRL_FB_BANK_REMAP_ENTRY: TypeAlias = struct_NV2080_CTRL_FB_BANK_REMAP_ENTRY +@c.record +class struct_NV2080_CTRL_FB_GET_REMAPPED_BANKS_PARAMS(c.Struct): + SIZE = 8208 + entryCount: int + flags: int + histogram: struct_NV2080_CTRL_FB_BANK_REMAP_HISTOGRAM + entries: c.Array[struct_NV2080_CTRL_FB_BANK_REMAP_ENTRY, Literal[512]] +struct_NV2080_CTRL_FB_GET_REMAPPED_BANKS_PARAMS.register_fields([('entryCount', NvU32, 0), ('flags', NvU8, 4), ('histogram', NV2080_CTRL_FB_BANK_REMAP_HISTOGRAM, 8), ('entries', c.Array[NV2080_CTRL_FB_BANK_REMAP_ENTRY, Literal[512]], 16)]) +NV2080_CTRL_FB_GET_REMAPPED_BANKS_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_REMAPPED_BANKS_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_UGPU_MEMORY_INFO_PARAMS(c.Struct): + SIZE = 48 + totalMemory: int + freeMemory: int + ugpuTotalMemory: c.Array[ctypes.c_uint64, Literal[2]] + ugpuFreeMemory: c.Array[ctypes.c_uint64, Literal[2]] +struct_NV2080_CTRL_FB_GET_UGPU_MEMORY_INFO_PARAMS.register_fields([('totalMemory', NvU64, 0), ('freeMemory', NvU64, 8), ('ugpuTotalMemory', c.Array[NvU64, Literal[2]], 16), ('ugpuFreeMemory', c.Array[NvU64, Literal[2]], 32)]) +NV2080_CTRL_FB_GET_UGPU_MEMORY_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_UGPU_MEMORY_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_CPU_COHERENT_RANGE_PARAMS(c.Struct): + SIZE = 16 + coherentCpuFbBase: int + coherentCpuFbEnd: int +struct_NV2080_CTRL_FB_GET_CPU_COHERENT_RANGE_PARAMS.register_fields([('coherentCpuFbBase', NvU64, 0), ('coherentCpuFbEnd', NvU64, 8)]) +NV2080_CTRL_FB_GET_CPU_COHERENT_RANGE_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_CPU_COHERENT_RANGE_PARAMS +@c.record +class struct_NV2080_CTRL_FB_GET_WPR_REGION_INFO_PARAMS(c.Struct): + SIZE = 40 + wpr1Start: int + wpr1End: int + wpr2Start: int + wpr2End: int + fbRegionOfWpr2Start: int +struct_NV2080_CTRL_FB_GET_WPR_REGION_INFO_PARAMS.register_fields([('wpr1Start', NvU64, 0), ('wpr1End', NvU64, 8), ('wpr2Start', NvU64, 16), ('wpr2End', NvU64, 24), ('fbRegionOfWpr2Start', NvU64, 32)]) +NV2080_CTRL_FB_GET_WPR_REGION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FB_GET_WPR_REGION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_SET_GPFIFO_PARAMS(c.Struct): + SIZE = 24 + hChannel: int + base: int + numEntries: int +struct_NV2080_CTRL_CMD_SET_GPFIFO_PARAMS.register_fields([('hChannel', NvHandle, 0), ('base', NvU64, 8), ('numEntries', NvU32, 16)]) +NV2080_CTRL_CMD_SET_GPFIFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_SET_GPFIFO_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_GET_PHYSICAL_CHANNEL_COUNT_PARAMS(c.Struct): + SIZE = 8 + physChannelCount: int + physChannelCountInUse: int +struct_NV2080_CTRL_FIFO_GET_PHYSICAL_CHANNEL_COUNT_PARAMS.register_fields([('physChannelCount', NvU32, 0), ('physChannelCountInUse', NvU32, 4)]) +NV2080_CTRL_FIFO_GET_PHYSICAL_CHANNEL_COUNT_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_GET_PHYSICAL_CHANNEL_COUNT_PARAMS +NV2080_CTRL_FIFO_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_FIFO_GET_INFO_PARAMS(c.Struct): + SIZE = 2056 + fifoInfoTblSize: int + fifoInfoTbl: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[256]] + engineType: int +struct_NV2080_CTRL_FIFO_GET_INFO_PARAMS.register_fields([('fifoInfoTblSize', NvU32, 0), ('fifoInfoTbl', c.Array[NV2080_CTRL_FIFO_INFO, Literal[256]], 4), ('engineType', NvU32, 2052)]) +NV2080_CTRL_FIFO_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS(c.Struct): + SIZE = 536 + bDisable: int + numChannels: int + bOnlyDisableScheduling: int + bRewindGpPut: int + pRunlistPreemptEvent: ctypes.c_void_p + hClientList: c.Array[ctypes.c_uint32, Literal[64]] + hChannelList: c.Array[ctypes.c_uint32, Literal[64]] +struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS.register_fields([('bDisable', NvBool, 0), ('numChannels', NvU32, 4), ('bOnlyDisableScheduling', NvBool, 8), ('bRewindGpPut', NvBool, 9), ('pRunlistPreemptEvent', NvP64, 16), ('hClientList', c.Array[NvHandle, Literal[64]], 24), ('hChannelList', c.Array[NvHandle, Literal[64]], 280)]) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_MEM_INFO(c.Struct): + SIZE = 24 + aperture: int + base: int + size: int +struct_NV2080_CTRL_FIFO_MEM_INFO.register_fields([('aperture', NvU32, 0), ('base', NvU64, 8), ('size', NvU64, 16)]) +NV2080_CTRL_FIFO_MEM_INFO: TypeAlias = struct_NV2080_CTRL_FIFO_MEM_INFO +@c.record +class struct_NV2080_CTRL_FIFO_CHANNEL_MEM_INFO(c.Struct): + SIZE = 104 + inst: struct_NV2080_CTRL_FIFO_MEM_INFO + ramfc: struct_NV2080_CTRL_FIFO_MEM_INFO + methodBuf: c.Array[struct_NV2080_CTRL_FIFO_MEM_INFO, Literal[2]] + methodBufCount: int +struct_NV2080_CTRL_FIFO_CHANNEL_MEM_INFO.register_fields([('inst', NV2080_CTRL_FIFO_MEM_INFO, 0), ('ramfc', NV2080_CTRL_FIFO_MEM_INFO, 24), ('methodBuf', c.Array[NV2080_CTRL_FIFO_MEM_INFO, Literal[2]], 48), ('methodBufCount', NvU32, 96)]) +NV2080_CTRL_FIFO_CHANNEL_MEM_INFO: TypeAlias = struct_NV2080_CTRL_FIFO_CHANNEL_MEM_INFO +@c.record +class struct_NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS(c.Struct): + SIZE = 112 + hChannel: int + chMemInfo: struct_NV2080_CTRL_FIFO_CHANNEL_MEM_INFO +struct_NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS.register_fields([('hChannel', NvHandle, 0), ('chMemInfo', NV2080_CTRL_FIFO_CHANNEL_MEM_INFO, 8)]) +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_PARAMS(c.Struct): + SIZE = 8 + aperture: int + attribute: int +struct_NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_PARAMS.register_fields([('aperture', NvU32, 0), ('attribute', NvU32, 4)]) +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_ENTRY(c.Struct): + SIZE = 120 + timestampNs: int + timeRunTotalNs: int + timeRunNs: int + swrlId: int + targetTimeSlice: int + cumulativePreemptionTime: int + cumulativeIdleTime: int + weight: int + counters: c.Array[ctypes.c_uint64, Literal[8]] +NvS64: TypeAlias = ctypes.c_int64 +struct_NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_ENTRY.register_fields([('timestampNs', NvU64, 0), ('timeRunTotalNs', NvS64, 8), ('timeRunNs', NvU32, 16), ('swrlId', NvU32, 20), ('targetTimeSlice', NvU32, 24), ('cumulativePreemptionTime', NvU64, 32), ('cumulativeIdleTime', NvU64, 40), ('weight', NvU32, 48), ('counters', c.Array[NvU64, Literal[8]], 56)]) +NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_ENTRY: TypeAlias = struct_NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_ENTRY +@c.record +class struct_NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_PARAMS(c.Struct): + SIZE = 24024 + engineId: int + count: int + entry: c.Array[struct_NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_ENTRY, Literal[200]] + schedPolicy: int + arrEnabled: int + arrAvgFactor: int + targetTimesliceNs: int +struct_NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_PARAMS.register_fields([('engineId', NvU32, 0), ('count', NvU32, 4), ('entry', c.Array[NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_ENTRY, Literal[200]], 8), ('schedPolicy', NvU32, 24008), ('arrEnabled', NvU32, 24012), ('arrAvgFactor', NvU32, 24016), ('targetTimesliceNs', NvU32, 24020)]) +NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_CONFIG_CTXSW_TIMEOUT_PARAMS(c.Struct): + SIZE = 8 + timeout: int + bEnable: int +struct_NV2080_CTRL_FIFO_CONFIG_CTXSW_TIMEOUT_PARAMS.register_fields([('timeout', NvU32, 0), ('bEnable', NvBool, 4)]) +NV2080_CTRL_FIFO_CONFIG_CTXSW_TIMEOUT_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_CONFIG_CTXSW_TIMEOUT_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_DEVICE_ENTRY(c.Struct): + SIZE = 104 + engineData: c.Array[ctypes.c_uint32, Literal[17]] + pbdmaIds: c.Array[ctypes.c_uint32, Literal[2]] + pbdmaFaultIds: c.Array[ctypes.c_uint32, Literal[2]] + numPbdmas: int + engineName: c.Array[ctypes.c_char, Literal[16]] +struct_NV2080_CTRL_FIFO_DEVICE_ENTRY.register_fields([('engineData', c.Array[NvU32, Literal[17]], 0), ('pbdmaIds', c.Array[NvU32, Literal[2]], 68), ('pbdmaFaultIds', c.Array[NvU32, Literal[2]], 76), ('numPbdmas', NvU32, 84), ('engineName', c.Array[ctypes.c_char, Literal[16]], 88)]) +NV2080_CTRL_FIFO_DEVICE_ENTRY: TypeAlias = struct_NV2080_CTRL_FIFO_DEVICE_ENTRY +@c.record +class struct_NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS(c.Struct): + SIZE = 3340 + baseIndex: int + numEntries: int + bMore: int + entries: c.Array[struct_NV2080_CTRL_FIFO_DEVICE_ENTRY, Literal[32]] +struct_NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS.register_fields([('baseIndex', NvU32, 0), ('numEntries', NvU32, 4), ('bMore', NvBool, 8), ('entries', c.Array[NV2080_CTRL_FIFO_DEVICE_ENTRY, Literal[32]], 12)]) +NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FIFO_CLEAR_FAULTED_BIT_PARAMS(c.Struct): + SIZE = 12 + engineType: int + vChid: int + faultType: int +struct_NV2080_CTRL_CMD_FIFO_CLEAR_FAULTED_BIT_PARAMS.register_fields([('engineType', NvU32, 0), ('vChid', NvU32, 4), ('faultType', NvU32, 8)]) +NV2080_CTRL_CMD_FIFO_CLEAR_FAULTED_BIT_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FIFO_CLEAR_FAULTED_BIT_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_RUNLIST_SET_SCHED_POLICY_PARAMS(c.Struct): + SIZE = 8 + flags: int + schedPolicy: int +struct_NV2080_CTRL_FIFO_RUNLIST_SET_SCHED_POLICY_PARAMS.register_fields([('flags', NvU32, 0), ('schedPolicy', NvU32, 4)]) +NV2080_CTRL_FIFO_RUNLIST_SET_SCHED_POLICY_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_RUNLIST_SET_SCHED_POLICY_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_UPDATE_CHANNEL_INFO_PARAMS(c.Struct): + SIZE = 32 + hClient: int + hChannel: int + hUserdMemory: int + gpFifoEntries: int + gpFifoOffset: int + userdOffset: int +struct_NV2080_CTRL_FIFO_UPDATE_CHANNEL_INFO_PARAMS.register_fields([('hClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('hUserdMemory', NvHandle, 8), ('gpFifoEntries', NvU32, 12), ('gpFifoOffset', NvU64, 16), ('userdOffset', NvU64, 24)]) +NV2080_CTRL_FIFO_UPDATE_CHANNEL_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_UPDATE_CHANNEL_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS(c.Struct): + SIZE = 1 + bDisable: int +struct_NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS.register_fields([('bDisable', NvBool, 0)]) +NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_SETUP_VF_ZOMBIE_SUBCTX_PDB_PARAMS(c.Struct): + SIZE = 24 + base: int + size: int + addressSpace: int + cacheAttrib: int +struct_NV2080_CTRL_FIFO_SETUP_VF_ZOMBIE_SUBCTX_PDB_PARAMS.register_fields([('base', NvU64, 0), ('size', NvU64, 8), ('addressSpace', NvU32, 16), ('cacheAttrib', NvU32, 20)]) +NV2080_CTRL_FIFO_SETUP_VF_ZOMBIE_SUBCTX_PDB_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_SETUP_VF_ZOMBIE_SUBCTX_PDB_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_GET_ALLOCATED_CHANNELS_PARAMS(c.Struct): + SIZE = 516 + runlistId: int + bitMask: c.Array[ctypes.c_uint32, Literal[128]] +struct_NV2080_CTRL_FIFO_GET_ALLOCATED_CHANNELS_PARAMS.register_fields([('runlistId', NvU32, 0), ('bitMask', c.Array[NvU32, Literal[128]], 4)]) +NV2080_CTRL_FIFO_GET_ALLOCATED_CHANNELS_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_GET_ALLOCATED_CHANNELS_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_PARAMS(c.Struct): + SIZE = 520 + numChannels: int + hClientList: c.Array[ctypes.c_uint32, Literal[64]] + hChannelList: c.Array[ctypes.c_uint32, Literal[64]] + bEnableAfterKeyRotation: int +struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_PARAMS.register_fields([('numChannels', NvU32, 0), ('hClientList', c.Array[NvHandle, Literal[64]], 4), ('hChannelList', c.Array[NvHandle, Literal[64]], 260), ('bEnableAfterKeyRotation', NvBool, 516)]) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_V2_PARAMS(c.Struct): + SIZE = 264 + numChannels: int + hChannelList: c.Array[ctypes.c_uint32, Literal[64]] + bEnableAfterKeyRotation: int +struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_V2_PARAMS.register_fields([('numChannels', NvU32, 0), ('hChannelList', c.Array[NvHandle, Literal[64]], 4), ('bEnableAfterKeyRotation', NvBool, 260)]) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_V2_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_FIFO_ROTATE_KEYS_PARAMS(c.Struct): + SIZE = 520 + numChannels: int + hClientList: c.Array[ctypes.c_uint32, Literal[64]] + hChannelList: c.Array[ctypes.c_uint32, Literal[64]] + bEnableAfterKeyRotation: int +struct_NV2080_CTRL_CMD_FIFO_ROTATE_KEYS_PARAMS.register_fields([('numChannels', NvU32, 0), ('hClientList', c.Array[NvHandle, Literal[64]], 4), ('hChannelList', c.Array[NvHandle, Literal[64]], 260), ('bEnableAfterKeyRotation', NvBool, 516)]) +NV2080_CTRL_CMD_FIFO_ROTATE_KEYS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FIFO_ROTATE_KEYS_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_OBJSCHED_GET_STATE_PARAMS(c.Struct): + SIZE = 20 + engineId: int + schedPolicy: int + arrEnabled: int + targetTimesliceNs: int + arrAvgFactor: int +struct_NV2080_CTRL_FIFO_OBJSCHED_GET_STATE_PARAMS.register_fields([('engineId', NvU32, 0), ('schedPolicy', NvU32, 4), ('arrEnabled', NvU32, 8), ('targetTimesliceNs', NvU32, 12), ('arrAvgFactor', NvU32, 16)]) +NV2080_CTRL_FIFO_OBJSCHED_GET_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_OBJSCHED_GET_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_OBJSCHED_SET_STATE_PARAMS(c.Struct): + SIZE = 24 + engineId: int + schedPolicy: int + enableArr: int + timesliceTargetNs: int + frequencyForARR: int + avgFactorForARR: int +struct_NV2080_CTRL_FIFO_OBJSCHED_SET_STATE_PARAMS.register_fields([('engineId', NvU32, 0), ('schedPolicy', NvU32, 4), ('enableArr', NvU32, 8), ('timesliceTargetNs', NvU32, 12), ('frequencyForARR', NvU32, 16), ('avgFactorForARR', NvU32, 20)]) +NV2080_CTRL_FIFO_OBJSCHED_SET_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_OBJSCHED_SET_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_OBJSCHED_GET_CAPS_PARAMS(c.Struct): + SIZE = 44 + engineId: int + supportedSchedulers: c.Array[ctypes.c_uint32, Literal[3]] + bIsArrModeSupported: int + maxTimesliceNs: int + minTimesliceNs: int + maxFrequencyForARR: int + minFrequencyForARR: int + maxAvgFactorForARR: int + minAvgFactorForARR: int +struct_NV2080_CTRL_FIFO_OBJSCHED_GET_CAPS_PARAMS.register_fields([('engineId', NvU32, 0), ('supportedSchedulers', c.Array[NvU32, Literal[3]], 4), ('bIsArrModeSupported', NvBool, 16), ('maxTimesliceNs', NvU32, 20), ('minTimesliceNs', NvU32, 24), ('maxFrequencyForARR', NvU32, 28), ('minFrequencyForARR', NvU32, 32), ('maxAvgFactorForARR', NvU32, 36), ('minAvgFactorForARR', NvU32, 40)]) +NV2080_CTRL_FIFO_OBJSCHED_GET_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_OBJSCHED_GET_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_GET_CHANNEL_GROUP_UNIQUE_ID_INFO_PARAMS(c.Struct): + SIZE = 1552 + hClient: int + hChannelOrTsg: int + tsgId: int + numChannels: int + channelUniqueID: c.Array[ctypes.c_uint32, Literal[128]] + vasUniqueID: c.Array[ctypes.c_uint32, Literal[128]] + veid: c.Array[ctypes.c_uint32, Literal[128]] +struct_NV2080_CTRL_FIFO_GET_CHANNEL_GROUP_UNIQUE_ID_INFO_PARAMS.register_fields([('hClient', NvHandle, 0), ('hChannelOrTsg', NvHandle, 4), ('tsgId', NvU32, 8), ('numChannels', NvU32, 12), ('channelUniqueID', c.Array[NvU32, Literal[128]], 16), ('vasUniqueID', c.Array[NvU32, Literal[128]], 528), ('veid', c.Array[NvU32, Literal[128]], 1040)]) +NV2080_CTRL_FIFO_GET_CHANNEL_GROUP_UNIQUE_ID_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_GET_CHANNEL_GROUP_UNIQUE_ID_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_QUERY_CHANNEL_UNIQUE_ID_PARAMS(c.Struct): + SIZE = 1540 + hClients: c.Array[ctypes.c_uint32, Literal[128]] + hChannels: c.Array[ctypes.c_uint32, Literal[128]] + numChannels: int + channelUniqueIDs: c.Array[ctypes.c_uint32, Literal[128]] +struct_NV2080_CTRL_FIFO_QUERY_CHANNEL_UNIQUE_ID_PARAMS.register_fields([('hClients', c.Array[NvHandle, Literal[128]], 0), ('hChannels', c.Array[NvHandle, Literal[128]], 512), ('numChannels', NvU32, 1024), ('channelUniqueIDs', c.Array[NvU32, Literal[128]], 1028)]) +NV2080_CTRL_FIFO_QUERY_CHANNEL_UNIQUE_ID_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_QUERY_CHANNEL_UNIQUE_ID_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_VGPU_SWRUNLIST_SUBMIT_PARAMS(c.Struct): + SIZE = 16 + engineType: int + submitRunlistOffset: int + bUpdateRunlist: int + bSkipSubmitRunlist: int + bSubmitLastUpdatedRunlist: int + numRunlistEntries: int +struct_NV2080_CTRL_FIFO_VGPU_SWRUNLIST_SUBMIT_PARAMS.register_fields([('engineType', NvU32, 0), ('submitRunlistOffset', NvU32, 4), ('bUpdateRunlist', NvBool, 8), ('bSkipSubmitRunlist', NvBool, 9), ('bSubmitLastUpdatedRunlist', NvBool, 10), ('numRunlistEntries', NvU32, 12)]) +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_SUBMIT_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_VGPU_SWRUNLIST_SUBMIT_PARAMS +@c.record +class struct_NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_PARAMS(c.Struct): + SIZE = 16 + engineType: int + vChId: int + channelLevel: int + operation: int +struct_NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_PARAMS.register_fields([('engineType', NvU32, 0), ('vChId', NvU32, 4), ('channelLevel', NvU32, 8), ('operation', NvU32, 12)]) +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_PARAMS: TypeAlias = struct_NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_PARAMS +@c.record +class struct_NV2080_CTRL_FLA_RANGE_PARAMS(c.Struct): + SIZE = 24 + base: int + size: int + mode: int + hVASpace: int +struct_NV2080_CTRL_FLA_RANGE_PARAMS.register_fields([('base', NvU64, 0), ('size', NvU64, 8), ('mode', NvU32, 16), ('hVASpace', NvHandle, 20)]) +NV2080_CTRL_FLA_RANGE_PARAMS: TypeAlias = struct_NV2080_CTRL_FLA_RANGE_PARAMS +enum_NV2080_CTRL_FLA_ADDRSPACE: dict[int, str] = {(NV2080_CTRL_FLA_ADDRSPACE_SYSMEM:=0): 'NV2080_CTRL_FLA_ADDRSPACE_SYSMEM', (NV2080_CTRL_FLA_ADDRSPACE_FBMEM:=1): 'NV2080_CTRL_FLA_ADDRSPACE_FBMEM'} +NV2080_CTRL_FLA_ADDRSPACE: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_FLA_ACTION: dict[int, str] = {(NV2080_CTRL_FLA_ACTION_BIND:=0): 'NV2080_CTRL_FLA_ACTION_BIND', (NV2080_CTRL_FLA_ACTION_UNBIND:=1): 'NV2080_CTRL_FLA_ACTION_UNBIND'} +NV2080_CTRL_FLA_ACTION: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_FLA_SETUP_INSTANCE_MEM_BLOCK_PARAMS(c.Struct): + SIZE = 16 + imbPhysAddr: int + addrSpace: int + flaAction: int +struct_NV2080_CTRL_FLA_SETUP_INSTANCE_MEM_BLOCK_PARAMS.register_fields([('imbPhysAddr', NvU64, 0), ('addrSpace', NV2080_CTRL_FLA_ADDRSPACE, 8), ('flaAction', NV2080_CTRL_FLA_ACTION, 12)]) +NV2080_CTRL_FLA_SETUP_INSTANCE_MEM_BLOCK_PARAMS: TypeAlias = struct_NV2080_CTRL_FLA_SETUP_INSTANCE_MEM_BLOCK_PARAMS +@c.record +class struct_NV2080_CTRL_FLA_GET_RANGE_PARAMS(c.Struct): + SIZE = 16 + base: int + size: int +struct_NV2080_CTRL_FLA_GET_RANGE_PARAMS.register_fields([('base', NvU64, 0), ('size', NvU64, 8)]) +NV2080_CTRL_FLA_GET_RANGE_PARAMS: TypeAlias = struct_NV2080_CTRL_FLA_GET_RANGE_PARAMS +@c.record +class struct_NV2080_CTRL_FLA_GET_FABRIC_MEM_STATS_PARAMS(c.Struct): + SIZE = 32 + totalSize: int + freeSize: int + emulatedHandleTotalSize: int + emulatedHandleFreeSize: int +struct_NV2080_CTRL_FLA_GET_FABRIC_MEM_STATS_PARAMS.register_fields([('totalSize', NvU64, 0), ('freeSize', NvU64, 8), ('emulatedHandleTotalSize', NvU64, 16), ('emulatedHandleFreeSize', NvU64, 24)]) +NV2080_CTRL_FLA_GET_FABRIC_MEM_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_FLA_GET_FABRIC_MEM_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_FLCN_GET_DMEM_USAGE_PARAMS(c.Struct): + SIZE = 12 + flcnID: int + heapSize: int + heapFree: int +struct_NV2080_CTRL_FLCN_GET_DMEM_USAGE_PARAMS.register_fields([('flcnID', NvU32, 0), ('heapSize', NvU32, 4), ('heapFree', NvU32, 8)]) +NV2080_CTRL_FLCN_GET_DMEM_USAGE_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_GET_DMEM_USAGE_PARAMS +@c.record +class struct_NV2080_CTRL_FLCN_GET_ENGINE_ARCH_PARAMS(c.Struct): + SIZE = 8 + engine: int + engineArch: int +struct_NV2080_CTRL_FLCN_GET_ENGINE_ARCH_PARAMS.register_fields([('engine', NvU32, 0), ('engineArch', NvU32, 4)]) +NV2080_CTRL_FLCN_GET_ENGINE_ARCH_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_GET_ENGINE_ARCH_PARAMS +@c.record +class struct_NV2080_CTRL_FLCN_USTREAMER_EVENT_FILTER(c.Struct): + SIZE = 36 + mask: c.Array[ctypes.c_ubyte, Literal[36]] +struct_NV2080_CTRL_FLCN_USTREAMER_EVENT_FILTER.register_fields([('mask', c.Array[NvU8, Literal[36]], 0)]) +NV2080_CTRL_FLCN_USTREAMER_EVENT_FILTER: TypeAlias = struct_NV2080_CTRL_FLCN_USTREAMER_EVENT_FILTER +@c.record +class struct_NV2080_CTRL_FLCN_USTREAMER_QUEUE_INFO_PARAMS(c.Struct): + SIZE = 24 + engine: int + pageSize: int + offset: int + size: int + queueFeatureId: int +NvUPtr: TypeAlias = ctypes.c_uint64 +struct_NV2080_CTRL_FLCN_USTREAMER_QUEUE_INFO_PARAMS.register_fields([('engine', NvU32, 0), ('pageSize', NvU32, 4), ('offset', NvUPtr, 8), ('size', NvU32, 16), ('queueFeatureId', NvU8, 20)]) +NV2080_CTRL_FLCN_USTREAMER_QUEUE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_USTREAMER_QUEUE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FLCN_USTREAMER_CONTROL_PARAMS(c.Struct): + SIZE = 44 + engine: int + eventFilter: struct_NV2080_CTRL_FLCN_USTREAMER_EVENT_FILTER + queueId: int +struct_NV2080_CTRL_FLCN_USTREAMER_CONTROL_PARAMS.register_fields([('engine', NvU32, 0), ('eventFilter', NV2080_CTRL_FLCN_USTREAMER_EVENT_FILTER, 4), ('queueId', NvU8, 40)]) +NV2080_CTRL_FLCN_USTREAMER_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_USTREAMER_CONTROL_PARAMS +NV2080_CTRL_FLCN_USTREAMER_CONTROL_GET_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_USTREAMER_CONTROL_PARAMS +NV2080_CTRL_FLCN_USTREAMER_CONTROL_SET_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_USTREAMER_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_FLCN_GET_CTX_BUFFER_INFO_PARAMS(c.Struct): + SIZE = 80 + hUserClient: int + hChannel: int + alignment: int + size: int + bufferHandle: ctypes.c_void_p + pageCount: int + physAddr: int + aperture: int + kind: int + pageSize: int + bIsContigous: int + bDeviceDescendant: int + uuid: c.Array[ctypes.c_ubyte, Literal[16]] +struct_NV2080_CTRL_FLCN_GET_CTX_BUFFER_INFO_PARAMS.register_fields([('hUserClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('alignment', NvU64, 8), ('size', NvU64, 16), ('bufferHandle', NvP64, 24), ('pageCount', NvU64, 32), ('physAddr', NvU64, 40), ('aperture', NvU32, 48), ('kind', NvU32, 52), ('pageSize', NvU32, 56), ('bIsContigous', NvBool, 60), ('bDeviceDescendant', NvBool, 61), ('uuid', c.Array[NvU8, Literal[16]], 62)]) +NV2080_CTRL_FLCN_GET_CTX_BUFFER_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_GET_CTX_BUFFER_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_FLCN_GET_CTX_BUFFER_SIZE_PARAMS(c.Struct): + SIZE = 16 + hChannel: int + totalBufferSize: int +struct_NV2080_CTRL_FLCN_GET_CTX_BUFFER_SIZE_PARAMS.register_fields([('hChannel', NvHandle, 0), ('totalBufferSize', NvU64, 8)]) +NV2080_CTRL_FLCN_GET_CTX_BUFFER_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_FLCN_GET_CTX_BUFFER_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPIO_PROGRAM_DIRECTION_PARAMS(c.Struct): + SIZE = 8 + gpioPin: int + bInput: int +struct_NV2080_CTRL_INTERNAL_GPIO_PROGRAM_DIRECTION_PARAMS.register_fields([('gpioPin', NvU32, 0), ('bInput', NvBool, 4)]) +NV2080_CTRL_INTERNAL_GPIO_PROGRAM_DIRECTION_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPIO_PROGRAM_DIRECTION_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPIO_PROGRAM_OUTPUT_PARAMS(c.Struct): + SIZE = 8 + gpioPin: int + value: int +struct_NV2080_CTRL_INTERNAL_GPIO_PROGRAM_OUTPUT_PARAMS.register_fields([('gpioPin', NvU32, 0), ('value', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GPIO_PROGRAM_OUTPUT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPIO_PROGRAM_OUTPUT_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPIO_READ_INPUT_PARAMS(c.Struct): + SIZE = 8 + gpioPin: int + value: int +struct_NV2080_CTRL_INTERNAL_GPIO_READ_INPUT_PARAMS.register_fields([('gpioPin', NvU32, 0), ('value', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GPIO_READ_INPUT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPIO_READ_INPUT_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPIO_ACTIVATE_HW_FUNCTION_PARAMS(c.Struct): + SIZE = 8 + function: int + pin: int +struct_NV2080_CTRL_INTERNAL_GPIO_ACTIVATE_HW_FUNCTION_PARAMS.register_fields([('function', NvU32, 0), ('pin', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GPIO_ACTIVATE_HW_FUNCTION_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPIO_ACTIVATE_HW_FUNCTION_PARAMS +NV2080_CTRL_GPU_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_INFO_PARAMS(c.Struct): + SIZE = 16 + gpuInfoListSize: int + gpuInfoList: ctypes.c_void_p +struct_NV2080_CTRL_GPU_GET_INFO_PARAMS.register_fields([('gpuInfoListSize', NvU32, 0), ('gpuInfoList', NvP64, 8)]) +NV2080_CTRL_GPU_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_INFO_V2_PARAMS(c.Struct): + SIZE = 580 + gpuInfoListSize: int + gpuInfoList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[72]] +struct_NV2080_CTRL_GPU_GET_INFO_V2_PARAMS.register_fields([('gpuInfoListSize', NvU32, 0), ('gpuInfoList', c.Array[NV2080_CTRL_GPU_INFO, Literal[72]], 4)]) +NV2080_CTRL_GPU_GET_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS(c.Struct): + SIZE = 68 + gpuNameStringFlags: int + gpuNameString: struct_NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS_gpuNameString +@c.record +class struct_NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS_gpuNameString(c.Struct): + SIZE = 64 + ascii: c.Array[ctypes.c_ubyte, Literal[64]] +struct_NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS_gpuNameString.register_fields([('ascii', c.Array[NvU8, Literal[64]], 0)]) +struct_NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS.register_fields([('gpuNameStringFlags', NvU32, 0), ('gpuNameString', struct_NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS_gpuNameString, 4)]) +NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_SHORT_NAME_STRING_PARAMS(c.Struct): + SIZE = 64 + gpuShortNameString: c.Array[ctypes.c_ubyte, Literal[64]] +struct_NV2080_CTRL_GPU_GET_SHORT_NAME_STRING_PARAMS.register_fields([('gpuShortNameString', c.Array[NvU8, Literal[64]], 0)]) +NV2080_CTRL_GPU_GET_SHORT_NAME_STRING_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_SHORT_NAME_STRING_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_POWER_PARAMS(c.Struct): + SIZE = 12 + target: int + newLevel: int + oldLevel: int +struct_NV2080_CTRL_GPU_SET_POWER_PARAMS.register_fields([('target', NvU32, 0), ('newLevel', NvU32, 4), ('oldLevel', NvU32, 8)]) +NV2080_CTRL_GPU_SET_POWER_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_POWER_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_SDM_PARAMS(c.Struct): + SIZE = 4 + subdeviceMask: int +struct_NV2080_CTRL_GPU_GET_SDM_PARAMS.register_fields([('subdeviceMask', NvU32, 0)]) +NV2080_CTRL_GPU_GET_SDM_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_SDM_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_SDM_PARAMS(c.Struct): + SIZE = 4 + subdeviceMask: int +struct_NV2080_CTRL_GPU_SET_SDM_PARAMS.register_fields([('subdeviceMask', NvU32, 0)]) +NV2080_CTRL_GPU_SET_SDM_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_SDM_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_SIMULATION_INFO_PARAMS(c.Struct): + SIZE = 4 + type: int +struct_NV2080_CTRL_GPU_GET_SIMULATION_INFO_PARAMS.register_fields([('type', NvU32, 0)]) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_SIMULATION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_REG_OP(c.Struct): + SIZE = 32 + regOp: int + regType: int + regStatus: int + regQuad: int + regGroupMask: int + regSubGroupMask: int + regOffset: int + regValueHi: int + regValueLo: int + regAndNMaskHi: int + regAndNMaskLo: int +struct_NV2080_CTRL_GPU_REG_OP.register_fields([('regOp', NvU8, 0), ('regType', NvU8, 1), ('regStatus', NvU8, 2), ('regQuad', NvU8, 3), ('regGroupMask', NvU32, 4), ('regSubGroupMask', NvU32, 8), ('regOffset', NvU32, 12), ('regValueHi', NvU32, 16), ('regValueLo', NvU32, 20), ('regAndNMaskHi', NvU32, 24), ('regAndNMaskLo', NvU32, 28)]) +NV2080_CTRL_GPU_REG_OP: TypeAlias = struct_NV2080_CTRL_GPU_REG_OP +@c.record +class struct_NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS(c.Struct): + SIZE = 48 + hClientTarget: int + hChannelTarget: int + bNonTransactional: int + reserved00: c.Array[ctypes.c_uint32, Literal[2]] + regOpCount: int + regOps: ctypes.c_void_p + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +NV2080_CTRL_GR_ROUTE_INFO: TypeAlias = struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS.register_fields([('hClientTarget', NvHandle, 0), ('hChannelTarget', NvHandle, 4), ('bNonTransactional', NvU32, 8), ('reserved00', c.Array[NvU32, Literal[2]], 12), ('regOpCount', NvU32, 20), ('regOps', NvP64, 24), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 32)]) +NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ENGINES_PARAMS(c.Struct): + SIZE = 16 + engineCount: int + engineList: ctypes.c_void_p +struct_NV2080_CTRL_GPU_GET_ENGINES_PARAMS.register_fields([('engineCount', NvU32, 0), ('engineList', NvP64, 8)]) +NV2080_CTRL_GPU_GET_ENGINES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENGINES_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ENGINES_V2_PARAMS(c.Struct): + SIZE = 340 + engineCount: int + engineList: c.Array[ctypes.c_uint32, Literal[84]] +struct_NV2080_CTRL_GPU_GET_ENGINES_V2_PARAMS.register_fields([('engineCount', NvU32, 0), ('engineList', c.Array[NvU32, Literal[84]], 4)]) +NV2080_CTRL_GPU_GET_ENGINES_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENGINES_V2_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ENGINE_CLASSLIST_PARAMS(c.Struct): + SIZE = 16 + engineType: int + numClasses: int + classList: ctypes.c_void_p +struct_NV2080_CTRL_GPU_GET_ENGINE_CLASSLIST_PARAMS.register_fields([('engineType', NvU32, 0), ('numClasses', NvU32, 4), ('classList', NvP64, 8)]) +NV2080_CTRL_GPU_GET_ENGINE_CLASSLIST_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENGINE_CLASSLIST_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ENGINE_FAULT_INFO_PARAMS(c.Struct): + SIZE = 12 + engineType: int + mmuFaultId: int + bSubcontextSupported: int +struct_NV2080_CTRL_GPU_GET_ENGINE_FAULT_INFO_PARAMS.register_fields([('engineType', NvU32, 0), ('mmuFaultId', NvU32, 4), ('bSubcontextSupported', NvBool, 8)]) +NV2080_CTRL_GPU_GET_ENGINE_FAULT_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENGINE_FAULT_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY(c.Struct): + SIZE = 32 + gpuPhysAddr: int + gpuVirtAddr: int + size: int + physAttr: int + bufferId: int + bInitialize: int + bNonmapped: int +struct_NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY.register_fields([('gpuPhysAddr', NvU64, 0), ('gpuVirtAddr', NvU64, 8), ('size', NvU64, 16), ('physAttr', NvU32, 24), ('bufferId', NvU16, 28), ('bInitialize', NvU8, 30), ('bNonmapped', NvU8, 31)]) +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY: TypeAlias = struct_NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY +@c.record +class struct_NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS(c.Struct): + SIZE = 560 + engineType: int + hClient: int + ChID: int + hChanClient: int + hObject: int + hVirtMemory: int + virtAddress: int + size: int + entryCount: int + promoteEntry: c.Array[struct_NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY, Literal[16]] +struct_NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.register_fields([('engineType', NvU32, 0), ('hClient', NvHandle, 4), ('ChID', NvU32, 8), ('hChanClient', NvHandle, 12), ('hObject', NvHandle, 16), ('hVirtMemory', NvHandle, 20), ('virtAddress', NvU64, 24), ('size', NvU64, 32), ('entryCount', NvU32, 40), ('promoteEntry', c.Array[NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY, Literal[16]], 48)]) +NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS +PNV2080_CTRL_GPU_PROMOTE_CTX_PARAMS: TypeAlias = c.POINTER[struct_NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS] +@c.record +class struct_NV2080_CTRL_GPU_EVICT_CTX_PARAMS(c.Struct): + SIZE = 20 + engineType: int + hClient: int + ChID: int + hChanClient: int + hObject: int +struct_NV2080_CTRL_GPU_EVICT_CTX_PARAMS.register_fields([('engineType', NvU32, 0), ('hClient', NvHandle, 4), ('ChID', NvU32, 8), ('hChanClient', NvHandle, 12), ('hObject', NvHandle, 16)]) +NV2080_CTRL_GPU_EVICT_CTX_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_EVICT_CTX_PARAMS +PNV2080_CTRL_GPU_EVICT_CTX_PARAMS: TypeAlias = c.POINTER[struct_NV2080_CTRL_GPU_EVICT_CTX_PARAMS] +@c.record +class struct_NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS(c.Struct): + SIZE = 56 + engineType: int + hClient: int + ChID: int + hChanClient: int + hObject: int + hVirtMemory: int + physAddress: int + physAttr: int + hDmaHandle: int + index: int + size: int +struct_NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS.register_fields([('engineType', NvU32, 0), ('hClient', NvHandle, 4), ('ChID', NvU32, 8), ('hChanClient', NvHandle, 12), ('hObject', NvHandle, 16), ('hVirtMemory', NvHandle, 20), ('physAddress', NvU64, 24), ('physAttr', NvU32, 32), ('hDmaHandle', NvHandle, 36), ('index', NvU32, 40), ('size', NvU64, 48)]) +NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS +PNV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS: TypeAlias = c.POINTER[struct_NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS] +@c.record +class struct_NV2080_CTRL_GPU_QUERY_ECC_INTR_PARAMS(c.Struct): + SIZE = 4 + eccIntrStatus: int +struct_NV2080_CTRL_GPU_QUERY_ECC_INTR_PARAMS.register_fields([('eccIntrStatus', NvU32, 0)]) +NV2080_CTRL_GPU_QUERY_ECC_INTR_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_QUERY_ECC_INTR_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS(c.Struct): + SIZE = 8 + count: int +struct_NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS.register_fields([('count', NvU64, 0)]) +NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS: TypeAlias = struct_NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS +@c.record +class struct_NV2080_CTRL_GPU_QUERY_ECC_UNIT_STATUS(c.Struct): + SIZE = 40 + enabled: int + scrubComplete: int + supported: int + dbe: struct_NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS + dbeNonResettable: struct_NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS + sbe: struct_NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS + sbeNonResettable: struct_NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS +struct_NV2080_CTRL_GPU_QUERY_ECC_UNIT_STATUS.register_fields([('enabled', NvBool, 0), ('scrubComplete', NvBool, 1), ('supported', NvBool, 2), ('dbe', NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS, 8), ('dbeNonResettable', NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS, 16), ('sbe', NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS, 24), ('sbeNonResettable', NV2080_CTRL_GPU_QUERY_ECC_EXCEPTION_STATUS, 32)]) +NV2080_CTRL_GPU_QUERY_ECC_UNIT_STATUS: TypeAlias = struct_NV2080_CTRL_GPU_QUERY_ECC_UNIT_STATUS +@c.record +class struct_NV2080_CTRL_GPU_QUERY_ECC_STATUS_PARAMS(c.Struct): + SIZE = 1664 + units: c.Array[struct_NV2080_CTRL_GPU_QUERY_ECC_UNIT_STATUS, Literal[41]] + bFatalPoisonError: int + uncorrectableError: int + flags: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GPU_QUERY_ECC_STATUS_PARAMS.register_fields([('units', c.Array[NV2080_CTRL_GPU_QUERY_ECC_UNIT_STATUS, Literal[41]], 0), ('bFatalPoisonError', NvBool, 1640), ('uncorrectableError', NvU8, 1641), ('flags', NvU32, 1644), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 1648)]) +NV2080_CTRL_GPU_QUERY_ECC_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_QUERY_ECC_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_COMPUTE_MODE_RULES_PARAMS(c.Struct): + SIZE = 8 + rules: int + flags: int +struct_NV2080_CTRL_GPU_SET_COMPUTE_MODE_RULES_PARAMS.register_fields([('rules', NvU32, 0), ('flags', NvU32, 4)]) +NV2080_CTRL_GPU_SET_COMPUTE_MODE_RULES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_COMPUTE_MODE_RULES_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_QUERY_COMPUTE_MODE_RULES_PARAMS(c.Struct): + SIZE = 4 + rules: int +struct_NV2080_CTRL_GPU_QUERY_COMPUTE_MODE_RULES_PARAMS.register_fields([('rules', NvU32, 0)]) +NV2080_CTRL_GPU_QUERY_COMPUTE_MODE_RULES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_QUERY_COMPUTE_MODE_RULES_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_QUERY_ECC_CONFIGURATION_PARAMS(c.Struct): + SIZE = 8 + currentConfiguration: int + defaultConfiguration: int +struct_NV2080_CTRL_GPU_QUERY_ECC_CONFIGURATION_PARAMS.register_fields([('currentConfiguration', NvU32, 0), ('defaultConfiguration', NvU32, 4)]) +NV2080_CTRL_GPU_QUERY_ECC_CONFIGURATION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_QUERY_ECC_CONFIGURATION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_ECC_CONFIGURATION_PARAMS(c.Struct): + SIZE = 4 + newConfiguration: int +struct_NV2080_CTRL_GPU_SET_ECC_CONFIGURATION_PARAMS.register_fields([('newConfiguration', NvU32, 0)]) +NV2080_CTRL_GPU_SET_ECC_CONFIGURATION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_ECC_CONFIGURATION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_PARAMS(c.Struct): + SIZE = 8 + statuses: int + flags: int +struct_NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_PARAMS.register_fields([('statuses', NvU32, 0), ('flags', NvU8, 4)]) +NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_FERMI_GPC_INFO_PARAMS(c.Struct): + SIZE = 4 + gpcMask: int +struct_NV2080_CTRL_GPU_GET_FERMI_GPC_INFO_PARAMS.register_fields([('gpcMask', NvU32, 0)]) +NV2080_CTRL_GPU_GET_FERMI_GPC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_FERMI_GPC_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_FERMI_TPC_INFO_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + tpcMask: int +struct_NV2080_CTRL_GPU_GET_FERMI_TPC_INFO_PARAMS.register_fields([('gpcId', NvU32, 0), ('tpcMask', NvU32, 4)]) +NV2080_CTRL_GPU_GET_FERMI_TPC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_FERMI_TPC_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_FERMI_ZCULL_INFO_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + zcullMask: int +struct_NV2080_CTRL_GPU_GET_FERMI_ZCULL_INFO_PARAMS.register_fields([('gpcId', NvU32, 0), ('zcullMask', NvU32, 4)]) +NV2080_CTRL_GPU_GET_FERMI_ZCULL_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_FERMI_ZCULL_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_OEM_BOARD_INFO_PARAMS(c.Struct): + SIZE = 136 + buildDate: int + marketingName: c.Array[ctypes.c_ubyte, Literal[24]] + serialNumber: c.Array[ctypes.c_ubyte, Literal[16]] + memoryManufacturer: int + memoryPartID: c.Array[ctypes.c_ubyte, Literal[20]] + memoryDateCode: c.Array[ctypes.c_ubyte, Literal[6]] + productPartNumber: c.Array[ctypes.c_ubyte, Literal[20]] + boardRevision: c.Array[ctypes.c_ubyte, Literal[3]] + boardType: int + board699PartNumber: c.Array[ctypes.c_ubyte, Literal[20]] + board965PartNumber: c.Array[ctypes.c_ubyte, Literal[20]] +struct_NV2080_CTRL_GPU_GET_OEM_BOARD_INFO_PARAMS.register_fields([('buildDate', NvU32, 0), ('marketingName', c.Array[NvU8, Literal[24]], 4), ('serialNumber', c.Array[NvU8, Literal[16]], 28), ('memoryManufacturer', NvU8, 44), ('memoryPartID', c.Array[NvU8, Literal[20]], 45), ('memoryDateCode', c.Array[NvU8, Literal[6]], 65), ('productPartNumber', c.Array[NvU8, Literal[20]], 71), ('boardRevision', c.Array[NvU8, Literal[3]], 91), ('boardType', NvU8, 94), ('board699PartNumber', c.Array[NvU8, Literal[20]], 95), ('board965PartNumber', c.Array[NvU8, Literal[20]], 115)]) +NV2080_CTRL_GPU_GET_OEM_BOARD_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_OEM_BOARD_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ID_PARAMS(c.Struct): + SIZE = 4 + gpuId: int +struct_NV2080_CTRL_GPU_GET_ID_PARAMS.register_fields([('gpuId', NvU32, 0)]) +NV2080_CTRL_GPU_GET_ID_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ID_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_GPU_DEBUG_MODE_PARAMS(c.Struct): + SIZE = 4 + mode: int +struct_NV2080_CTRL_GPU_SET_GPU_DEBUG_MODE_PARAMS.register_fields([('mode', NvU32, 0)]) +NV2080_CTRL_GPU_SET_GPU_DEBUG_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_GPU_DEBUG_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_GPU_DEBUG_MODE_PARAMS(c.Struct): + SIZE = 4 + currentMode: int +struct_NV2080_CTRL_GPU_GET_GPU_DEBUG_MODE_PARAMS.register_fields([('currentMode', NvU32, 0)]) +NV2080_CTRL_GPU_GET_GPU_DEBUG_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_GPU_DEBUG_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ENGINE_PARTNERLIST_PARAMS(c.Struct): + SIZE = 144 + engineType: int + partnershipClassId: int + runqueue: int + numPartners: int + partnerList: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV2080_CTRL_GPU_GET_ENGINE_PARTNERLIST_PARAMS.register_fields([('engineType', NvU32, 0), ('partnershipClassId', NvU32, 4), ('runqueue', NvU32, 8), ('numPartners', NvU32, 12), ('partnerList', c.Array[NvU32, Literal[32]], 16)]) +NV2080_CTRL_GPU_GET_ENGINE_PARTNERLIST_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENGINE_PARTNERLIST_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_GID_INFO_PARAMS(c.Struct): + SIZE = 268 + index: int + flags: int + length: int + data: c.Array[ctypes.c_ubyte, Literal[256]] +struct_NV2080_CTRL_GPU_GET_GID_INFO_PARAMS.register_fields([('index', NvU32, 0), ('flags', NvU32, 4), ('length', NvU32, 8), ('data', c.Array[NvU8, Literal[256]], 12)]) +NV2080_CTRL_GPU_GET_GID_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_GID_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_INFOROM_OBJECT_VERSION_PARAMS(c.Struct): + SIZE = 5 + objectType: c.Array[ctypes.c_char, Literal[3]] + version: int + subversion: int +struct_NV2080_CTRL_GPU_GET_INFOROM_OBJECT_VERSION_PARAMS.register_fields([('objectType', c.Array[ctypes.c_char, Literal[3]], 0), ('version', NvU8, 3), ('subversion', NvU8, 4)]) +NV2080_CTRL_GPU_GET_INFOROM_OBJECT_VERSION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_INFOROM_OBJECT_VERSION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_OPTIMUS_INFO_PARAMS(c.Struct): + SIZE = 1 + isOptimusEnabled: int +struct_NV2080_CTRL_GPU_OPTIMUS_INFO_PARAMS.register_fields([('isOptimusEnabled', NvBool, 0)]) +NV2080_CTRL_GPU_OPTIMUS_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_OPTIMUS_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_IP_VERSION_PARAMS(c.Struct): + SIZE = 8 + targetEngine: int + ipVersion: int +struct_NV2080_CTRL_GPU_GET_IP_VERSION_PARAMS.register_fields([('targetEngine', NvU32, 0), ('ipVersion', NvU32, 4)]) +NV2080_CTRL_GPU_GET_IP_VERSION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_IP_VERSION_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GPU_QUERY_ILLUM_SUPPORT_PARAMS(c.Struct): + SIZE = 8 + attribute: int + bSupported: int +struct_NV2080_CTRL_CMD_GPU_QUERY_ILLUM_SUPPORT_PARAMS.register_fields([('attribute', NvU32, 0), ('bSupported', NvBool, 4)]) +NV2080_CTRL_CMD_GPU_QUERY_ILLUM_SUPPORT_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_QUERY_ILLUM_SUPPORT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GPU_ILLUM_PARAMS(c.Struct): + SIZE = 8 + attribute: int + value: int +struct_NV2080_CTRL_CMD_GPU_ILLUM_PARAMS.register_fields([('attribute', NvU32, 0), ('value', NvU32, 4)]) +NV2080_CTRL_CMD_GPU_ILLUM_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_ILLUM_PARAMS +NV2080_CTRL_GPU_GET_ILLUM_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_ILLUM_PARAMS +NV2080_CTRL_GPU_SET_ILLUM_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_ILLUM_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_INFOROM_IMAGE_VERSION_PARAMS(c.Struct): + SIZE = 16 + version: c.Array[ctypes.c_ubyte, Literal[16]] +struct_NV2080_CTRL_GPU_GET_INFOROM_IMAGE_VERSION_PARAMS.register_fields([('version', c.Array[NvU8, Literal[16]], 0)]) +NV2080_CTRL_GPU_GET_INFOROM_IMAGE_VERSION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_INFOROM_IMAGE_VERSION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_PHYSICAL_BRIDGE_VERSION_PARAMS(c.Struct): + SIZE = 8 + fwVersion: int + oemVersion: int + siliconRevision: int + hwbcResourceType: int +struct_NV2080_CTRL_GPU_PHYSICAL_BRIDGE_VERSION_PARAMS.register_fields([('fwVersion', NvU32, 0), ('oemVersion', NvU8, 4), ('siliconRevision', NvU8, 5), ('hwbcResourceType', NvU8, 6)]) +NV2080_CTRL_GPU_PHYSICAL_BRIDGE_VERSION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_PHYSICAL_BRIDGE_VERSION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_PHYSICAL_BRIDGE_VERSION_INFO_PARAMS(c.Struct): + SIZE = 1204 + bridgeCount: int + hPhysicalBridges: c.Array[ctypes.c_uint32, Literal[100]] + bridgeList: c.Array[struct_NV2080_CTRL_GPU_PHYSICAL_BRIDGE_VERSION_PARAMS, Literal[100]] +struct_NV2080_CTRL_GPU_GET_PHYSICAL_BRIDGE_VERSION_INFO_PARAMS.register_fields([('bridgeCount', NvU8, 0), ('hPhysicalBridges', c.Array[NvHandle, Literal[100]], 4), ('bridgeList', c.Array[NV2080_CTRL_GPU_PHYSICAL_BRIDGE_VERSION_PARAMS, Literal[100]], 404)]) +NV2080_CTRL_GPU_GET_PHYSICAL_BRIDGE_VERSION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_PHYSICAL_BRIDGE_VERSION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_BRIDGE_VERSION_PARAMS(c.Struct): + SIZE = 16 + bus: int + device: int + func: int + oemVersion: int + siliconRevision: int + hwbcResourceType: int + domain: int + fwVersion: int +struct_NV2080_CTRL_GPU_BRIDGE_VERSION_PARAMS.register_fields([('bus', NvU8, 0), ('device', NvU8, 1), ('func', NvU8, 2), ('oemVersion', NvU8, 3), ('siliconRevision', NvU8, 4), ('hwbcResourceType', NvU8, 5), ('domain', NvU32, 8), ('fwVersion', NvU32, 12)]) +NV2080_CTRL_GPU_BRIDGE_VERSION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_BRIDGE_VERSION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ALL_BRIDGES_UPSTREAM_OF_GPU_PARAMS(c.Struct): + SIZE = 2004 + bridgeCount: int + physicalBridgeIds: c.Array[ctypes.c_uint32, Literal[100]] + bridgeList: c.Array[struct_NV2080_CTRL_GPU_BRIDGE_VERSION_PARAMS, Literal[100]] +struct_NV2080_CTRL_GPU_GET_ALL_BRIDGES_UPSTREAM_OF_GPU_PARAMS.register_fields([('bridgeCount', NvU8, 0), ('physicalBridgeIds', c.Array[NvU32, Literal[100]], 4), ('bridgeList', c.Array[NV2080_CTRL_GPU_BRIDGE_VERSION_PARAMS, Literal[100]], 404)]) +NV2080_CTRL_GPU_GET_ALL_BRIDGES_UPSTREAM_OF_GPU_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ALL_BRIDGES_UPSTREAM_OF_GPU_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_QUERY_SCRUBBER_STATUS_PARAMS(c.Struct): + SIZE = 24 + scrubberStatus: int + remainingTimeMs: int + scrubStartAddr: int + scrubEndAddr: int +struct_NV2080_CTRL_GPU_QUERY_SCRUBBER_STATUS_PARAMS.register_fields([('scrubberStatus', NvU32, 0), ('remainingTimeMs', NvU32, 4), ('scrubStartAddr', NvU64, 8), ('scrubEndAddr', NvU64, 16)]) +NV2080_CTRL_GPU_QUERY_SCRUBBER_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_QUERY_SCRUBBER_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_VPR_CAPS_PARAMS(c.Struct): + SIZE = 16 + minStartAddr: int + maxEndAddr: int +struct_NV2080_CTRL_GPU_GET_VPR_CAPS_PARAMS.register_fields([('minStartAddr', NvU64, 0), ('maxEndAddr', NvU64, 8)]) +NV2080_CTRL_GPU_GET_VPR_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_VPR_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_PES_INFO_PARAMS(c.Struct): + SIZE = 76 + gpcId: int + numPesInGpc: int + activePesMask: int + maxTpcPerGpcCount: int + tpcToPesMap: c.Array[ctypes.c_uint32, Literal[15]] +struct_NV2080_CTRL_GPU_GET_PES_INFO_PARAMS.register_fields([('gpcId', NvU32, 0), ('numPesInGpc', NvU32, 4), ('activePesMask', NvU32, 8), ('maxTpcPerGpcCount', NvU32, 12), ('tpcToPesMap', c.Array[NvU32, Literal[15]], 16)]) +NV2080_CTRL_GPU_GET_PES_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_PES_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_OEM_INFO_PARAMS(c.Struct): + SIZE = 504 + oemInfo: c.Array[ctypes.c_ubyte, Literal[504]] +struct_NV2080_CTRL_GPU_GET_OEM_INFO_PARAMS.register_fields([('oemInfo', c.Array[NvU8, Literal[504]], 0)]) +NV2080_CTRL_GPU_GET_OEM_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_OEM_INFO_PARAMS +enum_NV2080_CTRL_VPR_INFO_QUERY_TYPE: dict[int, str] = {(NV2080_CTRL_GPU_GET_VPR_INFO_QUERY_VPR_CAPS:=0): 'NV2080_CTRL_GPU_GET_VPR_INFO_QUERY_VPR_CAPS', (NV2080_CTRL_GPU_GET_VPR_INFO_QUERY_CUR_VPR_RANGE:=1): 'NV2080_CTRL_GPU_GET_VPR_INFO_QUERY_CUR_VPR_RANGE'} +NV2080_CTRL_VPR_INFO_QUERY_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GPU_GET_VPR_INFO_PARAMS(c.Struct): + SIZE = 24 + queryType: int + bIsVprEnabled: int + vprStartAddressInBytes: int + vprEndAddressInBytes: int +struct_NV2080_CTRL_GPU_GET_VPR_INFO_PARAMS.register_fields([('queryType', NV2080_CTRL_VPR_INFO_QUERY_TYPE, 0), ('bIsVprEnabled', NvBool, 4), ('vprStartAddressInBytes', NvU64, 8), ('vprEndAddressInBytes', NvU64, 16)]) +NV2080_CTRL_GPU_GET_VPR_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_VPR_INFO_PARAMS +enum_NV2080_CTRL_ENCODER_CAPACITY_QUERY_TYPE: dict[int, str] = {(NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_H264:=0): 'NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_H264', (NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_HEVC:=1): 'NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_HEVC', (NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_AV1:=2): 'NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_AV1'} +NV2080_CTRL_ENCODER_CAPACITY_QUERY_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_PARAMS(c.Struct): + SIZE = 8 + queryType: int + encoderCapacity: int +struct_NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_PARAMS.register_fields([('queryType', NV2080_CTRL_ENCODER_CAPACITY_QUERY_TYPE, 0), ('encoderCapacity', NvU32, 4)]) +NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_STATS_PARAMS(c.Struct): + SIZE = 12 + encoderSessionCount: int + averageEncodeFps: int + averageEncodeLatency: int +struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_STATS_PARAMS.register_fields([('encoderSessionCount', NvU32, 0), ('averageEncodeFps', NvU32, 4), ('averageEncodeLatency', NvU32, 8)]) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_NVENC_SW_SESSION_INFO(c.Struct): + SIZE = 32 + processId: int + subProcessId: int + sessionId: int + codecType: int + hResolution: int + vResolution: int + averageEncodeFps: int + averageEncodeLatency: int +struct_NV2080_CTRL_NVENC_SW_SESSION_INFO.register_fields([('processId', NvU32, 0), ('subProcessId', NvU32, 4), ('sessionId', NvU32, 8), ('codecType', NvU32, 12), ('hResolution', NvU32, 16), ('vResolution', NvU32, 20), ('averageEncodeFps', NvU32, 24), ('averageEncodeLatency', NvU32, 28)]) +NV2080_CTRL_NVENC_SW_SESSION_INFO: TypeAlias = struct_NV2080_CTRL_NVENC_SW_SESSION_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_PARAMS(c.Struct): + SIZE = 16 + sessionInfoTblEntry: int + sessionInfoTbl: ctypes.c_void_p +struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_PARAMS.register_fields([('sessionInfoTblEntry', NvU32, 0), ('sessionInfoTbl', NvP64, 8)]) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_FABRIC_BASE_ADDR_PARAMS(c.Struct): + SIZE = 8 + fabricBaseAddr: int +struct_NV2080_CTRL_GPU_SET_FABRIC_BASE_ADDR_PARAMS.register_fields([('fabricBaseAddr', NvU64, 0)]) +NV2080_CTRL_GPU_SET_FABRIC_BASE_ADDR_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_FABRIC_BASE_ADDR_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_VIRTUAL_INTERRUPT_PARAMS(c.Struct): + SIZE = 4 + handle: int +struct_NV2080_CTRL_GPU_VIRTUAL_INTERRUPT_PARAMS.register_fields([('handle', NvU32, 0)]) +NV2080_CTRL_GPU_VIRTUAL_INTERRUPT_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_VIRTUAL_INTERRUPT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GPU_QUERY_FUNCTION_STATUS_PARAMS(c.Struct): + SIZE = 12 + statusMask: int + xusbData: int + ppcData: int +struct_NV2080_CTRL_CMD_GPU_QUERY_FUNCTION_STATUS_PARAMS.register_fields([('statusMask', NvU32, 0), ('xusbData', NvU32, 4), ('ppcData', NvU32, 8)]) +NV2080_CTRL_CMD_GPU_QUERY_FUNCTION_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_QUERY_FUNCTION_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_PARTITION_SPAN(c.Struct): + SIZE = 16 + lo: int + hi: int +struct_NV2080_CTRL_GPU_PARTITION_SPAN.register_fields([('lo', NvU64, 0), ('hi', NvU64, 8)]) +NV2080_CTRL_GPU_PARTITION_SPAN: TypeAlias = struct_NV2080_CTRL_GPU_PARTITION_SPAN +@c.record +class struct_NV2080_CTRL_EXEC_PARTITION_SPAN(c.Struct): + SIZE = 16 + lo: int + hi: int +struct_NV2080_CTRL_EXEC_PARTITION_SPAN.register_fields([('lo', NvU64, 0), ('hi', NvU64, 8)]) +NV2080_CTRL_EXEC_PARTITION_SPAN: TypeAlias = struct_NV2080_CTRL_EXEC_PARTITION_SPAN +@c.record +class struct_NV2080_CTRL_GPU_SET_PARTITION_INFO(c.Struct): + SIZE = 48 + swizzId: int + uuid: c.Array[ctypes.c_ubyte, Literal[16]] + partitionFlag: int + bValid: int + placement: struct_NV2080_CTRL_GPU_PARTITION_SPAN +struct_NV2080_CTRL_GPU_SET_PARTITION_INFO.register_fields([('swizzId', NvU32, 0), ('uuid', c.Array[NvU8, Literal[16]], 4), ('partitionFlag', NvU32, 20), ('bValid', NvBool, 24), ('placement', NV2080_CTRL_GPU_PARTITION_SPAN, 32)]) +NV2080_CTRL_GPU_SET_PARTITION_INFO: TypeAlias = struct_NV2080_CTRL_GPU_SET_PARTITION_INFO +@c.record +class struct_NV2080_CTRL_GPU_SET_PARTITIONS_PARAMS(c.Struct): + SIZE = 392 + partitionCount: int + partitionInfo: c.Array[struct_NV2080_CTRL_GPU_SET_PARTITION_INFO, Literal[8]] +struct_NV2080_CTRL_GPU_SET_PARTITIONS_PARAMS.register_fields([('partitionCount', NvU32, 0), ('partitionInfo', c.Array[NV2080_CTRL_GPU_SET_PARTITION_INFO, Literal[8]], 8)]) +NV2080_CTRL_GPU_SET_PARTITIONS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_PARTITIONS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_PARTITION_INFO(c.Struct): + SIZE = 232 + swizzId: int + partitionFlag: int + grEngCount: int + veidCount: int + smCount: int + ceCount: int + nvEncCount: int + nvDecCount: int + nvJpgCount: int + nvOfaCount: int + gpcCount: int + virtualGpcCount: int + gfxGpcCount: int + gpcsPerGr: c.Array[ctypes.c_uint32, Literal[8]] + virtualGpcsPerGr: c.Array[ctypes.c_uint32, Literal[8]] + gfxGpcPerGr: c.Array[ctypes.c_uint32, Literal[8]] + veidsPerGr: c.Array[ctypes.c_uint32, Literal[8]] + memSize: int + span: struct_NV2080_CTRL_GPU_PARTITION_SPAN + bValid: int + bPartitionError: int + validCTSIdMask: int + validGfxCTSIdMask: int +struct_NV2080_CTRL_GPU_GET_PARTITION_INFO.register_fields([('swizzId', NvU32, 0), ('partitionFlag', NvU32, 4), ('grEngCount', NvU32, 8), ('veidCount', NvU32, 12), ('smCount', NvU32, 16), ('ceCount', NvU32, 20), ('nvEncCount', NvU32, 24), ('nvDecCount', NvU32, 28), ('nvJpgCount', NvU32, 32), ('nvOfaCount', NvU32, 36), ('gpcCount', NvU32, 40), ('virtualGpcCount', NvU32, 44), ('gfxGpcCount', NvU32, 48), ('gpcsPerGr', c.Array[NvU32, Literal[8]], 52), ('virtualGpcsPerGr', c.Array[NvU32, Literal[8]], 84), ('gfxGpcPerGr', c.Array[NvU32, Literal[8]], 116), ('veidsPerGr', c.Array[NvU32, Literal[8]], 148), ('memSize', NvU64, 184), ('span', NV2080_CTRL_GPU_PARTITION_SPAN, 192), ('bValid', NvBool, 208), ('bPartitionError', NvBool, 209), ('validCTSIdMask', NvU64, 216), ('validGfxCTSIdMask', NvU64, 224)]) +NV2080_CTRL_GPU_GET_PARTITION_INFO: TypeAlias = struct_NV2080_CTRL_GPU_GET_PARTITION_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_PARTITIONS_PARAMS(c.Struct): + SIZE = 1864 + queryPartitionInfo: c.Array[struct_NV2080_CTRL_GPU_GET_PARTITION_INFO, Literal[8]] + validPartitionCount: int + bGetAllPartitionInfo: int +struct_NV2080_CTRL_GPU_GET_PARTITIONS_PARAMS.register_fields([('queryPartitionInfo', c.Array[NV2080_CTRL_GPU_GET_PARTITION_INFO, Literal[8]], 0), ('validPartitionCount', NvU32, 1856), ('bGetAllPartitionInfo', NvBool, 1860)]) +NV2080_CTRL_GPU_GET_PARTITIONS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_PARTITIONS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_CONFIGURE_PARTITION_PARAMS(c.Struct): + SIZE = 44 + swizzId: int + gpcCountPerSmcEng: c.Array[ctypes.c_uint32, Literal[8]] + updateSmcEngMask: int + bUseAllGPCs: int +struct_NV2080_CTRL_GPU_CONFIGURE_PARTITION_PARAMS.register_fields([('swizzId', NvU32, 0), ('gpcCountPerSmcEng', c.Array[NvU32, Literal[8]], 4), ('updateSmcEngMask', NvU32, 36), ('bUseAllGPCs', NvBool, 40)]) +NV2080_CTRL_GPU_CONFIGURE_PARTITION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_CONFIGURE_PARTITION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_FAULT_PACKET(c.Struct): + SIZE = 32 + data: c.Array[ctypes.c_ubyte, Literal[32]] +struct_NV2080_CTRL_GPU_FAULT_PACKET.register_fields([('data', c.Array[NvU8, Literal[32]], 0)]) +NV2080_CTRL_GPU_FAULT_PACKET: TypeAlias = struct_NV2080_CTRL_GPU_FAULT_PACKET +@c.record +class struct_NV2080_CTRL_GPU_REPORT_NON_REPLAYABLE_FAULT_PARAMS(c.Struct): + SIZE = 32 + faultPacket: struct_NV2080_CTRL_GPU_FAULT_PACKET +struct_NV2080_CTRL_GPU_REPORT_NON_REPLAYABLE_FAULT_PARAMS.register_fields([('faultPacket', NV2080_CTRL_GPU_FAULT_PACKET, 0)]) +NV2080_CTRL_GPU_REPORT_NON_REPLAYABLE_FAULT_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_REPORT_NON_REPLAYABLE_FAULT_PARAMS +NV2080_CTRL_GPU_EXEC_REG_OPS_VGPU_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_PRI_BASE_PARAMS(c.Struct): + SIZE = 1008 + engineList: c.Array[ctypes.c_uint32, Literal[84]] + runlistPriBase: c.Array[ctypes.c_uint32, Literal[84]] + runlistId: c.Array[ctypes.c_uint32, Literal[84]] +struct_NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_PRI_BASE_PARAMS.register_fields([('engineList', c.Array[NvU32, Literal[84]], 0), ('runlistPriBase', c.Array[NvU32, Literal[84]], 336), ('runlistId', c.Array[NvU32, Literal[84]], 672)]) +NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_PRI_BASE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_PRI_BASE_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_HW_ENGINE_ID_PARAMS(c.Struct): + SIZE = 672 + engineList: c.Array[ctypes.c_uint32, Literal[84]] + hwEngineID: c.Array[ctypes.c_uint32, Literal[84]] +struct_NV2080_CTRL_GPU_GET_HW_ENGINE_ID_PARAMS.register_fields([('engineList', c.Array[NvU32, Literal[84]], 0), ('hwEngineID', c.Array[NvU32, Literal[84]], 336)]) +NV2080_CTRL_GPU_GET_HW_ENGINE_ID_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_HW_ENGINE_ID_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_STATS_PARAMS(c.Struct): + SIZE = 12 + sessionCount: int + averageFPS: int + averageLatency: int +struct_NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_STATS_PARAMS.register_fields([('sessionCount', NvU32, 0), ('averageFPS', NvU32, 4), ('averageLatency', NvU32, 8)]) +NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_NVFBC_SW_SESSION_INFO(c.Struct): + SIZE = 52 + processId: int + subProcessId: int + vgpuInstanceId: int + sessionId: int + sessionType: int + displayOrdinal: int + sessionFlags: int + hMaxResolution: int + vMaxResolution: int + hResolution: int + vResolution: int + averageFPS: int + averageLatency: int +struct_NV2080_CTRL_NVFBC_SW_SESSION_INFO.register_fields([('processId', NvU32, 0), ('subProcessId', NvU32, 4), ('vgpuInstanceId', NvU32, 8), ('sessionId', NvU32, 12), ('sessionType', NvU32, 16), ('displayOrdinal', NvU32, 20), ('sessionFlags', NvU32, 24), ('hMaxResolution', NvU32, 28), ('vMaxResolution', NvU32, 32), ('hResolution', NvU32, 36), ('vResolution', NvU32, 40), ('averageFPS', NvU32, 44), ('averageLatency', NvU32, 48)]) +NV2080_CTRL_NVFBC_SW_SESSION_INFO: TypeAlias = struct_NV2080_CTRL_NVFBC_SW_SESSION_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_INFO_PARAMS(c.Struct): + SIZE = 13316 + sessionInfoCount: int + sessionInfoTbl: c.Array[struct_NV2080_CTRL_NVFBC_SW_SESSION_INFO, Literal[256]] +struct_NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_INFO_PARAMS.register_fields([('sessionInfoCount', NvU32, 0), ('sessionInfoTbl', c.Array[NV2080_CTRL_NVFBC_SW_SESSION_INFO, Literal[256]], 4)]) +NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_FIRST_ASYNC_CE_IDX_PARAMS(c.Struct): + SIZE = 4 + firstAsyncCEIdx: int +struct_NV2080_CTRL_GPU_GET_FIRST_ASYNC_CE_IDX_PARAMS.register_fields([('firstAsyncCEIdx', NvU32, 0)]) +NV2080_CTRL_GPU_GET_FIRST_ASYNC_CE_IDX_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_FIRST_ASYNC_CE_IDX_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_VMMU_SEGMENT_SIZE_PARAMS(c.Struct): + SIZE = 8 + vmmuSegmentSize: int +struct_NV2080_CTRL_GPU_GET_VMMU_SEGMENT_SIZE_PARAMS.register_fields([('vmmuSegmentSize', NvU64, 0)]) +NV2080_CTRL_GPU_GET_VMMU_SEGMENT_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_VMMU_SEGMENT_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_PARTITION_CAPACITY_PARAMS(c.Struct): + SIZE = 280 + partitionFlag: int + partitionCount: int + availableSpans: c.Array[struct_NV2080_CTRL_GPU_PARTITION_SPAN, Literal[8]] + availableSpansCount: int + totalPartitionCount: int + totalSpans: c.Array[struct_NV2080_CTRL_GPU_PARTITION_SPAN, Literal[8]] + totalSpansCount: int + bStaticInfo: int +struct_NV2080_CTRL_GPU_GET_PARTITION_CAPACITY_PARAMS.register_fields([('partitionFlag', NvU32, 0), ('partitionCount', NvU32, 4), ('availableSpans', c.Array[NV2080_CTRL_GPU_PARTITION_SPAN, Literal[8]], 8), ('availableSpansCount', NvU32, 136), ('totalPartitionCount', NvU32, 140), ('totalSpans', c.Array[NV2080_CTRL_GPU_PARTITION_SPAN, Literal[8]], 144), ('totalSpansCount', NvU32, 272), ('bStaticInfo', NvBool, 276)]) +NV2080_CTRL_GPU_GET_PARTITION_CAPACITY_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_PARTITION_CAPACITY_PARAMS +NV2080_CTRL_GPU_GET_CACHED_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_PARTITIONING_MODE_PARAMS(c.Struct): + SIZE = 4 + partitioningMode: int +struct_NV2080_CTRL_GPU_SET_PARTITIONING_MODE_PARAMS.register_fields([('partitioningMode', NvU32, 0)]) +NV2080_CTRL_GPU_SET_PARTITIONING_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_PARTITIONING_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_INFO(c.Struct): + SIZE = 64 + partitionFlag: int + grCount: int + gfxGrCount: int + gpcCount: int + virtualGpcCount: int + gfxGpcCount: int + veidCount: int + smCount: int + ceCount: int + nvEncCount: int + nvDecCount: int + nvJpgCount: int + nvOfaCount: int + memorySize: int +struct_NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_INFO.register_fields([('partitionFlag', NvU32, 0), ('grCount', NvU32, 4), ('gfxGrCount', NvU32, 8), ('gpcCount', NvU32, 12), ('virtualGpcCount', NvU32, 16), ('gfxGpcCount', NvU32, 20), ('veidCount', NvU32, 24), ('smCount', NvU32, 28), ('ceCount', NvU32, 32), ('nvEncCount', NvU32, 36), ('nvDecCount', NvU32, 40), ('nvJpgCount', NvU32, 44), ('nvOfaCount', NvU32, 48), ('memorySize', NvU64, 56)]) +NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_INFO: TypeAlias = struct_NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_INFO +@c.record +class struct_NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_PARAMS(c.Struct): + SIZE = 5768 + descCount: int + partitionDescs: c.Array[struct_NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_INFO, Literal[90]] +struct_NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_PARAMS.register_fields([('descCount', NvU32, 0), ('partitionDescs', c.Array[NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_INFO, Literal[90]], 8)]) +NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_MAX_SUPPORTED_PAGE_SIZE_PARAMS(c.Struct): + SIZE = 8 + maxSupportedPageSize: int +struct_NV2080_CTRL_GPU_GET_MAX_SUPPORTED_PAGE_SIZE_PARAMS.register_fields([('maxSupportedPageSize', NvU64, 0)]) +NV2080_CTRL_GPU_GET_MAX_SUPPORTED_PAGE_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_MAX_SUPPORTED_PAGE_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_NUM_MMUS_PER_GPC_PARAMS(c.Struct): + SIZE = 24 + gpcId: int + count: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GPU_GET_NUM_MMUS_PER_GPC_PARAMS.register_fields([('gpcId', NvU32, 0), ('count', NvU32, 4), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 8)]) +NV2080_CTRL_GPU_GET_NUM_MMUS_PER_GPC_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_NUM_MMUS_PER_GPC_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ACTIVE_PARTITION_IDS_PARAMS(c.Struct): + SIZE = 40 + swizzId: c.Array[ctypes.c_uint32, Literal[9]] + partitionCount: int +struct_NV2080_CTRL_GPU_GET_ACTIVE_PARTITION_IDS_PARAMS.register_fields([('swizzId', c.Array[NvU32, Literal[9]], 0), ('partitionCount', NvU32, 36)]) +NV2080_CTRL_GPU_GET_ACTIVE_PARTITION_IDS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ACTIVE_PARTITION_IDS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_PIDS_PARAMS(c.Struct): + SIZE = 3812 + idType: int + id: int + pidTblCount: int + pidTbl: c.Array[ctypes.c_uint32, Literal[950]] +struct_NV2080_CTRL_GPU_GET_PIDS_PARAMS.register_fields([('idType', NvU32, 0), ('id', NvU32, 4), ('pidTblCount', NvU32, 8), ('pidTbl', c.Array[NvU32, Literal[950]], 12)]) +NV2080_CTRL_GPU_GET_PIDS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_PIDS_PARAMS +@c.record +class struct_NV2080_CTRL_SMC_SUBSCRIPTION_INFO(c.Struct): + SIZE = 8 + computeInstanceId: int + gpuInstanceId: int +struct_NV2080_CTRL_SMC_SUBSCRIPTION_INFO.register_fields([('computeInstanceId', NvU32, 0), ('gpuInstanceId', NvU32, 4)]) +NV2080_CTRL_SMC_SUBSCRIPTION_INFO: TypeAlias = struct_NV2080_CTRL_SMC_SUBSCRIPTION_INFO +@c.record +class struct_NV2080_CTRL_GPU_PID_INFO_VIDEO_MEMORY_USAGE_DATA(c.Struct): + SIZE = 48 + memPrivate: int + memSharedOwned: int + memSharedDuped: int + protectedMemPrivate: int + protectedMemSharedOwned: int + protectedMemSharedDuped: int +struct_NV2080_CTRL_GPU_PID_INFO_VIDEO_MEMORY_USAGE_DATA.register_fields([('memPrivate', NvU64, 0), ('memSharedOwned', NvU64, 8), ('memSharedDuped', NvU64, 16), ('protectedMemPrivate', NvU64, 24), ('protectedMemSharedOwned', NvU64, 32), ('protectedMemSharedDuped', NvU64, 40)]) +NV2080_CTRL_GPU_PID_INFO_VIDEO_MEMORY_USAGE_DATA: TypeAlias = struct_NV2080_CTRL_GPU_PID_INFO_VIDEO_MEMORY_USAGE_DATA +@c.record +class union_NV2080_CTRL_GPU_PID_INFO_DATA(c.Struct): + SIZE = 48 + vidMemUsage: struct_NV2080_CTRL_GPU_PID_INFO_VIDEO_MEMORY_USAGE_DATA +union_NV2080_CTRL_GPU_PID_INFO_DATA.register_fields([('vidMemUsage', NV2080_CTRL_GPU_PID_INFO_VIDEO_MEMORY_USAGE_DATA, 0)]) +NV2080_CTRL_GPU_PID_INFO_DATA: TypeAlias = union_NV2080_CTRL_GPU_PID_INFO_DATA +@c.record +class struct_NV2080_CTRL_GPU_PID_INFO(c.Struct): + SIZE = 72 + pid: int + index: int + result: int + data: union_NV2080_CTRL_GPU_PID_INFO_DATA + smcSubscription: struct_NV2080_CTRL_SMC_SUBSCRIPTION_INFO +struct_NV2080_CTRL_GPU_PID_INFO.register_fields([('pid', NvU32, 0), ('index', NvU32, 4), ('result', NvU32, 8), ('data', NV2080_CTRL_GPU_PID_INFO_DATA, 16), ('smcSubscription', NV2080_CTRL_SMC_SUBSCRIPTION_INFO, 64)]) +NV2080_CTRL_GPU_PID_INFO: TypeAlias = struct_NV2080_CTRL_GPU_PID_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_PID_INFO_PARAMS(c.Struct): + SIZE = 14408 + pidInfoListCount: int + pidInfoList: c.Array[struct_NV2080_CTRL_GPU_PID_INFO, Literal[200]] +struct_NV2080_CTRL_GPU_GET_PID_INFO_PARAMS.register_fields([('pidInfoListCount', NvU32, 0), ('pidInfoList', c.Array[NV2080_CTRL_GPU_PID_INFO, Literal[200]], 8)]) +NV2080_CTRL_GPU_GET_PID_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_PID_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_PARAMS(c.Struct): + SIZE = 4 + faultType: int +struct_NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_PARAMS.register_fields([('faultType', NvU32, 0)]) +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_PARAMS +enum_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_DATA_TIMESLICE: dict[int, str] = {(NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_DEFAULT:=0): 'NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_DEFAULT', (NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_SHORT:=1): 'NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_SHORT', (NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_MEDIUM:=2): 'NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_MEDIUM', (NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_LONG:=3): 'NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_LONG', (NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_MAX:=4): 'NV2080_CTRL_CMD_GPU_COMPUTE_TIMESLICE_MAX'} +NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_DATA_TIMESLICE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG(c.Struct): + SIZE = 8 + type: int + data: struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_data +@c.record +class struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_data(c.Struct): + SIZE = 4 + timeslice: int +struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_data.register_fields([('timeslice', NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_DATA_TIMESLICE, 0)]) +struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG.register_fields([('type', NvU32, 0), ('data', struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_data, 4)]) +NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG: TypeAlias = struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG +@c.record +class struct_NV2080_CTRL_GPU_SET_COMPUTE_POLICY_CONFIG_PARAMS(c.Struct): + SIZE = 8 + config: struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG +struct_NV2080_CTRL_GPU_SET_COMPUTE_POLICY_CONFIG_PARAMS.register_fields([('config', NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG, 0)]) +NV2080_CTRL_GPU_SET_COMPUTE_POLICY_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_COMPUTE_POLICY_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_COMPUTE_POLICY_CONFIG_PARAMS(c.Struct): + SIZE = 260 + numConfigs: int + configList: c.Array[struct_NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG, Literal[32]] +struct_NV2080_CTRL_GPU_GET_COMPUTE_POLICY_CONFIG_PARAMS.register_fields([('numConfigs', NvU32, 0), ('configList', c.Array[NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG, Literal[32]], 4)]) +NV2080_CTRL_GPU_GET_COMPUTE_POLICY_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_COMPUTE_POLICY_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_GFID_PARAMS(c.Struct): + SIZE = 16 + domain: int + bus: int + device: int + func: int + gfid: int + gfidMask: int +struct_NV2080_CTRL_GPU_GET_GFID_PARAMS.register_fields([('domain', NvU32, 0), ('bus', NvU8, 4), ('device', NvU8, 5), ('func', NvU8, 6), ('gfid', NvU32, 8), ('gfidMask', NvU32, 12)]) +NV2080_CTRL_GPU_GET_GFID_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_GFID_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GPU_UPDATE_GFID_P2P_CAPABILITY_PARAMS(c.Struct): + SIZE = 12 + gfid: int + bEnable: int + fabricPartitionId: int +struct_NV2080_CTRL_CMD_GPU_UPDATE_GFID_P2P_CAPABILITY_PARAMS.register_fields([('gfid', NvU32, 0), ('bEnable', NvBool, 4), ('fabricPartitionId', NvU32, 8)]) +NV2080_CTRL_CMD_GPU_UPDATE_GFID_P2P_CAPABILITY_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_UPDATE_GFID_P2P_CAPABILITY_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS(c.Struct): + SIZE = 24 + addressStart: int + addressLength: int + protection: int +struct_NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS.register_fields([('addressStart', NvU64, 0), ('addressLength', NvU64, 8), ('protection', NvU32, 16)]) +NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ENGINE_LOAD_TIMES_PARAMS(c.Struct): + SIZE = 2624 + engineCount: int + engineList: c.Array[ctypes.c_uint32, Literal[201]] + engineStateLoadTime: c.Array[ctypes.c_uint64, Literal[201]] + engineIsInit: c.Array[ctypes.c_ubyte, Literal[201]] +struct_NV2080_CTRL_GPU_GET_ENGINE_LOAD_TIMES_PARAMS.register_fields([('engineCount', NvU32, 0), ('engineList', c.Array[NvU32, Literal[201]], 4), ('engineStateLoadTime', c.Array[NvU64, Literal[201]], 808), ('engineIsInit', c.Array[NvBool, Literal[201]], 2416)]) +NV2080_CTRL_GPU_GET_ENGINE_LOAD_TIMES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ENGINE_LOAD_TIMES_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_ID_NAME_MAPPING_PARAMS(c.Struct): + SIZE = 20908 + engineCount: int + engineID: c.Array[ctypes.c_uint32, Literal[201]] + engineName: c.Array[c.Array[ctypes.c_char, Literal[100]], Literal[201]] +struct_NV2080_CTRL_GPU_GET_ID_NAME_MAPPING_PARAMS.register_fields([('engineCount', NvU32, 0), ('engineID', c.Array[NvU32, Literal[201]], 4), ('engineName', c.Array[c.Array[ctypes.c_char, Literal[100]], Literal[201]], 808)]) +NV2080_CTRL_GPU_GET_ID_NAME_MAPPING_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_ID_NAME_MAPPING_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_EXEC_REG_OPS_NOPTRS_PARAMS(c.Struct): + SIZE = 3240 + hClientTarget: int + hChannelTarget: int + bNonTransactional: int + reserved00: c.Array[ctypes.c_uint32, Literal[2]] + regOpCount: int + regOps: c.Array[struct_NV2080_CTRL_GPU_REG_OP, Literal[100]] + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GPU_EXEC_REG_OPS_NOPTRS_PARAMS.register_fields([('hClientTarget', NvHandle, 0), ('hChannelTarget', NvHandle, 4), ('bNonTransactional', NvU32, 8), ('reserved00', c.Array[NvU32, Literal[2]], 12), ('regOpCount', NvU32, 20), ('regOps', c.Array[NV2080_CTRL_GPU_REG_OP, Literal[100]], 24), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 3224)]) +NV2080_CTRL_GPU_EXEC_REG_OPS_NOPTRS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_EXEC_REG_OPS_NOPTRS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SKYLINE_INFO(c.Struct): + SIZE = 76 + skylineVgpcSize: c.Array[ctypes.c_ubyte, Literal[64]] + singletonVgpcMask: int + maxInstances: int + computeSizeFlag: int +struct_NV2080_CTRL_GPU_SKYLINE_INFO.register_fields([('skylineVgpcSize', c.Array[NvU8, Literal[64]], 0), ('singletonVgpcMask', NvU32, 64), ('maxInstances', NvU32, 68), ('computeSizeFlag', NvU32, 72)]) +NV2080_CTRL_GPU_SKYLINE_INFO: TypeAlias = struct_NV2080_CTRL_GPU_SKYLINE_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_SKYLINE_INFO_PARAMS(c.Struct): + SIZE = 688 + skylineTable: c.Array[struct_NV2080_CTRL_GPU_SKYLINE_INFO, Literal[9]] + validEntries: int +struct_NV2080_CTRL_GPU_GET_SKYLINE_INFO_PARAMS.register_fields([('skylineTable', c.Array[NV2080_CTRL_GPU_SKYLINE_INFO, Literal[9]], 0), ('validEntries', NvU32, 684)]) +NV2080_CTRL_GPU_GET_SKYLINE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_SKYLINE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO(c.Struct): + SIZE = 52 + gpuId: int + gpuUuid: c.Array[ctypes.c_ubyte, Literal[16]] + p2pCaps: int + p2pOptimalReadCEs: int + p2pOptimalWriteCEs: int + p2pCapsStatus: c.Array[ctypes.c_ubyte, Literal[9]] + busPeerId: int + busEgmPeerId: int +struct_NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO.register_fields([('gpuId', NvU32, 0), ('gpuUuid', c.Array[NvU8, Literal[16]], 4), ('p2pCaps', NvU32, 20), ('p2pOptimalReadCEs', NvU32, 24), ('p2pOptimalWriteCEs', NvU32, 28), ('p2pCapsStatus', c.Array[NvU8, Literal[9]], 32), ('busPeerId', NvU32, 44), ('busEgmPeerId', NvU32, 48)]) +NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO: TypeAlias = struct_NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO +@c.record +class struct_NV2080_CTRL_GET_P2P_CAPS_PARAMS(c.Struct): + SIZE = 1672 + bAllCaps: int + bUseUuid: int + peerGpuCount: int + peerGpuCaps: c.Array[struct_NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO, Literal[32]] +struct_NV2080_CTRL_GET_P2P_CAPS_PARAMS.register_fields([('bAllCaps', NvBool, 0), ('bUseUuid', NvBool, 1), ('peerGpuCount', NvU32, 4), ('peerGpuCaps', c.Array[NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO, Literal[32]], 8)]) +NV2080_CTRL_GET_P2P_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_GET_P2P_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_COMPUTE_PROFILE(c.Struct): + SIZE = 20 + computeSize: int + flags: int + gfxGpcCount: int + gpcCount: int + veidCount: int + smCount: int +struct_NV2080_CTRL_GPU_COMPUTE_PROFILE.register_fields([('computeSize', NvU8, 0), ('flags', NvU16, 2), ('gfxGpcCount', NvU32, 4), ('gpcCount', NvU32, 8), ('veidCount', NvU32, 12), ('smCount', NvU32, 16)]) +NV2080_CTRL_GPU_COMPUTE_PROFILE: TypeAlias = struct_NV2080_CTRL_GPU_COMPUTE_PROFILE +@c.record +class struct_NV2080_CTRL_GPU_GET_COMPUTE_PROFILES_PARAMS(c.Struct): + SIZE = 168 + partitionFlag: int + profileCount: int + profiles: c.Array[struct_NV2080_CTRL_GPU_COMPUTE_PROFILE, Literal[8]] +struct_NV2080_CTRL_GPU_GET_COMPUTE_PROFILES_PARAMS.register_fields([('partitionFlag', NvU32, 0), ('profileCount', NvU32, 4), ('profiles', c.Array[NV2080_CTRL_GPU_COMPUTE_PROFILE, Literal[8]], 8)]) +NV2080_CTRL_GPU_GET_COMPUTE_PROFILES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_COMPUTE_PROFILES_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_FABRIC_CLIQUE(c.Struct): + SIZE = 8 + type: int + id: int +struct_NV2080_CTRL_GPU_FABRIC_CLIQUE.register_fields([('type', NvU8, 0), ('id', NvU32, 4)]) +NV2080_CTRL_GPU_FABRIC_CLIQUE: TypeAlias = struct_NV2080_CTRL_GPU_FABRIC_CLIQUE +@c.record +class struct_NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO_PARAMS(c.Struct): + SIZE = 320 + state: int + status: int + clusterUuid: c.Array[ctypes.c_ubyte, Literal[16]] + fabricPartitionId: int + fabricCaps: int + fabricCliqueId: int + fabricCliques: c.Array[struct_NV2080_CTRL_GPU_FABRIC_CLIQUE, Literal[32]] + fabricNumCliques: int + fabricHealthMask: int + fabricHealthSummary: int + numProbeReqs: int +struct_NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO_PARAMS.register_fields([('state', NvU8, 0), ('status', NV_STATUS, 4), ('clusterUuid', c.Array[NvU8, Literal[16]], 8), ('fabricPartitionId', NvU16, 24), ('fabricCaps', NvU64, 32), ('fabricCliqueId', NvU32, 40), ('fabricCliques', c.Array[NV2080_CTRL_GPU_FABRIC_CLIQUE, Literal[32]], 44), ('fabricNumCliques', NvU8, 300), ('fabricHealthMask', NvU32, 304), ('fabricHealthSummary', NvU8, 308), ('numProbeReqs', NvU64, 312)]) +NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_CHIP_DETAILS_PARAMS(c.Struct): + SIZE = 16 + pciDevId: int + chipSku: c.Array[ctypes.c_ubyte, Literal[4]] + chipMajor: int + chipMinor: int +struct_NV2080_CTRL_GPU_GET_CHIP_DETAILS_PARAMS.register_fields([('pciDevId', NvU32, 0), ('chipSku', c.Array[NvU8, Literal[4]], 4), ('chipMajor', NvU32, 8), ('chipMinor', NvU32, 12)]) +NV2080_CTRL_GPU_GET_CHIP_DETAILS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_CHIP_DETAILS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_MOVE_RUNLISTS_ALLOCATION_TO_SUBHEAP_PARAMS(c.Struct): + SIZE = 4 + swizzId: int +struct_NV2080_CTRL_GPU_MOVE_RUNLISTS_ALLOCATION_TO_SUBHEAP_PARAMS.register_fields([('swizzId', NvU32, 0)]) +NV2080_CTRL_GPU_MOVE_RUNLISTS_ALLOCATION_TO_SUBHEAP_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_MOVE_RUNLISTS_ALLOCATION_TO_SUBHEAP_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_MIGRATABLE_OPS_CMN_PARAMS(c.Struct): + SIZE = 1832 + hClientTarget: int + hChannelTarget: int + bNonTransactional: int + regOpCount: int + smIds: c.Array[ctypes.c_uint32, Literal[50]] + regOps: c.Array[struct_NV2080_CTRL_GPU_REG_OP, Literal[50]] + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GPU_MIGRATABLE_OPS_CMN_PARAMS.register_fields([('hClientTarget', NvHandle, 0), ('hChannelTarget', NvHandle, 4), ('bNonTransactional', NvU32, 8), ('regOpCount', NvU32, 12), ('smIds', c.Array[NvU32, Literal[50]], 16), ('regOps', c.Array[NV2080_CTRL_GPU_REG_OP, Literal[50]], 216), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 1816)]) +NV2080_CTRL_GPU_MIGRATABLE_OPS_CMN_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_MIGRATABLE_OPS_CMN_PARAMS +NV2080_CTRL_GPU_MIGRATABLE_OPS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_MIGRATABLE_OPS_CMN_PARAMS +NV2080_CTRL_GPU_MIGRATABLE_OPS_GSP_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_MIGRATABLE_OPS_CMN_PARAMS +NV2080_CTRL_GPU_MIGRATABLE_OPS_VGPU_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_MIGRATABLE_OPS_CMN_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_RESET_STATUS_PARAMS(c.Struct): + SIZE = 1 + bResetRequired: int +struct_NV2080_CTRL_GPU_GET_RESET_STATUS_PARAMS.register_fields([('bResetRequired', NvBool, 0)]) +NV2080_CTRL_GPU_GET_RESET_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_RESET_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_DRAIN_AND_RESET_STATUS_PARAMS(c.Struct): + SIZE = 1 + bDrainRecommended: int +struct_NV2080_CTRL_GPU_GET_DRAIN_AND_RESET_STATUS_PARAMS.register_fields([('bDrainRecommended', NvBool, 0)]) +NV2080_CTRL_GPU_GET_DRAIN_AND_RESET_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_DRAIN_AND_RESET_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_V2_PARAMS(c.Struct): + SIZE = 16388 + sessionInfoTblEntry: int + sessionInfoTbl: c.Array[struct_NV2080_CTRL_NVENC_SW_SESSION_INFO, Literal[512]] +struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_V2_PARAMS.register_fields([('sessionInfoTblEntry', NvU32, 0), ('sessionInfoTbl', c.Array[NV2080_CTRL_NVENC_SW_SESSION_INFO, Literal[512]], 4)]) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_CONSTRUCTED_FALCON_INFO(c.Struct): + SIZE = 20 + engDesc: int + ctxAttr: int + ctxBufferSize: int + addrSpaceList: int + registerBase: int +struct_NV2080_CTRL_GPU_CONSTRUCTED_FALCON_INFO.register_fields([('engDesc', NvU32, 0), ('ctxAttr', NvU32, 4), ('ctxBufferSize', NvU32, 8), ('addrSpaceList', NvU32, 12), ('registerBase', NvU32, 16)]) +NV2080_CTRL_GPU_CONSTRUCTED_FALCON_INFO: TypeAlias = struct_NV2080_CTRL_GPU_CONSTRUCTED_FALCON_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO_PARAMS(c.Struct): + SIZE = 1284 + numConstructedFalcons: int + constructedFalconsTable: c.Array[struct_NV2080_CTRL_GPU_CONSTRUCTED_FALCON_INFO, Literal[64]] +struct_NV2080_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO_PARAMS.register_fields([('numConstructedFalcons', NvU32, 0), ('constructedFalconsTable', c.Array[NV2080_CTRL_GPU_CONSTRUCTED_FALCON_INFO, Literal[64]], 4)]) +NV2080_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO_PARAMS +@c.record +class struct_NV2080_VF_MSIX_CAPS(c.Struct): + SIZE = 12 + msix_header: int + msix_table: int + msix_pba: int +struct_NV2080_VF_MSIX_CAPS.register_fields([('msix_header', NvU32, 0), ('msix_table', NvU32, 4), ('msix_pba', NvU32, 8)]) +NV2080_VF_MSIX_CAPS: TypeAlias = struct_NV2080_VF_MSIX_CAPS +@c.record +class struct_NV2080_CTRL_GPU_GET_VF_CAPS_PARAMS(c.Struct): + SIZE = 16 + gfid: int + vfMsixCap: struct_NV2080_VF_MSIX_CAPS +struct_NV2080_CTRL_GPU_GET_VF_CAPS_PARAMS.register_fields([('gfid', NvU32, 0), ('vfMsixCap', NV2080_VF_MSIX_CAPS, 4)]) +NV2080_CTRL_GPU_GET_VF_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_VF_CAPS_PARAMS +enum_NV2080_CTRL_GPU_RECOVERY_ACTION: dict[int, str] = {(NV2080_CTRL_GPU_RECOVERY_ACTION_NONE:=0): 'NV2080_CTRL_GPU_RECOVERY_ACTION_NONE', (NV2080_CTRL_GPU_RECOVERY_ACTION_GPU_PF_FLR:=1): 'NV2080_CTRL_GPU_RECOVERY_ACTION_GPU_PF_FLR', (NV2080_CTRL_GPU_RECOVERY_ACTION_OS_REBOOT:=2): 'NV2080_CTRL_GPU_RECOVERY_ACTION_OS_REBOOT', (NV2080_CTRL_GPU_RECOVERY_ACTION_DRAIN_P2P:=3): 'NV2080_CTRL_GPU_RECOVERY_ACTION_DRAIN_P2P', (NV2080_CTRL_GPU_RECOVERY_ACTION_DRAIN_AND_RESET:=4): 'NV2080_CTRL_GPU_RECOVERY_ACTION_DRAIN_AND_RESET', (NV2080_CTRL_GPU_RECOVERY_ACTION_RECOVER_IMEX_DOMAIN:=5): 'NV2080_CTRL_GPU_RECOVERY_ACTION_RECOVER_IMEX_DOMAIN', (NV2080_CTRL_GPU_RECOVERY_ACTION_BUS_RESET:=6): 'NV2080_CTRL_GPU_RECOVERY_ACTION_BUS_RESET', (NV2080_CTRL_GPU_RECOVERY_ACTION_SYSTEM_REBOOT:=7): 'NV2080_CTRL_GPU_RECOVERY_ACTION_SYSTEM_REBOOT'} +NV2080_CTRL_GPU_RECOVERY_ACTION: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GPU_GET_RECOVERY_ACTION_PARAMS(c.Struct): + SIZE = 4 + action: int +struct_NV2080_CTRL_GPU_GET_RECOVERY_ACTION_PARAMS.register_fields([('action', NV2080_CTRL_GPU_RECOVERY_ACTION, 0)]) +NV2080_CTRL_GPU_GET_RECOVERY_ACTION_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_RECOVERY_ACTION_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_FIPS_STATUS_PARAMS(c.Struct): + SIZE = 1 + bFipsEnabled: int +struct_NV2080_CTRL_GPU_GET_FIPS_STATUS_PARAMS.register_fields([('bFipsEnabled', NvBool, 0)]) +NV2080_CTRL_GPU_GET_FIPS_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_FIPS_STATUS_PARAMS +enum_NV2080_RAFTS_FLOORSWEEP_UNIT_MASK_TYPE: dict[int, str] = {(NV2080_RAFTS_FLOORSWEEP_UNIT_TYPE_INVALID:=0): 'NV2080_RAFTS_FLOORSWEEP_UNIT_TYPE_INVALID', (NV2080_RAFTS_FLOORSWEEP_UNIT_TYPE_TPC:=1): 'NV2080_RAFTS_FLOORSWEEP_UNIT_TYPE_TPC', (NV2080_RAFTS_FLOORSWEEP_UNIT_TYPE_GPC:=2): 'NV2080_RAFTS_FLOORSWEEP_UNIT_TYPE_GPC'} +NV2080_RAFTS_FLOORSWEEP_UNIT_MASK_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_RAFTS_FLOORSWEEP_INFO(c.Struct): + SIZE = 12 + unitType: int + parentId: int + mask: int +struct_NV2080_RAFTS_FLOORSWEEP_INFO.register_fields([('unitType', NV2080_RAFTS_FLOORSWEEP_UNIT_MASK_TYPE, 0), ('parentId', NvU32, 4), ('mask', NvU32, 8)]) +NV2080_RAFTS_FLOORSWEEP_INFO: TypeAlias = struct_NV2080_RAFTS_FLOORSWEEP_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_RAFTS_FS_MASK_PARAMS(c.Struct): + SIZE = 344 + tpcCountMatrix: c.Array[c.Array[ctypes.c_ubyte, Literal[12]], Literal[2]] + bValid: int + floorSweepConfig: c.Array[struct_NV2080_RAFTS_FLOORSWEEP_INFO, Literal[26]] + gfxGpcCount: int + gfxTpcPerGpcCount: int + maxUgpuTpcDiff: int +struct_NV2080_CTRL_GPU_GET_RAFTS_FS_MASK_PARAMS.register_fields([('tpcCountMatrix', c.Array[c.Array[NvU8, Literal[12]], Literal[2]], 0), ('bValid', NvBool, 24), ('floorSweepConfig', c.Array[NV2080_RAFTS_FLOORSWEEP_INFO, Literal[26]], 28), ('gfxGpcCount', NvU8, 340), ('gfxTpcPerGpcCount', NvU8, 341), ('maxUgpuTpcDiff', NvU8, 342)]) +NV2080_CTRL_GPU_GET_RAFTS_FS_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_RAFTS_FS_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_COMPUTE_PROFILE_CAPACITY_PARAMS(c.Struct): + SIZE = 152 + partitionFlag: int + computeSize: int + totalProfileCount: int + totalSpans: c.Array[struct_NV2080_CTRL_EXEC_PARTITION_SPAN, Literal[8]] + totalSpansCount: int +struct_NV2080_CTRL_GPU_GET_COMPUTE_PROFILE_CAPACITY_PARAMS.register_fields([('partitionFlag', NvU32, 0), ('computeSize', NvU32, 4), ('totalProfileCount', NvU32, 8), ('totalSpans', c.Array[NV2080_CTRL_EXEC_PARTITION_SPAN, Literal[8]], 16), ('totalSpansCount', NvU32, 144)]) +NV2080_CTRL_GPU_GET_COMPUTE_PROFILE_CAPACITY_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_COMPUTE_PROFILE_CAPACITY_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_TPC_RECONFIG_MASK_PARAMS(c.Struct): + SIZE = 8 + gpc: int + tpcReconfigMask: int +struct_NV2080_CTRL_GPU_GET_TPC_RECONFIG_MASK_PARAMS.register_fields([('gpc', NvU32, 0), ('tpcReconfigMask', NvU32, 4)]) +NV2080_CTRL_GPU_GET_TPC_RECONFIG_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_TPC_RECONFIG_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_RPC_GSP_TEST_PARAMS(c.Struct): + SIZE = 32 + test: int + dataSize: int + startTimestamp: int + stopTimestamp: int + data: ctypes.c_void_p +struct_NV2080_CTRL_GPU_RPC_GSP_TEST_PARAMS.register_fields([('test', NvU8, 0), ('dataSize', NvU32, 4), ('startTimestamp', NvU64, 8), ('stopTimestamp', NvU64, 16), ('data', NvP64, 24)]) +NV2080_CTRL_GPU_RPC_GSP_TEST_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_RPC_GSP_TEST_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_RPC_GSP_QUERY_SIZES_PARAMS(c.Struct): + SIZE = 24 + maxRpcSize: int + finnRmapiSize: int + rpcGspControlSize: int + rpcMessageHeaderSize: int + timestampFreq: int +struct_NV2080_CTRL_GPU_RPC_GSP_QUERY_SIZES_PARAMS.register_fields([('maxRpcSize', NvU32, 0), ('finnRmapiSize', NvU32, 4), ('rpcGspControlSize', NvU32, 8), ('rpcMessageHeaderSize', NvU32, 12), ('timestampFreq', NvU64, 16)]) +NV2080_CTRL_GPU_RPC_GSP_QUERY_SIZES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_RPC_GSP_QUERY_SIZES_PARAMS +@c.record +class struct_NV2080_CTRL_RUSD_GET_SUPPORTED_FEATURES_PARAMS(c.Struct): + SIZE = 4 + supportedFeatures: int +struct_NV2080_CTRL_RUSD_GET_SUPPORTED_FEATURES_PARAMS.register_fields([('supportedFeatures', NvU32, 0)]) +NV2080_CTRL_RUSD_GET_SUPPORTED_FEATURES_PARAMS: TypeAlias = struct_NV2080_CTRL_RUSD_GET_SUPPORTED_FEATURES_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_RUSD_SET_FEATURES_PARAMS(c.Struct): + SIZE = 8 + permanentPolledDataMask: int +struct_NV2080_CTRL_GPU_RUSD_SET_FEATURES_PARAMS.register_fields([('permanentPolledDataMask', NvU64, 0)]) +NV2080_CTRL_GPU_RUSD_SET_FEATURES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_RUSD_SET_FEATURES_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_FORCE_GSP_UNLOAD_PARAMS(c.Struct): + SIZE = 4 + flags: int +struct_NV2080_CTRL_GPU_FORCE_GSP_UNLOAD_PARAMS.register_fields([('flags', NvU32, 0)]) +NV2080_CTRL_GPU_FORCE_GSP_UNLOAD_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_FORCE_GSP_UNLOAD_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_VGA_SR_READ_WRITE_PARAMS(c.Struct): + SIZE = 12 + bWrite: int + head: int + index: int + data: int +struct_NV2080_CTRL_GPU_VGA_SR_READ_WRITE_PARAMS.register_fields([('bWrite', NvBool, 0), ('head', NvU32, 4), ('index', NvU8, 8), ('data', NvU8, 9)]) +NV2080_CTRL_GPU_VGA_SR_READ_WRITE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_VGA_SR_READ_WRITE_PARAMS +enum_NV2080_CTRL_GPU_INJECT_ERROR_TYPE: dict[int, str] = {(NV2080_CTRL_GPU_INJECT_ERROR_MESSAGE_QUEUE:=0): 'NV2080_CTRL_GPU_INJECT_ERROR_MESSAGE_QUEUE', (NV2080_CTRL_GPU_INJECT_ERROR__COUNT:=1): 'NV2080_CTRL_GPU_INJECT_ERROR__COUNT'} +NV2080_CTRL_GPU_INJECT_ERROR_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GPU_ERROR_INJECTION_CONTROL_PARAMS(c.Struct): + SIZE = 4 + errorType: int +struct_NV2080_CTRL_GPU_ERROR_INJECTION_CONTROL_PARAMS.register_fields([('errorType', NvU32, 0)]) +NV2080_CTRL_GPU_ERROR_INJECTION_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_ERROR_INJECTION_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_CHECK_MEM_SUBSYS_ERROR_PARAMS(c.Struct): + SIZE = 4 + flags: int +struct_NV2080_CTRL_GPU_CHECK_MEM_SUBSYS_ERROR_PARAMS.register_fields([('flags', NvU32, 0)]) +NV2080_CTRL_GPU_CHECK_MEM_SUBSYS_ERROR_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_CHECK_MEM_SUBSYS_ERROR_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_DEFAULT_TIMEOUT_PARAMS(c.Struct): + SIZE = 4 + defaultTimeout: int +struct_NV2080_CTRL_GPU_GET_DEFAULT_TIMEOUT_PARAMS.register_fields([('defaultTimeout', NvU32, 0)]) +NV2080_CTRL_GPU_GET_DEFAULT_TIMEOUT_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_DEFAULT_TIMEOUT_PARAMS +enum_NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_TYPE: dict[int, str] = {(NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_RSA_DECRYPTION:=0): 'NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_RSA_DECRYPTION', (NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_ECDSA:=1): 'NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_ECDSA', (NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_RNG:=2): 'NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_RNG', (NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_SHA:=3): 'NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_SHA'} +NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GSP_CRYPTO_CONTROL_PARAMS(c.Struct): + SIZE = 120 + opcode: int + inputMessageSize: int + inputMessage: ctypes.c_void_p + outputMessageSize: int + outputMessage: ctypes.c_void_p + modulusSize: int + modulus: ctypes.c_void_p + exponent: c.Array[ctypes.c_ubyte, Literal[4]] + usePresetKey: int + keySize: int + key: ctypes.c_void_p + nonce: c.Array[ctypes.c_ubyte, Literal[48]] +struct_NV2080_CTRL_GSP_CRYPTO_CONTROL_PARAMS.register_fields([('opcode', NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_TYPE, 0), ('inputMessageSize', NvU32, 4), ('inputMessage', NvP64, 8), ('outputMessageSize', NvU32, 16), ('outputMessage', NvP64, 24), ('modulusSize', NvU32, 32), ('modulus', NvP64, 40), ('exponent', c.Array[NvU8, Literal[4]], 48), ('usePresetKey', NvBool, 52), ('keySize', NvU32, 56), ('key', NvP64, 64), ('nonce', c.Array[NvU8, Literal[48]], 72)]) +NV2080_CTRL_GSP_CRYPTO_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_GSP_CRYPTO_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GPU_IS_RESET_COUPLED_PARAMS(c.Struct): + SIZE = 1 + isResetCoupled: int +struct_NV2080_CTRL_CMD_GPU_IS_RESET_COUPLED_PARAMS.register_fields([('isResetCoupled', NvBool, 0)]) +NV2080_CTRL_CMD_GPU_IS_RESET_COUPLED_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GPU_IS_RESET_COUPLED_PARAMS +enum_NV2080_CTRL_GPU_DIELET_TYPE: dict[int, str] = {(NV2080_CTRL_GPU_DIELET_TYPE_UGD:=0): 'NV2080_CTRL_GPU_DIELET_TYPE_UGD', (NV2080_CTRL_GPU_DIELET_TYPE_IODN:=1): 'NV2080_CTRL_GPU_DIELET_TYPE_IODN', (NV2080_CTRL_GPU_DIELET_TYPE_IODS:=2): 'NV2080_CTRL_GPU_DIELET_TYPE_IODS', (NV2080_CTRL_GPU_DIELET_TYPE_LAST:=3): 'NV2080_CTRL_GPU_DIELET_TYPE_LAST'} +NV2080_CTRL_GPU_DIELET_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GPU_GET_DIELET_INFO(c.Struct): + SIZE = 8 + dieletType: int + typeSpecificId: int +struct_NV2080_CTRL_GPU_GET_DIELET_INFO.register_fields([('dieletType', NV2080_CTRL_GPU_DIELET_TYPE, 0), ('typeSpecificId', NvU32, 4)]) +NV2080_CTRL_GPU_GET_DIELET_INFO: TypeAlias = struct_NV2080_CTRL_GPU_GET_DIELET_INFO +@c.record +class struct_NV2080_CTRL_GPU_GET_DIELET_INFO_PARAMS(c.Struct): + SIZE = 2052 + dieletListSize: int + dielets: c.Array[struct_NV2080_CTRL_GPU_GET_DIELET_INFO, Literal[256]] +struct_NV2080_CTRL_GPU_GET_DIELET_INFO_PARAMS.register_fields([('dieletListSize', NvU32, 0), ('dielets', c.Array[NV2080_CTRL_GPU_GET_DIELET_INFO, Literal[256]], 4)]) +NV2080_CTRL_GPU_GET_DIELET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_DIELET_INFO_PARAMS +enum_NV2080_CTRL_GPU_UNIT_TYPE: dict[int, str] = {(NV2080_CTRL_GPU_UNIT_TYPE_SYSLTC:=0): 'NV2080_CTRL_GPU_UNIT_TYPE_SYSLTC', (NV2080_CTRL_GPU_UNIT_TYPE_SYSLTS:=1): 'NV2080_CTRL_GPU_UNIT_TYPE_SYSLTS', (NV2080_CTRL_GPU_UNIT_TYPE_HSHUBCHI:=2): 'NV2080_CTRL_GPU_UNIT_TYPE_HSHUBCHI', (NV2080_CTRL_GPU_UNIT_TYPE_HSHUBNVL:=3): 'NV2080_CTRL_GPU_UNIT_TYPE_HSHUBNVL', (NV2080_CTRL_GPU_UNIT_TYPE_LAST:=4): 'NV2080_CTRL_GPU_UNIT_TYPE_LAST'} +NV2080_CTRL_GPU_UNIT_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GPU_GET_UNIT_FS_INFO_FROM_DIELET_PARAMS(c.Struct): + SIZE = 16 + dieletType: int + dieletTypeSpecificId: int + unitType: int + unitMask: int +struct_NV2080_CTRL_GPU_GET_UNIT_FS_INFO_FROM_DIELET_PARAMS.register_fields([('dieletType', NV2080_CTRL_GPU_DIELET_TYPE, 0), ('dieletTypeSpecificId', NvU32, 4), ('unitType', NV2080_CTRL_GPU_UNIT_TYPE, 8), ('unitMask', NvU32, 12)]) +NV2080_CTRL_GPU_GET_UNIT_FS_INFO_FROM_DIELET_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_UNIT_FS_INFO_FROM_DIELET_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_SET_MIGRATION_BLOCK_PARAMS(c.Struct): + SIZE = 1 + bIsDevtoolInUse: int +struct_NV2080_CTRL_GPU_SET_MIGRATION_BLOCK_PARAMS.register_fields([('bIsDevtoolInUse', NvBool, 0)]) +NV2080_CTRL_GPU_SET_MIGRATION_BLOCK_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_MIGRATION_BLOCK_PARAMS +@c.record +class struct_NV2080_CTRL_GPU_GET_EGM_INFO_PARAMS(c.Struct): + SIZE = 24 + egmPhysAddr: int + egmSize: int + egmNodeId: int +struct_NV2080_CTRL_GPU_GET_EGM_INFO_PARAMS.register_fields([('egmPhysAddr', NvU64, 0), ('egmSize', NvU64, 8), ('egmNodeId', NvS32, 16)]) +NV2080_CTRL_GPU_GET_EGM_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_EGM_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GPUMON_SAMPLE(c.Struct): + SIZE = 8 + timeStamp: int +struct_NV2080_CTRL_GPUMON_SAMPLE.register_fields([('timeStamp', NvU64, 0)]) +NV2080_CTRL_GPUMON_SAMPLE: TypeAlias = struct_NV2080_CTRL_GPUMON_SAMPLE +@c.record +class struct_NV2080_CTRL_GPUMON_SAMPLES(c.Struct): + SIZE = 24 + type: int + bufSize: int + count: int + tracker: int + pSamples: ctypes.c_void_p +struct_NV2080_CTRL_GPUMON_SAMPLES.register_fields([('type', NvU8, 0), ('bufSize', NvU32, 4), ('count', NvU32, 8), ('tracker', NvU32, 12), ('pSamples', NvP64, 16)]) +NV2080_CTRL_GPUMON_SAMPLES: TypeAlias = struct_NV2080_CTRL_GPUMON_SAMPLES +NV2080_CTRL_GR_INFO: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_GR_GET_INFO_PARAMS(c.Struct): + SIZE = 32 + grInfoListSize: int + grInfoList: ctypes.c_void_p + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_GET_INFO_PARAMS.register_fields([('grInfoListSize', NvU32, 0), ('grInfoList', NvP64, 8), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 16)]) +NV2080_CTRL_GR_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GR_CTXSW_ZCULL_MODE_PARAMS(c.Struct): + SIZE = 16 + hChannel: int + hShareClient: int + hShareChannel: int + zcullMode: int +struct_NV2080_CTRL_GR_CTXSW_ZCULL_MODE_PARAMS.register_fields([('hChannel', NvHandle, 0), ('hShareClient', NvHandle, 4), ('hShareChannel', NvHandle, 8), ('zcullMode', NvU32, 12)]) +NV2080_CTRL_GR_CTXSW_ZCULL_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_ZCULL_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS(c.Struct): + SIZE = 40 + widthAlignPixels: int + heightAlignPixels: int + pixelSquaresByAliquots: int + aliquotTotal: int + zcullRegionByteMultiplier: int + zcullRegionHeaderSize: int + zcullSubregionHeaderSize: int + subregionCount: int + subregionWidthAlignPixels: int + subregionHeightAlignPixels: int +struct_NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS.register_fields([('widthAlignPixels', NvU32, 0), ('heightAlignPixels', NvU32, 4), ('pixelSquaresByAliquots', NvU32, 8), ('aliquotTotal', NvU32, 12), ('zcullRegionByteMultiplier', NvU32, 16), ('zcullRegionHeaderSize', NvU32, 20), ('zcullSubregionHeaderSize', NvU32, 24), ('subregionCount', NvU32, 28), ('subregionWidthAlignPixels', NvU32, 32), ('subregionHeightAlignPixels', NvU32, 36)]) +NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GR_CTXSW_PM_MODE_PARAMS(c.Struct): + SIZE = 24 + hChannel: int + pmMode: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_CTXSW_PM_MODE_PARAMS.register_fields([('hChannel', NvHandle, 0), ('pmMode', NvU32, 4), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 8)]) +NV2080_CTRL_GR_CTXSW_PM_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_PM_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_CTXSW_ZCULL_BIND_PARAMS(c.Struct): + SIZE = 24 + hClient: int + hChannel: int + vMemPtr: int + zcullMode: int +struct_NV2080_CTRL_GR_CTXSW_ZCULL_BIND_PARAMS.register_fields([('hClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('vMemPtr', NvU64, 8), ('zcullMode', NvU32, 16)]) +NV2080_CTRL_GR_CTXSW_ZCULL_BIND_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_ZCULL_BIND_PARAMS +@c.record +class struct_NV2080_CTRL_GR_CTXSW_PM_BIND_PARAMS(c.Struct): + SIZE = 40 + hClient: int + hChannel: int + vMemPtr: int + pmMode: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_CTXSW_PM_BIND_PARAMS.register_fields([('hClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('vMemPtr', NvU64, 8), ('pmMode', NvU32, 16), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 24)]) +NV2080_CTRL_GR_CTXSW_PM_BIND_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_PM_BIND_PARAMS +@c.record +class struct_NV2080_CTRL_GR_CTXSW_SETUP_BIND_PARAMS(c.Struct): + SIZE = 16 + hClient: int + hChannel: int + vMemPtr: int +struct_NV2080_CTRL_GR_CTXSW_SETUP_BIND_PARAMS.register_fields([('hClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('vMemPtr', NvU64, 8)]) +NV2080_CTRL_GR_CTXSW_SETUP_BIND_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_SETUP_BIND_PARAMS +@c.record +class struct_NV2080_CTRL_GR_SET_GPC_TILE_MAP_PARAMS(c.Struct): + SIZE = 152 + mapValueCount: int + mapValues: c.Array[ctypes.c_ubyte, Literal[128]] + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_SET_GPC_TILE_MAP_PARAMS.register_fields([('mapValueCount', NvU32, 0), ('mapValues', c.Array[NvU8, Literal[128]], 4), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 136)]) +NV2080_CTRL_GR_SET_GPC_TILE_MAP_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_SET_GPC_TILE_MAP_PARAMS +@c.record +class struct_NV2080_CTRL_GR_CTXSW_SMPC_MODE_PARAMS(c.Struct): + SIZE = 24 + hChannel: int + smpcMode: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_CTXSW_SMPC_MODE_PARAMS.register_fields([('hChannel', NvHandle, 0), ('smpcMode', NvU32, 4), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 8)]) +NV2080_CTRL_GR_CTXSW_SMPC_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_SMPC_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS(c.Struct): + SIZE = 4120 + smId: c.Array[struct_NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS_smId, Literal[512]] + smCount: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +@c.record +class struct_NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS_smId(c.Struct): + SIZE = 8 + gpcId: int + tpcId: int +struct_NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS_smId.register_fields([('gpcId', NvU32, 0), ('tpcId', NvU32, 4)]) +struct_NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS.register_fields([('smId', c.Array[struct_NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS_smId, Literal[512]], 0), ('smCount', NvU32, 4096), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 4104)]) +NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS +@c.record +class struct_NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_PARAMS(c.Struct): + SIZE = 32 + flags: int + hChannel: int + gfxpPreemptMode: int + cilpPreemptMode: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_PARAMS.register_fields([('flags', NvU32, 0), ('hChannel', NvHandle, 4), ('gfxpPreemptMode', NvU32, 8), ('cilpPreemptMode', NvU32, 12), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 16)]) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_PARAMS +enum_NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS: dict[int, str] = {(NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_MAIN:=0): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_MAIN', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_SPILL:=1): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_SPILL', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_PAGEPOOL:=2): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_PAGEPOOL', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_BETACB:=3): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_BETACB', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_RTV:=4): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_RTV', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL:=5): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL_CONTROL:=6): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL_CONTROL', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL_CONTROL_CPU:=7): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL_CONTROL_CPU', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_SETUP:=8): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_SETUP', (NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_END:=9): 'NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_END'} +NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GR_CTXSW_PREEMPTION_BIND_PARAMS(c.Struct): + SIZE = 112 + flags: int + hClient: int + hChannel: int + vMemPtrs: c.Array[ctypes.c_uint64, Literal[9]] + gfxpPreemptMode: int + cilpPreemptMode: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_CTXSW_PREEMPTION_BIND_PARAMS.register_fields([('flags', NvU32, 0), ('hClient', NvHandle, 4), ('hChannel', NvHandle, 8), ('vMemPtrs', c.Array[NvU64, Literal[9]], 16), ('gfxpPreemptMode', NvU32, 88), ('cilpPreemptMode', NvU32, 92), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 96)]) +NV2080_CTRL_GR_CTXSW_PREEMPTION_BIND_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_PREEMPTION_BIND_PARAMS +@c.record +class struct_NV2080_CTRL_GR_PC_SAMPLING_MODE_PARAMS(c.Struct): + SIZE = 24 + hChannel: int + samplingMode: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_PC_SAMPLING_MODE_PARAMS.register_fields([('hChannel', NvHandle, 0), ('samplingMode', NvU32, 4), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 8)]) +NV2080_CTRL_GR_PC_SAMPLING_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_PC_SAMPLING_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_ROP_INFO_PARAMS(c.Struct): + SIZE = 12 + ropUnitCount: int + ropOperationsFactor: int + ropOperationsCount: int +struct_NV2080_CTRL_GR_GET_ROP_INFO_PARAMS.register_fields([('ropUnitCount', NvU32, 0), ('ropOperationsFactor', NvU32, 4), ('ropOperationsCount', NvU32, 8)]) +NV2080_CTRL_GR_GET_ROP_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_ROP_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_CTXSW_STATS_PARAMS(c.Struct): + SIZE = 32 + hChannel: int + flags: int + saveCnt: int + restoreCnt: int + wfiSaveCnt: int + ctaSaveCnt: int + cilpSaveCnt: int + gfxpSaveCnt: int +struct_NV2080_CTRL_GR_GET_CTXSW_STATS_PARAMS.register_fields([('hChannel', NvHandle, 0), ('flags', NvU32, 4), ('saveCnt', NvU32, 8), ('restoreCnt', NvU32, 12), ('wfiSaveCnt', NvU32, 16), ('ctaSaveCnt', NvU32, 20), ('cilpSaveCnt', NvU32, 24), ('gfxpSaveCnt', NvU32, 28)]) +NV2080_CTRL_GR_GET_CTXSW_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_CTXSW_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_CTX_BUFFER_SIZE_PARAMS(c.Struct): + SIZE = 16 + hChannel: int + totalBufferSize: int +struct_NV2080_CTRL_GR_GET_CTX_BUFFER_SIZE_PARAMS.register_fields([('hChannel', NvHandle, 0), ('totalBufferSize', NvU64, 8)]) +NV2080_CTRL_GR_GET_CTX_BUFFER_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_CTX_BUFFER_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_CTX_BUFFER_INFO(c.Struct): + SIZE = 80 + alignment: int + size: int + bufferHandle: ctypes.c_void_p + pageCount: int + physAddr: int + bufferType: int + aperture: int + kind: int + pageSize: int + bIsContigous: int + bGlobalBuffer: int + bLocalBuffer: int + bDeviceDescendant: int + uuid: c.Array[ctypes.c_ubyte, Literal[16]] +struct_NV2080_CTRL_GR_CTX_BUFFER_INFO.register_fields([('alignment', NvU64, 0), ('size', NvU64, 8), ('bufferHandle', NvP64, 16), ('pageCount', NvU64, 24), ('physAddr', NvU64, 32), ('bufferType', NvU32, 40), ('aperture', NvU32, 44), ('kind', NvU32, 48), ('pageSize', NvU32, 52), ('bIsContigous', NvBool, 56), ('bGlobalBuffer', NvBool, 57), ('bLocalBuffer', NvBool, 58), ('bDeviceDescendant', NvBool, 59), ('uuid', c.Array[NvU8, Literal[16]], 60)]) +NV2080_CTRL_GR_CTX_BUFFER_INFO: TypeAlias = struct_NV2080_CTRL_GR_CTX_BUFFER_INFO +PNV2080_CTRL_GR_CTX_BUFFER_INFO: TypeAlias = c.POINTER[struct_NV2080_CTRL_GR_CTX_BUFFER_INFO] +@c.record +class struct_NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS(c.Struct): + SIZE = 5136 + hUserClient: int + hChannel: int + bufferCount: int + ctxBufferInfo: c.Array[struct_NV2080_CTRL_GR_CTX_BUFFER_INFO, Literal[64]] +struct_NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS.register_fields([('hUserClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('bufferCount', NvU32, 8), ('ctxBufferInfo', c.Array[NV2080_CTRL_GR_CTX_BUFFER_INFO, Literal[64]], 16)]) +NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS(c.Struct): + SIZE = 10264 + globalSmId: c.Array[struct_NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS_globalSmId, Literal[512]] + numSm: int + numTpc: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +@c.record +class struct_NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS_globalSmId(c.Struct): + SIZE = 20 + gpcId: int + localTpcId: int + localSmId: int + globalTpcId: int + virtualGpcId: int + virtualDpcId: int + migratableTpcId: int + ugpuId: int + physicalCpcId: int + virtualTpcId: int +struct_NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS_globalSmId.register_fields([('gpcId', NvU16, 0), ('localTpcId', NvU16, 2), ('localSmId', NvU16, 4), ('globalTpcId', NvU16, 6), ('virtualGpcId', NvU16, 8), ('virtualDpcId', NvU16, 10), ('migratableTpcId', NvU16, 12), ('ugpuId', NvU16, 14), ('physicalCpcId', NvU16, 16), ('virtualTpcId', NvU16, 18)]) +struct_NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS.register_fields([('globalSmId', c.Array[struct_NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS_globalSmId, Literal[512]], 0), ('numSm', NvU16, 10240), ('numTpc', NvU16, 10242), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 10248)]) +NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_CURRENT_RESIDENT_CHANNEL_PARAMS(c.Struct): + SIZE = 24 + chID: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_GET_CURRENT_RESIDENT_CHANNEL_PARAMS.register_fields([('chID', NvU32, 0), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 8)]) +NV2080_CTRL_GR_GET_CURRENT_RESIDENT_CHANNEL_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_CURRENT_RESIDENT_CHANNEL_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_ATTRIBUTE_BUFFER_SIZE_PARAMS(c.Struct): + SIZE = 4 + attribBufferSize: int +struct_NV2080_CTRL_GR_GET_ATTRIBUTE_BUFFER_SIZE_PARAMS.register_fields([('attribBufferSize', NvU32, 0)]) +NV2080_CTRL_GR_GET_ATTRIBUTE_BUFFER_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_ATTRIBUTE_BUFFER_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GFX_POOL_QUERY_SIZE_PARAMS(c.Struct): + SIZE = 40 + maxSlots: int + slotStride: int + ctrlStructSize: int + ctrlStructAlign: int + poolSize: int + poolAlign: int +struct_NV2080_CTRL_GR_GFX_POOL_QUERY_SIZE_PARAMS.register_fields([('maxSlots', NvU32, 0), ('slotStride', NvU32, 4), ('ctrlStructSize', NvU64, 8), ('ctrlStructAlign', NvU64, 16), ('poolSize', NvU64, 24), ('poolAlign', NvU64, 32)]) +NV2080_CTRL_GR_GFX_POOL_QUERY_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GFX_POOL_QUERY_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GFX_POOL_INITIALIZE_PARAMS(c.Struct): + SIZE = 16 + maxSlots: int + hMemory: int + offset: int + size: int +struct_NV2080_CTRL_GR_GFX_POOL_INITIALIZE_PARAMS.register_fields([('maxSlots', NvU32, 0), ('hMemory', NvHandle, 4), ('offset', NvU32, 8), ('size', NvU32, 12)]) +NV2080_CTRL_GR_GFX_POOL_INITIALIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GFX_POOL_INITIALIZE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GFX_POOL_ADD_SLOTS_PARAMS(c.Struct): + SIZE = 272 + numSlots: int + slots: c.Array[ctypes.c_uint32, Literal[64]] + hMemory: int + offset: int + size: int +struct_NV2080_CTRL_GR_GFX_POOL_ADD_SLOTS_PARAMS.register_fields([('numSlots', NvU32, 0), ('slots', c.Array[NvU32, Literal[64]], 4), ('hMemory', NvHandle, 260), ('offset', NvU32, 264), ('size', NvU32, 268)]) +NV2080_CTRL_GR_GFX_POOL_ADD_SLOTS_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GFX_POOL_ADD_SLOTS_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GFX_POOL_REMOVE_SLOTS_PARAMS(c.Struct): + SIZE = 276 + numSlots: int + slots: c.Array[ctypes.c_uint32, Literal[64]] + bRemoveSpecificSlots: int + hMemory: int + offset: int + size: int +struct_NV2080_CTRL_GR_GFX_POOL_REMOVE_SLOTS_PARAMS.register_fields([('numSlots', NvU32, 0), ('slots', c.Array[NvU32, Literal[64]], 4), ('bRemoveSpecificSlots', NvBool, 260), ('hMemory', NvHandle, 264), ('offset', NvU32, 268), ('size', NvU32, 272)]) +NV2080_CTRL_GR_GFX_POOL_REMOVE_SLOTS_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GFX_POOL_REMOVE_SLOTS_PARAMS +NV2080_CTRL_GR_GET_CAPS_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_GET_CAPS_V2_PARAMS +NV2080_CTRL_GR_GET_INFO_V2_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_GET_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_GPC_MASK_PARAMS(c.Struct): + SIZE = 24 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + gpcMask: int +struct_NV2080_CTRL_GR_GET_GPC_MASK_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('gpcMask', NvU32, 16)]) +NV2080_CTRL_GR_GET_GPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_GPC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_TPC_MASK_PARAMS(c.Struct): + SIZE = 24 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + gpcId: int + tpcMask: int +struct_NV2080_CTRL_GR_GET_TPC_MASK_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('gpcId', NvU32, 16), ('tpcMask', NvU32, 20)]) +NV2080_CTRL_GR_GET_TPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_TPC_MASK_PARAMS +NV2080_CTRL_GR_SET_TPC_PARTITION_MODE_PARAMS: TypeAlias = struct_NV0080_CTRL_GR_TPC_PARTITION_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS(c.Struct): + SIZE = 32 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + engineId: int + alignment: int + size: int + bInfoPopulated: int +struct_NV2080_CTRL_GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('engineId', NvU32, 16), ('alignment', NvU32, 20), ('size', NvU32, 24), ('bInfoPopulated', NvBool, 28)]) +NV2080_CTRL_GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS(c.Struct): + SIZE = 32 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + imla0: int + fmla16: int + dp: int + fmla32: int + ffma: int + imla1: int + imla2: int + imla3: int + imla4: int +struct_NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('imla0', NvU8, 16), ('fmla16', NvU8, 17), ('dp', NvU8, 18), ('fmla32', NvU8, 19), ('ffma', NvU8, 20), ('imla1', NvU8, 21), ('imla2', NvU8, 22), ('imla3', NvU8, 23), ('imla4', NvU8, 24)]) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS(c.Struct): + SIZE = 2044 + smIssueRateModifierListSize: int + smIssueRateModifierList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[255]] +struct_NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS.register_fields([('smIssueRateModifierListSize', NvU32, 0), ('smIssueRateModifierList', c.Array[NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2, Literal[255]], 4)]) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS +NV2080_CTRL_GR_SM_ISSUE_THROTTLE_CTRL: TypeAlias = struct_NVXXXX_CTRL_XXX_INFO +@c.record +class struct_NV2080_CTRL_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS(c.Struct): + SIZE = 2044 + smIssueThrottleCtrlListSize: int + smIssueThrottleCtrlList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[255]] +struct_NV2080_CTRL_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS.register_fields([('smIssueThrottleCtrlListSize', NvU32, 0), ('smIssueThrottleCtrlList', c.Array[NV2080_CTRL_GR_SM_ISSUE_THROTTLE_CTRL, Literal[255]], 4)]) +NV2080_CTRL_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS +enum_NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD: dict[int, str] = {(NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_FULL:=0): 'NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_FULL', (NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_SIMPLE:=1): 'NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_SIMPLE', (NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_COMPAT:=2): 'NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_COMPAT', (NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_CUSTOM:=3): 'NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_CUSTOM'} +NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_PARAMS(c.Struct): + SIZE = 20 + hEventBuffer: int + recordSize: int + levelOfDetail: int + eventFilter: int + bAllUsers: int +struct_NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_PARAMS.register_fields([('hEventBuffer', NvHandle, 0), ('recordSize', NvU32, 4), ('levelOfDetail', NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD, 8), ('eventFilter', NvU32, 12), ('bAllUsers', NvBool, 16)]) +NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_PHYS_GPC_MASK_PARAMS(c.Struct): + SIZE = 8 + physSyspipeId: int + gpcMask: int +struct_NV2080_CTRL_GR_GET_PHYS_GPC_MASK_PARAMS.register_fields([('physSyspipeId', NvU32, 0), ('gpcMask', NvU32, 4)]) +NV2080_CTRL_GR_GET_PHYS_GPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_PHYS_GPC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_PPC_MASK_PARAMS(c.Struct): + SIZE = 24 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + gpcId: int + ppcMask: int +struct_NV2080_CTRL_GR_GET_PPC_MASK_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('gpcId', NvU32, 16), ('ppcMask', NvU32, 20)]) +NV2080_CTRL_GR_GET_PPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_PPC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_NUM_TPCS_FOR_GPC_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + numTpcs: int +struct_NV2080_CTRL_GR_GET_NUM_TPCS_FOR_GPC_PARAMS.register_fields([('gpcId', NvU32, 0), ('numTpcs', NvU32, 4)]) +NV2080_CTRL_GR_GET_NUM_TPCS_FOR_GPC_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_NUM_TPCS_FOR_GPC_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_CTXSW_MODES_PARAMS(c.Struct): + SIZE = 24 + hChannel: int + zcullMode: int + pmMode: int + smpcMode: int + cilpPreemptMode: int + gfxpPreemptMode: int +struct_NV2080_CTRL_GR_GET_CTXSW_MODES_PARAMS.register_fields([('hChannel', NvHandle, 0), ('zcullMode', NvU32, 4), ('pmMode', NvU32, 8), ('smpcMode', NvU32, 12), ('cilpPreemptMode', NvU32, 16), ('gfxpPreemptMode', NvU32, 20)]) +NV2080_CTRL_GR_GET_CTXSW_MODES_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_CTXSW_MODES_PARAMS +NV2080_CTRL_GR_GET_GPC_TILE_MAP_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_SET_GPC_TILE_MAP_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_ZCULL_MASK_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + zcullMask: int +struct_NV2080_CTRL_GR_GET_ZCULL_MASK_PARAMS.register_fields([('gpcId', NvU32, 0), ('zcullMask', NvU32, 4)]) +NV2080_CTRL_GR_GET_ZCULL_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_ZCULL_MASK_PARAMS +enum_NV2080_CTRL_GR_FECS_BIND_EVTBUF_REASON_CODE: dict[int, str] = {(NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NONE:=0): 'NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NONE', (NV2080_CTRL_GR_FECS_BIND_REASON_CODE_GPU_TOO_OLD:=1): 'NV2080_CTRL_GR_FECS_BIND_REASON_CODE_GPU_TOO_OLD', (NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED_GPU:=2): 'NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED_GPU', (NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED:=3): 'NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED', (NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NEED_ADMIN:=4): 'NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NEED_ADMIN', (NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NEED_CAPABILITY:=5): 'NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NEED_CAPABILITY'} +NV2080_CTRL_GR_FECS_BIND_EVTBUF_REASON_CODE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_V2_PARAMS(c.Struct): + SIZE = 24 + hEventBuffer: int + recordSize: int + levelOfDetail: int + eventFilter: int + bAllUsers: int + reasonCode: int +struct_NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_V2_PARAMS.register_fields([('hEventBuffer', NvHandle, 0), ('recordSize', NvU32, 4), ('levelOfDetail', NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD, 8), ('eventFilter', NvU32, 12), ('bAllUsers', NvBool, 16), ('reasonCode', NvU32, 20)]) +NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_V2_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_GFX_GPC_AND_TPC_INFO_PARAMS(c.Struct): + SIZE = 24 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + physGfxGpcMask: int + numGfxTpc: int +struct_NV2080_CTRL_GR_GET_GFX_GPC_AND_TPC_INFO_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('physGfxGpcMask', NvU32, 16), ('numGfxTpc', NvU32, 20)]) +NV2080_CTRL_GR_GET_GFX_GPC_AND_TPC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_GFX_GPC_AND_TPC_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GR_GET_TPC_RECONFIG_MASK_PARAMS(c.Struct): + SIZE = 24 + gpc: int + tpcReconfigMask: int + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO +struct_NV2080_CTRL_GR_GET_TPC_RECONFIG_MASK_PARAMS.register_fields([('gpc', NvU32, 0), ('tpcReconfigMask', NvU32, 4), ('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 8)]) +NV2080_CTRL_GR_GET_TPC_RECONFIG_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_TPC_RECONFIG_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GR_TEST_CTXSW_ERROR_LOGS_PARAMS(c.Struct): + SIZE = 8 + expectedCharCount: int + bUseInterrupt: int +struct_NV2080_CTRL_GR_TEST_CTXSW_ERROR_LOGS_PARAMS.register_fields([('expectedCharCount', NvU32, 0), ('bUseInterrupt', NvBool, 4)]) +NV2080_CTRL_GR_TEST_CTXSW_ERROR_LOGS_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_TEST_CTXSW_ERROR_LOGS_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GPC_COUNT_PARAMS(c.Struct): + SIZE = 4 + gpcCount: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GPC_COUNT_PARAMS.register_fields([('gpcCount', NvU32, 0)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_GPC_COUNT_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GPC_COUNT_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GPC_MAP_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + chipletGpcMap: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GPC_MAP_PARAMS.register_fields([('gpcId', NvU32, 0), ('chipletGpcMap', NvU32, 4)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GPC_MAP_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GPC_MAP_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_TPC_MASK_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + tpcMask: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_TPC_MASK_PARAMS.register_fields([('gpcId', NvU32, 0), ('tpcMask', NvU32, 4)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_TPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_TPC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PPC_MASK_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + ppcMask: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PPC_MASK_PARAMS.register_fields([('gpcId', NvU32, 0), ('ppcMask', NvU32, 4)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_PPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PPC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_GPC_MAP_PARAMS(c.Struct): + SIZE = 12 + swizzId: int + gpcId: int + chipletGpcMap: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_GPC_MAP_PARAMS.register_fields([('swizzId', NvU32, 0), ('gpcId', NvU32, 4), ('chipletGpcMap', NvU32, 8)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_GPC_MAP_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_GPC_MAP_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_ROP_MASK_PARAMS(c.Struct): + SIZE = 8 + gpcId: int + ropMask: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_ROP_MASK_PARAMS.register_fields([('gpcId', NvU32, 0), ('ropMask', NvU32, 4)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_ROP_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_ROP_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_SYSPIPE_MASK_PARAMS(c.Struct): + SIZE = 4 + chipletSyspipeMask: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_SYSPIPE_MASK_PARAMS.register_fields([('chipletSyspipeMask', NvU32, 0)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_SYSPIPE_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_SYSPIPE_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_SYSPIPE_IDS_PARAMS(c.Struct): + SIZE = 12 + swizzId: int + physSyspipeIdCount: int + physSyspipeId: c.Array[ctypes.c_ubyte, Literal[8]] +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_SYSPIPE_IDS_PARAMS.register_fields([('swizzId', NvU16, 0), ('physSyspipeIdCount', NvU16, 2), ('physSyspipeId', c.Array[NvU8, Literal[8]], 4)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_SYSPIPE_IDS_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_SYSPIPE_IDS_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PROFILER_MON_GPC_MASK_PARAMS(c.Struct): + SIZE = 12 + swizzId: int + grIdx: int + gpcEnMask: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PROFILER_MON_GPC_MASK_PARAMS.register_fields([('swizzId', NvU32, 0), ('grIdx', NvU32, 4), ('gpcEnMask', NvU32, 8)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_PROFILER_MON_GPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PROFILER_MON_GPC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_SYSPIPE_ID_PARAMS(c.Struct): + SIZE = 4 + syspipeId: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_SYSPIPE_ID_PARAMS.register_fields([('syspipeId', NvU32, 0)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_SYSPIPE_ID_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_SYSPIPE_ID_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GRAPHICS_SYSPIPE_MASK_PARAMS(c.Struct): + SIZE = 4 + chipletSyspipeMask: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GRAPHICS_SYSPIPE_MASK_PARAMS.register_fields([('chipletSyspipeMask', NvU32, 0)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GRAPHICS_SYSPIPE_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GRAPHICS_SYSPIPE_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GFX_CAPABLE_GPC_MASK_PARAMS(c.Struct): + SIZE = 12 + swizzId: int + grIdx: int + gpcEnMask: int +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GFX_CAPABLE_GPC_MASK_PARAMS.register_fields([('swizzId', NvU32, 0), ('grIdx', NvU32, 4), ('gpcEnMask', NvU32, 8)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_GFX_CAPABLE_GPC_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GFX_CAPABLE_GPC_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS(c.Struct): + SIZE = 20 + queryType: int + reserved: c.Array[ctypes.c_ubyte, Literal[2]] + status: int + queryData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS_queryData +@c.record +class struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS_queryData(c.Struct): + SIZE = 12 + gpcCountData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GPC_COUNT_PARAMS + chipletGpcMapData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GPC_MAP_PARAMS + tpcMaskData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_TPC_MASK_PARAMS + ppcMaskData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PPC_MASK_PARAMS + partitionGpcMapData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_GPC_MAP_PARAMS + syspipeMaskData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_SYSPIPE_MASK_PARAMS + partitionChipletSyspipeData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_SYSPIPE_IDS_PARAMS + dmGpcMaskData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PROFILER_MON_GPC_MASK_PARAMS + partitionSyspipeIdData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_SYSPIPE_ID_PARAMS + ropMaskData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_ROP_MASK_PARAMS + gfxSyspipeMaskData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GRAPHICS_SYSPIPE_MASK_PARAMS + gfxGpcMaskData: struct_NV2080_CTRL_GRMGR_GR_FS_INFO_GFX_CAPABLE_GPC_MASK_PARAMS +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS_queryData.register_fields([('gpcCountData', NV2080_CTRL_GRMGR_GR_FS_INFO_GPC_COUNT_PARAMS, 0), ('chipletGpcMapData', NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GPC_MAP_PARAMS, 0), ('tpcMaskData', NV2080_CTRL_GRMGR_GR_FS_INFO_TPC_MASK_PARAMS, 0), ('ppcMaskData', NV2080_CTRL_GRMGR_GR_FS_INFO_PPC_MASK_PARAMS, 0), ('partitionGpcMapData', NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_GPC_MAP_PARAMS, 0), ('syspipeMaskData', NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_SYSPIPE_MASK_PARAMS, 0), ('partitionChipletSyspipeData', NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_CHIPLET_SYSPIPE_IDS_PARAMS, 0), ('dmGpcMaskData', NV2080_CTRL_GRMGR_GR_FS_INFO_PROFILER_MON_GPC_MASK_PARAMS, 0), ('partitionSyspipeIdData', NV2080_CTRL_GRMGR_GR_FS_INFO_PARTITION_SYSPIPE_ID_PARAMS, 0), ('ropMaskData', NV2080_CTRL_GRMGR_GR_FS_INFO_ROP_MASK_PARAMS, 0), ('gfxSyspipeMaskData', NV2080_CTRL_GRMGR_GR_FS_INFO_CHIPLET_GRAPHICS_SYSPIPE_MASK_PARAMS, 0), ('gfxGpcMaskData', NV2080_CTRL_GRMGR_GR_FS_INFO_GFX_CAPABLE_GPC_MASK_PARAMS, 0)]) +struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS.register_fields([('queryType', NvU16, 0), ('reserved', c.Array[NvU8, Literal[2]], 2), ('status', NvU32, 4), ('queryData', struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS_queryData, 8)]) +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS +@c.record +class struct_NV2080_CTRL_GRMGR_GET_GR_FS_INFO_PARAMS(c.Struct): + SIZE = 1928 + numQueries: int + reserved: c.Array[ctypes.c_ubyte, Literal[6]] + queries: c.Array[struct_NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS, Literal[96]] +struct_NV2080_CTRL_GRMGR_GET_GR_FS_INFO_PARAMS.register_fields([('numQueries', NvU16, 0), ('reserved', c.Array[NvU8, Literal[6]], 2), ('queries', c.Array[NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARAMS, Literal[96]], 8)]) +NV2080_CTRL_GRMGR_GET_GR_FS_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_GRMGR_GET_GR_FS_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_GSP_GET_FEATURES_PARAMS(c.Struct): + SIZE = 72 + gspFeatures: int + bValid: int + bDefaultGspRmGpu: int + firmwareVersion: c.Array[ctypes.c_ubyte, Literal[64]] +struct_NV2080_CTRL_GSP_GET_FEATURES_PARAMS.register_fields([('gspFeatures', NvU32, 0), ('bValid', NvBool, 4), ('bDefaultGspRmGpu', NvBool, 5), ('firmwareVersion', c.Array[NvU8, Literal[64]], 6)]) +NV2080_CTRL_GSP_GET_FEATURES_PARAMS: TypeAlias = struct_NV2080_CTRL_GSP_GET_FEATURES_PARAMS +@c.record +class struct_NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT(c.Struct): + SIZE = 32 + allocatedSize: int + usableSize: int + memTrackOverhead: int + allocationCount: int +struct_NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT.register_fields([('allocatedSize', NvU64, 0), ('usableSize', NvU64, 8), ('memTrackOverhead', NvU64, 16), ('allocationCount', NvU32, 24)]) +NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT: TypeAlias = struct_NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT +@c.record +class struct_NV2080_CTRL_GSP_GET_RM_HEAP_STATS_PARAMS(c.Struct): + SIZE = 88 + gfid: int + managedSize: int + largestFreeChunkSize: int + current: struct_NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT + peak: struct_NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT +struct_NV2080_CTRL_GSP_GET_RM_HEAP_STATS_PARAMS.register_fields([('gfid', NvU32, 0), ('managedSize', NvU64, 8), ('largestFreeChunkSize', NvU64, 16), ('current', NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT, 24), ('peak', NV2080_CTRL_GSP_RM_HEAP_STATS_SNAPSHOT, 56)]) +NV2080_CTRL_GSP_GET_RM_HEAP_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_GSP_GET_RM_HEAP_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GSP_GET_VGPU_HEAP_STATS_PARAMS(c.Struct): + SIZE = 40 + allocatedSize: int + peakAllocatedSize: int + managedSize: int + allocationCount: int + peakAllocationCount: int + largestFreeChunkSize: int +struct_NV2080_CTRL_CMD_GSP_GET_VGPU_HEAP_STATS_PARAMS.register_fields([('allocatedSize', NvU64, 0), ('peakAllocatedSize', NvU64, 8), ('managedSize', NvU64, 16), ('allocationCount', NvU32, 24), ('peakAllocationCount', NvU32, 28), ('largestFreeChunkSize', NvU64, 32)]) +NV2080_CTRL_CMD_GSP_GET_VGPU_HEAP_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GSP_GET_VGPU_HEAP_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_GSP_LIBOS_POOL_STATS(c.Struct): + SIZE = 16 + allocations: int + peakAllocations: int + objectSize: int +struct_NV2080_CTRL_GSP_LIBOS_POOL_STATS.register_fields([('allocations', NvU32, 0), ('peakAllocations', NvU32, 4), ('objectSize', NvU64, 8)]) +NV2080_CTRL_GSP_LIBOS_POOL_STATS: TypeAlias = struct_NV2080_CTRL_GSP_LIBOS_POOL_STATS +@c.record +class struct_NV2080_CTRL_CMD_GSP_GET_LIBOS_HEAP_STATS_PARAMS(c.Struct): + SIZE = 1040 + poolStats: c.Array[struct_NV2080_CTRL_GSP_LIBOS_POOL_STATS, Literal[64]] + totalHeapSize: int + poolCount: int +struct_NV2080_CTRL_CMD_GSP_GET_LIBOS_HEAP_STATS_PARAMS.register_fields([('poolStats', c.Array[NV2080_CTRL_GSP_LIBOS_POOL_STATS, Literal[64]], 0), ('totalHeapSize', NvU64, 1024), ('poolCount', NvU8, 1032)]) +NV2080_CTRL_CMD_GSP_GET_LIBOS_HEAP_STATS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GSP_GET_LIBOS_HEAP_STATS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_GSP_GDMA_FUZZ_TEST_PARAMS(c.Struct): + SIZE = 16 + iters: int + pSysmemPA: int +struct_NV2080_CTRL_CMD_GSP_GDMA_FUZZ_TEST_PARAMS.register_fields([('iters', NvU64, 0), ('pSysmemPA', NvU64, 8)]) +NV2080_CTRL_CMD_GSP_GDMA_FUZZ_TEST_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GSP_GDMA_FUZZ_TEST_PARAMS +enum_NV2080_CTRL_GSP_CORE_TEST_OP: dict[int, str] = {(NV2080_CTRL_GSP_CORE_TEST_OP_WRITE_LATENCY:=0): 'NV2080_CTRL_GSP_CORE_TEST_OP_WRITE_LATENCY', (NV2080_CTRL_GSP_CORE_TEST_OP_READ_LATENCY:=1): 'NV2080_CTRL_GSP_CORE_TEST_OP_READ_LATENCY', (NV2080_CTRL_GSP_CORE_TEST_OP_WRITE_BLOCK:=2): 'NV2080_CTRL_GSP_CORE_TEST_OP_WRITE_BLOCK', (NV2080_CTRL_GSP_CORE_TEST_OP_READ_BLOCK:=3): 'NV2080_CTRL_GSP_CORE_TEST_OP_READ_BLOCK', (NV2080_CTRL_GSP_CORE_TEST_SYSMEM_TO_FB_OP_COPY_NAIVE:=4): 'NV2080_CTRL_GSP_CORE_TEST_SYSMEM_TO_FB_OP_COPY_NAIVE', (NV2080_CTRL_GSP_CORE_TEST_FB_TO_FB_OP_COPY_NAIVE:=5): 'NV2080_CTRL_GSP_CORE_TEST_FB_TO_FB_OP_COPY_NAIVE', (NV2080_CTRL_GSP_CORE_TEST_DMEM_TO_FB_OP_PORT_MEM_COPY:=6): 'NV2080_CTRL_GSP_CORE_TEST_DMEM_TO_FB_OP_PORT_MEM_COPY', (NV2080_CTRL_GSP_CORE_TEST_FB_TO_DMEM_OP_PORT_MEM_COPY:=7): 'NV2080_CTRL_GSP_CORE_TEST_FB_TO_DMEM_OP_PORT_MEM_COPY', (NV2080_CTRL_GSP_CORE_TEST_FB_TO_FB_OP_PORT_MEM_COPY:=8): 'NV2080_CTRL_GSP_CORE_TEST_FB_TO_FB_OP_PORT_MEM_COPY', (NV2080_CTRL_GSP_CORE_TEST_DMEM_TO_FB_OP_DMA:=9): 'NV2080_CTRL_GSP_CORE_TEST_DMEM_TO_FB_OP_DMA', (NV2080_CTRL_GSP_CORE_TEST_FB_TO_DMEM_OP_DMA:=10): 'NV2080_CTRL_GSP_CORE_TEST_FB_TO_DMEM_OP_DMA', (NV2080_CTRL_GSP_CORE_TEST_FB_TO_FB_OP_DMA:=11): 'NV2080_CTRL_GSP_CORE_TEST_FB_TO_FB_OP_DMA', (NV2080_CTRL_GSP_CORE_TEST_L1D_INVAL_TIME:=12): 'NV2080_CTRL_GSP_CORE_TEST_L1D_INVAL_TIME', (NV2080_CTRL_GSP_CORE_TEST_OP_POSTED_NON_POSTED:=13): 'NV2080_CTRL_GSP_CORE_TEST_OP_POSTED_NON_POSTED'} +NV2080_CTRL_GSP_CORE_TEST_OP: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_GSP_CORE_TEST_MEM: dict[int, str] = {(NV2080_CTRL_GSP_CORE_TEST_MEM_DMEM:=0): 'NV2080_CTRL_GSP_CORE_TEST_MEM_DMEM', (NV2080_CTRL_GSP_CORE_TEST_MEM_FB:=1): 'NV2080_CTRL_GSP_CORE_TEST_MEM_FB', (NV2080_CTRL_GSP_CORE_TEST_MEM_SYSMEM:=2): 'NV2080_CTRL_GSP_CORE_TEST_MEM_SYSMEM', (NV2080_CTRL_GSP_CORE_TEST_MEM_PRI:=3): 'NV2080_CTRL_GSP_CORE_TEST_MEM_PRI'} +NV2080_CTRL_GSP_CORE_TEST_MEM: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_CMD_GSP_CORE_TEST_PARAMS(c.Struct): + SIZE = 48 + op: int + mem: int + flags: int + size: int + iters: int + timeNs: int + cycles: int +struct_NV2080_CTRL_CMD_GSP_CORE_TEST_PARAMS.register_fields([('op', NV2080_CTRL_GSP_CORE_TEST_OP, 0), ('mem', NV2080_CTRL_GSP_CORE_TEST_MEM, 4), ('flags', NvU64, 8), ('size', NvU64, 16), ('iters', NvU64, 24), ('timeNs', NvU64, 32), ('cycles', NvU64, 40)]) +NV2080_CTRL_CMD_GSP_CORE_TEST_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_GSP_CORE_TEST_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_HSHUB_GET_AVAILABLE_MASK_PARAMS(c.Struct): + SIZE = 8 + hshubNcisocMask: int + hshubNvlMask: int +struct_NV2080_CTRL_CMD_HSHUB_GET_AVAILABLE_MASK_PARAMS.register_fields([('hshubNcisocMask', NvU32, 0), ('hshubNvlMask', NvU32, 4)]) +NV2080_CTRL_CMD_HSHUB_GET_AVAILABLE_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_HSHUB_GET_AVAILABLE_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_HSHUB_SET_EC_THROTTLE_MODE_PARAMS(c.Struct): + SIZE = 8 + ecMode: int + status: int +struct_NV2080_CTRL_CMD_HSHUB_SET_EC_THROTTLE_MODE_PARAMS.register_fields([('ecMode', NvU32, 0), ('status', NvU32, 4)]) +NV2080_CTRL_CMD_HSHUB_SET_EC_THROTTLE_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_HSHUB_SET_EC_THROTTLE_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_I2C_READ_BUFFER_PARAMS(c.Struct): + SIZE = 532 + version: int + port: int + flags: int + inputCount: int + inputBuffer: c.Array[ctypes.c_ubyte, Literal[256]] + outputCount: int + outputBuffer: c.Array[ctypes.c_ubyte, Literal[256]] +struct_NV2080_CTRL_I2C_READ_BUFFER_PARAMS.register_fields([('version', NvU32, 0), ('port', NvU32, 4), ('flags', NvU32, 8), ('inputCount', NvU32, 12), ('inputBuffer', c.Array[NvU8, Literal[256]], 16), ('outputCount', NvU32, 272), ('outputBuffer', c.Array[NvU8, Literal[256]], 276)]) +NV2080_CTRL_I2C_READ_BUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_I2C_READ_BUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_I2C_WRITE_BUFFER_PARAMS(c.Struct): + SIZE = 276 + version: int + port: int + flags: int + inputCount: int + inputBuffer: c.Array[ctypes.c_ubyte, Literal[256]] + encrClientID: int +struct_NV2080_CTRL_I2C_WRITE_BUFFER_PARAMS.register_fields([('version', NvU32, 0), ('port', NvU32, 4), ('flags', NvU32, 8), ('inputCount', NvU32, 12), ('inputBuffer', c.Array[NvU8, Literal[256]], 16), ('encrClientID', NvU32, 272)]) +NV2080_CTRL_I2C_WRITE_BUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_I2C_WRITE_BUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_I2C_RW_REG_PARAMS(c.Struct): + SIZE = 276 + version: int + port: int + flags: int + addr: int + reg: int + bufsize: int + buffer: c.Array[ctypes.c_ubyte, Literal[255]] +struct_NV2080_CTRL_I2C_RW_REG_PARAMS.register_fields([('version', NvU32, 0), ('port', NvU32, 4), ('flags', NvU32, 8), ('addr', NvU32, 12), ('reg', NvU8, 16), ('bufsize', NvU8, 17), ('buffer', c.Array[NvU8, Literal[255]], 18)]) +NV2080_CTRL_I2C_RW_REG_PARAMS: TypeAlias = struct_NV2080_CTRL_I2C_RW_REG_PARAMS +NV2080_CTRL_I2C_READ_REG_PARAMS: TypeAlias = struct_NV2080_CTRL_I2C_RW_REG_PARAMS +NV2080_CTRL_I2C_WRITE_REG_PARAMS: TypeAlias = struct_NV2080_CTRL_I2C_RW_REG_PARAMS +@c.record +class struct_NV2080_CTRL_I2C_ACCESS_PARAMS(c.Struct): + SIZE = 40 + token: int + cmd: int + port: int + flags: int + data: ctypes.c_void_p + status: int + dataBuffSize: int + speed: int + encrClientID: int +struct_NV2080_CTRL_I2C_ACCESS_PARAMS.register_fields([('token', NvU32, 0), ('cmd', NvU32, 4), ('port', NvU32, 8), ('flags', NvU32, 12), ('data', NvP64, 16), ('status', NvU32, 24), ('dataBuffSize', NvU32, 28), ('speed', NvU32, 32), ('encrClientID', NvU32, 36)]) +NV2080_CTRL_I2C_ACCESS_PARAMS: TypeAlias = struct_NV2080_CTRL_I2C_ACCESS_PARAMS +@c.record +class struct_NV2080_CTRL_I2C_ENABLE_MONITOR_3D_MODE_PARAMS(c.Struct): + SIZE = 12 + head: int + authType: int + status: int +struct_NV2080_CTRL_I2C_ENABLE_MONITOR_3D_MODE_PARAMS.register_fields([('head', NvU32, 0), ('authType', NvU32, 4), ('status', NvU32, 8)]) +NV2080_CTRL_I2C_ENABLE_MONITOR_3D_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_I2C_ENABLE_MONITOR_3D_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO_PARAMS(c.Struct): + SIZE = 32 + feHwSysCap: int + windowPresentMask: int + numHeads: int + i2cPort: int + internalDispActiveMask: int + embeddedDisplayPortMask: int + bExternalMuxSupported: int + bInternalMuxSupported: int + numDispChannels: int +struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO_PARAMS.register_fields([('feHwSysCap', NvU32, 0), ('windowPresentMask', NvU32, 4), ('numHeads', NvU32, 8), ('i2cPort', NvU32, 12), ('internalDispActiveMask', NvU32, 16), ('embeddedDisplayPortMask', NvU32, 20), ('bExternalMuxSupported', NvBool, 24), ('bInternalMuxSupported', NvBool, 25), ('numDispChannels', NvU32, 28)]) +NV2080_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS(c.Struct): + SIZE = 40 + bOneToOneComptagLineAllocation: int + bUseOneToFourComptagLineAllocation: int + bUseRawModeComptaglineAllocation: int + bDisableCompbitBacking: int + bDisablePostL2Compression: int + bEnabledEccFBPA: int + bL2PreFill: int + l2CacheSize: int + bFbpaPresent: int + comprPageSize: int + comprPageShift: int + ramType: int + ltcCount: int + ltsPerLtcCount: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS.register_fields([('bOneToOneComptagLineAllocation', NvBool, 0), ('bUseOneToFourComptagLineAllocation', NvBool, 1), ('bUseRawModeComptaglineAllocation', NvBool, 2), ('bDisableCompbitBacking', NvBool, 3), ('bDisablePostL2Compression', NvBool, 4), ('bEnabledEccFBPA', NvBool, 5), ('bL2PreFill', NvBool, 6), ('l2CacheSize', NvU64, 8), ('bFbpaPresent', NvBool, 16), ('comprPageSize', NvU32, 20), ('comprPageShift', NvU32, 24), ('ramType', NvU32, 28), ('ltcCount', NvU32, 32), ('ltsPerLtcCount', NvU32, 36)]) +NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_UVM_REGISTER_ACCESS_CNTR_BUFFER_PARAMS(c.Struct): + SIZE = 520 + accessCounterIndex: int + bufferSize: int + bufferPteArray: c.Array[ctypes.c_uint64, Literal[64]] +struct_NV2080_CTRL_INTERNAL_UVM_REGISTER_ACCESS_CNTR_BUFFER_PARAMS.register_fields([('accessCounterIndex', NvU32, 0), ('bufferSize', NvU32, 4), ('bufferPteArray', c.Array[NvU64, Literal[64]], 8)]) +NV2080_CTRL_INTERNAL_UVM_REGISTER_ACCESS_CNTR_BUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_UVM_REGISTER_ACCESS_CNTR_BUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_UVM_UNREGISTER_ACCESS_CNTR_BUFFER_PARAMS(c.Struct): + SIZE = 4 + accessCounterIndex: int +struct_NV2080_CTRL_INTERNAL_UVM_UNREGISTER_ACCESS_CNTR_BUFFER_PARAMS.register_fields([('accessCounterIndex', NvU32, 0)]) +NV2080_CTRL_INTERNAL_UVM_UNREGISTER_ACCESS_CNTR_BUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_UVM_UNREGISTER_ACCESS_CNTR_BUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_CAPS(c.Struct): + SIZE = 23 + capsTbl: c.Array[ctypes.c_ubyte, Literal[23]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_CAPS.register_fields([('capsTbl', c.Array[NvU8, Literal[23]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_CAPS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_CAPS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS(c.Struct): + SIZE = 184 + engineCaps: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_CAPS, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS.register_fields([('engineCaps', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_CAPS, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER(c.Struct): + SIZE = 10244 + globalSmId: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER_globalSmId, Literal[512]] + numSm: int + numTpc: int +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER_globalSmId(c.Struct): + SIZE = 20 + gpcId: int + localTpcId: int + localSmId: int + globalTpcId: int + virtualGpcId: int + virtualDpcId: int + migratableTpcId: int + ugpuId: int + physicalCpcId: int + virtualTpcId: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER_globalSmId.register_fields([('gpcId', NvU16, 0), ('localTpcId', NvU16, 2), ('localSmId', NvU16, 4), ('globalTpcId', NvU16, 6), ('virtualGpcId', NvU16, 8), ('virtualDpcId', NvU16, 10), ('migratableTpcId', NvU16, 12), ('ugpuId', NvU16, 14), ('physicalCpcId', NvU16, 16), ('virtualTpcId', NvU16, 18)]) +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER.register_fields([('globalSmId', c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER_globalSmId, Literal[512]], 0), ('numSm', NvU16, 10240), ('numTpc', NvU16, 10242)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS(c.Struct): + SIZE = 81952 + globalSmOrder: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS.register_fields([('globalSmOrder', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_GLOBAL_SM_ORDER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVDEC_CAPS(c.Struct): + SIZE = 8 + capsTbl: c.Array[ctypes.c_ubyte, Literal[8]] +struct_NV2080_CTRL_INTERNAL_NVDEC_CAPS.register_fields([('capsTbl', c.Array[NvU8, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_NVDEC_CAPS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVDEC_CAPS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVDEC_GET_CAPS_PARAMS(c.Struct): + SIZE = 72 + caps: c.Array[struct_NV2080_CTRL_INTERNAL_NVDEC_CAPS, Literal[8]] + valid: c.Array[ctypes.c_ubyte, Literal[8]] +struct_NV2080_CTRL_INTERNAL_NVDEC_GET_CAPS_PARAMS.register_fields([('caps', c.Array[NV2080_CTRL_INTERNAL_NVDEC_CAPS, Literal[8]], 0), ('valid', c.Array[NvBool, Literal[8]], 64)]) +NV2080_CTRL_INTERNAL_NVDEC_GET_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVDEC_GET_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVENC_CAPS(c.Struct): + SIZE = 6 + capsTbl: c.Array[ctypes.c_ubyte, Literal[6]] +struct_NV2080_CTRL_INTERNAL_NVENC_CAPS.register_fields([('capsTbl', c.Array[NvU8, Literal[6]], 0)]) +NV2080_CTRL_INTERNAL_NVENC_CAPS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVENC_CAPS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVENC_GET_CAPS_PARAMS(c.Struct): + SIZE = 56 + caps: c.Array[struct_NV2080_CTRL_INTERNAL_NVENC_CAPS, Literal[8]] + valid: c.Array[ctypes.c_ubyte, Literal[8]] +struct_NV2080_CTRL_INTERNAL_NVENC_GET_CAPS_PARAMS.register_fields([('caps', c.Array[NV2080_CTRL_INTERNAL_NVENC_CAPS, Literal[8]], 0), ('valid', c.Array[NvBool, Literal[8]], 48)]) +NV2080_CTRL_INTERNAL_NVENC_GET_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVENC_GET_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_FLOORSWEEPING_MASKS(c.Struct): + SIZE = 1356 + gpcMask: int + tpcMask: c.Array[ctypes.c_uint32, Literal[64]] + tpcCount: c.Array[ctypes.c_uint32, Literal[64]] + physGpcMask: int + mmuPerGpc: c.Array[ctypes.c_uint32, Literal[64]] + tpcToPesMap: c.Array[ctypes.c_uint32, Literal[15]] + numPesPerGpc: c.Array[ctypes.c_uint32, Literal[64]] + zcullMask: c.Array[ctypes.c_uint32, Literal[64]] + physGfxGpcMask: int + numGfxTpc: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_FLOORSWEEPING_MASKS.register_fields([('gpcMask', NvU32, 0), ('tpcMask', c.Array[NvU32, Literal[64]], 4), ('tpcCount', c.Array[NvU32, Literal[64]], 260), ('physGpcMask', NvU32, 516), ('mmuPerGpc', c.Array[NvU32, Literal[64]], 520), ('tpcToPesMap', c.Array[NvU32, Literal[15]], 776), ('numPesPerGpc', c.Array[NvU32, Literal[64]], 836), ('zcullMask', c.Array[NvU32, Literal[64]], 1092), ('physGfxGpcMask', NvU32, 1348), ('numGfxTpc', NvU32, 1352)]) +NV2080_CTRL_INTERNAL_STATIC_GR_FLOORSWEEPING_MASKS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_FLOORSWEEPING_MASKS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS(c.Struct): + SIZE = 10848 + floorsweepingMasks: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_FLOORSWEEPING_MASKS, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS.register_fields([('floorsweepingMasks', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_FLOORSWEEPING_MASKS, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_FLOORSWEEPING_MASKS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS +@c.record +class struct_NV2080_CTRL_KGR_GET_CTX_BUFFER_PTES_PARAMS(c.Struct): + SIZE = 1056 + hUserClient: int + hChannel: int + bufferType: int + firstPage: int + numPages: int + physAddrs: c.Array[ctypes.c_uint64, Literal[128]] + bNoMorePages: int +struct_NV2080_CTRL_KGR_GET_CTX_BUFFER_PTES_PARAMS.register_fields([('hUserClient', NvHandle, 0), ('hChannel', NvHandle, 4), ('bufferType', NvU32, 8), ('firstPage', NvU32, 12), ('numPages', NvU32, 16), ('physAddrs', c.Array[NvU64, Literal[128]], 24), ('bNoMorePages', NvBool, 1048)]) +NV2080_CTRL_KGR_GET_CTX_BUFFER_PTES_PARAMS: TypeAlias = struct_NV2080_CTRL_KGR_GET_CTX_BUFFER_PTES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GR_INFO(c.Struct): + SIZE = 8 + index: int + data: int +struct_NV2080_CTRL_INTERNAL_GR_INFO.register_fields([('index', NvU32, 0), ('data', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GR_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_INFO(c.Struct): + SIZE = 504 + infoList: c.Array[struct_NV2080_CTRL_INTERNAL_GR_INFO, Literal[63]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_INFO.register_fields([('infoList', c.Array[NV2080_CTRL_INTERNAL_GR_INFO, Literal[63]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS(c.Struct): + SIZE = 4032 + engineInfo: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_INFO, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS.register_fields([('engineInfo', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_INFO, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_ZCULL_INFO(c.Struct): + SIZE = 40 + widthAlignPixels: int + heightAlignPixels: int + pixelSquaresByAliquots: int + aliquotTotal: int + zcullRegionByteMultiplier: int + zcullRegionHeaderSize: int + zcullSubregionHeaderSize: int + subregionCount: int + subregionWidthAlignPixels: int + subregionHeightAlignPixels: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_ZCULL_INFO.register_fields([('widthAlignPixels', NvU32, 0), ('heightAlignPixels', NvU32, 4), ('pixelSquaresByAliquots', NvU32, 8), ('aliquotTotal', NvU32, 12), ('zcullRegionByteMultiplier', NvU32, 16), ('zcullRegionHeaderSize', NvU32, 20), ('zcullSubregionHeaderSize', NvU32, 24), ('subregionCount', NvU32, 28), ('subregionWidthAlignPixels', NvU32, 32), ('subregionHeightAlignPixels', NvU32, 36)]) +NV2080_CTRL_INTERNAL_STATIC_GR_ZCULL_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_ZCULL_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS(c.Struct): + SIZE = 320 + engineZcullInfo: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_ZCULL_INFO, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS.register_fields([('engineZcullInfo', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_ZCULL_INFO, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_ZCULL_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_ROP_INFO(c.Struct): + SIZE = 12 + ropUnitCount: int + ropOperationsFactor: int + ropOperationsCount: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_ROP_INFO.register_fields([('ropUnitCount', NvU32, 0), ('ropOperationsFactor', NvU32, 4), ('ropOperationsCount', NvU32, 8)]) +NV2080_CTRL_INTERNAL_STATIC_GR_ROP_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_ROP_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS(c.Struct): + SIZE = 96 + engineRopInfo: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_ROP_INFO, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS.register_fields([('engineRopInfo', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_ROP_INFO, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_ROP_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_PPC_MASKS(c.Struct): + SIZE = 256 + mask: c.Array[ctypes.c_uint32, Literal[64]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_PPC_MASKS.register_fields([('mask', c.Array[NvU32, Literal[64]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_PPC_MASKS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_PPC_MASKS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS(c.Struct): + SIZE = 2048 + enginePpcMasks: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_PPC_MASKS, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS.register_fields([('enginePpcMasks', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_PPC_MASKS, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_PPC_MASKS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO(c.Struct): + SIZE = 8 + size: int + alignment: int +struct_NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO.register_fields([('size', NvU32, 0), ('alignment', NvU32, 4)]) +NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_CONTEXT_BUFFERS_INFO(c.Struct): + SIZE = 208 + engine: c.Array[struct_NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO, Literal[26]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_CONTEXT_BUFFERS_INFO.register_fields([('engine', c.Array[NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO, Literal[26]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_CONTEXT_BUFFERS_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_CONTEXT_BUFFERS_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS(c.Struct): + SIZE = 1664 + engineContextBuffersInfo: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_CONTEXT_BUFFERS_INFO, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS.register_fields([('engineContextBuffersInfo', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_CONTEXT_BUFFERS_INFO, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER(c.Struct): + SIZE = 9 + imla0: int + fmla16: int + dp: int + fmla32: int + ffma: int + imla1: int + imla2: int + imla3: int + imla4: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER.register_fields([('imla0', NvU8, 0), ('fmla16', NvU8, 1), ('dp', NvU8, 2), ('fmla32', NvU8, 3), ('ffma', NvU8, 4), ('imla1', NvU8, 5), ('imla2', NvU8, 6), ('imla3', NvU8, 7), ('imla4', NvU8, 8)]) +NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS(c.Struct): + SIZE = 72 + smIssueRateModifier: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS.register_fields([('smIssueRateModifier', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_GPU_SYNC_FABRIC_PROBE_INFO_PARAMS(c.Struct): + SIZE = 16 + linkMaskToBeReduced: struct_NV2080_CTRL_NVLINK_LINK_MASK +NV2080_CTRL_NVLINK_LINK_MASK: TypeAlias = struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_CMD_INTERNAL_GPU_SYNC_FABRIC_PROBE_INFO_PARAMS.register_fields([('linkMaskToBeReduced', NV2080_CTRL_NVLINK_LINK_MASK, 0)]) +NV2080_CTRL_CMD_INTERNAL_GPU_SYNC_FABRIC_PROBE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_GPU_SYNC_FABRIC_PROBE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_HW_ENABLE_PARAMS(c.Struct): + SIZE = 24 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + bEnable: int +struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_HW_ENABLE_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('bEnable', NvBool, 16)]) +NV2080_CTRL_INTERNAL_GR_FECS_TRACE_HW_ENABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_HW_ENABLE_PARAMS +NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_HW_ENABLE_PARAMS +NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_HW_ENABLE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_OFFSET_PARAMS(c.Struct): + SIZE = 24 + grRouteInfo: struct_NV0080_CTRL_GR_ROUTE_INFO + offset: int +struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_OFFSET_PARAMS.register_fields([('grRouteInfo', NV2080_CTRL_GR_ROUTE_INFO, 0), ('offset', NvU32, 16)]) +NV2080_CTRL_INTERNAL_GR_FECS_TRACE_OFFSET_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_OFFSET_PARAMS +NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_OFFSET_PARAMS +NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_OFFSET_PARAMS +NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GR_FECS_TRACE_OFFSET_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE(c.Struct): + SIZE = 4 + fecsRecordSize: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE.register_fields([('fecsRecordSize', NvU32, 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS(c.Struct): + SIZE = 32 + fecsRecordSize: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS.register_fields([('fecsRecordSize', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_RECORD_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES(c.Struct): + SIZE = 32 + fecsRecordSize: int + timestampHiTagMask: int + timestampHiTagShift: int + timestampVMask: int + numLowerBitsZeroShift: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES.register_fields([('fecsRecordSize', NvU32, 0), ('timestampHiTagMask', NvU32, 4), ('timestampHiTagShift', NvU8, 8), ('timestampVMask', NvU64, 16), ('numLowerBitsZeroShift', NvU8, 24)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS(c.Struct): + SIZE = 256 + fecsTraceDefines: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS.register_fields([('fecsTraceDefines', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_FECS_TRACE_DEFINES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DEVICE_INFO(c.Struct): + SIZE = 48 + faultId: int + instanceId: int + typeEnum: int + resetId: int + devicePriBase: int + isEngine: int + rlEngId: int + runlistPriBase: int + groupId: int + ginTargetId: int + deviceBroadcastPriBase: int + groupLocalInstanceId: int +struct_NV2080_CTRL_INTERNAL_DEVICE_INFO.register_fields([('faultId', NvU32, 0), ('instanceId', NvU32, 4), ('typeEnum', NvU32, 8), ('resetId', NvU32, 12), ('devicePriBase', NvU32, 16), ('isEngine', NvU32, 20), ('rlEngId', NvU32, 24), ('runlistPriBase', NvU32, 28), ('groupId', NvU32, 32), ('ginTargetId', NvU32, 36), ('deviceBroadcastPriBase', NvU32, 40), ('groupLocalInstanceId', NvU32, 44)]) +NV2080_CTRL_INTERNAL_DEVICE_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_DEVICE_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS(c.Struct): + SIZE = 24580 + numEntries: int + deviceInfoTable: c.Array[struct_NV2080_CTRL_INTERNAL_DEVICE_INFO, Literal[512]] +struct_NV2080_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS.register_fields([('numEntries', NvU32, 0), ('deviceInfoTable', c.Array[NV2080_CTRL_INTERNAL_DEVICE_INFO, Literal[512]], 4)]) +NV2080_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS(c.Struct): + SIZE = 20492 + userRegisterAccessMapSize: int + compressedSize: int + compressedData: c.Array[ctypes.c_ubyte, Literal[16384]] + profilingRangesSize: int + profilingRanges: c.Array[ctypes.c_ubyte, Literal[4096]] +struct_NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS.register_fields([('userRegisterAccessMapSize', NvU32, 0), ('compressedSize', NvU32, 4), ('compressedData', c.Array[NvU8, Literal[16384]], 8), ('profilingRangesSize', NvU32, 16392), ('profilingRanges', c.Array[NvU8, Literal[4096]], 16396)]) +NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NV_RANGE(c.Struct): + SIZE = 16 + lo: int + hi: int +struct_NV2080_CTRL_INTERNAL_NV_RANGE.register_fields([('lo', NvU64, 0), ('hi', NvU64, 8)]) +NV2080_CTRL_INTERNAL_NV_RANGE: TypeAlias = struct_NV2080_CTRL_INTERNAL_NV_RANGE +@c.record +class struct_NV2080_CTRL_INTERNAL_MIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS(c.Struct): + SIZE = 24 + swizzId: int + memAddrRange: struct_NV2080_CTRL_INTERNAL_NV_RANGE +struct_NV2080_CTRL_INTERNAL_MIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS.register_fields([('swizzId', NvU32, 0), ('memAddrRange', NV2080_CTRL_INTERNAL_NV_RANGE, 8)]) +NV2080_CTRL_INTERNAL_MIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS +NV2080_CTRL_INTERNAL_KMIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_PDB_PROPERTIES(c.Struct): + SIZE = 1 + bPerSubCtxheaderSupported: int +struct_NV2080_CTRL_INTERNAL_STATIC_GR_PDB_PROPERTIES.register_fields([('bPerSubCtxheaderSupported', NvBool, 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_PDB_PROPERTIES: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_PDB_PROPERTIES +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_CHECK_CTS_ID_VALID_PARAMS(c.Struct): + SIZE = 16 + ctsId: int + bCheckClientGI: int + giComputeSize: int + bCtsIdValid: int +struct_NV2080_CTRL_INTERNAL_GPU_CHECK_CTS_ID_VALID_PARAMS.register_fields([('ctsId', NvU32, 0), ('bCheckClientGI', NvBool, 4), ('giComputeSize', NvU32, 8), ('bCtsIdValid', NvBool, 12)]) +NV2080_CTRL_INTERNAL_GPU_CHECK_CTS_ID_VALID_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_CHECK_CTS_ID_VALID_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS(c.Struct): + SIZE = 8 + pdbTable: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_PDB_PROPERTIES, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS.register_fields([('pdbTable', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_PDB_PROPERTIES, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_PDB_PROPERTIES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISPLAY_WRITE_INST_MEM_PARAMS(c.Struct): + SIZE = 24 + instMemPhysAddr: int + instMemSize: int + instMemAddrSpace: int + instMemCpuCacheAttr: int +struct_NV2080_CTRL_INTERNAL_DISPLAY_WRITE_INST_MEM_PARAMS.register_fields([('instMemPhysAddr', NvU64, 0), ('instMemSize', NvU64, 8), ('instMemAddrSpace', NvU32, 16), ('instMemCpuCacheAttr', NvU32, 20)]) +NV2080_CTRL_INTERNAL_DISPLAY_WRITE_INST_MEM_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISPLAY_WRITE_INST_MEM_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION_PARAMS(c.Struct): + SIZE = 4 + ipVersion: int +struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION_PARAMS.register_fields([('ipVersion', NvU32, 0)]) +NV2080_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_GET_SMC_MODE_PARAMS(c.Struct): + SIZE = 4 + smcMode: int +struct_NV2080_CTRL_INTERNAL_GPU_GET_SMC_MODE_PARAMS.register_fields([('smcMode', NvU32, 0)]) +NV2080_CTRL_INTERNAL_GPU_GET_SMC_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_GET_SMC_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISPLAY_SETUP_RG_LINE_INTR_PARAMS(c.Struct): + SIZE = 16 + head: int + rgLineNum: int + intrLine: int + bEnable: int +struct_NV2080_CTRL_INTERNAL_DISPLAY_SETUP_RG_LINE_INTR_PARAMS.register_fields([('head', NvU32, 0), ('rgLineNum', NvU32, 4), ('intrLine', NvU32, 8), ('bEnable', NvBool, 12)]) +NV2080_CTRL_INTERNAL_DISPLAY_SETUP_RG_LINE_INTR_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISPLAY_SETUP_RG_LINE_INTR_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MIGMGR_PROFILE_INFO(c.Struct): + SIZE = 72 + partitionFlag: int + grCount: int + gfxGrCount: int + gpcCount: int + virtualGpcCount: int + gfxGpcCount: int + veidCount: int + smCount: int + ceCount: int + nvEncCount: int + nvDecCount: int + nvJpgCount: int + nvOfaCount: int + validCTSIdMask: int + validGfxCTSIdMask: int +struct_NV2080_CTRL_INTERNAL_MIGMGR_PROFILE_INFO.register_fields([('partitionFlag', NvU32, 0), ('grCount', NvU32, 4), ('gfxGrCount', NvU32, 8), ('gpcCount', NvU32, 12), ('virtualGpcCount', NvU32, 16), ('gfxGpcCount', NvU32, 20), ('veidCount', NvU32, 24), ('smCount', NvU32, 28), ('ceCount', NvU32, 32), ('nvEncCount', NvU32, 36), ('nvDecCount', NvU32, 40), ('nvJpgCount', NvU32, 44), ('nvOfaCount', NvU32, 48), ('validCTSIdMask', NvU64, 56), ('validGfxCTSIdMask', NvU64, 64)]) +NV2080_CTRL_INTERNAL_MIGMGR_PROFILE_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_MIGMGR_PROFILE_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PROFILES_PARAMS(c.Struct): + SIZE = 6488 + count: int + table: c.Array[struct_NV2080_CTRL_INTERNAL_MIGMGR_PROFILE_INFO, Literal[90]] +struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PROFILES_PARAMS.register_fields([('count', NvU32, 0), ('table', c.Array[NV2080_CTRL_INTERNAL_MIGMGR_PROFILE_INFO, Literal[90]], 8)]) +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PROFILES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PROFILES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM_PARAMS(c.Struct): + SIZE = 40 + partitionableMemSize: int + bottomRsvdSize: int + topRsvdSize: int + partitionableStartAddr: int + partitionableEndAddr: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM_PARAMS.register_fields([('partitionableMemSize', NvU64, 0), ('bottomRsvdSize', NvU64, 8), ('topRsvdSize', NvU64, 16), ('partitionableStartAddr', NvU64, 24), ('partitionableEndAddr', NvU64, 32)]) +NV2080_CTRL_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS(c.Struct): + SIZE = 32 + engineMask: c.Array[ctypes.c_uint64, Literal[4]] +struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS.register_fields([('engineMask', c.Array[NvU64, Literal[4]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_FIFO_PROMOTE_RUNLIST_BUFFERS_PARAMS(c.Struct): + SIZE = 4112 + rlBuffers: c.Array[c.Array[struct_NV2080_CTRL_INTERNAL_MEMDESC_INFO, Literal[2]], Literal[64]] + runlistIdMask: int + swizzId: int +struct_NV2080_CTRL_INTERNAL_FIFO_PROMOTE_RUNLIST_BUFFERS_PARAMS.register_fields([('rlBuffers', c.Array[c.Array[NV2080_CTRL_INTERNAL_MEMDESC_INFO, Literal[2]], Literal[64]], 0), ('runlistIdMask', NvU64, 4096), ('swizzId', NvU32, 4104)]) +NV2080_CTRL_INTERNAL_FIFO_PROMOTE_RUNLIST_BUFFERS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_FIFO_PROMOTE_RUNLIST_BUFFERS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISPLAY_SET_IMP_INIT_INFO_PARAMS(c.Struct): + SIZE = 492 + tegraImpImportData: struct_TEGRA_IMP_IMPORT_DATA +@c.record +class struct_TEGRA_IMP_IMPORT_DATA(c.Struct): + SIZE = 492 + max_iso_bw_kbps: int + num_dram_channels: int + num_dram_clk_entries: int + dram_clk_instance: c.Array[struct_DRAM_CLK_INSTANCE, Literal[24]] +TEGRA_IMP_IMPORT_DATA: TypeAlias = struct_TEGRA_IMP_IMPORT_DATA +@c.record +class struct_DRAM_CLK_INSTANCE(c.Struct): + SIZE = 20 + dram_clk_freq_khz: int + mchub_clk_khz: int + mc_clk_khz: int + max_iso_bw_kbps: int + switch_latency_ns: int +DRAM_CLK_INSTANCE: TypeAlias = struct_DRAM_CLK_INSTANCE +struct_DRAM_CLK_INSTANCE.register_fields([('dram_clk_freq_khz', NvU32, 0), ('mchub_clk_khz', NvU32, 4), ('mc_clk_khz', NvU32, 8), ('max_iso_bw_kbps', NvU32, 12), ('switch_latency_ns', NvU32, 16)]) +struct_TEGRA_IMP_IMPORT_DATA.register_fields([('max_iso_bw_kbps', NvU32, 0), ('num_dram_channels', NvU32, 4), ('num_dram_clk_entries', NvU32, 8), ('dram_clk_instance', c.Array[DRAM_CLK_INSTANCE, Literal[24]], 12)]) +struct_NV2080_CTRL_INTERNAL_DISPLAY_SET_IMP_INIT_INFO_PARAMS.register_fields([('tegraImpImportData', TEGRA_IMP_IMPORT_DATA, 0)]) +NV2080_CTRL_INTERNAL_DISPLAY_SET_IMP_INIT_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISPLAY_SET_IMP_INIT_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS(c.Struct): + SIZE = 6 + pciDeviceId: int + pciSubDeviceId: int + iseGPUBridge: int + approvedBusType: int +struct_NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS.register_fields([('pciDeviceId', NvU16, 0), ('pciSubDeviceId', NvU16, 2), ('iseGPUBridge', NvBool, 4), ('approvedBusType', NvU8, 5)]) +NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS(c.Struct): + SIZE = 48 + local2Remote: int + remote2Local: int + localP2PDomainRemoteAddr: int + remoteP2PDomainLocalAddr: int + remoteWMBoxLocalAddr: int + p2pWmbTag: int + bNeedWarBug999673: int +struct_NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS.register_fields([('local2Remote', NvU32, 0), ('remote2Local', NvU32, 4), ('localP2PDomainRemoteAddr', NvU64, 8), ('remoteP2PDomainLocalAddr', NvU64, 16), ('remoteWMBoxLocalAddr', NvU64, 24), ('p2pWmbTag', NvU64, 32), ('bNeedWarBug999673', NvBool, 40)]) +NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS(c.Struct): + SIZE = 40 + local2Remote: int + remote2Local: int + localP2PDomainRemoteAddr: int + remoteP2PDomainLocalAddr: int + remoteWMBoxAddrU64: int + p2pWmbTag: int +struct_NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS.register_fields([('local2Remote', NvU32, 0), ('remote2Local', NvU32, 4), ('localP2PDomainRemoteAddr', NvU64, 8), ('remoteP2PDomainLocalAddr', NvU64, 16), ('remoteWMBoxAddrU64', NvU64, 24), ('p2pWmbTag', NvU64, 32)]) +NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_BUS_DESTROY_P2P_MAILBOX_PARAMS(c.Struct): + SIZE = 8 + peerIdx: int + bNeedWarBug999673: int +struct_NV2080_CTRL_INTERNAL_BUS_DESTROY_P2P_MAILBOX_PARAMS.register_fields([('peerIdx', NvU32, 0), ('bNeedWarBug999673', NvBool, 4)]) +NV2080_CTRL_INTERNAL_BUS_DESTROY_P2P_MAILBOX_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_BUS_DESTROY_P2P_MAILBOX_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_BUS_CREATE_C2C_PEER_MAPPING_PARAMS(c.Struct): + SIZE = 4 + peerId: int +struct_NV2080_CTRL_INTERNAL_BUS_CREATE_C2C_PEER_MAPPING_PARAMS.register_fields([('peerId', NvU32, 0)]) +NV2080_CTRL_INTERNAL_BUS_CREATE_C2C_PEER_MAPPING_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_BUS_CREATE_C2C_PEER_MAPPING_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_BUS_REMOVE_C2C_PEER_MAPPING_PARAMS(c.Struct): + SIZE = 4 + peerId: int +struct_NV2080_CTRL_INTERNAL_BUS_REMOVE_C2C_PEER_MAPPING_PARAMS.register_fields([('peerId', NvU32, 0)]) +NV2080_CTRL_INTERNAL_BUS_REMOVE_C2C_PEER_MAPPING_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_BUS_REMOVE_C2C_PEER_MAPPING_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES_PARAMS(c.Struct): + SIZE = 2056 + gfid: int + numEntries: int + gpaEntries: c.Array[ctypes.c_uint64, Literal[128]] + spaEntries: c.Array[ctypes.c_uint64, Literal[128]] +struct_NV2080_CTRL_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES_PARAMS.register_fields([('gfid', NvU32, 0), ('numEntries', NvU32, 4), ('gpaEntries', c.Array[NvU64, Literal[128]], 8), ('spaEntries', c.Array[NvU64, Literal[128]], 1032)]) +NV2080_CTRL_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER_PARAMS(c.Struct): + SIZE = 56 + addressSpace: int + physicalAddr: int + limit: int + cacheSnoop: int + hclass: int + channelInstance: int + valid: int + pbTargetAperture: int + channelPBSize: int + subDeviceId: int +struct_NV2080_CTRL_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER_PARAMS.register_fields([('addressSpace', NvU32, 0), ('physicalAddr', NvU64, 8), ('limit', NvU64, 16), ('cacheSnoop', NvU32, 24), ('hclass', NvU32, 28), ('channelInstance', NvU32, 32), ('valid', NvBool, 36), ('pbTargetAperture', NvU32, 40), ('channelPBSize', NvU32, 44), ('subDeviceId', NvU32, 48)]) +NV2080_CTRL_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GMMU_GET_STATIC_INFO_PARAMS(c.Struct): + SIZE = 16 + replayableFaultBufferSize: int + replayableShadowFaultBufferMetadataSize: int + nonReplayableFaultBufferSize: int + nonReplayableShadowFaultBufferMetadataSize: int +struct_NV2080_CTRL_INTERNAL_GMMU_GET_STATIC_INFO_PARAMS.register_fields([('replayableFaultBufferSize', NvU32, 0), ('replayableShadowFaultBufferMetadataSize', NvU32, 4), ('nonReplayableFaultBufferSize', NvU32, 8), ('nonReplayableShadowFaultBufferMetadataSize', NvU32, 12)]) +NV2080_CTRL_INTERNAL_GMMU_GET_STATIC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GMMU_GET_STATIC_INFO_PARAMS +NV2080_CTRL_INTERNAL_GR_GET_CTXSW_MODES_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_GET_CTXSW_MODES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_FB_GET_HEAP_RESERVATION_SIZE_PARAMS(c.Struct): + SIZE = 8 + moduleIndex: int + size: int +struct_NV2080_CTRL_INTERNAL_FB_GET_HEAP_RESERVATION_SIZE_PARAMS.register_fields([('moduleIndex', NvU32, 0), ('size', NvU32, 4)]) +NV2080_CTRL_INTERNAL_FB_GET_HEAP_RESERVATION_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_FB_GET_HEAP_RESERVATION_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_ENTRY(c.Struct): + SIZE = 16 + engineIdx: int + pmcIntrMask: int + vectorStall: int + vectorNonStall: int +struct_NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_ENTRY.register_fields([('engineIdx', NvU16, 0), ('pmcIntrMask', NvU32, 4), ('vectorStall', NvU32, 8), ('vectorNonStall', NvU32, 12)]) +NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_ENTRY: TypeAlias = struct_NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_ENTRY +@c.record +class struct_NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS(c.Struct): + SIZE = 2112 + tableLen: int + table: c.Array[struct_NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_ENTRY, Literal[128]] + subtreeMap: c.Array[struct_NV2080_INTR_CATEGORY_SUBTREE_MAP, Literal[7]] +@c.record +class struct_NV2080_INTR_CATEGORY_SUBTREE_MAP(c.Struct): + SIZE = 8 + subtreeMask: int +NV2080_INTR_CATEGORY_SUBTREE_MAP: TypeAlias = struct_NV2080_INTR_CATEGORY_SUBTREE_MAP +struct_NV2080_INTR_CATEGORY_SUBTREE_MAP.register_fields([('subtreeMask', NvU64, 0)]) +struct_NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS.register_fields([('tableLen', NvU32, 0), ('table', c.Array[NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_ENTRY, Literal[128]], 4), ('subtreeMap', c.Array[NV2080_INTR_CATEGORY_SUBTREE_MAP, Literal[7]], 2056)]) +NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_CHECK_PARAMS(c.Struct): + SIZE = 1 + bReservation: int +struct_NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_CHECK_PARAMS.register_fields([('bReservation', NvBool, 0)]) +NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_CHECK_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_CHECK_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_SET_PARAMS(c.Struct): + SIZE = 3 + bReservation: int + bClientHandlesGrGating: int + bRmHandlesIdleSlow: int +struct_NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_SET_PARAMS.register_fields([('bReservation', NvBool, 0), ('bClientHandlesGrGating', NvBool, 1), ('bRmHandlesIdleSlow', NvBool, 2)]) +NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_SET_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_SET_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES_PARAMS(c.Struct): + SIZE = 8 + displayMask: int + numHeads: int +struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES_PARAMS.register_fields([('displayMask', NvU32, 0), ('numHeads', NvU32, 4)]) +NV2080_CTRL_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS(c.Struct): + SIZE = 240 + fbMemPageRanges: c.Array[struct_NV2080_CTRL_INTERNAL_NV_RANGE, Literal[15]] +struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS.register_fields([('fbMemPageRanges', c.Array[NV2080_CTRL_INTERNAL_NV_RANGE, Literal[15]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS(c.Struct): + SIZE = 8 + runlistId: int + numChannels: int +struct_NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS.register_fields([('runlistId', NvU32, 0), ('numChannels', NvU32, 4)]) +NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_PROFILES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PROFILES_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS(c.Struct): + SIZE = 32 + memBoundaryCfgA: int + memBoundaryCfgB: int + memBoundaryCfgC: int + memBoundaryCfg: int + memBoundaryCfgValInit: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS.register_fields([('memBoundaryCfgA', NvU64, 0), ('memBoundaryCfgB', NvU64, 8), ('memBoundaryCfgC', NvU32, 16), ('memBoundaryCfg', NvU32, 20), ('memBoundaryCfgValInit', NvU32, 24)]) +NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS +NV2080_CTRL_INTERNAL_KMEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE_PARAMS(c.Struct): + SIZE = 32 + data: c.Array[ctypes.c_uint32, Literal[8]] +struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE_PARAMS.register_fields([('data', c.Array[NvU32, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_GET_AUX_POWER_STATE_PARAMS(c.Struct): + SIZE = 4 + powerState: int +struct_NV2080_CTRL_INTERNAL_PERF_GET_AUX_POWER_STATE_PARAMS.register_fields([('powerState', NvU32, 0)]) +NV2080_CTRL_INTERNAL_PERF_GET_AUX_POWER_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_GET_AUX_POWER_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_2X(c.Struct): + SIZE = 8 + flags: int + duration: int +struct_NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_2X.register_fields([('flags', NvBool, 0), ('duration', NvU32, 4)]) +NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_2X: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_2X +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_CONTROL_PARAMS(c.Struct): + SIZE = 1 + bActivate: int +struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_CONTROL_PARAMS.register_fields([('bActivate', NvBool, 0)]) +NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS(c.Struct): + SIZE = 16 + flags: int + bBridgeless: int + currLimits: c.Array[ctypes.c_uint32, Literal[2]] +struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS.register_fields([('flags', NvU32, 0), ('bBridgeless', NvBool, 4), ('currLimits', c.Array[NvU32, Literal[2]], 8)]) +NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO_PARAMS(c.Struct): + SIZE = 16 + hysteresisus: int + bHystersisEnable: int + bSliGpuBoostSyncEnable: int +struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO_PARAMS.register_fields([('hysteresisus', NvU64, 0), ('bHystersisEnable', NvBool, 8), ('bSliGpuBoostSyncEnable', NvBool, 9)]) +NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GMMU_REGISTER_FAULT_BUFFER_PARAMS(c.Struct): + SIZE = 2064 + hClient: int + hObject: int + faultBufferSize: int + faultBufferPteArray: c.Array[ctypes.c_uint64, Literal[256]] +struct_NV2080_CTRL_INTERNAL_GMMU_REGISTER_FAULT_BUFFER_PARAMS.register_fields([('hClient', NvHandle, 0), ('hObject', NvHandle, 4), ('faultBufferSize', NvU32, 8), ('faultBufferPteArray', c.Array[NvU64, Literal[256]], 16)]) +NV2080_CTRL_INTERNAL_GMMU_REGISTER_FAULT_BUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GMMU_REGISTER_FAULT_BUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GMMU_REGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS(c.Struct): + SIZE = 24032 + shadowFaultBufferQueuePhysAddr: int + shadowFaultBufferSize: int + shadowFaultBufferMetadataSize: int + shadowFaultBufferPteArray: c.Array[ctypes.c_uint64, Literal[3000]] + shadowFaultBufferType: int + faultBufferSharedMemoryPhysAddr: int +struct_NV2080_CTRL_INTERNAL_GMMU_REGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS.register_fields([('shadowFaultBufferQueuePhysAddr', NvU64, 0), ('shadowFaultBufferSize', NvU32, 8), ('shadowFaultBufferMetadataSize', NvU32, 12), ('shadowFaultBufferPteArray', c.Array[NvU64, Literal[3000]], 16), ('shadowFaultBufferType', NvU32, 24016), ('faultBufferSharedMemoryPhysAddr', NvU64, 24024)]) +NV2080_CTRL_INTERNAL_GMMU_REGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GMMU_REGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GMMU_UNREGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS(c.Struct): + SIZE = 4 + shadowFaultBufferType: int +struct_NV2080_CTRL_INTERNAL_GMMU_UNREGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS.register_fields([('shadowFaultBufferType', NvU32, 0)]) +NV2080_CTRL_INTERNAL_GMMU_UNREGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GMMU_UNREGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GMMU_COPY_RESERVED_SPLIT_GVASPACE_PDES_TO_SERVER_PARAMS(c.Struct): + SIZE = 184 + PdeCopyParams: struct_NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS +struct_NV2080_CTRL_INTERNAL_GMMU_COPY_RESERVED_SPLIT_GVASPACE_PDES_TO_SERVER_PARAMS.register_fields([('PdeCopyParams', NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS, 0)]) +NV2080_CTRL_INTERNAL_GMMU_COPY_RESERVED_SPLIT_GVASPACE_PDES_TO_SERVER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GMMU_COPY_RESERVED_SPLIT_GVASPACE_PDES_TO_SERVER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_3X(c.Struct): + SIZE = 16 + flags: int + boostDuration: int + gfId: int + bOverrideInfinite: int +struct_NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_3X.register_fields([('flags', NvU32, 0), ('boostDuration', NvU32, 4), ('gfId', NvU32, 8), ('bOverrideInfinite', NvBool, 12)]) +NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_3X: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_3X +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_BOOST_CLEAR_PARAMS_3X(c.Struct): + SIZE = 8 + bIsCudaClient: int + gfId: int +struct_NV2080_CTRL_INTERNAL_PERF_BOOST_CLEAR_PARAMS_3X.register_fields([('bIsCudaClient', NvBool, 0), ('gfId', NvU32, 4)]) +NV2080_CTRL_INTERNAL_PERF_BOOST_CLEAR_PARAMS_3X: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_BOOST_CLEAR_PARAMS_3X +@c.record +class struct_NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO(c.Struct): + SIZE = 80 + skylineVgpcSize: c.Array[ctypes.c_ubyte, Literal[64]] + singletonVgpcMask: int + maxInstances: int + computeSizeFlag: int + numNonSingletonVgpcs: int +struct_NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO.register_fields([('skylineVgpcSize', c.Array[NvU8, Literal[64]], 0), ('singletonVgpcMask', NvU32, 64), ('maxInstances', NvU32, 68), ('computeSizeFlag', NvU32, 72), ('numNonSingletonVgpcs', NvU32, 76)]) +NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GRMGR_GET_SKYLINE_INFO_PARAMS(c.Struct): + SIZE = 724 + skylineTable: c.Array[struct_NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO, Literal[9]] + validEntries: int +struct_NV2080_CTRL_INTERNAL_STATIC_GRMGR_GET_SKYLINE_INFO_PARAMS.register_fields([('skylineTable', c.Array[NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO, Literal[9]], 0), ('validEntries', NvU32, 720)]) +NV2080_CTRL_INTERNAL_STATIC_GRMGR_GET_SKYLINE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GRMGR_GET_SKYLINE_INFO_PARAMS +NV2080_CTRL_INTERNAL_MIGMGR_SET_PARTITIONING_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_PARTITIONING_MODE_PARAMS +NV2080_CTRL_INTERNAL_MIGMGR_CONFIGURE_GPU_INSTANCE_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_CONFIGURE_PARTITION_PARAMS +NV2080_CTRL_INTERNAL_MIGMGR_SET_GPU_INSTANCES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_SET_PARTITIONS_PARAMS +NV2080_CTRL_INTERNAL_MIGMGR_GET_GPU_INSTANCES_PARAMS: TypeAlias = struct_NV2080_CTRL_GPU_GET_PARTITIONS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS(c.Struct): + SIZE = 1 + bZbcSurfacesExist: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS.register_fields([('bZbcSurfacesExist', NvBool, 0)]) +NV2080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_KMIGMGR_EXPORTED_GPU_INSTANCE_INFO(c.Struct): + SIZE = 56 + enginesMask: c.Array[ctypes.c_uint64, Literal[4]] + partitionFlags: int + gpcMask: int + virtualGpcCount: int + veidOffset: int + veidCount: int +struct_NV2080_CTRL_INTERNAL_KMIGMGR_EXPORTED_GPU_INSTANCE_INFO.register_fields([('enginesMask', c.Array[NvU64, Literal[4]], 0), ('partitionFlags', NvU32, 32), ('gpcMask', NvU32, 36), ('virtualGpcCount', NvU32, 40), ('veidOffset', NvU32, 44), ('veidCount', NvU32, 48)]) +NV2080_CTRL_INTERNAL_KMIGMGR_EXPORTED_GPU_INSTANCE_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_KMIGMGR_EXPORTED_GPU_INSTANCE_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS(c.Struct): + SIZE = 80 + swizzId: int + uuid: c.Array[ctypes.c_ubyte, Literal[16]] + info: struct_NV2080_CTRL_INTERNAL_KMIGMGR_EXPORTED_GPU_INSTANCE_INFO +struct_NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS.register_fields([('swizzId', NvU32, 0), ('uuid', c.Array[NvU8, Literal[16]], 4), ('info', NV2080_CTRL_INTERNAL_KMIGMGR_EXPORTED_GPU_INSTANCE_INFO, 24)]) +NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS +NV2080_CTRL_INTERNAL_KMIGMGR_EXPORT_GPU_INSTANCE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS +NV2080_CTRL_INTERNAL_MIGMGR_EXPORT_GPU_INSTANCE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS +NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_GPU_INSTANCE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS +NV2080_CTRL_INTERNAL_MIGMGR_IMPORT_GPU_INSTANCE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_EXPORT_GPU_INSTANCE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_PARAMS(c.Struct): + SIZE = 4 + flags: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_PARAMS.register_fields([('flags', NvU32, 0)]) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_BIF_GET_STATIC_INFO_PARAMS(c.Struct): + SIZE = 4 + bPcieGen4Capable: int + bIsC2CLinkUp: int + bIsDeviceMultiFunction: int + bGcxPmuCfgSpaceRestore: int +struct_NV2080_CTRL_INTERNAL_BIF_GET_STATIC_INFO_PARAMS.register_fields([('bPcieGen4Capable', NvBool, 0), ('bIsC2CLinkUp', NvBool, 1), ('bIsDeviceMultiFunction', NvBool, 2), ('bGcxPmuCfgSpaceRestore', NvBool, 3)]) +NV2080_CTRL_INTERNAL_BIF_GET_STATIC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_BIF_GET_STATIC_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS(c.Struct): + SIZE = 12 + programPeerMask: int + invalidatePeerMask: int + programPciePeerMask: int +struct_NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS.register_fields([('programPeerMask', NvU32, 0), ('invalidatePeerMask', NvU32, 4), ('programPciePeerMask', NvU32, 8)]) +NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS(c.Struct): + SIZE = 36 + linkMask: int + hshubIds: c.Array[ctypes.c_ubyte, Literal[32]] +struct_NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS.register_fields([('linkMask', NvU32, 0), ('hshubIds', c.Array[NvU8, Literal[32]], 4)]) +NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_HSHUB_GET_NUM_UNITS_PARAMS(c.Struct): + SIZE = 4 + numHshubs: int +struct_NV2080_CTRL_INTERNAL_HSHUB_GET_NUM_UNITS_PARAMS.register_fields([('numHshubs', NvU32, 0)]) +NV2080_CTRL_INTERNAL_HSHUB_GET_NUM_UNITS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_HSHUB_GET_NUM_UNITS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_HSHUB_NEXT_HSHUB_ID_PARAMS(c.Struct): + SIZE = 1 + hshubId: int +struct_NV2080_CTRL_INTERNAL_HSHUB_NEXT_HSHUB_ID_PARAMS.register_fields([('hshubId', NvU8, 0)]) +NV2080_CTRL_INTERNAL_HSHUB_NEXT_HSHUB_ID_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_HSHUB_NEXT_HSHUB_ID_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_HSHUB_EGM_CONFIG_PARAMS(c.Struct): + SIZE = 4 + egmPeerId: int +struct_NV2080_CTRL_INTERNAL_HSHUB_EGM_CONFIG_PARAMS.register_fields([('egmPeerId', NvU32, 0)]) +NV2080_CTRL_INTERNAL_HSHUB_EGM_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_HSHUB_EGM_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR_PARAMS(c.Struct): + SIZE = 16 + bGet: int + addr: int +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR_PARAMS.register_fields([('bGet', NvBool, 0), ('addr', NvU64, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_CF_CONTROLLERS_SET_MAX_VGPU_VM_COUNT_PARAMS(c.Struct): + SIZE = 1 + maxActiveVGpuVMCount: int +struct_NV2080_CTRL_INTERNAL_PERF_CF_CONTROLLERS_SET_MAX_VGPU_VM_COUNT_PARAMS.register_fields([('maxActiveVGpuVMCount', NvU8, 0)]) +NV2080_CTRL_INTERNAL_PERF_CF_CONTROLLERS_SET_MAX_VGPU_VM_COUNT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_CF_CONTROLLERS_SET_MAX_VGPU_VM_COUNT_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS(c.Struct): + SIZE = 1 + bRawMode: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS.register_fields([('bRawMode', NvBool, 0)]) +NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CCU_SAMPLE_INFO_PARAMS(c.Struct): + SIZE = 4 + ccuSampleSize: int +struct_NV2080_CTRL_INTERNAL_CCU_SAMPLE_INFO_PARAMS.register_fields([('ccuSampleSize', NvU32, 0)]) +NV2080_CTRL_INTERNAL_CCU_SAMPLE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CCU_SAMPLE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CCU_MAP_INFO(c.Struct): + SIZE = 16 + phyAddr: int + shrBufSize: int + cntBlkSize: int +struct_NV2080_CTRL_INTERNAL_CCU_MAP_INFO.register_fields([('phyAddr', NvU64, 0), ('shrBufSize', NvU32, 8), ('cntBlkSize', NvU32, 12)]) +NV2080_CTRL_INTERNAL_CCU_MAP_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_CCU_MAP_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_CCU_MAP_INFO_PARAMS(c.Struct): + SIZE = 144 + mapInfo: c.Array[struct_NV2080_CTRL_INTERNAL_CCU_MAP_INFO, Literal[9]] +struct_NV2080_CTRL_INTERNAL_CCU_MAP_INFO_PARAMS.register_fields([('mapInfo', c.Array[NV2080_CTRL_INTERNAL_CCU_MAP_INFO, Literal[9]], 0)]) +NV2080_CTRL_INTERNAL_CCU_MAP_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CCU_MAP_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CCU_UNMAP_INFO_PARAMS(c.Struct): + SIZE = 2 + bDevShrBuf: int + bMigShrBuf: int +struct_NV2080_CTRL_INTERNAL_CCU_UNMAP_INFO_PARAMS.register_fields([('bDevShrBuf', NvBool, 0), ('bMigShrBuf', NvBool, 1)]) +NV2080_CTRL_INTERNAL_CCU_UNMAP_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CCU_UNMAP_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PEER_INFO(c.Struct): + SIZE = 40 + gpuId: int + gpuInstance: int + p2pCaps: int + p2pOptimalReadCEs: int + p2pOptimalWriteCEs: int + p2pCapsStatus: c.Array[ctypes.c_ubyte, Literal[9]] + busPeerId: int + busEgmPeerId: int +struct_NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PEER_INFO.register_fields([('gpuId', NvU32, 0), ('gpuInstance', NvU32, 4), ('p2pCaps', NvU32, 8), ('p2pOptimalReadCEs', NvU32, 12), ('p2pOptimalWriteCEs', NvU32, 16), ('p2pCapsStatus', c.Array[NvU8, Literal[9]], 20), ('busPeerId', NvU32, 32), ('busEgmPeerId', NvU32, 36)]) +NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PEER_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PEER_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PARAMS(c.Struct): + SIZE = 1284 + peerGpuCount: int + peerGpuInfos: c.Array[struct_NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PEER_INFO, Literal[32]] +struct_NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PARAMS.register_fields([('peerGpuCount', NvU32, 0), ('peerGpuInfos', c.Array[NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PEER_INFO, Literal[32]], 4)]) +NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_REMOVE_P2P_CAPS_PARAMS(c.Struct): + SIZE = 132 + peerGpuIdCount: int + peerGpuIds: c.Array[ctypes.c_uint32, Literal[32]] +struct_NV2080_CTRL_INTERNAL_REMOVE_P2P_CAPS_PARAMS.register_fields([('peerGpuIdCount', NvU32, 0), ('peerGpuIds', c.Array[NvU32, Literal[32]], 4)]) +NV2080_CTRL_INTERNAL_REMOVE_P2P_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_REMOVE_P2P_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GET_PCIE_P2P_CAPS_PARAMS(c.Struct): + SIZE = 4 + bCommonPciSwitchFound: int + p2pReadCapsStatus: int + p2pWriteCapsStatus: int + bSkipChipsetCheck: int +struct_NV2080_CTRL_INTERNAL_GET_PCIE_P2P_CAPS_PARAMS.register_fields([('bCommonPciSwitchFound', NvBool, 0), ('p2pReadCapsStatus', NvU8, 1), ('p2pWriteCapsStatus', NvU8, 2), ('bSkipChipsetCheck', NvBool, 3)]) +NV2080_CTRL_INTERNAL_GET_PCIE_P2P_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GET_PCIE_P2P_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_BIF_SET_PCIE_RO_PARAMS(c.Struct): + SIZE = 1 + enableRo: int +struct_NV2080_CTRL_INTERNAL_BIF_SET_PCIE_RO_PARAMS.register_fields([('enableRo', NvBool, 0)]) +NV2080_CTRL_INTERNAL_BIF_SET_PCIE_RO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_BIF_SET_PCIE_RO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS(c.Struct): + SIZE = 3 + bSave: int + bUseVbios: int + bReturnEarly: int +struct_NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS.register_fields([('bSave', NvBool, 0), ('bUseVbios', NvBool, 1), ('bReturnEarly', NvBool, 2)]) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS(c.Struct): + SIZE = 3 + bSave: int + bUseVbios: int + bVbiosCallSuccessful: int +struct_NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS.register_fields([('bSave', NvBool, 0), ('bUseVbios', NvBool, 1), ('bVbiosCallSuccessful', NvBool, 2)]) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MIGMGR_COMPUTE_PROFILE(c.Struct): + SIZE = 24 + computeSize: int + flags: int + gfxGpcCount: int + gpcCount: int + veidCount: int + smCount: int + physicalSlots: int +struct_NV2080_CTRL_INTERNAL_MIGMGR_COMPUTE_PROFILE.register_fields([('computeSize', NvU8, 0), ('flags', NvU16, 2), ('gfxGpcCount', NvU32, 4), ('gpcCount', NvU32, 8), ('veidCount', NvU32, 12), ('smCount', NvU32, 16), ('physicalSlots', NvU32, 20)]) +NV2080_CTRL_INTERNAL_MIGMGR_COMPUTE_PROFILE: TypeAlias = struct_NV2080_CTRL_INTERNAL_MIGMGR_COMPUTE_PROFILE +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_COMPUTE_PROFILES_PARAMS(c.Struct): + SIZE = 196 + profileCount: int + profiles: c.Array[struct_NV2080_CTRL_INTERNAL_MIGMGR_COMPUTE_PROFILE, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_COMPUTE_PROFILES_PARAMS.register_fields([('profileCount', NvU32, 0), ('profiles', c.Array[NV2080_CTRL_INTERNAL_MIGMGR_COMPUTE_PROFILE, Literal[8]], 4)]) +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_COMPUTE_PROFILES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_COMPUTE_PROFILES_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_COMPUTE_PROFILES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_COMPUTE_PROFILES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CCU_STREAM_STATE_PARAMS(c.Struct): + SIZE = 1 + bStreamState: int +struct_NV2080_CTRL_INTERNAL_CCU_STREAM_STATE_PARAMS.register_fields([('bStreamState', NvBool, 0)]) +NV2080_CTRL_INTERNAL_CCU_STREAM_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CCU_STREAM_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_ATTACH_AND_INIT_PARAMS(c.Struct): + SIZE = 1 + bExtDevFound: int +struct_NV2080_CTRL_INTERNAL_GSYNC_ATTACH_AND_INIT_PARAMS.register_fields([('bExtDevFound', NvBool, 0)]) +NV2080_CTRL_INTERNAL_GSYNC_ATTACH_AND_INIT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_ATTACH_AND_INIT_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_OPTIMIZE_TIMING_PARAMETERS_PARAMS(c.Struct): + SIZE = 96 + timingParameters: struct_NV30F1_CTRL_GSYNC_GET_OPTIMIZED_TIMING_PARAMS +@c.record +class struct_NV30F1_CTRL_GSYNC_GET_OPTIMIZED_TIMING_PARAMS(c.Struct): + SIZE = 96 + gpuId: int + output: int + protocol: int + structure: int + adjust: int + hDeltaStep: int + hDeltaMax: int + vDeltaStep: int + vDeltaMax: int + hSyncEnd: int + hBlankEnd: int + hBlankStart: int + hTotal: int + vSyncEnd: int + vBlankEnd: int + vBlankStart: int + vInterlacedBlankEnd: int + vInterlacedBlankStart: int + vTotal: int + refreshX10K: int + pixelClockHz: int + bOptimized: int +NV30F1_CTRL_GSYNC_GET_OPTIMIZED_TIMING_PARAMS: TypeAlias = struct_NV30F1_CTRL_GSYNC_GET_OPTIMIZED_TIMING_PARAMS +struct_NV30F1_CTRL_GSYNC_GET_OPTIMIZED_TIMING_PARAMS.register_fields([('gpuId', NvU32, 0), ('output', NvU32, 4), ('protocol', NvU32, 8), ('structure', NvU32, 12), ('adjust', NvU32, 16), ('hDeltaStep', NvU32, 20), ('hDeltaMax', NvU32, 24), ('vDeltaStep', NvU32, 28), ('vDeltaMax', NvU32, 32), ('hSyncEnd', NvU32, 36), ('hBlankEnd', NvU32, 40), ('hBlankStart', NvU32, 44), ('hTotal', NvU32, 48), ('vSyncEnd', NvU32, 52), ('vBlankEnd', NvU32, 56), ('vBlankStart', NvU32, 60), ('vInterlacedBlankEnd', NvU32, 64), ('vInterlacedBlankStart', NvU32, 68), ('vTotal', NvU32, 72), ('refreshX10K', NvU32, 76), ('pixelClockHz', NvU64, 80), ('bOptimized', NvBool, 88)]) +struct_NV2080_CTRL_INTERNAL_GSYNC_OPTIMIZE_TIMING_PARAMETERS_PARAMS.register_fields([('timingParameters', NV30F1_CTRL_GSYNC_GET_OPTIMIZED_TIMING_PARAMS, 0)]) +NV2080_CTRL_INTERNAL_GSYNC_OPTIMIZE_TIMING_PARAMETERS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_OPTIMIZE_TIMING_PARAMETERS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_GET_DISPLAY_IDS_PARAMS(c.Struct): + SIZE = 16 + displayIds: c.Array[ctypes.c_uint32, Literal[4]] +struct_NV2080_CTRL_INTERNAL_GSYNC_GET_DISPLAY_IDS_PARAMS.register_fields([('displayIds', c.Array[NvU32, Literal[4]], 0)]) +NV2080_CTRL_INTERNAL_GSYNC_GET_DISPLAY_IDS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_GET_DISPLAY_IDS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_SET_STREO_SYNC_PARAMS(c.Struct): + SIZE = 52 + slave: c.Array[ctypes.c_uint32, Literal[4]] + localSlave: c.Array[ctypes.c_uint32, Literal[4]] + master: c.Array[ctypes.c_uint32, Literal[4]] + regStatus: int +struct_NV2080_CTRL_INTERNAL_GSYNC_SET_STREO_SYNC_PARAMS.register_fields([('slave', c.Array[NvU32, Literal[4]], 0), ('localSlave', c.Array[NvU32, Literal[4]], 16), ('master', c.Array[NvU32, Literal[4]], 32), ('regStatus', NvU32, 48)]) +NV2080_CTRL_INTERNAL_GSYNC_SET_STREO_SYNC_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_SET_STREO_SYNC_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_GET_VERTICAL_ACTIVE_LINES_PARAMS(c.Struct): + SIZE = 8 + headIdx: int + vActiveLines: int +struct_NV2080_CTRL_INTERNAL_GSYNC_GET_VERTICAL_ACTIVE_LINES_PARAMS.register_fields([('headIdx', NvU32, 0), ('vActiveLines', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GSYNC_GET_VERTICAL_ACTIVE_LINES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_GET_VERTICAL_ACTIVE_LINES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_IS_DISPLAYID_VALID_PARAMS(c.Struct): + SIZE = 8 + displays: int + displayId: int +struct_NV2080_CTRL_INTERNAL_GSYNC_IS_DISPLAYID_VALID_PARAMS.register_fields([('displays', NvU32, 0), ('displayId', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GSYNC_IS_DISPLAYID_VALID_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_IS_DISPLAYID_VALID_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_SET_OR_RESTORE_RASTER_SYNC_PARAMS(c.Struct): + SIZE = 8 + bEnableMaster: int + bRasterSyncGpioSaved: int + bRasterSyncGpioDirection: int +struct_NV2080_CTRL_INTERNAL_GSYNC_SET_OR_RESTORE_RASTER_SYNC_PARAMS.register_fields([('bEnableMaster', NvBool, 0), ('bRasterSyncGpioSaved', NvBool, 1), ('bRasterSyncGpioDirection', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GSYNC_SET_OR_RESTORE_RASTER_SYNC_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_SET_OR_RESTORE_RASTER_SYNC_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS(c.Struct): + SIZE = 24 + hClient: int + hSysMem: int + bEnteringGcoffState: int + sysmemAddrOfSuspendResumeData: int +struct_NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS.register_fields([('hClient', NvHandle, 0), ('hSysMem', NvHandle, 4), ('bEnteringGcoffState', NvBool, 8), ('sysmemAddrOfSuspendResumeData', NvU64, 16)]) +NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_FIFO_TOGGLE_ACTIVE_CHANNEL_SCHEDULING_PARAMS(c.Struct): + SIZE = 1 + bDisableActiveChannels: int +struct_NV2080_CTRL_CMD_INTERNAL_FIFO_TOGGLE_ACTIVE_CHANNEL_SCHEDULING_PARAMS.register_fields([('bDisableActiveChannels', NvBool, 0)]) +NV2080_CTRL_CMD_INTERNAL_FIFO_TOGGLE_ACTIVE_CHANNEL_SCHEDULING_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_FIFO_TOGGLE_ACTIVE_CHANNEL_SCHEDULING_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB_PARAMS(c.Struct): + SIZE = 16 + hostReservedFb: int + vgpuTypeId: int +struct_NV2080_CTRL_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB_PARAMS.register_fields([('hostReservedFb', NvU64, 0), ('vgpuTypeId', NvU32, 8)]) +NV2080_CTRL_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_INIT_BRIGHTC_STATE_LOAD_PARAMS(c.Struct): + SIZE = 4104 + status: int + bNitsPwmFallbackEnabled: int + backLightDataSize: int + backLightData: c.Array[ctypes.c_ubyte, Literal[4096]] +struct_NV2080_CTRL_INTERNAL_INIT_BRIGHTC_STATE_LOAD_PARAMS.register_fields([('status', NvU32, 0), ('bNitsPwmFallbackEnabled', NvBool, 4), ('backLightDataSize', NvU16, 6), ('backLightData', c.Array[NvU8, Literal[4096]], 8)]) +NV2080_CTRL_INTERNAL_INIT_BRIGHTC_STATE_LOAD_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_INIT_BRIGHTC_STATE_LOAD_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL_PARAMS(c.Struct): + SIZE = 4 + numActiveLinksPerIoctrl: int +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL_PARAMS.register_fields([('numActiveLinksPerIoctrl', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL_PARAMS(c.Struct): + SIZE = 4 + numLinksPerIoctrl: int +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL_PARAMS.register_fields([('numLinksPerIoctrl', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_SMBPBI_PFM_REQ_HNDLR_CAP_UPDATE_PARAMS(c.Struct): + SIZE = 2 + bIsSysCtrlSupported: int + bIsPlatformLegacy: int +struct_NV2080_CTRL_INTERNAL_SMBPBI_PFM_REQ_HNDLR_CAP_UPDATE_PARAMS.register_fields([('bIsSysCtrlSupported', NvBool, 0), ('bIsPlatformLegacy', NvBool, 1)]) +NV2080_CTRL_INTERNAL_SMBPBI_PFM_REQ_HNDLR_CAP_UPDATE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_SMBPBI_PFM_REQ_HNDLR_CAP_UPDATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_SMBPBI(c.Struct): + SIZE = 8 + sensorId: int + limit: int +struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_SMBPBI.register_fields([('sensorId', NvU32, 0), ('limit', NvU32, 4)]) +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_SMBPBI: TypeAlias = struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_SMBPBI +@c.record +class struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA(c.Struct): + SIZE = 12 + type: int + data: struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_data +@c.record +class struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_data(c.Struct): + SIZE = 8 + smbpbi: struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_SMBPBI +struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_data.register_fields([('smbpbi', NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_SMBPBI, 0)]) +struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA.register_fields([('type', NvU8, 0), ('data', struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_data, 4)]) +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA: TypeAlias = struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA +@c.record +class struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS(c.Struct): + SIZE = 16 + flags: int + syncData: struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA +struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS.register_fields([('flags', NvU8, 0), ('syncData', NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA, 4)]) +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS +NV2080_CTRL_INTERNAL_PMGR_PFM_REQ_HNDLR_STATE_LOAD_SYNC_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS +NV2080_CTRL_INTERNAL_THERM_PFM_REQ_HNDLR_STATE_INIT_SYNC_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_PM1_STATE_PARAMS(c.Struct): + SIZE = 2 + flag: int + bStatus: int +struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_PM1_STATE_PARAMS.register_fields([('flag', NvU8, 0), ('bStatus', NvBool, 1)]) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_PM1_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_PM1_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_PM1_STATE_PARAMS(c.Struct): + SIZE = 1 + bEnable: int +struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_PM1_STATE_PARAMS.register_fields([('bEnable', NvBool, 0)]) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_PM1_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_PM1_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_UPDATE_EDPP_LIMIT_PARAMS(c.Struct): + SIZE = 8 + bEnable: int + clientLimit: int +struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_UPDATE_EDPP_LIMIT_PARAMS.register_fields([('bEnable', NvBool, 0), ('clientLimit', NvU32, 4)]) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_UPDATE_EDPP_LIMIT_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_UPDATE_EDPP_LIMIT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_THERM_PFM_REQ_HNDLR_UPDATE_TGPU_LIMIT_PARAMS(c.Struct): + SIZE = 4 + targetTemp: int +struct_NV2080_CTRL_CMD_INTERNAL_THERM_PFM_REQ_HNDLR_UPDATE_TGPU_LIMIT_PARAMS.register_fields([('targetTemp', NvS32, 0)]) +NV2080_CTRL_CMD_INTERNAL_THERM_PFM_REQ_HNDLR_UPDATE_TGPU_LIMIT_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_THERM_PFM_REQ_HNDLR_UPDATE_TGPU_LIMIT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TGP_MODE_PARAMS(c.Struct): + SIZE = 1 + bEnable: int +struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TGP_MODE_PARAMS.register_fields([('bEnable', NvBool, 0)]) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TGP_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TGP_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TURBO_V2_PARAMS(c.Struct): + SIZE = 4 + ctgpOffsetmW: int +struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TURBO_V2_PARAMS.register_fields([('ctgpOffsetmW', NvU32, 0)]) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TURBO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TURBO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_INFO_PARAMS(c.Struct): + SIZE = 8 + bVpsPs20Supported: int + vPstateIdxHighest: int +struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_INFO_PARAMS.register_fields([('bVpsPs20Supported', NvBool, 0), ('vPstateIdxHighest', NvU32, 4)]) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_MAPPING_PARAMS(c.Struct): + SIZE = 8 + pStateIdx: int + vPstateIdxMapping: int +struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_MAPPING_PARAMS.register_fields([('pStateIdx', NvU32, 0), ('vPstateIdxMapping', NvU32, 4)]) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_MAPPING_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_MAPPING_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_VPSTATE_PARAMS(c.Struct): + SIZE = 4 + vPstateIdx: int +struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_VPSTATE_PARAMS.register_fields([('vPstateIdx', NvU32, 0)]) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_VPSTATE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_VPSTATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS(c.Struct): + SIZE = 2 + bIsGC6Satisfied: int + bIsGCOFFSatisfied: int +struct_NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS.register_fields([('bIsGC6Satisfied', NvBool, 0), ('bIsGCOFFSatisfied', NvBool, 1)]) +NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS_PARAMS(c.Struct): + SIZE = 8 + maxSec2SecureChannels: int + maxCeSecureChannels: int +struct_NV2080_CTRL_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS_PARAMS.register_fields([('maxSec2SecureChannels', NvU32, 0), ('maxCeSecureChannels', NvU32, 4)]) +NV2080_CTRL_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_BIF_DISABLE_SYSTEM_MEMORY_ACCESS_PARAMS(c.Struct): + SIZE = 1 + bDisable: int +struct_NV2080_CTRL_INTERNAL_BIF_DISABLE_SYSTEM_MEMORY_ACCESS_PARAMS.register_fields([('bDisable', NvBool, 0)]) +NV2080_CTRL_INTERNAL_BIF_DISABLE_SYSTEM_MEMORY_ACCESS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_BIF_DISABLE_SYSTEM_MEMORY_ACCESS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISP_PINSETS_TO_LOCKPINS_PARAMS(c.Struct): + SIZE = 24 + pinSetIn: int + pinSetOut: int + bMasterScanLock: int + masterScanLockPin: int + bSlaveScanLock: int + slaveScanLockPin: int +struct_NV2080_CTRL_INTERNAL_DISP_PINSETS_TO_LOCKPINS_PARAMS.register_fields([('pinSetIn', NvU32, 0), ('pinSetOut', NvU32, 4), ('bMasterScanLock', NvBool, 8), ('masterScanLockPin', NvU32, 12), ('bSlaveScanLock', NvBool, 16), ('slaveScanLockPin', NvU32, 20)]) +NV2080_CTRL_INTERNAL_DISP_PINSETS_TO_LOCKPINS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISP_PINSETS_TO_LOCKPINS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_DISP_SET_SLI_LINK_GPIO_SW_CONTROL_PARAMS(c.Struct): + SIZE = 16 + pinSet: int + gpioFunction: int + gpioPin: int + gpioDirection: int +struct_NV2080_CTRL_INTERNAL_DISP_SET_SLI_LINK_GPIO_SW_CONTROL_PARAMS.register_fields([('pinSet', NvU32, 0), ('gpioFunction', NvU32, 4), ('gpioPin', NvU32, 8), ('gpioDirection', NvBool, 12)]) +NV2080_CTRL_INTERNAL_DISP_SET_SLI_LINK_GPIO_SW_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_DISP_SET_SLI_LINK_GPIO_SW_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_EDID_DATA(c.Struct): + SIZE = 524 + status: int + acpiId: int + bufferSize: int + edidBuffer: c.Array[ctypes.c_ubyte, Literal[512]] +struct_NV2080_CTRL_INTERNAL_EDID_DATA.register_fields([('status', NvU32, 0), ('acpiId', NvU32, 4), ('bufferSize', NvU32, 8), ('edidBuffer', c.Array[NvU8, Literal[512]], 12)]) +NV2080_CTRL_INTERNAL_EDID_DATA: TypeAlias = struct_NV2080_CTRL_INTERNAL_EDID_DATA +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_SET_STATIC_EDID_DATA_PARAMS(c.Struct): + SIZE = 8388 + tableLen: int + edidTable: c.Array[struct_NV2080_CTRL_INTERNAL_EDID_DATA, Literal[16]] +struct_NV2080_CTRL_CMD_INTERNAL_SET_STATIC_EDID_DATA_PARAMS.register_fields([('tableLen', NvU32, 0), ('edidTable', c.Array[NV2080_CTRL_INTERNAL_EDID_DATA, Literal[16]], 4)]) +NV2080_CTRL_CMD_INTERNAL_SET_STATIC_EDID_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_SET_STATIC_EDID_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_GET_GPU_FABRIC_PROBE_INFO_PARAMS(c.Struct): + SIZE = 8 + numProbes: int +struct_NV2080_CTRL_CMD_INTERNAL_GET_GPU_FABRIC_PROBE_INFO_PARAMS.register_fields([('numProbes', NvU64, 0)]) +NV2080_CTRL_CMD_INTERNAL_GET_GPU_FABRIC_PROBE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_GET_GPU_FABRIC_PROBE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_START_GPU_FABRIC_PROBE_INFO_PARAMS(c.Struct): + SIZE = 4 + bwMode: int + bLocalEgmEnabled: int +struct_NV2080_CTRL_CMD_INTERNAL_START_GPU_FABRIC_PROBE_INFO_PARAMS.register_fields([('bwMode', NvU16, 0), ('bLocalEgmEnabled', NvBool, 2)]) +NV2080_CTRL_CMD_INTERNAL_START_GPU_FABRIC_PROBE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_START_GPU_FABRIC_PROBE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_RESUME_GPU_FABRIC_PROBE_INFO_PARAMS(c.Struct): + SIZE = 2 + bwMode: int +struct_NV2080_CTRL_CMD_INTERNAL_RESUME_GPU_FABRIC_PROBE_INFO_PARAMS.register_fields([('bwMode', NvU16, 0)]) +NV2080_CTRL_CMD_INTERNAL_RESUME_GPU_FABRIC_PROBE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_RESUME_GPU_FABRIC_PROBE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_GET_STATIC_INFO_PARAMS(c.Struct): + SIZE = 2 + bIsBar1Trusted: int + bIsPcieTrusted: int +struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_GET_STATIC_INFO_PARAMS.register_fields([('bIsBar1Trusted', NvBool, 0), ('bIsPcieTrusted', NvBool, 1)]) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_GET_STATIC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_GET_STATIC_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK(c.Struct): + SIZE = 12 + ivMask: c.Array[ctypes.c_uint32, Literal[3]] +struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK.register_fields([('ivMask', c.Array[NvU32, Literal[3]], 0)]) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK: TypeAlias = struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK +@c.record +class struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_SWL_KEYS_PARAMS(c.Struct): + SIZE = 40 + engineId: int + ivMaskSet: c.Array[struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK, Literal[3]] +struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_SWL_KEYS_PARAMS.register_fields([('engineId', NvU32, 0), ('ivMaskSet', c.Array[NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK, Literal[3]], 4)]) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_SWL_KEYS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_SWL_KEYS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_LCE_KEYS_PARAMS(c.Struct): + SIZE = 76 + engineId: int + ivMaskSet: c.Array[struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK, Literal[6]] +struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_LCE_KEYS_PARAMS.register_fields([('engineId', NvU32, 0), ('ivMaskSet', c.Array[NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK, Literal[6]], 4)]) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_LCE_KEYS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_LCE_KEYS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_ROTATE_KEYS_PARAMS(c.Struct): + SIZE = 28 + globalH2DKey: int + updatedEncryptIVMask: c.Array[ctypes.c_uint32, Literal[3]] + updatedDecryptIVMask: c.Array[ctypes.c_uint32, Literal[3]] +struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_ROTATE_KEYS_PARAMS.register_fields([('globalH2DKey', NvU32, 0), ('updatedEncryptIVMask', c.Array[NvU32, Literal[3]], 4), ('updatedDecryptIVMask', c.Array[NvU32, Literal[3]], 16)]) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_ROTATE_KEYS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_ROTATE_KEYS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_RC_CHANNELS_FOR_KEY_ROTATION_PARAMS(c.Struct): + SIZE = 8 + exceptionType: int + globalH2DKey: int +struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_RC_CHANNELS_FOR_KEY_ROTATION_PARAMS.register_fields([('exceptionType', NvU32, 0), ('globalH2DKey', NvU32, 4)]) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_RC_CHANNELS_FOR_KEY_ROTATION_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_CONF_COMPUTE_RC_CHANNELS_FOR_KEY_ROTATION_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS(c.Struct): + SIZE = 1 + bAcceptClientRequest: int +struct_NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS.register_fields([('bAcceptClientRequest', NvBool, 0)]) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS(c.Struct): + SIZE = 8 + attackerAdvantage: int +struct_NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS.register_fields([('attackerAdvantage', NvU64, 0)]) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_FIFO_GET_LOGICAL_UPROC_ID_PARAMS(c.Struct): + SIZE = 1 + logicalUprocId: int +struct_NV2080_CTRL_INTERNAL_FIFO_GET_LOGICAL_UPROC_ID_PARAMS.register_fields([('logicalUprocId', NvU8, 0)]) +NV2080_CTRL_INTERNAL_FIFO_GET_LOGICAL_UPROC_ID_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_FIFO_GET_LOGICAL_UPROC_ID_PARAMS +enum_NV2080_CTRL_MEMMGR_MEMORY_OP: dict[int, str] = {(NV2080_CTRL_MEMMGR_MEMORY_OP_MEMCPY:=0): 'NV2080_CTRL_MEMMGR_MEMORY_OP_MEMCPY', (NV2080_CTRL_MEMMGR_MEMORY_OP_MEMSET:=1): 'NV2080_CTRL_MEMMGR_MEMORY_OP_MEMSET'} +NV2080_CTRL_MEMMGR_MEMORY_OP: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO(c.Struct): + SIZE = 32 + baseAddr: int + size: int + offset: int + aperture: int + cpuCacheAttrib: int +struct_NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO.register_fields([('baseAddr', NvU64, 0), ('size', NvU64, 8), ('offset', NvU64, 16), ('aperture', NvU32, 24), ('cpuCacheAttrib', NvU32, 28)]) +NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMMGR_MEMORY_TRANSFER_WITH_GSP_PARAMS(c.Struct): + SIZE = 96 + src: struct_NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO + authTag: c.Array[ctypes.c_ubyte, Literal[16]] + dst: struct_NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO + transferSize: int + value: int + memop: int +struct_NV2080_CTRL_INTERNAL_MEMMGR_MEMORY_TRANSFER_WITH_GSP_PARAMS.register_fields([('src', NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO, 0), ('authTag', c.Array[NvU8, Literal[16]], 32), ('dst', NV2080_CTRL_INTERNAL_TRANSFER_SURFACE_INFO, 48), ('transferSize', NvU64, 80), ('value', NvU32, 88), ('memop', NV2080_CTRL_MEMMGR_MEMORY_OP, 92)]) +NV2080_CTRL_INTERNAL_MEMMGR_MEMORY_TRANSFER_WITH_GSP_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMMGR_MEMORY_TRANSFER_WITH_GSP_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS(c.Struct): + SIZE = 24 + addrSysPhys: int + addrWidth: int + mask: int + maskWidth: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS.register_fields([('addrSysPhys', NvU64, 0), ('addrWidth', NvU32, 8), ('mask', NvU32, 12), ('maskWidth', NvU32, 16)]) +NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS(c.Struct): + SIZE = 32 + peerId: int + addrSysPhys: int + addrWidth: int + mask: int + maskWidth: int +struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS.register_fields([('peerId', NvU32, 0), ('addrSysPhys', NvU64, 8), ('addrWidth', NvU32, 16), ('mask', NvU32, 20), ('maskWidth', NvU32, 24)]) +NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_GET_EDPP_LIMIT_INFO_PARAMS(c.Struct): + SIZE = 24 + limitMin: int + limitRated: int + limitMax: int + limitCurr: int + limitBattRated: int + limitBattMax: int +struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_GET_EDPP_LIMIT_INFO_PARAMS.register_fields([('limitMin', NvU32, 0), ('limitRated', NvU32, 4), ('limitMax', NvU32, 8), ('limitCurr', NvU32, 12), ('limitBattRated', NvU32, 16), ('limitBattMax', NvU32, 20)]) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_GET_EDPP_LIMIT_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_GET_EDPP_LIMIT_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS(c.Struct): + SIZE = 16 + bInit: int + physAddr: int +struct_NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS.register_fields([('bInit', NvBool, 0), ('physAddr', NvU64, 8)]) +NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_USER_SHARED_DATA_SET_DATA_POLL_PARAMS(c.Struct): + SIZE = 16 + polledDataMask: int + pollIntervalMs: int +struct_NV2080_CTRL_INTERNAL_USER_SHARED_DATA_SET_DATA_POLL_PARAMS.register_fields([('polledDataMask', NvU64, 0), ('pollIntervalMs', NvU32, 8)]) +NV2080_CTRL_INTERNAL_USER_SHARED_DATA_SET_DATA_POLL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_USER_SHARED_DATA_SET_DATA_POLL_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_PARAMS(c.Struct): + SIZE = 32 + tracepointMask: int + bufferAddr: int + bufferSize: int + bufferWatermark: int + targetTask: int + flag: int +struct_NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_PARAMS.register_fields([('tracepointMask', NvU64, 0), ('bufferAddr', NvU64, 8), ('bufferSize', NvU32, 16), ('bufferWatermark', NvU32, 20), ('targetTask', NvU16, 24), ('flag', NvU8, 26)]) +NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_INTERNAL_GET_ENABLED_SEC2_CLASSES_PARAMS(c.Struct): + SIZE = 3 + bMaxwellSec2Enabled: int + bNv95A1TsecEnabled: int + bHopperSec2WorkLaunchAEnabled: int +struct_NV2080_CTRL_CMD_INTERNAL_GET_ENABLED_SEC2_CLASSES_PARAMS.register_fields([('bMaxwellSec2Enabled', NvBool, 0), ('bNv95A1TsecEnabled', NvBool, 1), ('bHopperSec2WorkLaunchAEnabled', NvBool, 2)]) +NV2080_CTRL_CMD_INTERNAL_GET_ENABLED_SEC2_CLASSES_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_INTERNAL_GET_ENABLED_SEC2_CLASSES_PARAMS +NV2080_CTRL_INTERNAL_GR_CTXSW_SETUP_BIND_PARAMS: TypeAlias = struct_NV2080_CTRL_GR_CTXSW_SETUP_BIND_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_CLIENT_LOW_POWER_MODE_ENTER_PARAMS(c.Struct): + SIZE = 8 + bInPMTransition: int + newPMLevel: int +struct_NV2080_CTRL_INTERNAL_GPU_CLIENT_LOW_POWER_MODE_ENTER_PARAMS.register_fields([('bInPMTransition', NvBool, 0), ('newPMLevel', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GPU_CLIENT_LOW_POWER_MODE_ENTER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_CLIENT_LOW_POWER_MODE_ENTER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_POST_FATAL_ERROR_RECOVERY_PARAMS(c.Struct): + SIZE = 1 + bSuccessful: int +struct_NV2080_CTRL_INTERNAL_NVLINK_POST_FATAL_ERROR_RECOVERY_PARAMS.register_fields([('bSuccessful', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_POST_FATAL_ERROR_RECOVERY_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_POST_FATAL_ERROR_RECOVERY_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_PARAMS(c.Struct): + SIZE = 4 + action: int +struct_NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_PARAMS.register_fields([('action', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_GET_GSP_RM_FREE_HEAP_PARAMS(c.Struct): + SIZE = 8 + freeHeapSize: int +struct_NV2080_CTRL_INTERNAL_GPU_GET_GSP_RM_FREE_HEAP_PARAMS.register_fields([('freeHeapSize', NvU64, 0)]) +NV2080_CTRL_INTERNAL_GPU_GET_GSP_RM_FREE_HEAP_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_GET_GSP_RM_FREE_HEAP_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_SET_ILLUM_PARAMS(c.Struct): + SIZE = 8 + attribute: int + value: int +struct_NV2080_CTRL_INTERNAL_GPU_SET_ILLUM_PARAMS.register_fields([('attribute', NvU32, 0), ('value', NvU32, 4)]) +NV2080_CTRL_INTERNAL_GPU_SET_ILLUM_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_SET_ILLUM_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_APPLY_STEREO_PIN_ALWAYS_HI_WAR_PARAMS(c.Struct): + SIZE = 1 + bApplyStereoPinAlwaysHiWar: int +struct_NV2080_CTRL_INTERNAL_GSYNC_APPLY_STEREO_PIN_ALWAYS_HI_WAR_PARAMS.register_fields([('bApplyStereoPinAlwaysHiWar', NvBool, 0)]) +NV2080_CTRL_INTERNAL_GSYNC_APPLY_STEREO_PIN_ALWAYS_HI_WAR_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_APPLY_STEREO_PIN_ALWAYS_HI_WAR_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_HSHUB_GET_MAX_HSHUBS_PER_SHIM_PARAMS(c.Struct): + SIZE = 4 + maxHshubs: int +struct_NV2080_CTRL_INTERNAL_HSHUB_GET_MAX_HSHUBS_PER_SHIM_PARAMS.register_fields([('maxHshubs', NvU32, 0)]) +NV2080_CTRL_INTERNAL_HSHUB_GET_MAX_HSHUBS_PER_SHIM_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_HSHUB_GET_MAX_HSHUBS_PER_SHIM_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_GET_RASTER_SYNC_DECODE_MODE_PARAMS(c.Struct): + SIZE = 4 + rasterSyncDecodeMode: int +struct_NV2080_CTRL_INTERNAL_GSYNC_GET_RASTER_SYNC_DECODE_MODE_PARAMS.register_fields([('rasterSyncDecodeMode', NvU32, 0)]) +NV2080_CTRL_INTERNAL_GSYNC_GET_RASTER_SYNC_DECODE_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_GET_RASTER_SYNC_DECODE_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_GET_PF_BAR1_SPA_PARAMS(c.Struct): + SIZE = 8 + spaValue: int +struct_NV2080_CTRL_INTERNAL_GPU_GET_PF_BAR1_SPA_PARAMS.register_fields([('spaValue', NvU64, 0)]) +NV2080_CTRL_INTERNAL_GPU_GET_PF_BAR1_SPA_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_GET_PF_BAR1_SPA_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_NVLINK_PEER_PARAMS(c.Struct): + SIZE = 8 + peerMask: int + bEnable: int +struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_NVLINK_PEER_PARAMS.register_fields([('peerMask', NvU32, 0), ('bEnable', NvBool, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_ENABLE_NVLINK_PEER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_NVLINK_PEER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_DL_LINK_MODE_PARAMS(c.Struct): + SIZE = 4 + mode: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_DL_LINK_MODE_PARAMS.register_fields([('mode', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_DL_LINK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_DL_LINK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_OFF_PARAMS(c.Struct): + SIZE = 28 + seedData: c.Array[ctypes.c_uint32, Literal[7]] +struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_OFF_PARAMS.register_fields([('seedData', c.Array[NvU32, Literal[7]], 0)]) +NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_OFF_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_OFF_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_PRE_HS_PARAMS(c.Struct): + SIZE = 8 + remoteDeviceType: int + ipVerDlPl: int +struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_PRE_HS_PARAMS.register_fields([('remoteDeviceType', NvU32, 0), ('ipVerDlPl', NvU32, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_PRE_HS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_PRE_HS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_INIT_PHASE1_PARAMS(c.Struct): + SIZE = 28 + seedData: c.Array[ctypes.c_uint32, Literal[7]] +struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_INIT_PHASE1_PARAMS.register_fields([('seedData', c.Array[NvU32, Literal[7]], 0)]) +NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_INIT_PHASE1_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_INIT_PHASE1_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO(c.Struct): + SIZE = 24 + remoteSid: int + remoteDeviceType: int + remoteLinkId: int + localSid: int +struct_NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO.register_fields([('remoteSid', NvU64, 0), ('remoteDeviceType', NvU32, 8), ('remoteLinkId', NvU32, 12), ('localSid', NvU64, 16)]) +NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITNEGOTIATE_PARAMS(c.Struct): + SIZE = 32 + bInitnegotiateConfigGood: int + remoteLocalSidInfo: struct_NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO +struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITNEGOTIATE_PARAMS.register_fields([('bInitnegotiateConfigGood', NvBool, 0), ('remoteLocalSidInfo', NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITNEGOTIATE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITNEGOTIATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITOPTIMIZE_PARAMS(c.Struct): + SIZE = 1 + bPollDone: int +struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITOPTIMIZE_PARAMS.register_fields([('bPollDone', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITOPTIMIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITOPTIMIZE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS(c.Struct): + SIZE = 48 + mode: int + bSync: int + linkMode: int + linkModeParams: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS_linkModeParams +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS_linkModeParams(c.Struct): + SIZE = 32 + linkModeOffParams: struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_OFF_PARAMS + linkModePreHsParams: struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_PRE_HS_PARAMS + linkModeInitPhase1Params: struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_INIT_PHASE1_PARAMS + linkModePostInitNegotiateParams: struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITNEGOTIATE_PARAMS + linkModePostInitOptimizeParams: struct_NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITOPTIMIZE_PARAMS +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS_linkModeParams.register_fields([('linkModeOffParams', NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_OFF_PARAMS, 0), ('linkModePreHsParams', NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_PRE_HS_PARAMS, 0), ('linkModeInitPhase1Params', NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_INIT_PHASE1_PARAMS, 0), ('linkModePostInitNegotiateParams', NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITNEGOTIATE_PARAMS, 0), ('linkModePostInitOptimizeParams', NV2080_CTRL_INTERNAL_NVLINK_SET_DL_LINK_MODE_POST_INITOPTIMIZE_PARAMS, 0)]) +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS.register_fields([('mode', NvU64, 0), ('bSync', NvBool, 8), ('linkMode', NvU32, 12), ('linkModeParams', struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS_linkModeParams, 16)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_TL_LINK_MODE_PARAMS(c.Struct): + SIZE = 4 + mode: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_TL_LINK_MODE_PARAMS.register_fields([('mode', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_TL_LINK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_TL_LINK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TL_LINK_MODE_PARAMS(c.Struct): + SIZE = 16 + mode: int + bSync: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TL_LINK_MODE_PARAMS.register_fields([('mode', NvU64, 0), ('bSync', NvBool, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TL_LINK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TL_LINK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS(c.Struct): + SIZE = 8 + sublinkMode: int + sublinkSubMode: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS.register_fields([('sublinkMode', NvU32, 0), ('sublinkSubMode', NvU32, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TX_SUBLINK_MODE_PARAMS(c.Struct): + SIZE = 16 + mode: int + bSync: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TX_SUBLINK_MODE_PARAMS.register_fields([('mode', NvU64, 0), ('bSync', NvBool, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TX_SUBLINK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TX_SUBLINK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_SUBLINK_MODE_PARAMS(c.Struct): + SIZE = 16 + mode: int + bSync: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_SUBLINK_MODE_PARAMS.register_fields([('mode', NvU64, 0), ('bSync', NvBool, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_SUBLINK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_SUBLINK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_RX_DETECT_PARAMS(c.Struct): + SIZE = 4 + laneRxdetStatusMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_RX_DETECT_PARAMS.register_fields([('laneRxdetStatusMask', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_RX_DETECT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_RX_DETECT_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_DETECT_PARAMS(c.Struct): + SIZE = 1 + bSync: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_DETECT_PARAMS.register_fields([('bSync', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_DETECT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_DETECT_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS(c.Struct): + SIZE = 16 + ipVerDlPl: int + token: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS.register_fields([('ipVerDlPl', NvU32, 0), ('token', NvU64, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_UPHY_LOAD_PARAMS(c.Struct): + SIZE = 1 + bUnlocked: int +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_UPHY_LOAD_PARAMS.register_fields([('bUnlocked', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_UPHY_LOAD_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_UPHY_LOAD_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE(c.Struct): + SIZE = 56 + type: int + callbackParams: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_callbackParams +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_callbackParams(c.Struct): + SIZE = 48 + getDlLinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_DL_LINK_MODE_PARAMS + setDlLinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS + getTlLinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_TL_LINK_MODE_PARAMS + setTlLinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TL_LINK_MODE_PARAMS + getTxSublinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS + setTxSublinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TX_SUBLINK_MODE_PARAMS + getRxSublinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS + setRxSublinkMode: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_SUBLINK_MODE_PARAMS + getRxSublinkDetect: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_RX_DETECT_PARAMS + setRxSublinkDetect: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_DETECT_PARAMS + writeDiscoveryToken: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS + readDiscoveryToken: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS + getUphyLoad: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_UPHY_LOAD_PARAMS +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_callbackParams.register_fields([('getDlLinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_DL_LINK_MODE_PARAMS, 0), ('setDlLinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_DL_LINK_MODE_PARAMS, 0), ('getTlLinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_TL_LINK_MODE_PARAMS, 0), ('setTlLinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TL_LINK_MODE_PARAMS, 0), ('getTxSublinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS, 0), ('setTxSublinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_TX_SUBLINK_MODE_PARAMS, 0), ('getRxSublinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_SUBLINK_MODE_PARAMS, 0), ('setRxSublinkMode', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_SUBLINK_MODE_PARAMS, 0), ('getRxSublinkDetect', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_RX_DETECT_PARAMS, 0), ('setRxSublinkDetect', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_SET_RX_DETECT_PARAMS, 0), ('writeDiscoveryToken', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS, 0), ('readDiscoveryToken', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_RD_WR_DISCOVERY_TOKEN_PARAMS, 0), ('getUphyLoad', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_GET_UPHY_LOAD_PARAMS, 0)]) +struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE.register_fields([('type', NvU8, 0), ('callbackParams', struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_callbackParams, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_CORE_CALLBACK_PARAMS(c.Struct): + SIZE = 64 + linkId: int + callbackType: struct_NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE +struct_NV2080_CTRL_INTERNAL_NVLINK_CORE_CALLBACK_PARAMS.register_fields([('linkId', NvU32, 0), ('callbackType', NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_CORE_CALLBACK_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_CORE_CALLBACK_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_REMOTE_LOCAL_SID_PARAMS(c.Struct): + SIZE = 32 + linkId: int + remoteLocalSidInfo: struct_NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO +struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_REMOTE_LOCAL_SID_PARAMS.register_fields([('linkId', NvU32, 0), ('remoteLocalSidInfo', NV2080_CTRL_INTERNAL_NVLINK_REMOTE_LOCAL_SID_INFO, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_REMOTE_LOCAL_SID_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_REMOTE_LOCAL_SID_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_ALI_ENABLED_PARAMS(c.Struct): + SIZE = 1 + bEnableAli: int +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_ALI_ENABLED_PARAMS.register_fields([('bEnableAli', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_ALI_ENABLED_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_ALI_ENABLED_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_HSHUB_MUX_PARAMS(c.Struct): + SIZE = 8 + updateType: int + bSysMem: int + peerMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_HSHUB_MUX_PARAMS.register_fields([('updateType', NvBool, 0), ('bSysMem', NvBool, 1), ('peerMask', NvU32, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_HSHUB_MUX_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_HSHUB_MUX_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_PRE_SETUP_NVLINK_PEER_PARAMS(c.Struct): + SIZE = 24 + peerId: int + peerLinkMask: int + bEgmPeer: int + bNvswitchConn: int +struct_NV2080_CTRL_INTERNAL_NVLINK_PRE_SETUP_NVLINK_PEER_PARAMS.register_fields([('peerId', NvU32, 0), ('peerLinkMask', NvU64, 8), ('bEgmPeer', NvBool, 16), ('bNvswitchConn', NvBool, 17)]) +NV2080_CTRL_INTERNAL_NVLINK_PRE_SETUP_NVLINK_PEER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_PRE_SETUP_NVLINK_PEER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_POST_SETUP_NVLINK_PEER_PARAMS(c.Struct): + SIZE = 4 + peerMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_POST_SETUP_NVLINK_PEER_PARAMS.register_fields([('peerMask', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_POST_SETUP_NVLINK_PEER_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_POST_SETUP_NVLINK_PEER_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_LOG_OOB_XID_PARAMS(c.Struct): + SIZE = 92 + xid: int + message: c.Array[ctypes.c_ubyte, Literal[81]] + len: int +struct_NV2080_CTRL_INTERNAL_LOG_OOB_XID_PARAMS.register_fields([('xid', NvU32, 0), ('message', c.Array[NvU8, Literal[81]], 4), ('len', NvU32, 88)]) +NV2080_CTRL_INTERNAL_LOG_OOB_XID_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_LOG_OOB_XID_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING_PARAMS(c.Struct): + SIZE = 12 + mapTypeMask: int + peerMask: int + bL2Entry: int +struct_NV2080_CTRL_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING_PARAMS.register_fields([('mapTypeMask', NvU32, 0), ('peerMask', NvU32, 4), ('bL2Entry', NvBool, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SAVE_RESTORE_HSHUB_STATE_PARAMS(c.Struct): + SIZE = 24 + bSave: int + linkMask: struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_INTERNAL_NVLINK_SAVE_RESTORE_HSHUB_STATE_PARAMS.register_fields([('bSave', NvBool, 0), ('linkMask', NV2080_CTRL_NVLINK_LINK_MASK, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_SAVE_RESTORE_HSHUB_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SAVE_RESTORE_HSHUB_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_PARAMS(c.Struct): + SIZE = 16 + flags: int + bSysmem: int + peerLinkMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_PARAMS.register_fields([('flags', NvU32, 0), ('bSysmem', NvBool, 4), ('peerLinkMask', NvU64, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_CURRENT_CONFIG_PARAMS(c.Struct): + SIZE = 1 + bNvlinkSysmemEnabled: int +struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_CURRENT_CONFIG_PARAMS.register_fields([('bNvlinkSysmemEnabled', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_CURRENT_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_CURRENT_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_PEER_LINK_MASK_PARAMS(c.Struct): + SIZE = 16 + gpuInst: int + peerLinkMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_PEER_LINK_MASK_PARAMS.register_fields([('gpuInst', NvU32, 0), ('peerLinkMask', NvU64, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_PEER_LINK_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_PEER_LINK_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_LINK_CONNECTION_PARAMS(c.Struct): + SIZE = 40 + remoteDeviceType: int + remoteChipSid: int + linkId: int + laneRxdetStatusMask: int + remoteLinkNumber: int + remotePciDeviceId: int + remoteDomain: int + remoteBus: int + remoteDevice: int + remoteFunction: int + bConnected: int +struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_LINK_CONNECTION_PARAMS.register_fields([('remoteDeviceType', NvU64, 0), ('remoteChipSid', NvU64, 8), ('linkId', NvU32, 16), ('laneRxdetStatusMask', NvU32, 20), ('remoteLinkNumber', NvU32, 24), ('remotePciDeviceId', NvU32, 28), ('remoteDomain', NvU32, 32), ('remoteBus', NvU8, 36), ('remoteDevice', NvU8, 37), ('remoteFunction', NvU8, 38), ('bConnected', NvBool, 39)]) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_LINK_CONNECTION_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_UPDATE_LINK_CONNECTION_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_LINKS_POST_TOPOLOGY_PARAMS(c.Struct): + SIZE = 8 + linkMask: int + initializedLinks: int +struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_LINKS_POST_TOPOLOGY_PARAMS.register_fields([('linkMask', NvU32, 0), ('initializedLinks', NvU32, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_ENABLE_LINKS_POST_TOPOLOGY_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_LINKS_POST_TOPOLOGY_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_PRE_LINK_TRAIN_ALI_PARAMS(c.Struct): + SIZE = 8 + linkMask: int + bSync: int +struct_NV2080_CTRL_INTERNAL_NVLINK_PRE_LINK_TRAIN_ALI_PARAMS.register_fields([('linkMask', NvU32, 0), ('bSync', NvBool, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_PRE_LINK_TRAIN_ALI_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_PRE_LINK_TRAIN_ALI_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_MASK_POST_RX_DET_PARAMS(c.Struct): + SIZE = 264 + postRxDetLinkMask: int + laneRxdetStatusMask: c.Array[ctypes.c_uint32, Literal[64]] +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_MASK_POST_RX_DET_PARAMS.register_fields([('postRxDetLinkMask', NvU64, 0), ('laneRxdetStatusMask', c.Array[NvU32, Literal[64]], 8)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_MASK_POST_RX_DET_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_MASK_POST_RX_DET_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_LINK_TRAIN_ALI_PARAMS(c.Struct): + SIZE = 32 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + bSync: int +struct_NV2080_CTRL_INTERNAL_NVLINK_LINK_TRAIN_ALI_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('bSync', NvBool, 24)]) +NV2080_CTRL_INTERNAL_NVLINK_LINK_TRAIN_ALI_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_LINK_TRAIN_ALI_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_LINK_VALUES(c.Struct): + SIZE = 16 + bValid: int + linkId: int + ioctrlId: int + pllMasterLinkId: int + pllSlaveLinkId: int + ipVerDlPl: int +struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_LINK_VALUES.register_fields([('bValid', NvBool, 0), ('linkId', NvU8, 1), ('ioctrlId', NvU32, 4), ('pllMasterLinkId', NvU8, 8), ('pllSlaveLinkId', NvU8, 9), ('ipVerDlPl', NvU32, 12)]) +NV2080_CTRL_INTERNAL_NVLINK_DEVICE_LINK_VALUES: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_LINK_VALUES +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_NVLINK_DEVICE_INFO_PARAMS(c.Struct): + SIZE = 1072 + ioctrlMask: int + ioctrlNumEntries: int + ioctrlSize: int + discoveredLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK + ipVerNvlink: int + maxSupportedLinks: int + probeRequestTimeMs: int + linkStateChangeTimeMs: int + linkInfo: c.Array[struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_LINK_VALUES, Literal[64]] +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_NVLINK_DEVICE_INFO_PARAMS.register_fields([('ioctrlMask', NvU32, 0), ('ioctrlNumEntries', NvU8, 4), ('ioctrlSize', NvU32, 8), ('discoveredLinks', NV2080_CTRL_NVLINK_LINK_MASK, 16), ('ipVerNvlink', NvU32, 32), ('maxSupportedLinks', NvU32, 36), ('probeRequestTimeMs', NvU32, 40), ('linkStateChangeTimeMs', NvU32, 44), ('linkInfo', c.Array[NV2080_CTRL_INTERNAL_NVLINK_DEVICE_LINK_VALUES, Literal[64]], 48)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_NVLINK_DEVICE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_NVLINK_DEVICE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_IP_REVISION_VALUES(c.Struct): + SIZE = 8 + ipVerIoctrl: int + ipVerMinion: int +struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_IP_REVISION_VALUES.register_fields([('ipVerIoctrl', NvU32, 0), ('ipVerMinion', NvU32, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_DEVICE_IP_REVISION_VALUES: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_IP_REVISION_VALUES +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_IOCTRL_DEVICE_INFO_PARAMS(c.Struct): + SIZE = 32 + ioctrlIdx: int + PublicId: int + localDiscoveredLinks: int + localGlobalLinkOffset: int + ioctrlDiscoverySize: int + numDevices: int + ipRevisions: struct_NV2080_CTRL_INTERNAL_NVLINK_DEVICE_IP_REVISION_VALUES +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_IOCTRL_DEVICE_INFO_PARAMS.register_fields([('ioctrlIdx', NvU32, 0), ('PublicId', NvU32, 4), ('localDiscoveredLinks', NvU32, 8), ('localGlobalLinkOffset', NvU32, 12), ('ioctrlDiscoverySize', NvU32, 16), ('numDevices', NvU8, 20), ('ipRevisions', NV2080_CTRL_INTERNAL_NVLINK_DEVICE_IP_REVISION_VALUES, 24)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_IOCTRL_DEVICE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_IOCTRL_DEVICE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_LINK_SPEED_PARAMS(c.Struct): + SIZE = 12 + bPlatformLinerateDefined: int + platformLineRate: int + nvlinkLinkSpeed: int +struct_NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_LINK_SPEED_PARAMS.register_fields([('bPlatformLinerateDefined', NvBool, 0), ('platformLineRate', NvU32, 4), ('nvlinkLinkSpeed', NvU32, 8)]) +NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_LINK_SPEED_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_LINK_SPEED_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_ARE_LINKS_TRAINED_PARAMS(c.Struct): + SIZE = 88 + linkMask: struct_NV2080_CTRL_NVLINK_LINK_MASK + bActiveOnly: int + bIsLinkActive: c.Array[ctypes.c_ubyte, Literal[64]] +struct_NV2080_CTRL_INTERNAL_NVLINK_ARE_LINKS_TRAINED_PARAMS.register_fields([('linkMask', NV2080_CTRL_NVLINK_LINK_MASK, 0), ('bActiveOnly', NvBool, 16), ('bIsLinkActive', c.Array[NvBool, Literal[64]], 17)]) +NV2080_CTRL_INTERNAL_NVLINK_ARE_LINKS_TRAINED_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_ARE_LINKS_TRAINED_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_RESET_LINKS_PARAMS(c.Struct): + SIZE = 8 + linkMask: int + flags: int +struct_NV2080_CTRL_INTERNAL_NVLINK_RESET_LINKS_PARAMS.register_fields([('linkMask', NvU32, 0), ('flags', NvU32, 4)]) +NV2080_CTRL_INTERNAL_NVLINK_RESET_LINKS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_RESET_LINKS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_DISABLE_DL_INTERRUPTS_PARAMS(c.Struct): + SIZE = 4 + linkMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_DISABLE_DL_INTERRUPTS_PARAMS.register_fields([('linkMask', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_DISABLE_DL_INTERRUPTS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_DISABLE_DL_INTERRUPTS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_VALUES(c.Struct): + SIZE = 60 + bLinkConnectedToSystem: int + bLinkConnectedToPeer: int + bLinkReset: int + subLinkWidth: int + linkState: int + txSublinkState: int + rxSublinkState: int + bLaneReversal: int + nvlinkLinkClockKHz: int + nvlinkLineRateMbps: int + nvlinkLinkClockMhz: int + nvlinkLinkDataRateKiBps: int + nvlinkRefClkType: int + nvlinkReqLinkClockMhz: int + nvlinkMinL1Threshold: int + nvlinkMaxL1Threshold: int + nvlinkL1ThresholdUnits: int + nvlpwIdx: int +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_VALUES.register_fields([('bLinkConnectedToSystem', NvBool, 0), ('bLinkConnectedToPeer', NvBool, 1), ('bLinkReset', NvBool, 2), ('subLinkWidth', NvU8, 3), ('linkState', NvU32, 4), ('txSublinkState', NvU32, 8), ('rxSublinkState', NvU32, 12), ('bLaneReversal', NvBool, 16), ('nvlinkLinkClockKHz', NvU32, 20), ('nvlinkLineRateMbps', NvU32, 24), ('nvlinkLinkClockMhz', NvU32, 28), ('nvlinkLinkDataRateKiBps', NvU32, 32), ('nvlinkRefClkType', NvU8, 36), ('nvlinkReqLinkClockMhz', NvU32, 40), ('nvlinkMinL1Threshold', NvU32, 44), ('nvlinkMaxL1Threshold', NvU32, 48), ('nvlinkL1ThresholdUnits', NvU32, 52), ('nvlpwIdx', NvU32, 56)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_VALUES: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_VALUES +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_INFO_PARAMS(c.Struct): + SIZE = 3880 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + enabledNvlpwMask: int + nvlinkRefClkSpeedKHz: int + bSublinkStateInst: int + linkInfo: c.Array[struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_VALUES, Literal[64]] +struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_INFO_PARAMS.register_fields([('linkMask', NvU64, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('enabledNvlpwMask', NvU64, 24), ('nvlinkRefClkSpeedKHz', NvU32, 32), ('bSublinkStateInst', NvBool, 36), ('linkInfo', c.Array[NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_VALUES, Literal[64]], 40)]) +NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SETUP_NVLINK_SYSMEM_PARAMS(c.Struct): + SIZE = 4 + sysmemLinkMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_SETUP_NVLINK_SYSMEM_PARAMS.register_fields([('sysmemLinkMask', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_SETUP_NVLINK_SYSMEM_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SETUP_NVLINK_SYSMEM_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_PROCESS_FORCED_CONFIGS_PARAMS(c.Struct): + SIZE = 264 + bLegacyForcedConfig: int + bOverrideComputePeerMode: int + phase: int + linkConnection: c.Array[ctypes.c_uint32, Literal[64]] +struct_NV2080_CTRL_INTERNAL_NVLINK_PROCESS_FORCED_CONFIGS_PARAMS.register_fields([('bLegacyForcedConfig', NvBool, 0), ('bOverrideComputePeerMode', NvBool, 1), ('phase', NvU32, 4), ('linkConnection', c.Array[NvU32, Literal[64]], 8)]) +NV2080_CTRL_INTERNAL_NVLINK_PROCESS_FORCED_CONFIGS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_PROCESS_FORCED_CONFIGS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_FORCE_DEBUG_STALL_AND_UPDATE_AMAP_PARAMS(c.Struct): + SIZE = 16 + peerLinkMask: struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_INTERNAL_NVLINK_FORCE_DEBUG_STALL_AND_UPDATE_AMAP_PARAMS.register_fields([('peerLinkMask', NV2080_CTRL_NVLINK_LINK_MASK, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_FORCE_DEBUG_STALL_AND_UPDATE_AMAP_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_FORCE_DEBUG_STALL_AND_UPDATE_AMAP_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS_PARAMS(c.Struct): + SIZE = 1 + bLaneShutdownOnUnload: int +struct_NV2080_CTRL_INTERNAL_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS_PARAMS.register_fields([('bLaneShutdownOnUnload', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_SYSMEM_NVLINK_ATS_PARAMS(c.Struct): + SIZE = 4 + notUsed: int +struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_SYSMEM_NVLINK_ATS_PARAMS.register_fields([('notUsed', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_ENABLE_SYSMEM_NVLINK_ATS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_ENABLE_SYSMEM_NVLINK_ATS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK_PARAMS(c.Struct): + SIZE = 4 + sysmemLinkMask: int +struct_NV2080_CTRL_INTERNAL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK_PARAMS.register_fields([('sysmemLinkMask', NvU32, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_SEND_CMC_LIBOS_BUFFER_INFO_PARAMS(c.Struct): + SIZE = 24 + PublicId: int + logBufferSize: int + logBufferAddr: int +struct_NV2080_CTRL_INTERNAL_SEND_CMC_LIBOS_BUFFER_INFO_PARAMS.register_fields([('PublicId', NvU32, 0), ('logBufferSize', NvU64, 8), ('logBufferAddr', NvU64, 16)]) +NV2080_CTRL_INTERNAL_SEND_CMC_LIBOS_BUFFER_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_SEND_CMC_LIBOS_BUFFER_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GPU_GET_HFRP_INFO_PARAMS(c.Struct): + SIZE = 40 + hfrpPrivBase: c.Array[ctypes.c_uint32, Literal[5]] + hfrpIntrCtrlReg: c.Array[ctypes.c_uint32, Literal[5]] +struct_NV2080_CTRL_INTERNAL_GPU_GET_HFRP_INFO_PARAMS.register_fields([('hfrpPrivBase', c.Array[NvU32, Literal[5]], 0), ('hfrpIntrCtrlReg', c.Array[NvU32, Literal[5]], 20)]) +NV2080_CTRL_INTERNAL_GPU_GET_HFRP_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GPU_GET_HFRP_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_SEND_CMC_UMD_API_OP_PARAMS(c.Struct): + SIZE = 72 + opType: int + flags: int + ringBufferSize: int + ringBufferOffset: int + userdPa: int + userdVa: int + usermodeGpuVa: int + instBlkAddr: int + instBlkAperture: int + userdAperture: int + inputWorkSubmitToken: int + cmcWorkSubmitToken: int + cmcHandle: int +struct_NV2080_CTRL_INTERNAL_SEND_CMC_UMD_API_OP_PARAMS.register_fields([('opType', NvU32, 0), ('flags', NvU32, 4), ('ringBufferSize', NvU32, 8), ('ringBufferOffset', NvU64, 16), ('userdPa', NvU64, 24), ('userdVa', NvU64, 32), ('usermodeGpuVa', NvU64, 40), ('instBlkAddr', NvU64, 48), ('instBlkAperture', NvU8, 56), ('userdAperture', NvU8, 57), ('inputWorkSubmitToken', NvU32, 60), ('cmcWorkSubmitToken', NvU32, 64), ('cmcHandle', NvU32, 68)]) +NV2080_CTRL_INTERNAL_SEND_CMC_UMD_API_OP_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_SEND_CMC_UMD_API_OP_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER_V2(c.Struct): + SIZE = 2044 + smIssueRateModifierListSize: int + smIssueRateModifierList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[255]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER_V2.register_fields([('smIssueRateModifierListSize', NvU32, 0), ('smIssueRateModifierList', c.Array[NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2, Literal[255]], 4)]) +NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER_V2: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER_V2 +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS(c.Struct): + SIZE = 16352 + smIssueRateModifierV2: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER_V2, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS.register_fields([('smIssueRateModifierV2', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_RATE_MODIFIER_V2, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_THROTTLE_CTRL(c.Struct): + SIZE = 2044 + smIssueThrottleCtrlListSize: int + smIssueThrottleCtrlList: c.Array[struct_NVXXXX_CTRL_XXX_INFO, Literal[255]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_THROTTLE_CTRL.register_fields([('smIssueThrottleCtrlListSize', NvU32, 0), ('smIssueThrottleCtrlList', c.Array[NV2080_CTRL_GR_SM_ISSUE_THROTTLE_CTRL, Literal[255]], 4)]) +NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_THROTTLE_CTRL: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_THROTTLE_CTRL +@c.record +class struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS(c.Struct): + SIZE = 16352 + smIssueThrottleCtrl: c.Array[struct_NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_THROTTLE_CTRL, Literal[8]] +struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS.register_fields([('smIssueThrottleCtrl', c.Array[NV2080_CTRL_INTERNAL_STATIC_GR_SM_ISSUE_THROTTLE_CTRL, Literal[8]], 0)]) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_FLCN_SET_VIDEO_EVENT_BUFFER_MEMORY_PARAMS(c.Struct): + SIZE = 40 + memDescInfo: struct_NV2080_CTRL_INTERNAL_MEMDESC_INFO + engDesc: int + bEngineFound: int +struct_NV2080_CTRL_INTERNAL_FLCN_SET_VIDEO_EVENT_BUFFER_MEMORY_PARAMS.register_fields([('memDescInfo', NV2080_CTRL_INTERNAL_MEMDESC_INFO, 0), ('engDesc', NvU32, 32), ('bEngineFound', NvBool, 36)]) +NV2080_CTRL_INTERNAL_FLCN_SET_VIDEO_EVENT_BUFFER_MEMORY_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_FLCN_SET_VIDEO_EVENT_BUFFER_MEMORY_PARAMS +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_ROTATE_PER_CHANNEL_KEYS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_FIFO_ROTATE_KEYS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_NVLINK_RESUME_DISABLE_CHANNELS_PARAMS(c.Struct): + SIZE = 1 + bEnable: int +struct_NV2080_CTRL_INTERNAL_NVLINK_RESUME_DISABLE_CHANNELS_PARAMS.register_fields([('bEnable', NvBool, 0)]) +NV2080_CTRL_INTERNAL_NVLINK_RESUME_DISABLE_CHANNELS_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_NVLINK_RESUME_DISABLE_CHANNELS_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS(c.Struct): + SIZE = 80024 + params: struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS +@c.record +class struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS(c.Struct): + SIZE = 80024 + gpuId: int + hTotal: int + vTotal: int + minRefreshRatex10000: int + maxRefreshRatex10000: int + numFractions: int + fractions: c.Array[struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS_fraction, Literal[5000]] +NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS: TypeAlias = struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS +@c.record +class struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS_fraction(c.Struct): + SIZE = 16 + numerator: int + denominator: int +struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS_fraction.register_fields([('numerator', NvU64, 0), ('denominator', NvU64, 8)]) +struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS.register_fields([('gpuId', NvU32, 0), ('hTotal', NvU32, 4), ('vTotal', NvU32, 8), ('minRefreshRatex10000', NvU32, 12), ('maxRefreshRatex10000', NvU32, 16), ('numFractions', NvU32, 20), ('fractions', c.Array[struct_NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS_fraction, Literal[5000]], 24)]) +struct_NV2080_CTRL_INTERNAL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS.register_fields([('params', NV30F1_CTRL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS, 0)]) +NV2080_CTRL_INTERNAL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_STATE_PARAMS(c.Struct): + SIZE = 16 + ucode: int + gfid: int + bEnabled: int + bClear: int + instrumentationType: int +struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_STATE_PARAMS.register_fields([('ucode', NvU32, 0), ('gfid', NvU32, 4), ('bEnabled', NvBool, 8), ('bClear', NvBool, 9), ('instrumentationType', NvU32, 12)]) +NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_SET_STATE_PARAMS(c.Struct): + SIZE = 16 + ucode: int + gfid: int + bEnabled: int + bClear: int + instrumentationType: int +struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_SET_STATE_PARAMS.register_fields([('ucode', NvU32, 0), ('gfid', NvU32, 4), ('bEnabled', NvBool, 8), ('bClear', NvBool, 9), ('instrumentationType', NvU32, 12)]) +NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_SET_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_SET_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_DATA_PARAMS(c.Struct): + SIZE = 65560 + ucode: int + gfid: int + offset: int + instrumentationType: int + data: c.Array[ctypes.c_ubyte, Literal[65536]] + dataSize: int + bComplete: int +struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_DATA_PARAMS.register_fields([('ucode', NvU32, 0), ('gfid', NvU32, 4), ('offset', NvU32, 8), ('instrumentationType', NvU32, 12), ('data', c.Array[NvU8, Literal[65536]], 16), ('dataSize', NvU32, 65552), ('bComplete', NvBool, 65556)]) +NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_PERF_CLIENT_PERF_MODES_SET_CONTROL_PARAMS(c.Struct): + SIZE = 32 + activeModes: struct_NV2080_CTRL_BOARDOBJGRP_MASK_E255 +struct_NV2080_CTRL_INTERNAL_PERF_CLIENT_PERF_MODES_SET_CONTROL_PARAMS.register_fields([('activeModes', NV2080_CTRL_BOARDOBJGRP_MASK_E255, 0)]) +NV2080_CTRL_INTERNAL_PERF_CLIENT_PERF_MODES_SET_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_PERF_CLIENT_PERF_MODES_SET_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_LPWR_DIFR_CTRL_PARAMS(c.Struct): + SIZE = 4 + ctrlParamVal: int +struct_NV2080_CTRL_CMD_LPWR_DIFR_CTRL_PARAMS.register_fields([('ctrlParamVal', NvU32, 0)]) +NV2080_CTRL_CMD_LPWR_DIFR_CTRL_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_LPWR_DIFR_CTRL_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_LPWR_DIFR_PREFETCH_RESPONSE_PARAMS(c.Struct): + SIZE = 4 + responseVal: int +struct_NV2080_CTRL_CMD_LPWR_DIFR_PREFETCH_RESPONSE_PARAMS.register_fields([('responseVal', NvU32, 0)]) +NV2080_CTRL_CMD_LPWR_DIFR_PREFETCH_RESPONSE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_LPWR_DIFR_PREFETCH_RESPONSE_PARAMS +@c.record +class struct_NV2080_CTRL_MC_GET_ARCH_INFO_PARAMS(c.Struct): + SIZE = 16 + architecture: int + implementation: int + revision: int + subRevision: int +struct_NV2080_CTRL_MC_GET_ARCH_INFO_PARAMS.register_fields([('architecture', NvU32, 0), ('implementation', NvU32, 4), ('revision', NvU32, 8), ('subRevision', NvU8, 12)]) +NV2080_CTRL_MC_GET_ARCH_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_MC_GET_ARCH_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_MC_SERVICE_INTERRUPTS_PARAMS(c.Struct): + SIZE = 4 + engines: int +struct_NV2080_CTRL_MC_SERVICE_INTERRUPTS_PARAMS.register_fields([('engines', NvU32, 0)]) +NV2080_CTRL_MC_SERVICE_INTERRUPTS_PARAMS: TypeAlias = struct_NV2080_CTRL_MC_SERVICE_INTERRUPTS_PARAMS +@c.record +class struct_NV2080_CTRL_MC_GET_MANUFACTURER_PARAMS(c.Struct): + SIZE = 4 + manufacturer: int +struct_NV2080_CTRL_MC_GET_MANUFACTURER_PARAMS.register_fields([('manufacturer', NvU32, 0)]) +NV2080_CTRL_MC_GET_MANUFACTURER_PARAMS: TypeAlias = struct_NV2080_CTRL_MC_GET_MANUFACTURER_PARAMS +@c.record +class struct_NV2080_CTRL_MC_CHANGE_REPLAYABLE_FAULT_OWNERSHIP_PARAMS(c.Struct): + SIZE = 1 + bOwnedByRm: int +struct_NV2080_CTRL_MC_CHANGE_REPLAYABLE_FAULT_OWNERSHIP_PARAMS.register_fields([('bOwnedByRm', NvBool, 0)]) +NV2080_CTRL_MC_CHANGE_REPLAYABLE_FAULT_OWNERSHIP_PARAMS: TypeAlias = struct_NV2080_CTRL_MC_CHANGE_REPLAYABLE_FAULT_OWNERSHIP_PARAMS +@c.record +class struct_NV2080_CTRL_MC_ENGINE_NOTIFICATION_INTR_VECTOR_ENTRY(c.Struct): + SIZE = 8 + nv2080EngineType: int + notificationIntrVector: int +struct_NV2080_CTRL_MC_ENGINE_NOTIFICATION_INTR_VECTOR_ENTRY.register_fields([('nv2080EngineType', NvU32, 0), ('notificationIntrVector', NvU32, 4)]) +NV2080_CTRL_MC_ENGINE_NOTIFICATION_INTR_VECTOR_ENTRY: TypeAlias = struct_NV2080_CTRL_MC_ENGINE_NOTIFICATION_INTR_VECTOR_ENTRY +@c.record +class struct_NV2080_CTRL_MC_GET_ENGINE_NOTIFICATION_INTR_VECTORS_PARAMS(c.Struct): + SIZE = 2052 + numEntries: int + entries: c.Array[struct_NV2080_CTRL_MC_ENGINE_NOTIFICATION_INTR_VECTOR_ENTRY, Literal[256]] +struct_NV2080_CTRL_MC_GET_ENGINE_NOTIFICATION_INTR_VECTORS_PARAMS.register_fields([('numEntries', NvU32, 0), ('entries', c.Array[NV2080_CTRL_MC_ENGINE_NOTIFICATION_INTR_VECTOR_ENTRY, Literal[256]], 4)]) +NV2080_CTRL_MC_GET_ENGINE_NOTIFICATION_INTR_VECTORS_PARAMS: TypeAlias = struct_NV2080_CTRL_MC_GET_ENGINE_NOTIFICATION_INTR_VECTORS_PARAMS +@c.record +class struct_NV2080_CTRL_MC_STATIC_INTR_ENTRY(c.Struct): + SIZE = 16 + nv2080IntrType: int + pmcIntrMask: int + intrVectorStall: int + intrVectorNonStall: int +struct_NV2080_CTRL_MC_STATIC_INTR_ENTRY.register_fields([('nv2080IntrType', NvU32, 0), ('pmcIntrMask', NvU32, 4), ('intrVectorStall', NvU32, 8), ('intrVectorNonStall', NvU32, 12)]) +NV2080_CTRL_MC_STATIC_INTR_ENTRY: TypeAlias = struct_NV2080_CTRL_MC_STATIC_INTR_ENTRY +@c.record +class struct_NV2080_CTRL_MC_GET_STATIC_INTR_TABLE_PARAMS(c.Struct): + SIZE = 516 + numEntries: int + entries: c.Array[struct_NV2080_CTRL_MC_STATIC_INTR_ENTRY, Literal[32]] +struct_NV2080_CTRL_MC_GET_STATIC_INTR_TABLE_PARAMS.register_fields([('numEntries', NvU32, 0), ('entries', c.Array[NV2080_CTRL_MC_STATIC_INTR_ENTRY, Literal[32]], 4)]) +NV2080_CTRL_MC_GET_STATIC_INTR_TABLE_PARAMS: TypeAlias = struct_NV2080_CTRL_MC_GET_STATIC_INTR_TABLE_PARAMS +enum_NV2080_INTR_CATEGORY: dict[int, str] = {(NV2080_INTR_CATEGORY_DEFAULT:=0): 'NV2080_INTR_CATEGORY_DEFAULT', (NV2080_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE:=1): 'NV2080_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE', (NV2080_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION:=2): 'NV2080_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION', (NV2080_INTR_CATEGORY_RUNLIST:=3): 'NV2080_INTR_CATEGORY_RUNLIST', (NV2080_INTR_CATEGORY_RUNLIST_NOTIFICATION:=4): 'NV2080_INTR_CATEGORY_RUNLIST_NOTIFICATION', (NV2080_INTR_CATEGORY_UVM_OWNED:=5): 'NV2080_INTR_CATEGORY_UVM_OWNED', (NV2080_INTR_CATEGORY_UVM_SHARED:=6): 'NV2080_INTR_CATEGORY_UVM_SHARED', (NV2080_INTR_CATEGORY_ENUM_COUNT:=7): 'NV2080_INTR_CATEGORY_ENUM_COUNT'} +NV2080_INTR_CATEGORY: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_MC_GET_INTR_CATEGORY_SUBTREE_MAP_PARAMS(c.Struct): + SIZE = 56 + subtreeMap: c.Array[struct_NV2080_INTR_CATEGORY_SUBTREE_MAP, Literal[7]] +struct_NV2080_CTRL_MC_GET_INTR_CATEGORY_SUBTREE_MAP_PARAMS.register_fields([('subtreeMap', c.Array[NV2080_INTR_CATEGORY_SUBTREE_MAP, Literal[7]], 0)]) +NV2080_CTRL_MC_GET_INTR_CATEGORY_SUBTREE_MAP_PARAMS: TypeAlias = struct_NV2080_CTRL_MC_GET_INTR_CATEGORY_SUBTREE_MAP_PARAMS +@c.record +class struct_NV2080_CTRL_NVD_GET_DUMP_SIZE_PARAMS(c.Struct): + SIZE = 8 + component: int + size: int +struct_NV2080_CTRL_NVD_GET_DUMP_SIZE_PARAMS.register_fields([('component', NvU32, 0), ('size', NvU32, 4)]) +NV2080_CTRL_NVD_GET_DUMP_SIZE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVD_GET_DUMP_SIZE_PARAMS +@c.record +class struct_NV2080_CTRL_NVD_GET_DUMP_PARAMS(c.Struct): + SIZE = 16 + pBuffer: ctypes.c_void_p + component: int + size: int +struct_NV2080_CTRL_NVD_GET_DUMP_PARAMS.register_fields([('pBuffer', NvP64, 0), ('component', NvU32, 8), ('size', NvU32, 12)]) +NV2080_CTRL_NVD_GET_DUMP_PARAMS: TypeAlias = struct_NV2080_CTRL_NVD_GET_DUMP_PARAMS +@c.record +class struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS(c.Struct): + SIZE = 32 + userMinOffset: int + userMaxOffset: int + factoryMinOffset: int + factoryMaxOffset: int + lastActiveClock: int + lastActiveVolt: int + lastActivePoint: int + kappa: int +struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS.register_fields([('userMinOffset', NvS32, 0), ('userMaxOffset', NvS32, 4), ('factoryMinOffset', NvU32, 8), ('factoryMaxOffset', NvU32, 12), ('lastActiveClock', NvU32, 16), ('lastActiveVolt', NvU32, 20), ('lastActivePoint', NvU32, 24), ('kappa', NvU32, 28)]) +NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS: TypeAlias = struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS +@c.record +class struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_CFG(c.Struct): + SIZE = 72 + pstateVer: int + gpcOverclock: struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS + mclkOverclock: struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS + bUserOverclocked: int + bFactoryOverclocked: int +struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_CFG.register_fields([('pstateVer', NvU32, 0), ('gpcOverclock', NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS, 4), ('mclkOverclock', NV2080_NOCAT_JOURNAL_OVERCLOCK_DETAILS, 36), ('bUserOverclocked', NvBool, 68), ('bFactoryOverclocked', NvBool, 69)]) +NV2080_NOCAT_JOURNAL_OVERCLOCK_CFG: TypeAlias = struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_CFG +@c.record +class struct_NV2080_NOCAT_JOURNAL_GPU_STATE(c.Struct): + SIZE = 240 + bValid: int + strap: int + deviceId: int + vendorId: int + subsystemVendor: int + subsystemId: int + revision: int + type: int + vbiosVersion: int + bOptimus: int + bMsHybrid: int + bFullPower: int + vbiosOemVersion: int + memoryType: int + tag: c.Array[ctypes.c_ubyte, Literal[65]] + vbiosProject: c.Array[ctypes.c_ubyte, Literal[65]] + bInFullchipReset: int + bInSecBusReset: int + bInGc6Reset: int + overclockCfg: struct_NV2080_NOCAT_JOURNAL_OVERCLOCK_CFG +struct_NV2080_NOCAT_JOURNAL_GPU_STATE.register_fields([('bValid', NvBool, 0), ('strap', NvU32, 4), ('deviceId', NvU16, 8), ('vendorId', NvU16, 10), ('subsystemVendor', NvU16, 12), ('subsystemId', NvU16, 14), ('revision', NvU16, 16), ('type', NvU16, 18), ('vbiosVersion', NvU32, 20), ('bOptimus', NvBool, 24), ('bMsHybrid', NvBool, 25), ('bFullPower', NvBool, 26), ('vbiosOemVersion', NvU32, 28), ('memoryType', NvU16, 32), ('tag', c.Array[NvU8, Literal[65]], 34), ('vbiosProject', c.Array[NvU8, Literal[65]], 99), ('bInFullchipReset', NvBool, 164), ('bInSecBusReset', NvBool, 165), ('bInGc6Reset', NvBool, 166), ('overclockCfg', NV2080_NOCAT_JOURNAL_OVERCLOCK_CFG, 168)]) +NV2080_NOCAT_JOURNAL_GPU_STATE: TypeAlias = struct_NV2080_NOCAT_JOURNAL_GPU_STATE +@c.record +class struct_NV2080_NOCAT_JOURNAL_ENTRY(c.Struct): + SIZE = 1272 + recType: int + bugcheck: int + tdrBucketId: int + source: c.Array[ctypes.c_ubyte, Literal[65]] + subsystem: int + errorCode: int + diagBufferLen: int + diagBuffer: c.Array[ctypes.c_ubyte, Literal[1024]] + faultingEngine: c.Array[ctypes.c_ubyte, Literal[65]] + mmuFaultType: int + mmuErrorSrc: int + tdrReason: c.Array[ctypes.c_ubyte, Literal[65]] +struct_NV2080_NOCAT_JOURNAL_ENTRY.register_fields([('recType', NvU8, 0), ('bugcheck', NvU32, 4), ('tdrBucketId', NvU32, 8), ('source', c.Array[NvU8, Literal[65]], 12), ('subsystem', NvU32, 80), ('errorCode', NvU64, 88), ('diagBufferLen', NvU32, 96), ('diagBuffer', c.Array[NvU8, Literal[1024]], 100), ('faultingEngine', c.Array[NvU8, Literal[65]], 1124), ('mmuFaultType', NvU32, 1192), ('mmuErrorSrc', NvU32, 1196), ('tdrReason', c.Array[NvU8, Literal[65]], 1200)]) +NV2080_NOCAT_JOURNAL_ENTRY: TypeAlias = struct_NV2080_NOCAT_JOURNAL_ENTRY +@c.record +class struct_NV2080_NOCAT_JOURNAL_RECORD(c.Struct): + SIZE = 1544 + GPUTag: int + loadAddress: int + timeStamp: int + stateMask: int + nocatGpuState: struct_NV2080_NOCAT_JOURNAL_GPU_STATE + nocatJournalEntry: struct_NV2080_NOCAT_JOURNAL_ENTRY +struct_NV2080_NOCAT_JOURNAL_RECORD.register_fields([('GPUTag', NvU32, 0), ('loadAddress', NvU64, 8), ('timeStamp', NvU64, 16), ('stateMask', NvU64, 24), ('nocatGpuState', NV2080_NOCAT_JOURNAL_GPU_STATE, 32), ('nocatJournalEntry', NV2080_NOCAT_JOURNAL_ENTRY, 272)]) +NV2080_NOCAT_JOURNAL_RECORD: TypeAlias = struct_NV2080_NOCAT_JOURNAL_RECORD +@c.record +class struct_NV2080_CTRL_NVD_GET_NOCAT_JOURNAL_PARAMS(c.Struct): + SIZE = 15656 + flags: int + nocatRecordCount: int + nocatOutstandingRecordCount: int + journalRecords: c.Array[struct_NV2080_NOCAT_JOURNAL_RECORD, Literal[10]] + activityCounters: c.Array[ctypes.c_uint32, Literal[32]] + reserved: c.Array[ctypes.c_ubyte, Literal[65]] +struct_NV2080_CTRL_NVD_GET_NOCAT_JOURNAL_PARAMS.register_fields([('flags', NvU32, 0), ('nocatRecordCount', NvU32, 4), ('nocatOutstandingRecordCount', NvU32, 8), ('journalRecords', c.Array[NV2080_NOCAT_JOURNAL_RECORD, Literal[10]], 16), ('activityCounters', c.Array[NvU32, Literal[32]], 15456), ('reserved', c.Array[NvU8, Literal[65]], 15584)]) +NV2080_CTRL_NVD_GET_NOCAT_JOURNAL_PARAMS: TypeAlias = struct_NV2080_CTRL_NVD_GET_NOCAT_JOURNAL_PARAMS +@c.record +class struct_NV2080CtrlNocatJournalDataTdrReason(c.Struct): + SIZE = 96 + flags: int + source: c.Array[ctypes.c_ubyte, Literal[65]] + subsystem: int + errorCode: int + reasonCode: int +struct_NV2080CtrlNocatJournalDataTdrReason.register_fields([('flags', NvU32, 0), ('source', c.Array[NvU8, Literal[65]], 4), ('subsystem', NvU32, 72), ('errorCode', NvU64, 80), ('reasonCode', NvU32, 88)]) +NV2080CtrlNocatJournalDataTdrReason: TypeAlias = struct_NV2080CtrlNocatJournalDataTdrReason +@c.record +class struct_NV2080CtrlNocatJournalSetTag(c.Struct): + SIZE = 72 + flags: int + tag: c.Array[ctypes.c_ubyte, Literal[65]] +struct_NV2080CtrlNocatJournalSetTag.register_fields([('flags', NvU32, 0), ('tag', c.Array[NvU8, Literal[65]], 4)]) +NV2080CtrlNocatJournalSetTag: TypeAlias = struct_NV2080CtrlNocatJournalSetTag +@c.record +class struct_NV2080CtrlNocatJournalRclog(c.Struct): + SIZE = 92 + flags: int + rclogSize: int + rmGpuId: int + APIType: int + contextType: int + exceptType: int + processImageName: c.Array[ctypes.c_ubyte, Literal[65]] +struct_NV2080CtrlNocatJournalRclog.register_fields([('flags', NvU32, 0), ('rclogSize', NvU32, 4), ('rmGpuId', NvU32, 8), ('APIType', NvU32, 12), ('contextType', NvU32, 16), ('exceptType', NvU32, 20), ('processImageName', c.Array[NvU8, Literal[65]], 24)]) +NV2080CtrlNocatJournalRclog: TypeAlias = struct_NV2080CtrlNocatJournalRclog +@c.record +class struct_NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS(c.Struct): + SIZE = 104 + dataType: int + targetRecordType: int + nocatJournalData: struct_NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS_nocatJournalData +@c.record +class struct_NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS_nocatJournalData(c.Struct): + SIZE = 96 + tdrReason: struct_NV2080CtrlNocatJournalDataTdrReason + tagData: struct_NV2080CtrlNocatJournalSetTag + rclog: struct_NV2080CtrlNocatJournalRclog +struct_NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS_nocatJournalData.register_fields([('tdrReason', NV2080CtrlNocatJournalDataTdrReason, 0), ('tagData', NV2080CtrlNocatJournalSetTag, 0), ('rclog', NV2080CtrlNocatJournalRclog, 0)]) +struct_NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS.register_fields([('dataType', NvU32, 0), ('targetRecordType', NvU32, 4), ('nocatJournalData', struct_NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS_nocatJournalData, 8)]) +NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS +@c.record +class struct_NV2080CtrlNocatJournalInsertRecord(c.Struct): + SIZE = 1208 + flags: int + timestamp: int + recType: int + bugcheck: int + source: c.Array[ctypes.c_char, Literal[65]] + subsystem: int + errorCode: int + faultingEngine: c.Array[ctypes.c_char, Literal[65]] + tdrReason: int + diagBufferLen: int + diagBuffer: c.Array[ctypes.c_ubyte, Literal[1024]] +struct_NV2080CtrlNocatJournalInsertRecord.register_fields([('flags', NvU32, 0), ('timestamp', NvU64, 8), ('recType', NvU8, 16), ('bugcheck', NvU32, 20), ('source', c.Array[ctypes.c_char, Literal[65]], 24), ('subsystem', NvU32, 92), ('errorCode', NvU64, 96), ('faultingEngine', c.Array[ctypes.c_char, Literal[65]], 104), ('tdrReason', NvU32, 172), ('diagBufferLen', NvU32, 176), ('diagBuffer', c.Array[NvU8, Literal[1024]], 180)]) +NV2080CtrlNocatJournalInsertRecord: TypeAlias = struct_NV2080CtrlNocatJournalInsertRecord +@c.record +class struct_NV2080_CTRL_CMD_NVD_INSERT_NOCAT_JOURNAL_RECORD_PARAMS(c.Struct): + SIZE = 1208 + nocatJournalRecord: struct_NV2080CtrlNocatJournalInsertRecord +struct_NV2080_CTRL_CMD_NVD_INSERT_NOCAT_JOURNAL_RECORD_PARAMS.register_fields([('nocatJournalRecord', NV2080CtrlNocatJournalInsertRecord, 0)]) +NV2080_CTRL_CMD_NVD_INSERT_NOCAT_JOURNAL_RECORD_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_NVD_INSERT_NOCAT_JOURNAL_RECORD_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS_PARAMS(c.Struct): + SIZE = 48 + capsTbl: int + lowestNvlinkVersion: int + highestNvlinkVersion: int + lowestNciVersion: int + highestNciVersion: int + discoveredLinkMask: int + enabledLinkMask: int + discoveredLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK + enabledLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS_PARAMS.register_fields([('capsTbl', NvU32, 0), ('lowestNvlinkVersion', NvU8, 4), ('highestNvlinkVersion', NvU8, 5), ('lowestNciVersion', NvU8, 6), ('highestNciVersion', NvU8, 7), ('discoveredLinkMask', NvU32, 8), ('enabledLinkMask', NvU32, 12), ('discoveredLinks', NV2080_CTRL_NVLINK_LINK_MASK, 16), ('enabledLinks', NV2080_CTRL_NVLINK_LINK_MASK, 32)]) +NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_DEVICE_INFO(c.Struct): + SIZE = 56 + deviceIdFlags: int + domain: int + bus: int + device: int + function: int + pciDeviceId: int + deviceType: int + deviceUUID: c.Array[ctypes.c_ubyte, Literal[16]] + fabricRecoveryStatusMask: int +struct_NV2080_CTRL_NVLINK_DEVICE_INFO.register_fields([('deviceIdFlags', NvU32, 0), ('domain', NvU32, 4), ('bus', NvU16, 8), ('device', NvU16, 10), ('function', NvU16, 12), ('pciDeviceId', NvU32, 16), ('deviceType', NvU64, 24), ('deviceUUID', c.Array[NvU8, Literal[16]], 32), ('fabricRecoveryStatusMask', NvU32, 48)]) +NV2080_CTRL_NVLINK_DEVICE_INFO: TypeAlias = struct_NV2080_CTRL_NVLINK_DEVICE_INFO +@c.record +class struct_NV2080_CTRL_NVLINK_LINK_STATUS_INFO(c.Struct): + SIZE = 216 + capsTbl: int + phyType: int + subLinkWidth: int + linkState: int + rxSublinkStatus: int + txSublinkStatus: int + bLaneReversal: int + nvlinkVersion: int + nciVersion: int + phyVersion: int + nvlinkLinkClockKHz: int + nvlinkCommonClockSpeedKHz: int + nvlinkRefClkSpeedKHz: int + nvlinkCommonClockSpeedMhz: int + nvlinkLineRateMbps: int + nvlinkLinkClockMhz: int + nvlinkRefClkType: int + nvlinkLinkDataRateKiBps: int + nvlinkRefClkSpeedMhz: int + connected: int + loopProperty: int + remoteDeviceLinkNumber: int + localDeviceLinkNumber: int + remoteLinkSid: int + localLinkSid: int + laneRxdetStatusMask: int + nvlinkMinL1Threshold: int + nvlinkMaxL1Threshold: int + nvlinkL1ThresholdUnits: int + remoteDeviceInfo: struct_NV2080_CTRL_NVLINK_DEVICE_INFO + localDeviceInfo: struct_NV2080_CTRL_NVLINK_DEVICE_INFO + nvlpwIdx: int +struct_NV2080_CTRL_NVLINK_LINK_STATUS_INFO.register_fields([('capsTbl', NvU32, 0), ('phyType', NvU8, 4), ('subLinkWidth', NvU8, 5), ('linkState', NvU32, 8), ('rxSublinkStatus', NvU8, 12), ('txSublinkStatus', NvU8, 13), ('bLaneReversal', NvBool, 14), ('nvlinkVersion', NvU8, 15), ('nciVersion', NvU8, 16), ('phyVersion', NvU8, 17), ('nvlinkLinkClockKHz', NvU32, 20), ('nvlinkCommonClockSpeedKHz', NvU32, 24), ('nvlinkRefClkSpeedKHz', NvU32, 28), ('nvlinkCommonClockSpeedMhz', NvU32, 32), ('nvlinkLineRateMbps', NvU32, 36), ('nvlinkLinkClockMhz', NvU32, 40), ('nvlinkRefClkType', NvU8, 44), ('nvlinkLinkDataRateKiBps', NvU32, 48), ('nvlinkRefClkSpeedMhz', NvU32, 52), ('connected', NvBool, 56), ('loopProperty', NvU8, 57), ('remoteDeviceLinkNumber', NvU8, 58), ('localDeviceLinkNumber', NvU8, 59), ('remoteLinkSid', NvU64, 64), ('localLinkSid', NvU64, 72), ('laneRxdetStatusMask', NvU32, 80), ('nvlinkMinL1Threshold', NvU32, 84), ('nvlinkMaxL1Threshold', NvU32, 88), ('nvlinkL1ThresholdUnits', NvU32, 92), ('remoteDeviceInfo', NV2080_CTRL_NVLINK_DEVICE_INFO, 96), ('localDeviceInfo', NV2080_CTRL_NVLINK_DEVICE_INFO, 152), ('nvlpwIdx', NvU32, 208)]) +NV2080_CTRL_NVLINK_LINK_STATUS_INFO: TypeAlias = struct_NV2080_CTRL_NVLINK_LINK_STATUS_INFO +enum_NV2080_CTRL_NVLINK_STATUS_L1_THRESHOLD_UNIT: dict[int, str] = {(NV2080_CTRL_NVLINK_STATUS_L1_THRESHOLD_UNIT_100US:=0): 'NV2080_CTRL_NVLINK_STATUS_L1_THRESHOLD_UNIT_100US', (NV2080_CTRL_NVLINK_STATUS_L1_THRESHOLD_UNIT_50US:=1): 'NV2080_CTRL_NVLINK_STATUS_L1_THRESHOLD_UNIT_50US'} +NV2080_CTRL_NVLINK_STATUS_L1_THRESHOLD_UNIT: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS_PARAMS(c.Struct): + SIZE = 13864 + enabledLinkMask: int + enabledLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK + bSublinkStateInst: int + bNvleModeEnabled: int + linkInfo: c.Array[struct_NV2080_CTRL_NVLINK_LINK_STATUS_INFO, Literal[64]] + enabledNvlpwMask: int +struct_NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS_PARAMS.register_fields([('enabledLinkMask', NvU32, 0), ('enabledLinks', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('bSublinkStateInst', NvBool, 24), ('bNvleModeEnabled', NvBool, 25), ('linkInfo', c.Array[NV2080_CTRL_NVLINK_LINK_STATUS_INFO, Literal[64]], 32), ('enabledNvlpwMask', NvU64, 13856)]) +NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_ERR_INFO(c.Struct): + SIZE = 112 + TLErrlog: int + TLIntrEn: int + TLCTxErrStatus0: int + TLCTxErrStatus1: int + TLCTxSysErrStatus0: int + TLCRxErrStatus0: int + TLCRxErrStatus1: int + TLCRxSysErrStatus0: int + TLCTxErrLogEn0: int + TLCTxErrLogEn1: int + TLCTxSysErrLogEn0: int + TLCRxErrLogEn0: int + TLCRxErrLogEn1: int + TLCRxSysErrLogEn0: int + MIFTxErrStatus0: int + MIFRxErrStatus0: int + NVLIPTLnkErrStatus0: int + NVLIPTLnkErrLogEn0: int + NVLIPTLnkCtrlLinkStateRequest: int + DLSpeedStatusTx: int + DLSpeedStatusRx: int + NVLDLRxSlsmErrCntl: int + NVLDLTopLinkState: int + NVLDLTopIntr: int + DLStatMN00: int + DLStatUC01: int + MinionNvlinkLinkIntr: int + bExcessErrorDL: int +struct_NV2080_CTRL_NVLINK_ERR_INFO.register_fields([('TLErrlog', NvU32, 0), ('TLIntrEn', NvU32, 4), ('TLCTxErrStatus0', NvU32, 8), ('TLCTxErrStatus1', NvU32, 12), ('TLCTxSysErrStatus0', NvU32, 16), ('TLCRxErrStatus0', NvU32, 20), ('TLCRxErrStatus1', NvU32, 24), ('TLCRxSysErrStatus0', NvU32, 28), ('TLCTxErrLogEn0', NvU32, 32), ('TLCTxErrLogEn1', NvU32, 36), ('TLCTxSysErrLogEn0', NvU32, 40), ('TLCRxErrLogEn0', NvU32, 44), ('TLCRxErrLogEn1', NvU32, 48), ('TLCRxSysErrLogEn0', NvU32, 52), ('MIFTxErrStatus0', NvU32, 56), ('MIFRxErrStatus0', NvU32, 60), ('NVLIPTLnkErrStatus0', NvU32, 64), ('NVLIPTLnkErrLogEn0', NvU32, 68), ('NVLIPTLnkCtrlLinkStateRequest', NvU32, 72), ('DLSpeedStatusTx', NvU32, 76), ('DLSpeedStatusRx', NvU32, 80), ('NVLDLRxSlsmErrCntl', NvU32, 84), ('NVLDLTopLinkState', NvU32, 88), ('NVLDLTopIntr', NvU32, 92), ('DLStatMN00', NvU32, 96), ('DLStatUC01', NvU32, 100), ('MinionNvlinkLinkIntr', NvU32, 104), ('bExcessErrorDL', NvBool, 108)]) +NV2080_CTRL_NVLINK_ERR_INFO: TypeAlias = struct_NV2080_CTRL_NVLINK_ERR_INFO +@c.record +class struct_NV2080_CTRL_NVLINK_COMMON_ERR_INFO(c.Struct): + SIZE = 8 + NVLIPTErrStatus0: int + NVLIPTErrLogEn0: int +struct_NV2080_CTRL_NVLINK_COMMON_ERR_INFO.register_fields([('NVLIPTErrStatus0', NvU32, 0), ('NVLIPTErrLogEn0', NvU32, 4)]) +NV2080_CTRL_NVLINK_COMMON_ERR_INFO: TypeAlias = struct_NV2080_CTRL_NVLINK_COMMON_ERR_INFO +@c.record +class struct_NV2080_CTRL_NVLINK_GET_ERR_INFO_PARAMS(c.Struct): + SIZE = 7224 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + linkErrInfo: c.Array[struct_NV2080_CTRL_NVLINK_ERR_INFO, Literal[64]] + ioctrlMask: int + commonErrInfo: c.Array[struct_NV2080_CTRL_NVLINK_COMMON_ERR_INFO, Literal[3]] + ErrInfoFlags: int +struct_NV2080_CTRL_NVLINK_GET_ERR_INFO_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('linkErrInfo', c.Array[NV2080_CTRL_NVLINK_ERR_INFO, Literal[64]], 24), ('ioctrlMask', NvU32, 7192), ('commonErrInfo', c.Array[NV2080_CTRL_NVLINK_COMMON_ERR_INFO, Literal[3]], 7196), ('ErrInfoFlags', NvU8, 7220)]) +NV2080_CTRL_NVLINK_GET_ERR_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_ERR_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_COUNTERS_VALUES(c.Struct): + SIZE = 264 + bTx0TlCounterOverflow: int + bTx1TlCounterOverflow: int + bRx0TlCounterOverflow: int + bRx1TlCounterOverflow: int + value: c.Array[ctypes.c_uint64, Literal[32]] +struct_NV2080_CTRL_NVLINK_GET_COUNTERS_VALUES.register_fields([('bTx0TlCounterOverflow', NvBool, 0), ('bTx1TlCounterOverflow', NvBool, 1), ('bRx0TlCounterOverflow', NvBool, 2), ('bRx1TlCounterOverflow', NvBool, 3), ('value', c.Array[NvU64, Literal[32]], 8)]) +NV2080_CTRL_NVLINK_GET_COUNTERS_VALUES: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_COUNTERS_VALUES +@c.record +class struct_NV2080_CTRL_NVLINK_GET_COUNTERS_PARAMS(c.Struct): + SIZE = 16928 + counterMask: int + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + counters: c.Array[struct_NV2080_CTRL_NVLINK_GET_COUNTERS_VALUES, Literal[64]] +struct_NV2080_CTRL_NVLINK_GET_COUNTERS_PARAMS.register_fields([('counterMask', NvU32, 0), ('linkMask', NvU64, 8), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 16), ('counters', c.Array[NV2080_CTRL_NVLINK_GET_COUNTERS_VALUES, Literal[64]], 32)]) +NV2080_CTRL_NVLINK_GET_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_CLEAR_COUNTERS_PARAMS(c.Struct): + SIZE = 32 + counterMask: int + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_NVLINK_CLEAR_COUNTERS_PARAMS.register_fields([('counterMask', NvU32, 0), ('linkMask', NvU64, 8), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 16)]) +NV2080_CTRL_NVLINK_CLEAR_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_CLEAR_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_COUNTERS_V2_VALUES(c.Struct): + SIZE = 16 + overFlow: int + value: int +struct_NV2080_CTRL_NVLINK_COUNTERS_V2_VALUES.register_fields([('overFlow', NvBool, 0), ('value', NvU64, 8)]) +NV2080_CTRL_NVLINK_COUNTERS_V2_VALUES: TypeAlias = struct_NV2080_CTRL_NVLINK_COUNTERS_V2_VALUES +@c.record +class struct_NV2080_CTRL_NVLINK_GET_COUNTERS_V2_PARAMS(c.Struct): + SIZE = 28720 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + counterMask: c.Array[ctypes.c_uint64, Literal[3]] + counter: c.Array[c.Array[struct_NV2080_CTRL_NVLINK_COUNTERS_V2_VALUES, Literal[28]], Literal[64]] +struct_NV2080_CTRL_NVLINK_GET_COUNTERS_V2_PARAMS.register_fields([('linkMask', NvU64, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('counterMask', c.Array[NvU64, Literal[3]], 24), ('counter', c.Array[c.Array[NV2080_CTRL_NVLINK_COUNTERS_V2_VALUES, Literal[28]], Literal[64]], 48)]) +NV2080_CTRL_NVLINK_GET_COUNTERS_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_COUNTERS_V2_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_CLEAR_COUNTERS_V2_PARAMS(c.Struct): + SIZE = 48 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + counterMask: c.Array[ctypes.c_uint64, Literal[3]] +struct_NV2080_CTRL_NVLINK_CLEAR_COUNTERS_V2_PARAMS.register_fields([('linkMask', NvU64, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('counterMask', c.Array[NvU64, Literal[3]], 24)]) +NV2080_CTRL_NVLINK_CLEAR_COUNTERS_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_CLEAR_COUNTERS_V2_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_INJECT_ERROR_PARAMS(c.Struct): + SIZE = 32 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + bFatalError: int +struct_NV2080_CTRL_NVLINK_INJECT_ERROR_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('bFatalError', NvBool, 24)]) +NV2080_CTRL_NVLINK_INJECT_ERROR_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_INJECT_ERROR_PARAMS +enum_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE: dict[int, str] = {(NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_TX_ERR:=1): 'NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_TX_ERR', (NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_PKT_ERR:=2): 'NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_PKT_ERR', (NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_AUTH_TAG_ERR:=3): 'NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_AUTH_TAG_ERR', (NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_LINK_ERR:=4): 'NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_LINK_ERR', (NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_MAX:=5): 'NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE_MAX'} +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_CFG(c.Struct): + SIZE = 16 + errType: int + errSettings: int +struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_CFG.register_fields([('errType', NV2080_CTRL_NVLINK_HW_ERROR_INJECT_ERR_TYPE, 0), ('errSettings', NvU64, 8)]) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_CFG: TypeAlias = struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_CFG +@c.record +class struct_NV2080_CTRL_NVLINK_SET_HW_ERROR_INJECT_PARAMS(c.Struct): + SIZE = 1048 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + errCfg: c.Array[struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_CFG, Literal[64]] +struct_NV2080_CTRL_NVLINK_SET_HW_ERROR_INJECT_PARAMS.register_fields([('linkMask', NvU64, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('errCfg', c.Array[NV2080_CTRL_NVLINK_HW_ERROR_INJECT_CFG, Literal[64]], 24)]) +NV2080_CTRL_NVLINK_SET_HW_ERROR_INJECT_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_HW_ERROR_INJECT_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_INFO(c.Struct): + SIZE = 20 + txErrInfo: int + packetErrInfo: int + authErrInfo: int + linkStatus: int + errInjectStatus: int +struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_INFO.register_fields([('txErrInfo', NvU32, 0), ('packetErrInfo', NvU32, 4), ('authErrInfo', NvU32, 8), ('linkStatus', NvU32, 12), ('errInjectStatus', NvU32, 16)]) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_INFO: TypeAlias = struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_INFO +@c.record +class struct_NV2080_CTRL_NVLINK_GET_HW_ERROR_INJECT_PARAMS(c.Struct): + SIZE = 1304 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + errInfo: c.Array[struct_NV2080_CTRL_NVLINK_HW_ERROR_INJECT_INFO, Literal[64]] +struct_NV2080_CTRL_NVLINK_GET_HW_ERROR_INJECT_PARAMS.register_fields([('linkMask', NvU64, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('errInfo', c.Array[NV2080_CTRL_NVLINK_HW_ERROR_INJECT_INFO, Literal[64]], 24)]) +NV2080_CTRL_NVLINK_GET_HW_ERROR_INJECT_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_HW_ERROR_INJECT_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_NVLINK_GET_ERROR_RECOVERIES_PARAMS(c.Struct): + SIZE = 280 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + numRecoveries: c.Array[ctypes.c_uint32, Literal[64]] +struct_NV2080_CTRL_CMD_NVLINK_GET_ERROR_RECOVERIES_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('numRecoveries', c.Array[NvU32, Literal[64]], 24)]) +NV2080_CTRL_CMD_NVLINK_GET_ERROR_RECOVERIES_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_NVLINK_GET_ERROR_RECOVERIES_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LINK_LAST_ERROR_REMOTE_TYPE_PARAMS(c.Struct): + SIZE = 8 + linkId: int + remoteType: int +struct_NV2080_CTRL_NVLINK_GET_LINK_LAST_ERROR_REMOTE_TYPE_PARAMS.register_fields([('linkId', NvU32, 0), ('remoteType', NvU32, 4)]) +NV2080_CTRL_NVLINK_GET_LINK_LAST_ERROR_REMOTE_TYPE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LINK_LAST_ERROR_REMOTE_TYPE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LINK_FATAL_ERROR_COUNTS_PARAMS(c.Struct): + SIZE = 80 + linkId: int + supportedCounts: int + fatalErrorCounts: c.Array[ctypes.c_ubyte, Literal[63]] +struct_NV2080_CTRL_NVLINK_GET_LINK_FATAL_ERROR_COUNTS_PARAMS.register_fields([('linkId', NvU32, 0), ('supportedCounts', NvU64, 8), ('fatalErrorCounts', c.Array[NvU8, Literal[63]], 16)]) +NV2080_CTRL_NVLINK_GET_LINK_FATAL_ERROR_COUNTS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LINK_FATAL_ERROR_COUNTS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE(c.Struct): + SIZE = 8 + errorsPerMinute: int + timestamp: int +struct_NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE.register_fields([('errorsPerMinute', NvU32, 0), ('timestamp', NvU32, 4)]) +NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE: TypeAlias = struct_NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LINK_NONFATAL_ERROR_RATES_PARAMS(c.Struct): + SIZE = 92 + linkId: int + numDailyMaxNonfatalErrorRates: int + dailyMaxNonfatalErrorRates: c.Array[struct_NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE, Literal[5]] + numMonthlyMaxNonfatalErrorRates: int + monthlyMaxNonfatalErrorRates: c.Array[struct_NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE, Literal[5]] +struct_NV2080_CTRL_NVLINK_GET_LINK_NONFATAL_ERROR_RATES_PARAMS.register_fields([('linkId', NvU32, 0), ('numDailyMaxNonfatalErrorRates', NvU32, 4), ('dailyMaxNonfatalErrorRates', c.Array[NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE, Literal[5]], 8), ('numMonthlyMaxNonfatalErrorRates', NvU32, 48), ('monthlyMaxNonfatalErrorRates', c.Array[NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE, Literal[5]], 52)]) +NV2080_CTRL_NVLINK_GET_LINK_NONFATAL_ERROR_RATES_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LINK_NONFATAL_ERROR_RATES_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_ERROR_INJECTION_MODE_PARAMS(c.Struct): + SIZE = 1 + bEnabled: int +struct_NV2080_CTRL_NVLINK_SET_ERROR_INJECTION_MODE_PARAMS.register_fields([('bEnabled', NvBool, 0)]) +NV2080_CTRL_NVLINK_SET_ERROR_INJECTION_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_ERROR_INJECTION_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_NVLINK_SETUP_EOM_PARAMS(c.Struct): + SIZE = 8 + linkId: int + params: int +struct_NV2080_CTRL_CMD_NVLINK_SETUP_EOM_PARAMS.register_fields([('linkId', NvU8, 0), ('params', NvU32, 4)]) +NV2080_CTRL_CMD_NVLINK_SETUP_EOM_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_NVLINK_SETUP_EOM_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_POWER_STATE_PARAMS(c.Struct): + SIZE = 32 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + powerState: int +struct_NV2080_CTRL_NVLINK_SET_POWER_STATE_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('powerState', NvU32, 24)]) +NV2080_CTRL_NVLINK_SET_POWER_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_POWER_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_POWER_STATE_PARAMS(c.Struct): + SIZE = 8 + linkId: int + powerState: int +struct_NV2080_CTRL_NVLINK_GET_POWER_STATE_PARAMS.register_fields([('linkId', NvU32, 0), ('powerState', NvU32, 4)]) +NV2080_CTRL_NVLINK_GET_POWER_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_POWER_STATE_PARAMS +enum_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_DEVICE: dict[int, str] = {(TLC_RX_LNK:=0): 'TLC_RX_LNK', (TLC_TX_SYS:=1): 'TLC_TX_SYS'} +NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_DEVICE: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_NVLINK_INJECT_TLC_TX_SYS_REPORT_ERROR_TYPE: dict[int, str] = {(TX_SYS_TX_RSP_STATUS_HW_ERR:=0): 'TX_SYS_TX_RSP_STATUS_HW_ERR', (TX_SYS_TX_RSP_STATUS_UR_ERR:=1): 'TX_SYS_TX_RSP_STATUS_UR_ERR', (TX_SYS_TX_RSP_STATUS_PRIV_ERR:=2): 'TX_SYS_TX_RSP_STATUS_PRIV_ERR'} +NV2080_CTRL_NVLINK_INJECT_TLC_TX_SYS_REPORT_ERROR_TYPE: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_NVLINK_INJECT_TLC_RX_LNK_REPORT_ERROR_TYPE: dict[int, str] = {(RX_LNK_RX_RSP_STATUS_HW_ERR:=0): 'RX_LNK_RX_RSP_STATUS_HW_ERR', (RX_LNK_RX_RSP_STATUS_UR_ERR:=1): 'RX_LNK_RX_RSP_STATUS_UR_ERR', (RX_LNK_RX_RSP_STATUS_PRIV_ERR:=2): 'RX_LNK_RX_RSP_STATUS_PRIV_ERR'} +NV2080_CTRL_NVLINK_INJECT_TLC_RX_LNK_REPORT_ERROR_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class union_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_TYPE(c.Struct): + SIZE = 4 + txSysErrorType: int + rxLnkErrorType: int +union_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_TYPE.register_fields([('txSysErrorType', NV2080_CTRL_NVLINK_INJECT_TLC_TX_SYS_REPORT_ERROR_TYPE, 0), ('rxLnkErrorType', NV2080_CTRL_NVLINK_INJECT_TLC_RX_LNK_REPORT_ERROR_TYPE, 0)]) +NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_TYPE: TypeAlias = union_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_TYPE +@c.record +class struct_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_PARAMS(c.Struct): + SIZE = 16 + linkId: int + device: int + bBroadcast: int + errorType: union_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_TYPE +struct_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_PARAMS.register_fields([('linkId', NvU32, 0), ('device', NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_DEVICE, 4), ('bBroadcast', NvBool, 8), ('errorType', NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_TYPE, 12)]) +NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_NVLINK_GET_LINK_FOM_VALUES_PARAMS(c.Struct): + SIZE = 16 + linkId: int + numLanes: int + figureOfMeritValues: c.Array[ctypes.c_uint16, Literal[4]] +struct_NV2080_CTRL_CMD_NVLINK_GET_LINK_FOM_VALUES_PARAMS.register_fields([('linkId', NvU32, 0), ('numLanes', NvU8, 4), ('figureOfMeritValues', c.Array[NvU16, Literal[4]], 6)]) +NV2080_CTRL_CMD_NVLINK_GET_LINK_FOM_VALUES_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_NVLINK_GET_LINK_FOM_VALUES_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_NVLINK_PEER_PARAMS(c.Struct): + SIZE = 8 + peerMask: int + bEnable: int +struct_NV2080_CTRL_NVLINK_SET_NVLINK_PEER_PARAMS.register_fields([('peerMask', NvU32, 0), ('bEnable', NvBool, 4)]) +NV2080_CTRL_NVLINK_SET_NVLINK_PEER_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_NVLINK_PEER_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_READ_UPHY_PAD_LANE_REG_PARAMS(c.Struct): + SIZE = 8 + linkId: int + lane: int + addr: int + phyConfigData: int +struct_NV2080_CTRL_NVLINK_READ_UPHY_PAD_LANE_REG_PARAMS.register_fields([('linkId', NvU8, 0), ('lane', NvU8, 1), ('addr', NvU16, 2), ('phyConfigData', NvU32, 4)]) +NV2080_CTRL_NVLINK_READ_UPHY_PAD_LANE_REG_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_READ_UPHY_PAD_LANE_REG_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_LANE_ERROR(c.Struct): + SIZE = 12 + bValid: int + eccErrorValue: int + overflowed: int +struct_NV2080_CTRL_NVLINK_LANE_ERROR.register_fields([('bValid', NvBool, 0), ('eccErrorValue', NvU32, 4), ('overflowed', NvBool, 8)]) +NV2080_CTRL_NVLINK_LANE_ERROR: TypeAlias = struct_NV2080_CTRL_NVLINK_LANE_ERROR +@c.record +class struct_NV2080_CTRL_NVLINK_LINK_ECC_ERROR(c.Struct): + SIZE = 56 + errorLane: c.Array[struct_NV2080_CTRL_NVLINK_LANE_ERROR, Literal[4]] + eccDecFailed: int + eccDecFailedOverflowed: int +struct_NV2080_CTRL_NVLINK_LINK_ECC_ERROR.register_fields([('errorLane', c.Array[NV2080_CTRL_NVLINK_LANE_ERROR, Literal[4]], 0), ('eccDecFailed', NvU32, 48), ('eccDecFailedOverflowed', NvBool, 52)]) +NV2080_CTRL_NVLINK_LINK_ECC_ERROR: TypeAlias = struct_NV2080_CTRL_NVLINK_LINK_ECC_ERROR +@c.record +class struct_NV2080_CTRL_NVLINK_GET_NVLINK_ECC_ERRORS_PARAMS(c.Struct): + SIZE = 3608 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + errorLink: c.Array[struct_NV2080_CTRL_NVLINK_LINK_ECC_ERROR, Literal[64]] +struct_NV2080_CTRL_NVLINK_GET_NVLINK_ECC_ERRORS_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('errorLink', c.Array[NV2080_CTRL_NVLINK_LINK_ECC_ERROR, Literal[64]], 24)]) +NV2080_CTRL_NVLINK_GET_NVLINK_ECC_ERRORS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_NVLINK_ECC_ERRORS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_READ_TP_COUNTERS_VALUES(c.Struct): + SIZE = 32 + value: c.Array[ctypes.c_uint64, Literal[4]] +struct_NV2080_CTRL_NVLINK_READ_TP_COUNTERS_VALUES.register_fields([('value', c.Array[NvU64, Literal[4]], 0)]) +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_VALUES: TypeAlias = struct_NV2080_CTRL_NVLINK_READ_TP_COUNTERS_VALUES +@c.record +class struct_NV2080_CTRL_NVLINK_READ_TP_COUNTERS_PARAMS(c.Struct): + SIZE = 2080 + counterMask: int + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + counters: c.Array[struct_NV2080_CTRL_NVLINK_READ_TP_COUNTERS_VALUES, Literal[64]] +struct_NV2080_CTRL_NVLINK_READ_TP_COUNTERS_PARAMS.register_fields([('counterMask', NvU16, 0), ('linkMask', NvU64, 8), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 16), ('counters', c.Array[NV2080_CTRL_NVLINK_READ_TP_COUNTERS_VALUES, Literal[64]], 32)]) +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_READ_TP_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_NVLINK_LOCK_LINK_POWER_STATE_PARAMS(c.Struct): + SIZE = 1 + bLockPowerMode: int +struct_NV2080_CTRL_CMD_NVLINK_LOCK_LINK_POWER_STATE_PARAMS.register_fields([('bLockPowerMode', NvBool, 0)]) +NV2080_CTRL_CMD_NVLINK_LOCK_LINK_POWER_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_NVLINK_LOCK_LINK_POWER_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LP_COUNTERS_PARAMS(c.Struct): + SIZE = 72 + linkId: int + counterValidMask: int + counterValues: c.Array[ctypes.c_uint32, Literal[16]] +struct_NV2080_CTRL_NVLINK_GET_LP_COUNTERS_PARAMS.register_fields([('linkId', NvU32, 0), ('counterValidMask', NvU32, 4), ('counterValues', c.Array[NvU32, Literal[16]], 8)]) +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LP_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_PARAMS(c.Struct): + SIZE = 8 + linkId: int + loopbackMode: int +struct_NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_PARAMS.register_fields([('linkId', NvU32, 0), ('loopbackMode', NvU8, 4)]) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_PHY_REFRESH_STATUS_INFO(c.Struct): + SIZE = 6 + bValid: int + passCount: int + failCount: int +struct_NV2080_CTRL_NVLINK_PHY_REFRESH_STATUS_INFO.register_fields([('bValid', NvBool, 0), ('passCount', NvU16, 2), ('failCount', NvU16, 4)]) +NV2080_CTRL_NVLINK_PHY_REFRESH_STATUS_INFO: TypeAlias = struct_NV2080_CTRL_NVLINK_PHY_REFRESH_STATUS_INFO +@c.record +class struct_NV2080_CTRL_NVLINK_GET_REFRESH_COUNTERS_PARAMS(c.Struct): + SIZE = 216 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + refreshCount: c.Array[struct_NV2080_CTRL_NVLINK_PHY_REFRESH_STATUS_INFO, Literal[32]] +struct_NV2080_CTRL_NVLINK_GET_REFRESH_COUNTERS_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('refreshCount', c.Array[NV2080_CTRL_NVLINK_PHY_REFRESH_STATUS_INFO, Literal[32]], 24)]) +NV2080_CTRL_NVLINK_GET_REFRESH_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_REFRESH_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_CLEAR_REFRESH_COUNTERS_PARAMS(c.Struct): + SIZE = 24 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_NVLINK_CLEAR_REFRESH_COUNTERS_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8)]) +NV2080_CTRL_NVLINK_CLEAR_REFRESH_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_CLEAR_REFRESH_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_SET_NVSWITCH_FLA_ADDR_PARAMS(c.Struct): + SIZE = 16 + bGet: int + addr: int +struct_NV2080_CTRL_NVLINK_GET_SET_NVSWITCH_FLA_ADDR_PARAMS.register_fields([('bGet', NvBool, 0), ('addr', NvU64, 8)]) +NV2080_CTRL_NVLINK_GET_SET_NVSWITCH_FLA_ADDR_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_SET_NVSWITCH_FLA_ADDR_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO_PARAMS(c.Struct): + SIZE = 176 + discoveredLinks: int + discoveredLinkMasks: struct_NV2080_CTRL_NVLINK_LINK_MASK + connectedLinksMask: int + connectedLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK + bridgeSensableLinks: int + bridgeSensableLinkMasks: struct_NV2080_CTRL_NVLINK_LINK_MASK + bridgedLinks: int + bridgedLinkMasks: struct_NV2080_CTRL_NVLINK_LINK_MASK + initDisabledLinksMask: int + initDisabledLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK + vbiosDisabledLinkMask: int + vbiosDisabledLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK + initializedLinks: int + initializedLinkMasks: struct_NV2080_CTRL_NVLINK_LINK_MASK + bEnableTrainingAtLoad: int + bEnableSafeModeAtLoad: int +struct_NV2080_CTRL_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO_PARAMS.register_fields([('discoveredLinks', NvU64, 0), ('discoveredLinkMasks', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('connectedLinksMask', NvU32, 24), ('connectedLinks', NV2080_CTRL_NVLINK_LINK_MASK, 32), ('bridgeSensableLinks', NvU64, 48), ('bridgeSensableLinkMasks', NV2080_CTRL_NVLINK_LINK_MASK, 56), ('bridgedLinks', NvU32, 72), ('bridgedLinkMasks', NV2080_CTRL_NVLINK_LINK_MASK, 80), ('initDisabledLinksMask', NvU32, 96), ('initDisabledLinks', NV2080_CTRL_NVLINK_LINK_MASK, 104), ('vbiosDisabledLinkMask', NvU64, 120), ('vbiosDisabledLinks', NV2080_CTRL_NVLINK_LINK_MASK, 128), ('initializedLinks', NvU32, 144), ('initializedLinkMasks', NV2080_CTRL_NVLINK_LINK_MASK, 152), ('bEnableTrainingAtLoad', NvBool, 168), ('bEnableSafeModeAtLoad', NvBool, 169)]) +NV2080_CTRL_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_PROCESS_INIT_DISABLED_LINKS_PARAMS(c.Struct): + SIZE = 24 + initDisabledLinksMask: int + initDisabledLinks: struct_NV2080_CTRL_NVLINK_LINK_MASK +struct_NV2080_CTRL_NVLINK_PROCESS_INIT_DISABLED_LINKS_PARAMS.register_fields([('initDisabledLinksMask', NvU32, 0), ('initDisabledLinks', NV2080_CTRL_NVLINK_LINK_MASK, 8)]) +NV2080_CTRL_NVLINK_PROCESS_INIT_DISABLED_LINKS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_PROCESS_INIT_DISABLED_LINKS_PARAMS +enum_NV2080_CTRL_CMD_NVLINK_EOM_CONTROL_PARAMS_COMMAND: dict[int, str] = {(NVLINK_EOM_CONTROL_START_EOM:=0): 'NVLINK_EOM_CONTROL_START_EOM', (NVLINK_EOM_CONTROL_END_EOM:=1): 'NVLINK_EOM_CONTROL_END_EOM', (NVLINK_EOM_CONTROL_CONFIG_EOM:=2): 'NVLINK_EOM_CONTROL_CONFIG_EOM', (NVLINK_EOM_CONTROL_FULL_EOM_SEQUENCE:=3): 'NVLINK_EOM_CONTROL_FULL_EOM_SEQUENCE'} +NV2080_CTRL_CMD_NVLINK_EOM_CONTROL_PARAMS_COMMAND: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_NVLINK_EOM_MEASUREMENT(c.Struct): + SIZE = 4 + upper: int + middle: int + lower: int + composite: int +struct_NV2080_CTRL_NVLINK_EOM_MEASUREMENT.register_fields([('upper', NvU8, 0), ('middle', NvU8, 1), ('lower', NvU8, 2), ('composite', NvU8, 3)]) +NV2080_CTRL_NVLINK_EOM_MEASUREMENT: TypeAlias = struct_NV2080_CTRL_NVLINK_EOM_MEASUREMENT +@c.record +class struct_NV2080_CTRL_NVLINK_EOM_CONTROL_PARAMS(c.Struct): + SIZE = 28 + cmd: int + linkId: int + params: int + measurements: c.Array[struct_NV2080_CTRL_NVLINK_EOM_MEASUREMENT, Literal[4]] +struct_NV2080_CTRL_NVLINK_EOM_CONTROL_PARAMS.register_fields([('cmd', NV2080_CTRL_CMD_NVLINK_EOM_CONTROL_PARAMS_COMMAND, 0), ('linkId', NvU32, 4), ('params', NvU32, 8), ('measurements', c.Array[NV2080_CTRL_NVLINK_EOM_MEASUREMENT, Literal[4]], 12)]) +NV2080_CTRL_NVLINK_EOM_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_EOM_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS(c.Struct): + SIZE = 5124 + data: c.Array[ctypes.c_ubyte, Literal[5120]] + dataSize: int +struct_NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS.register_fields([('data', c.Array[NvU8, Literal[5120]], 0), ('dataSize', NvU32, 5120)]) +NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_L1_THRESHOLD_PARAMS(c.Struct): + SIZE = 8 + l1Threshold: int + l1ExitThreshold: int +struct_NV2080_CTRL_NVLINK_SET_L1_THRESHOLD_PARAMS.register_fields([('l1Threshold', NvU32, 0), ('l1ExitThreshold', NvU32, 4)]) +NV2080_CTRL_NVLINK_SET_L1_THRESHOLD_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_L1_THRESHOLD_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_L1_THRESHOLD_PARAMS(c.Struct): + SIZE = 12 + l1Threshold: int + l1ExitThreshold: int + bL1Enabled: int + bL1ThermThrottlingEnabled: int +struct_NV2080_CTRL_NVLINK_GET_L1_THRESHOLD_PARAMS.register_fields([('l1Threshold', NvU32, 0), ('l1ExitThreshold', NvU32, 4), ('bL1Enabled', NvBool, 8), ('bL1ThermThrottlingEnabled', NvBool, 9)]) +NV2080_CTRL_NVLINK_GET_L1_THRESHOLD_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_L1_THRESHOLD_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS(c.Struct): + SIZE = 5124 + buffer: c.Array[ctypes.c_ubyte, Literal[5120]] + dataSize: int +struct_NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS.register_fields([('buffer', c.Array[NvU8, Literal[5120]], 0), ('dataSize', NvU32, 5120)]) +NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_IS_GPU_DEGRADED_PARAMS(c.Struct): + SIZE = 8 + linkId: int + bIsGpuDegraded: int +struct_NV2080_CTRL_NVLINK_IS_GPU_DEGRADED_PARAMS.register_fields([('linkId', NvU32, 0), ('bIsGpuDegraded', NvBool, 4)]) +NV2080_CTRL_NVLINK_IS_GPU_DEGRADED_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_IS_GPU_DEGRADED_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_DIRECT_CONNECT_CHECK_PARAMS(c.Struct): + SIZE = 12 + bIsEnoughNvLink: int + numBridge: int + bridgePresenceMask: int +struct_NV2080_CTRL_NVLINK_DIRECT_CONNECT_CHECK_PARAMS.register_fields([('bIsEnoughNvLink', NvBool, 0), ('numBridge', NvU32, 4), ('bridgePresenceMask', NvU32, 8)]) +NV2080_CTRL_NVLINK_DIRECT_CONNECT_CHECK_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_DIRECT_CONNECT_CHECK_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_POST_FAULT_UP_PARAMS(c.Struct): + SIZE = 4 + linkId: int +struct_NV2080_CTRL_NVLINK_POST_FAULT_UP_PARAMS.register_fields([('linkId', NvU32, 0)]) +NV2080_CTRL_NVLINK_POST_FAULT_UP_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_POST_FAULT_UP_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_PORT_EVENT(c.Struct): + SIZE = 24 + portEventType: int + gpuId: int + linkId: int + time: int +struct_NV2080_CTRL_NVLINK_PORT_EVENT.register_fields([('portEventType', NvU32, 0), ('gpuId', NvU32, 4), ('linkId', NvU32, 8), ('time', NvU64, 16)]) +NV2080_CTRL_NVLINK_PORT_EVENT: TypeAlias = struct_NV2080_CTRL_NVLINK_PORT_EVENT +@c.record +class struct_NV2080_CTRL_NVLINK_GET_PORT_EVENTS_PARAMS(c.Struct): + SIZE = 1560 + portEventIndex: int + nextPortEventIndex: int + portEventCount: int + bOverflow: int + portEvent: c.Array[struct_NV2080_CTRL_NVLINK_PORT_EVENT, Literal[64]] +struct_NV2080_CTRL_NVLINK_GET_PORT_EVENTS_PARAMS.register_fields([('portEventIndex', NvU64, 0), ('nextPortEventIndex', NvU64, 8), ('portEventCount', NvU32, 16), ('bOverflow', NvBool, 20), ('portEvent', c.Array[NV2080_CTRL_NVLINK_PORT_EVENT, Literal[64]], 24)]) +NV2080_CTRL_NVLINK_GET_PORT_EVENTS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_PORT_EVENTS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_CYCLE_LINK_PARAMS(c.Struct): + SIZE = 4 + linkId: int +struct_NV2080_CTRL_NVLINK_CYCLE_LINK_PARAMS.register_fields([('linkId', NvU32, 0)]) +NV2080_CTRL_NVLINK_CYCLE_LINK_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_CYCLE_LINK_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_IS_REDUCED_CONFIG_PARAMS(c.Struct): + SIZE = 1 + bReducedNvlinkConfig: int +struct_NV2080_CTRL_NVLINK_IS_REDUCED_CONFIG_PARAMS.register_fields([('bReducedNvlinkConfig', NvBool, 0)]) +NV2080_CTRL_NVLINK_IS_REDUCED_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_IS_REDUCED_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_PRM_DATA(c.Struct): + SIZE = 496 + data: c.Array[ctypes.c_ubyte, Literal[496]] +struct_NV2080_CTRL_NVLINK_PRM_DATA.register_fields([('data', c.Array[NvU8, Literal[496]], 0)]) +NV2080_CTRL_NVLINK_PRM_DATA: TypeAlias = struct_NV2080_CTRL_NVLINK_PRM_DATA +@c.record +class struct_NV2080_CTRL_NVLINK_GET_SUPPORTED_COUNTERS_PARAMS(c.Struct): + SIZE = 24 + counterMask: c.Array[ctypes.c_uint64, Literal[3]] +struct_NV2080_CTRL_NVLINK_GET_SUPPORTED_COUNTERS_PARAMS.register_fields([('counterMask', c.Array[NvU64, Literal[3]], 0)]) +NV2080_CTRL_NVLINK_GET_SUPPORTED_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_SUPPORTED_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS(c.Struct): + SIZE = 39 + ibGuid: c.Array[ctypes.c_ubyte, Literal[16]] + rackGuid: c.Array[ctypes.c_ubyte, Literal[16]] + chassisPhysicalSlotNumber: int + computeSlotIndex: int + nodeIndex: int + peerType: int + moduleId: int + nvlinkSignalingProtocol: int + linksPerSpigot: int +struct_NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS.register_fields([('ibGuid', c.Array[NvU8, Literal[16]], 0), ('rackGuid', c.Array[NvU8, Literal[16]], 16), ('chassisPhysicalSlotNumber', NvU8, 32), ('computeSlotIndex', NvU8, 33), ('nodeIndex', NvU8, 34), ('peerType', NvU8, 35), ('moduleId', NvU8, 36), ('nvlinkSignalingProtocol', NvU8, 37), ('linksPerSpigot', NvU8, 38)]) +NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_UPHY_CLN_CMD(c.Struct): + SIZE = 4 + pllIndex: int + address: int +struct_NV2080_CTRL_NVLINK_UPHY_CLN_CMD.register_fields([('pllIndex', NvU8, 0), ('address', NvU16, 2)]) +NV2080_CTRL_NVLINK_UPHY_CLN_CMD: TypeAlias = struct_NV2080_CTRL_NVLINK_UPHY_CLN_CMD +@c.record +class struct_NV2080_CTRL_NVLINK_READ_UPHY_CLN_REG_PARAMS(c.Struct): + SIZE = 536 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + uphyCmd: c.Array[struct_NV2080_CTRL_NVLINK_UPHY_CLN_CMD, Literal[64]] + data: c.Array[ctypes.c_uint32, Literal[64]] +struct_NV2080_CTRL_NVLINK_READ_UPHY_CLN_REG_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('uphyCmd', c.Array[NV2080_CTRL_NVLINK_UPHY_CLN_CMD, Literal[64]], 24), ('data', c.Array[NvU32, Literal[64]], 280)]) +NV2080_CTRL_NVLINK_READ_UPHY_CLN_REG_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_READ_UPHY_CLN_REG_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_SUPPORTED_BW_MODE_PARAMS(c.Struct): + SIZE = 48 + rbmModesList: c.Array[ctypes.c_uint16, Literal[23]] + rbmTotalModes: int +struct_NV2080_CTRL_NVLINK_GET_SUPPORTED_BW_MODE_PARAMS.register_fields([('rbmModesList', c.Array[NvU16, Literal[23]], 0), ('rbmTotalModes', NvU8, 46)]) +NV2080_CTRL_NVLINK_GET_SUPPORTED_BW_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_SUPPORTED_BW_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_BW_MODE_PARAMS(c.Struct): + SIZE = 2 + rbmMode: int +struct_NV2080_CTRL_NVLINK_SET_BW_MODE_PARAMS.register_fields([('rbmMode', NvU16, 0)]) +NV2080_CTRL_NVLINK_SET_BW_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_BW_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_BW_MODE_ASYNC_PARAMS(c.Struct): + SIZE = 8 + rbmMode: int + rbmSetPollTimeoutMs: int +struct_NV2080_CTRL_NVLINK_SET_BW_MODE_ASYNC_PARAMS.register_fields([('rbmMode', NvU16, 0), ('rbmSetPollTimeoutMs', NvU32, 4)]) +NV2080_CTRL_NVLINK_SET_BW_MODE_ASYNC_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_BW_MODE_ASYNC_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_BW_MODE_PARAMS(c.Struct): + SIZE = 2 + rbmMode: int +struct_NV2080_CTRL_NVLINK_GET_BW_MODE_PARAMS.register_fields([('rbmMode', NvU16, 0)]) +NV2080_CTRL_NVLINK_GET_BW_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_BW_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LOCAL_DEVICE_INFO_PARAMS(c.Struct): + SIZE = 56 + localDeviceInfo: struct_NV2080_CTRL_NVLINK_DEVICE_INFO +struct_NV2080_CTRL_NVLINK_GET_LOCAL_DEVICE_INFO_PARAMS.register_fields([('localDeviceInfo', NV2080_CTRL_NVLINK_DEVICE_INFO, 0)]) +NV2080_CTRL_NVLINK_GET_LOCAL_DEVICE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LOCAL_DEVICE_INFO_PARAMS +enum_NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY: dict[int, str] = {(NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_NONFATAL:=0): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_NONFATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_APP_FATAL:=1): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_APP_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_FATAL:=2): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_DEGRADATION:=3): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_DEGRADATION', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_WATCHDOG_TIMEOUT:=4): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_WATCHDOG_TIMEOUT', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_SAW_MVB_NON_FATAL:=5): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_SAW_MVB_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_SAW_MSE:=6): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_SAW_MSE', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_RLW_NON_FATAL:=7): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_RLW_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_RLW_FATAL:=8): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_RLW_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_RLW_PRIV_ERR:=9): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_RLW_PRIV_ERR', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_TLW_NON_FATAL:=10): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_TLW_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_TLW_FATAL:=11): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_TLW_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_TREX_NON_FATAL:=12): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_TREX_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_NETIR_NON_FATAL:=13): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_NETIR_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_NETIR_FATAL:=14): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_NETIR_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_SAW_MVB_NON_FATAL:=15): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_SAW_MVB_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_SAW_MVB_FATAL:=16): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_SAW_MVB_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_RLW_NON_FATAL:=17): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_RLW_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_RLW_FATAL:=18): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_RLW_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_TLW_NON_FATAL:=19): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_TLW_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_TLW_FATAL:=20): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_TLW_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_SAW_MVB_FATAL:=21): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_SAW_MVB_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MAX:=22): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MAX', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_MVB_NON_FATAL:=23): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_MVB_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_MVB_FATAL:=24): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_GIN_MVB_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_MVB_NON_FATAL:=25): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_MVB_NON_FATAL', (NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_MVB_FATAL:=26): 'NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY_MSE_ECC_INJECT_MVB_FATAL'} +NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_NVLINK_INJECT_SW_ERROR_PARAMS(c.Struct): + SIZE = 32 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + severity: int +struct_NV2080_CTRL_NVLINK_INJECT_SW_ERROR_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('severity', NV2080_CTRL_NVLINK_INJECT_SW_ERROR_SEVERITY, 24)]) +NV2080_CTRL_NVLINK_INJECT_SW_ERROR_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_INJECT_SW_ERROR_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_UPDATE_NVLE_TOPOLOGY_PARAMS(c.Struct): + SIZE = 96 + localGpuAlid: int + localGpuClid: int + remoteGpuAlid: int + remoteGpuClid: int + bNvleQualMode: int + localGpuPlatformInfo: struct_NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS + remoteGpuPlatformInfo: struct_NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS +struct_NV2080_CTRL_NVLINK_UPDATE_NVLE_TOPOLOGY_PARAMS.register_fields([('localGpuAlid', NvU32, 0), ('localGpuClid', NvU32, 4), ('remoteGpuAlid', NvU32, 8), ('remoteGpuClid', NvU32, 12), ('bNvleQualMode', NvBool, 16), ('localGpuPlatformInfo', NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS, 17), ('remoteGpuPlatformInfo', NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS, 56)]) +NV2080_CTRL_NVLINK_UPDATE_NVLE_TOPOLOGY_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_UPDATE_NVLE_TOPOLOGY_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP(c.Struct): + SIZE = 8 + alid: int + clid: int +struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP.register_fields([('alid', NvU32, 0), ('clid', NvU32, 4)]) +NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP: TypeAlias = struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP +@c.record +class struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE(c.Struct): + SIZE = 260 + alidClidMap: c.Array[struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP, Literal[32]] + numEntries: int +struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE.register_fields([('alidClidMap', c.Array[NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP, Literal[32]], 0), ('numEntries', NvU32, 256)]) +NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE: TypeAlias = struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE +@c.record +class struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE_V2(c.Struct): + SIZE = 1028 + alidClidMap: c.Array[struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP, Literal[128]] + numEntries: int +struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE_V2.register_fields([('alidClidMap', c.Array[NV2080_CTRL_NVLINK_NVLE_ALID_CLID_MAP, Literal[128]], 0), ('numEntries', NvU32, 1024)]) +NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE_V2: TypeAlias = struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE_V2 +@c.record +class struct_NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_PARAMS(c.Struct): + SIZE = 276 + bGet: int + alid: int + clid: int + alidClidTable: struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE + bClidUpdated: int +struct_NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_PARAMS.register_fields([('bGet', NvBool, 0), ('alid', NvU32, 4), ('clid', NvU32, 8), ('alidClidTable', NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE, 12), ('bClidUpdated', NvBool, 272)]) +NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_PARAMS +enum_NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE: dict[int, str] = {(NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_DISABLED:=0): 'NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_DISABLED', (NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_TRIGGER_ONCE:=1): 'NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_TRIGGER_ONCE', (NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_FORCE_EXITED:=2): 'NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_FORCE_EXITED', (NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_FORCE_ENTERED:=3): 'NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_FORCE_ENTERED', (NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_DUTY_CYCLE:=4): 'NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE_DUTY_CYCLE'} +NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_NVLINK_L1_FORCE_CONFIG(c.Struct): + SIZE = 8 + mode: int + toggleActiveTime: int + toggleInactiveTime: int + bTrigger: int +struct_NV2080_CTRL_NVLINK_L1_FORCE_CONFIG.register_fields([('mode', NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_MODE, 0), ('toggleActiveTime', NvU8, 4), ('toggleInactiveTime', NvU8, 5), ('bTrigger', NvBool, 6)]) +NV2080_CTRL_NVLINK_L1_FORCE_CONFIG: TypeAlias = struct_NV2080_CTRL_NVLINK_L1_FORCE_CONFIG +@c.record +class struct_NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_PARAMS(c.Struct): + SIZE = 32 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + config: struct_NV2080_CTRL_NVLINK_L1_FORCE_CONFIG +struct_NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('config', NV2080_CTRL_NVLINK_L1_FORCE_CONFIG, 24)]) +NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_L1_TOGGLE_PARAMS(c.Struct): + SIZE = 536 + linkMask: int + links: struct_NV2080_CTRL_NVLINK_LINK_MASK + config: c.Array[struct_NV2080_CTRL_NVLINK_L1_FORCE_CONFIG, Literal[64]] +struct_NV2080_CTRL_NVLINK_GET_L1_TOGGLE_PARAMS.register_fields([('linkMask', NvU32, 0), ('links', NV2080_CTRL_NVLINK_LINK_MASK, 8), ('config', c.Array[NV2080_CTRL_NVLINK_L1_FORCE_CONFIG, Literal[64]], 24)]) +NV2080_CTRL_NVLINK_GET_L1_TOGGLE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_L1_TOGGLE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_NVLE_ENCRYPT_EN_INFO_PARAMS(c.Struct): + SIZE = 2 + bMmuNvlinkEncryptEn: int + bNvlinkTlwEncryptEn: int +struct_NV2080_CTRL_NVLINK_GET_NVLE_ENCRYPT_EN_INFO_PARAMS.register_fields([('bMmuNvlinkEncryptEn', NvBool, 0), ('bNvlinkTlwEncryptEn', NvBool, 1)]) +NV2080_CTRL_NVLINK_GET_NVLE_ENCRYPT_EN_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_NVLE_ENCRYPT_EN_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SEMANTIC_VERSION(c.Struct): + SIZE = 16 + ucodeType: int + major: int + minor: int + subMinor: int +struct_NV2080_CTRL_NVLINK_SEMANTIC_VERSION.register_fields([('ucodeType', NvU8, 0), ('major', NvU32, 4), ('minor', NvU32, 8), ('subMinor', NvU32, 12)]) +NV2080_CTRL_NVLINK_SEMANTIC_VERSION: TypeAlias = struct_NV2080_CTRL_NVLINK_SEMANTIC_VERSION +@c.record +class struct_NV2080_CTRL_NVLINK_GET_FIRMWARE_VERSION_INFO_PARAMS(c.Struct): + SIZE = 264 + firmwareVersion: c.Array[struct_NV2080_CTRL_NVLINK_SEMANTIC_VERSION, Literal[16]] + chipTypeArch: int + numValidEntries: int +struct_NV2080_CTRL_NVLINK_GET_FIRMWARE_VERSION_INFO_PARAMS.register_fields([('firmwareVersion', c.Array[NV2080_CTRL_NVLINK_SEMANTIC_VERSION, Literal[16]], 0), ('chipTypeArch', NvU8, 256), ('numValidEntries', NvU32, 260)]) +NV2080_CTRL_NVLINK_GET_FIRMWARE_VERSION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_FIRMWARE_VERSION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SET_NVLE_ENABLED_STATE_PARAMS(c.Struct): + SIZE = 1 + bIsNvleEnabled: int +struct_NV2080_CTRL_NVLINK_SET_NVLE_ENABLED_STATE_PARAMS.register_fields([('bIsNvleEnabled', NvBool, 0)]) +NV2080_CTRL_NVLINK_SET_NVLE_ENABLED_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SET_NVLE_ENABLED_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_PRM_ACCESS_PARAMS(c.Struct): + SIZE = 496 + prm: struct_NV2080_CTRL_NVLINK_PRM_DATA +struct_NV2080_CTRL_NVLINK_PRM_ACCESS_PARAMS.register_fields([('prm', NV2080_CTRL_NVLINK_PRM_DATA, 0)]) +NV2080_CTRL_NVLINK_PRM_ACCESS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_PRM_ACCESS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM(c.Struct): + SIZE = 36 + data: c.Array[ctypes.c_ubyte, Literal[32]] + size: int +NvUFXP0_8: TypeAlias = ctypes.c_ubyte +struct_NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM.register_fields([('data', c.Array[NvUFXP0_8, Literal[32]], 0), ('size', NvU32, 32)]) +NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM: TypeAlias = struct_NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM +@c.record +class struct_NV2080_CTRL_NVLINK_PLR_TX_BW_LOSS_LOW_FREQUENCY(c.Struct): + SIZE = 12 + lastPlrTxBwLossLf: int + meanPlrTxBwLossLf: int + maxPlrTxBwLossLf: int + consBelowPlrTxBwLossLfThreshold: int + numOfPlrTxBwLossLfWarnings: int + numOfPlrTxBwLossLfAlarms: int +struct_NV2080_CTRL_NVLINK_PLR_TX_BW_LOSS_LOW_FREQUENCY.register_fields([('lastPlrTxBwLossLf', NvU16, 0), ('meanPlrTxBwLossLf', NvU16, 2), ('maxPlrTxBwLossLf', NvU16, 4), ('consBelowPlrTxBwLossLfThreshold', NvU16, 6), ('numOfPlrTxBwLossLfWarnings', NvU16, 8), ('numOfPlrTxBwLossLfAlarms', NvU16, 10)]) +NV2080_CTRL_NVLINK_PLR_TX_BW_LOSS_LOW_FREQUENCY: TypeAlias = struct_NV2080_CTRL_NVLINK_PLR_TX_BW_LOSS_LOW_FREQUENCY +@c.record +class struct_NV2080_CTRL_NVLINK_RECOVERY_BW_LOSS_LOW_FREQUENCY(c.Struct): + SIZE = 12 + lastRecoveryBwLossLf: int + meanRecoveryBwLossLf: int + maxRecoveryBwLossLf: int + consBelowRecoveryBwLossLfThreshold: int + numOfRecoveryBwLossLfWarnings: int + numOfRecoveryBwLossLfAlarms: int +struct_NV2080_CTRL_NVLINK_RECOVERY_BW_LOSS_LOW_FREQUENCY.register_fields([('lastRecoveryBwLossLf', NvU16, 0), ('meanRecoveryBwLossLf', NvU16, 2), ('maxRecoveryBwLossLf', NvU16, 4), ('consBelowRecoveryBwLossLfThreshold', NvU16, 6), ('numOfRecoveryBwLossLfWarnings', NvU16, 8), ('numOfRecoveryBwLossLfAlarms', NvU16, 10)]) +NV2080_CTRL_NVLINK_RECOVERY_BW_LOSS_LOW_FREQUENCY: TypeAlias = struct_NV2080_CTRL_NVLINK_RECOVERY_BW_LOSS_LOW_FREQUENCY +@c.record +class struct_NV2080_CTRL_NVLINK_EFFECTIVE_BER_LOW_FREQUENCY(c.Struct): + SIZE = 10 + consBelowEffBerLfThreshold: int + numOfEffBerLfWarnings: int + numOfEffBerLfAlarms: int + meanEffBerLfMagnitude: int + lastEffBerLfMagnitude: int + maxEffBerLfMagnitude: int + maxEffBerLfCoef: int +struct_NV2080_CTRL_NVLINK_EFFECTIVE_BER_LOW_FREQUENCY.register_fields([('consBelowEffBerLfThreshold', NvU16, 0), ('numOfEffBerLfWarnings', NvU16, 2), ('numOfEffBerLfAlarms', NvU16, 4), ('meanEffBerLfMagnitude', NvU8, 6), ('lastEffBerLfMagnitude', NvU8, 7), ('maxEffBerLfMagnitude', NvU8, 8), ('maxEffBerLfCoef', NvU8, 9)]) +NV2080_CTRL_NVLINK_EFFECTIVE_BER_LOW_FREQUENCY: TypeAlias = struct_NV2080_CTRL_NVLINK_EFFECTIVE_BER_LOW_FREQUENCY +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LINK_ACCUMULATIVE_METRIC_DATA_PARAMS(c.Struct): + SIZE = 84 + linkId: int + globalMaxRawBERExp: int + globalMaxEffBERExp: int + linkDownCount: int + downReasonHistogram: struct_NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM + linkXidHistogram: struct_NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM +struct_NV2080_CTRL_NVLINK_GET_LINK_ACCUMULATIVE_METRIC_DATA_PARAMS.register_fields([('linkId', NvU32, 0), ('globalMaxRawBERExp', NvU8, 4), ('globalMaxEffBERExp', NvU8, 5), ('linkDownCount', NvU32, 8), ('downReasonHistogram', NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM, 12), ('linkXidHistogram', NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM, 48)]) +NV2080_CTRL_NVLINK_GET_LINK_ACCUMULATIVE_METRIC_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LINK_ACCUMULATIVE_METRIC_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LINK_RECORD_METRIC_DATA_PARAMS(c.Struct): + SIZE = 60 + linkId: int + initialRawBERExp: int + initialEffBERExp: int + lastDownReason: int + uptime: int + localMaxRawBERExp: int + localMaxEffBERExp: int + currEffBERMonitor: int + plrXmitRetryWithinTSecMaxLo: int + totalSuccessfulRecoveryEvents: int + plrTxBwLossLf: struct_NV2080_CTRL_NVLINK_PLR_TX_BW_LOSS_LOW_FREQUENCY + recoveryBwLossLf: struct_NV2080_CTRL_NVLINK_RECOVERY_BW_LOSS_LOW_FREQUENCY + effBerLf: struct_NV2080_CTRL_NVLINK_EFFECTIVE_BER_LOW_FREQUENCY + lastDownAdvancedReason: int +struct_NV2080_CTRL_NVLINK_GET_LINK_RECORD_METRIC_DATA_PARAMS.register_fields([('linkId', NvU32, 0), ('initialRawBERExp', NvU8, 4), ('initialEffBERExp', NvU8, 5), ('lastDownReason', NvU8, 6), ('uptime', NvU32, 8), ('localMaxRawBERExp', NvU8, 12), ('localMaxEffBERExp', NvU8, 13), ('currEffBERMonitor', NvU16, 14), ('plrXmitRetryWithinTSecMaxLo', NvU32, 16), ('totalSuccessfulRecoveryEvents', NvU32, 20), ('plrTxBwLossLf', NV2080_CTRL_NVLINK_PLR_TX_BW_LOSS_LOW_FREQUENCY, 24), ('recoveryBwLossLf', NV2080_CTRL_NVLINK_RECOVERY_BW_LOSS_LOW_FREQUENCY, 36), ('effBerLf', NV2080_CTRL_NVLINK_EFFECTIVE_BER_LOW_FREQUENCY, 48), ('lastDownAdvancedReason', NvU16, 58)]) +NV2080_CTRL_NVLINK_GET_LINK_RECORD_METRIC_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LINK_RECORD_METRIC_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_DEVICE_RECORD_METRIC_DATA_PARAMS(c.Struct): + SIZE = 496 + MFDE: struct_NV2080_CTRL_NVLINK_PRM_DATA +struct_NV2080_CTRL_NVLINK_GET_DEVICE_RECORD_METRIC_DATA_PARAMS.register_fields([('MFDE', NV2080_CTRL_NVLINK_PRM_DATA, 0)]) +NV2080_CTRL_NVLINK_GET_DEVICE_RECORD_METRIC_DATA_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_DEVICE_RECORD_METRIC_DATA_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_NODE_HOSTNAME(c.Struct): + SIZE = 64 + name: c.Array[ctypes.c_ubyte, Literal[64]] +struct_NV2080_CTRL_NVLINK_NODE_HOSTNAME.register_fields([('name', c.Array[NvU8, Literal[64]], 0)]) +NV2080_CTRL_NVLINK_NODE_HOSTNAME: TypeAlias = struct_NV2080_CTRL_NVLINK_NODE_HOSTNAME +@c.record +class struct_NV2080_CTRL_NVLINK_SAVE_NODE_HOSTNAME_PARAMS(c.Struct): + SIZE = 64 + hostname: struct_NV2080_CTRL_NVLINK_NODE_HOSTNAME +struct_NV2080_CTRL_NVLINK_SAVE_NODE_HOSTNAME_PARAMS.register_fields([('hostname', NV2080_CTRL_NVLINK_NODE_HOSTNAME, 0)]) +NV2080_CTRL_NVLINK_SAVE_NODE_HOSTNAME_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SAVE_NODE_HOSTNAME_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_SAVED_NODE_HOSTNAME_PARAMS(c.Struct): + SIZE = 64 + hostname: struct_NV2080_CTRL_NVLINK_NODE_HOSTNAME +struct_NV2080_CTRL_NVLINK_GET_SAVED_NODE_HOSTNAME_PARAMS.register_fields([('hostname', NV2080_CTRL_NVLINK_NODE_HOSTNAME, 0)]) +NV2080_CTRL_NVLINK_GET_SAVED_NODE_HOSTNAME_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_SAVED_NODE_HOSTNAME_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_UPDATE_CLID_PARAMS(c.Struct): + SIZE = 4 + clid: int +struct_NV2080_CTRL_NVLINK_UPDATE_CLID_PARAMS.register_fields([('clid', NvU32, 0)]) +NV2080_CTRL_NVLINK_UPDATE_CLID_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_UPDATE_CLID_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_LOCK_REMAP_TABLE_AND_MSE_PARAMS(c.Struct): + SIZE = 16 + linkMask: int + bRemapTableMseLocked: int + bRemapTableLockDisabled: int +struct_NV2080_CTRL_NVLINK_LOCK_REMAP_TABLE_AND_MSE_PARAMS.register_fields([('linkMask', NvU64, 0), ('bRemapTableMseLocked', NvBool, 8), ('bRemapTableLockDisabled', NvBool, 9)]) +NV2080_CTRL_NVLINK_LOCK_REMAP_TABLE_AND_MSE_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_LOCK_REMAP_TABLE_AND_MSE_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_PARAMS(c.Struct): + SIZE = 260 + flaRemapTabAddr: c.Array[ctypes.c_uint32, Literal[32]] + gpaRemapTabAddr: c.Array[ctypes.c_uint32, Literal[32]] + remapTabSize: int +struct_NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_PARAMS.register_fields([('flaRemapTabAddr', c.Array[NvU32, Literal[32]], 0), ('gpaRemapTabAddr', c.Array[NvU32, Literal[32]], 128), ('remapTabSize', NvU32, 256)]) +NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_NVLE_PKT_CNTRS(c.Struct): + SIZE = 32 + txPipe0Cntr: int + txPipe1Cntr: int + rxPipe0Cntr: int + rxPipe1Cntr: int +struct_NV2080_CTRL_NVLINK_NVLE_PKT_CNTRS.register_fields([('txPipe0Cntr', NvU64, 0), ('txPipe1Cntr', NvU64, 8), ('rxPipe0Cntr', NvU64, 16), ('rxPipe1Cntr', NvU64, 24)]) +NV2080_CTRL_NVLINK_NVLE_PKT_CNTRS: TypeAlias = struct_NV2080_CTRL_NVLINK_NVLE_PKT_CNTRS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_NVLE_PKT_COUNTERS_PARAMS(c.Struct): + SIZE = 2064 + linkMask: struct_NV2080_CTRL_NVLINK_LINK_MASK + nvleCounters: c.Array[struct_NV2080_CTRL_NVLINK_NVLE_PKT_CNTRS, Literal[64]] +struct_NV2080_CTRL_NVLINK_GET_NVLE_PKT_COUNTERS_PARAMS.register_fields([('linkMask', NV2080_CTRL_NVLINK_LINK_MASK, 0), ('nvleCounters', c.Array[NV2080_CTRL_NVLINK_NVLE_PKT_CNTRS, Literal[64]], 16)]) +NV2080_CTRL_NVLINK_GET_NVLE_PKT_COUNTERS_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_NVLE_PKT_COUNTERS_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_SETUP_NVLE_ENCRYPTION_KEY_PARAMS(c.Struct): + SIZE = 120 + nvleKey: c.Array[ctypes.c_ubyte, Literal[32]] + localGpuAlid: int + remoteGpuAlid: int + localGpuPlatformInfo: struct_NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS + remoteGpuPlatformInfo: struct_NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS +struct_NV2080_CTRL_NVLINK_SETUP_NVLE_ENCRYPTION_KEY_PARAMS.register_fields([('nvleKey', c.Array[NvU8, Literal[32]], 0), ('localGpuAlid', NvU32, 32), ('remoteGpuAlid', NvU32, 36), ('localGpuPlatformInfo', NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS, 40), ('remoteGpuPlatformInfo', NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS, 79)]) +NV2080_CTRL_NVLINK_SETUP_NVLE_ENCRYPTION_KEY_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_SETUP_NVLE_ENCRYPTION_KEY_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_V2_PARAMS(c.Struct): + SIZE = 1036 + flaRemapTabAddr: c.Array[ctypes.c_uint32, Literal[128]] + gpaRemapTabAddr: c.Array[ctypes.c_uint32, Literal[128]] + remapTabSize: int + remapEntryStart: int + remapEntryEnd: int +struct_NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_V2_PARAMS.register_fields([('flaRemapTabAddr', c.Array[NvU32, Literal[128]], 0), ('gpaRemapTabAddr', c.Array[NvU32, Literal[128]], 512), ('remapTabSize', NvU32, 1024), ('remapEntryStart', NvU32, 1028), ('remapEntryEnd', NvU32, 1032)]) +NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_V2_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_V2_PARAMS(c.Struct): + SIZE = 1044 + bGet: int + alid: int + clid: int + alidClidTable: struct_NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE_V2 + bClidUpdated: int +struct_NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_V2_PARAMS.register_fields([('bGet', NvBool, 0), ('alid', NvU32, 4), ('clid', NvU32, 8), ('alidClidTable', NV2080_CTRL_NVLINK_NVLE_ALID_CLID_TABLE_V2, 12), ('bClidUpdated', NvBool, 1040)]) +NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_V2_PARAMS +@c.record +class struct_NV2080_CTRL_NVLINK_GET_LINK_TRAINING_TIME_PARAMS(c.Struct): + SIZE = 4 + linkStateChangeTimeMs: int +struct_NV2080_CTRL_NVLINK_GET_LINK_TRAINING_TIME_PARAMS.register_fields([('linkStateChangeTimeMs', NvU32, 0)]) +NV2080_CTRL_NVLINK_GET_LINK_TRAINING_TIME_PARAMS: TypeAlias = struct_NV2080_CTRL_NVLINK_GET_LINK_TRAINING_TIME_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_BOOST_PARAMS(c.Struct): + SIZE = 8 + flags: int + duration: int +struct_NV2080_CTRL_PERF_BOOST_PARAMS.register_fields([('flags', NvU32, 0), ('duration', NvU32, 4)]) +NV2080_CTRL_PERF_BOOST_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_BOOST_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_RESERVE_PERFMON_HW_PARAMS(c.Struct): + SIZE = 1 + bAcquire: int +struct_NV2080_CTRL_PERF_RESERVE_PERFMON_HW_PARAMS.register_fields([('bAcquire', NvBool, 0)]) +NV2080_CTRL_PERF_RESERVE_PERFMON_HW_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_RESERVE_PERFMON_HW_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_POWERSTATE_PARAMS(c.Struct): + SIZE = 4 + powerState: int +struct_NV2080_CTRL_PERF_POWERSTATE_PARAMS.register_fields([('powerState', NvU32, 0)]) +NV2080_CTRL_PERF_POWERSTATE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_POWERSTATE_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS(c.Struct): + SIZE = 4 + powerStateInfo: struct_NV2080_CTRL_PERF_POWERSTATE_PARAMS +struct_NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS.register_fields([('powerStateInfo', NV2080_CTRL_PERF_POWERSTATE_PARAMS, 0)]) +NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS(c.Struct): + SIZE = 4 + powerState: int +struct_NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS.register_fields([('powerState', NvU32, 0)]) +NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS +enum_NV2080_CTRL_PERF_RATED_TDP_CLIENT: dict[int, str] = {(NV2080_CTRL_PERF_RATED_TDP_CLIENT_RM:=0): 'NV2080_CTRL_PERF_RATED_TDP_CLIENT_RM', (NV2080_CTRL_PERF_RATED_TDP_CLIENT_WAR_BUG_1785342:=1): 'NV2080_CTRL_PERF_RATED_TDP_CLIENT_WAR_BUG_1785342', (NV2080_CTRL_PERF_RATED_TDP_CLIENT_GLOBAL:=2): 'NV2080_CTRL_PERF_RATED_TDP_CLIENT_GLOBAL', (NV2080_CTRL_PERF_RATED_TDP_CLIENT_OS:=3): 'NV2080_CTRL_PERF_RATED_TDP_CLIENT_OS', (NV2080_CTRL_PERF_RATED_TDP_CLIENT_PROFILE:=4): 'NV2080_CTRL_PERF_RATED_TDP_CLIENT_PROFILE', (NV2080_CTRL_PERF_RATED_TDP_CLIENT_NUM_CLIENTS:=5): 'NV2080_CTRL_PERF_RATED_TDP_CLIENT_NUM_CLIENTS'} +NV2080_CTRL_PERF_RATED_TDP_CLIENT: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_PERF_RATED_TDP_ACTION: dict[int, str] = {(NV2080_CTRL_PERF_RATED_TDP_ACTION_DEFAULT:=0): 'NV2080_CTRL_PERF_RATED_TDP_ACTION_DEFAULT', (NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_EXCEED:=1): 'NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_EXCEED', (NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_LIMIT:=2): 'NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_LIMIT', (NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_LOCK:=3): 'NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_LOCK', (NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_FLOOR:=4): 'NV2080_CTRL_PERF_RATED_TDP_ACTION_FORCE_FLOOR'} +NV2080_CTRL_PERF_RATED_TDP_ACTION: TypeAlias = ctypes.c_uint32 +enum_NV2080_CTRL_PERF_RATED_TDP_VPSTATE_TYPE: dict[int, str] = {(NV2080_CTRL_PERF_VPSTATE_RATED_TDP:=0): 'NV2080_CTRL_PERF_VPSTATE_RATED_TDP', (NV2080_CTRL_PERF_VPSTATE_TURBO_BOOST:=1): 'NV2080_CTRL_PERF_VPSTATE_TURBO_BOOST', (NV2080_CTRL_PERF_VPSTATE_NUM_VPSTATES:=2): 'NV2080_CTRL_PERF_VPSTATE_NUM_VPSTATES'} +NV2080_CTRL_PERF_RATED_TDP_VPSTATE_TYPE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_PERF_RATED_TDP_CLIENT_REQUEST(c.Struct): + SIZE = 8 + action: int + vPstateType: int +struct_NV2080_CTRL_PERF_RATED_TDP_CLIENT_REQUEST.register_fields([('action', NV2080_CTRL_PERF_RATED_TDP_ACTION, 0), ('vPstateType', NV2080_CTRL_PERF_RATED_TDP_VPSTATE_TYPE, 4)]) +NV2080_CTRL_PERF_RATED_TDP_CLIENT_REQUEST: TypeAlias = struct_NV2080_CTRL_PERF_RATED_TDP_CLIENT_REQUEST +@c.record +class struct_NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS(c.Struct): + SIZE = 56 + rm: struct_NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS_rm + output: int + outputVPstate: int + inputs: c.Array[ctypes.c_uint32, Literal[5]] + vPstateTypes: c.Array[ctypes.c_uint32, Literal[5]] +@c.record +class struct_NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS_rm(c.Struct): + SIZE = 8 + clientActiveMask: int + bRegkeyLimitRatedTdp: int +struct_NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS_rm.register_fields([('clientActiveMask', NvU32, 0), ('bRegkeyLimitRatedTdp', NvU8, 4)]) +struct_NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS.register_fields([('rm', struct_NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS_rm, 0), ('output', NV2080_CTRL_PERF_RATED_TDP_ACTION, 8), ('outputVPstate', NV2080_CTRL_PERF_RATED_TDP_VPSTATE_TYPE, 12), ('inputs', c.Array[NV2080_CTRL_PERF_RATED_TDP_ACTION, Literal[5]], 16), ('vPstateTypes', c.Array[NV2080_CTRL_PERF_RATED_TDP_VPSTATE_TYPE, Literal[5]], 36)]) +NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_RATED_TDP_CONTROL_PARAMS(c.Struct): + SIZE = 12 + client: int + input: int + vPstateType: int +struct_NV2080_CTRL_PERF_RATED_TDP_CONTROL_PARAMS.register_fields([('client', NV2080_CTRL_PERF_RATED_TDP_CLIENT, 0), ('input', NV2080_CTRL_PERF_RATED_TDP_ACTION, 4), ('vPstateType', NV2080_CTRL_PERF_RATED_TDP_VPSTATE_TYPE, 8)]) +NV2080_CTRL_PERF_RATED_TDP_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_RATED_TDP_CONTROL_PARAMS +NV2080_CTRL_PERF_RATED_TDP_GET_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_RATED_TDP_CONTROL_PARAMS +NV2080_CTRL_PERF_RATED_TDP_SET_CONTROL_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_RATED_TDP_CONTROL_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE(c.Struct): + SIZE = 128 + util: int + vgpuScale: int + procId: int + subProcessID: int + subProcessName: c.Array[ctypes.c_char, Literal[100]] + pOsPidInfo: int +struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE.register_fields([('util', NvU32, 0), ('vgpuScale', NvU32, 4), ('procId', NvU32, 8), ('subProcessID', NvU32, 12), ('subProcessName', c.Array[ctypes.c_char, Literal[100]], 16), ('pOsPidInfo', NvU64, 120)]) +NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE: TypeAlias = struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE +@c.record +class struct_NV2080_CTRL_PERF_GPUMON_PERFMON_UTIL_SAMPLE(c.Struct): + SIZE = 776 + base: struct_NV2080_CTRL_GPUMON_SAMPLE + fb: struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE + gr: struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE + nvenc: struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE + nvdec: struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE + nvjpg: struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE + nvofa: struct_NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE +struct_NV2080_CTRL_PERF_GPUMON_PERFMON_UTIL_SAMPLE.register_fields([('base', NV2080_CTRL_GPUMON_SAMPLE, 0), ('fb', NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE, 8), ('gr', NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE, 136), ('nvenc', NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE, 264), ('nvdec', NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE, 392), ('nvjpg', NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE, 520), ('nvofa', NV2080_CTRL_PERF_GPUMON_ENGINE_UTIL_SAMPLE, 648)]) +NV2080_CTRL_PERF_GPUMON_PERFMON_UTIL_SAMPLE: TypeAlias = struct_NV2080_CTRL_PERF_GPUMON_PERFMON_UTIL_SAMPLE +NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_PARAM: TypeAlias = struct_NV2080_CTRL_GPUMON_SAMPLES +@c.record +class struct_NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS(c.Struct): + SIZE = 55888 + type: int + bufSize: int + count: int + tracker: int + samples: c.Array[struct_NV2080_CTRL_PERF_GPUMON_PERFMON_UTIL_SAMPLE, Literal[72]] +struct_NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS.register_fields([('type', NvU8, 0), ('bufSize', NvU32, 4), ('count', NvU32, 8), ('tracker', NvU32, 12), ('samples', c.Array[NV2080_CTRL_PERF_GPUMON_PERFMON_UTIL_SAMPLE, Literal[72]], 16)]) +NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_GPU_IS_IDLE_PARAMS(c.Struct): + SIZE = 8 + prevPstate: int + action: int +struct_NV2080_CTRL_PERF_GPU_IS_IDLE_PARAMS.register_fields([('prevPstate', NvU32, 0), ('action', NvU32, 4)]) +NV2080_CTRL_PERF_GPU_IS_IDLE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GPU_IS_IDLE_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_AGGRESSIVE_PSTATE_NOTIFY_PARAMS(c.Struct): + SIZE = 24 + bGpuIsIdle: int + bRestoreToMax: int + idleTimeUs: int + busyTimeUs: int +struct_NV2080_CTRL_PERF_AGGRESSIVE_PSTATE_NOTIFY_PARAMS.register_fields([('bGpuIsIdle', NvBool, 0), ('bRestoreToMax', NvBool, 1), ('idleTimeUs', NvU64, 8), ('busyTimeUs', NvU64, 16)]) +NV2080_CTRL_PERF_AGGRESSIVE_PSTATE_NOTIFY_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_AGGRESSIVE_PSTATE_NOTIFY_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_GET_CLK_INFO(c.Struct): + SIZE = 24 + flags: int + domain: int + currentFreq: int + defaultFreq: int + minFreq: int + maxFreq: int +struct_NV2080_CTRL_PERF_GET_CLK_INFO.register_fields([('flags', NvU32, 0), ('domain', NvU32, 4), ('currentFreq', NvU32, 8), ('defaultFreq', NvU32, 12), ('minFreq', NvU32, 16), ('maxFreq', NvU32, 20)]) +NV2080_CTRL_PERF_GET_CLK_INFO: TypeAlias = struct_NV2080_CTRL_PERF_GET_CLK_INFO +@c.record +class struct_NV2080_CTRL_PERF_GET_LEVEL_INFO_PARAMS(c.Struct): + SIZE = 24 + level: int + flags: int + perfGetClkInfoList: ctypes.c_void_p + perfGetClkInfoListSize: int +struct_NV2080_CTRL_PERF_GET_LEVEL_INFO_PARAMS.register_fields([('level', NvU32, 0), ('flags', NvU32, 4), ('perfGetClkInfoList', NvP64, 8), ('perfGetClkInfoListSize', NvU32, 16)]) +NV2080_CTRL_PERF_GET_LEVEL_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GET_LEVEL_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_GET_LEVEL_INFO_V2_PARAMS(c.Struct): + SIZE = 780 + level: int + flags: int + perfGetClkInfoList: c.Array[struct_NV2080_CTRL_PERF_GET_CLK_INFO, Literal[32]] + perfGetClkInfoListSize: int +struct_NV2080_CTRL_PERF_GET_LEVEL_INFO_V2_PARAMS.register_fields([('level', NvU32, 0), ('flags', NvU32, 4), ('perfGetClkInfoList', c.Array[NV2080_CTRL_PERF_GET_CLK_INFO, Literal[32]], 8), ('perfGetClkInfoListSize', NvU32, 776)]) +NV2080_CTRL_PERF_GET_LEVEL_INFO_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GET_LEVEL_INFO_V2_PARAMS +enum_NV2080_CTRL_CMD_PERF_VID_ENG: dict[int, str] = {(NV2080_CTRL_CMD_PERF_VID_ENG_NVENC:=1): 'NV2080_CTRL_CMD_PERF_VID_ENG_NVENC', (NV2080_CTRL_CMD_PERF_VID_ENG_NVDEC:=2): 'NV2080_CTRL_CMD_PERF_VID_ENG_NVDEC', (NV2080_CTRL_CMD_PERF_VID_ENG_NVJPG:=3): 'NV2080_CTRL_CMD_PERF_VID_ENG_NVJPG', (NV2080_CTRL_CMD_PERF_VID_ENG_NVOFA:=4): 'NV2080_CTRL_CMD_PERF_VID_ENG_NVOFA'} +NV2080_CTRL_CMD_PERF_VID_ENG: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_PERF_GET_VID_ENG_PERFMON_SAMPLE_PARAMS(c.Struct): + SIZE = 12 + engineType: int + clkPercentBusy: int + samplingPeriodUs: int +struct_NV2080_CTRL_PERF_GET_VID_ENG_PERFMON_SAMPLE_PARAMS.register_fields([('engineType', NV2080_CTRL_CMD_PERF_VID_ENG, 0), ('clkPercentBusy', NvU32, 4), ('samplingPeriodUs', NvU32, 8)]) +NV2080_CTRL_PERF_GET_VID_ENG_PERFMON_SAMPLE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GET_VID_ENG_PERFMON_SAMPLE_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_GET_POWERSTATE_PARAMS(c.Struct): + SIZE = 4 + powerStateInfo: struct_NV2080_CTRL_PERF_POWERSTATE_PARAMS +struct_NV2080_CTRL_PERF_GET_POWERSTATE_PARAMS.register_fields([('powerStateInfo', NV2080_CTRL_PERF_POWERSTATE_PARAMS, 0)]) +NV2080_CTRL_PERF_GET_POWERSTATE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GET_POWERSTATE_PARAMS +@c.record +class struct_NV2080_CTRL_PERF_NOTIFY_VIDEOEVENT_PARAMS(c.Struct): + SIZE = 4 + videoEvent: int +struct_NV2080_CTRL_PERF_NOTIFY_VIDEOEVENT_PARAMS.register_fields([('videoEvent', NvU32, 0)]) +NV2080_CTRL_PERF_NOTIFY_VIDEOEVENT_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_NOTIFY_VIDEOEVENT_PARAMS +NV2080_CTRL_PERF_PSTATES_ID: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_PERF_GET_CURRENT_PSTATE_PARAMS(c.Struct): + SIZE = 4 + currPstate: int +struct_NV2080_CTRL_PERF_GET_CURRENT_PSTATE_PARAMS.register_fields([('currPstate', NvU32, 0)]) +NV2080_CTRL_PERF_GET_CURRENT_PSTATE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GET_CURRENT_PSTATE_PARAMS +NV2080_CTRL_CLK_DOMAIN_TEGRA: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_PERF_GET_TEGRA_PERFMON_SAMPLE_PARAMS(c.Struct): + SIZE = 8 + clkDomain: int + clkPercentBusy: int +struct_NV2080_CTRL_PERF_GET_TEGRA_PERFMON_SAMPLE_PARAMS.register_fields([('clkDomain', NV2080_CTRL_CLK_DOMAIN_TEGRA, 0), ('clkPercentBusy', NvU32, 4)]) +NV2080_CTRL_PERF_GET_TEGRA_PERFMON_SAMPLE_PARAMS: TypeAlias = struct_NV2080_CTRL_PERF_GET_TEGRA_PERFMON_SAMPLE_PARAMS +@c.record +class struct_NV2080_CTRL_PMGR_MODULE_INFO_PARAMS(c.Struct): + SIZE = 8 + moduleId: int + nvswitchSupport: int +struct_NV2080_CTRL_PMGR_MODULE_INFO_PARAMS.register_fields([('moduleId', NvU32, 0), ('nvswitchSupport', NvU8, 4)]) +NV2080_CTRL_PMGR_MODULE_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_PMGR_MODULE_INFO_PARAMS +enum_NV2080_CTRL_GC6_FLAVOR_ID: dict[int, str] = {(NV2080_CTRL_GC6_FLAVOR_ID_MSHYBRID:=0): 'NV2080_CTRL_GC6_FLAVOR_ID_MSHYBRID', (NV2080_CTRL_GC6_FLAVOR_ID_OPTIMUS:=1): 'NV2080_CTRL_GC6_FLAVOR_ID_OPTIMUS', (NV2080_CTRL_GC6_FLAVOR_ID_MAX:=4): 'NV2080_CTRL_GC6_FLAVOR_ID_MAX'} +NV2080_CTRL_GC6_FLAVOR_ID: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GC6_ENTRY_PARAMS(c.Struct): + SIZE = 12 + flavorId: int + stepMask: int + params: struct_NV2080_CTRL_GC6_ENTRY_PARAMS_params +@c.record +class struct_NV2080_CTRL_GC6_ENTRY_PARAMS_params(c.Struct): + SIZE = 3 + bIsRTD3Transition: int + bIsRTD3CoreRailPowerCut: int + bSkipPstateSanity: int +struct_NV2080_CTRL_GC6_ENTRY_PARAMS_params.register_fields([('bIsRTD3Transition', NvBool, 0), ('bIsRTD3CoreRailPowerCut', NvBool, 1), ('bSkipPstateSanity', NvBool, 2)]) +struct_NV2080_CTRL_GC6_ENTRY_PARAMS.register_fields([('flavorId', NV2080_CTRL_GC6_FLAVOR_ID, 0), ('stepMask', NvU32, 4), ('params', struct_NV2080_CTRL_GC6_ENTRY_PARAMS_params, 8)]) +NV2080_CTRL_GC6_ENTRY_PARAMS: TypeAlias = struct_NV2080_CTRL_GC6_ENTRY_PARAMS +@c.record +class struct_NV2080_CTRL_GC6_EXIT_PARAMS(c.Struct): + SIZE = 8 + flavorId: int + params: struct_NV2080_CTRL_GC6_EXIT_PARAMS_params +@c.record +class struct_NV2080_CTRL_GC6_EXIT_PARAMS_params(c.Struct): + SIZE = 3 + bIsGpuSelfWake: int + bIsRTD3Transition: int + bIsRTD3HotTransition: int +struct_NV2080_CTRL_GC6_EXIT_PARAMS_params.register_fields([('bIsGpuSelfWake', NvBool, 0), ('bIsRTD3Transition', NvBool, 1), ('bIsRTD3HotTransition', NvBool, 2)]) +struct_NV2080_CTRL_GC6_EXIT_PARAMS.register_fields([('flavorId', NV2080_CTRL_GC6_FLAVOR_ID, 0), ('params', struct_NV2080_CTRL_GC6_EXIT_PARAMS_params, 4)]) +NV2080_CTRL_GC6_EXIT_PARAMS: TypeAlias = struct_NV2080_CTRL_GC6_EXIT_PARAMS +enum_NV2080_CTRL_GC6_STEP_ID: dict[int, str] = {(NV2080_CTRL_GC6_STEP_ID_SR_ENTRY:=0): 'NV2080_CTRL_GC6_STEP_ID_SR_ENTRY', (NV2080_CTRL_GC6_STEP_ID_GPU_OFF:=1): 'NV2080_CTRL_GC6_STEP_ID_GPU_OFF', (NV2080_CTRL_GC6_STEP_ID_MAX:=2): 'NV2080_CTRL_GC6_STEP_ID_MAX'} +NV2080_CTRL_GC6_STEP_ID: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NV2080_CTRL_GC6_FLAVOR_INFO(c.Struct): + SIZE = 8 + flavorId: int + stepMask: int +struct_NV2080_CTRL_GC6_FLAVOR_INFO.register_fields([('flavorId', NV2080_CTRL_GC6_FLAVOR_ID, 0), ('stepMask', NvU32, 4)]) +NV2080_CTRL_GC6_FLAVOR_INFO: TypeAlias = struct_NV2080_CTRL_GC6_FLAVOR_INFO +@c.record +class struct_NV2080_CTRL_RC_READ_VIRTUAL_MEM_PARAMS(c.Struct): + SIZE = 32 + hChannel: int + virtAddress: int + bufferPtr: ctypes.c_void_p + bufferSize: int +struct_NV2080_CTRL_RC_READ_VIRTUAL_MEM_PARAMS.register_fields([('hChannel', NvHandle, 0), ('virtAddress', NvU64, 8), ('bufferPtr', NvP64, 16), ('bufferSize', NvU32, 24)]) +NV2080_CTRL_RC_READ_VIRTUAL_MEM_PARAMS: TypeAlias = struct_NV2080_CTRL_RC_READ_VIRTUAL_MEM_PARAMS +@c.record +class struct_NV2080_CTRL_RC_GET_ERROR_COUNT_PARAMS(c.Struct): + SIZE = 4 + errorCount: int +struct_NV2080_CTRL_RC_GET_ERROR_COUNT_PARAMS.register_fields([('errorCount', NvU32, 0)]) +NV2080_CTRL_RC_GET_ERROR_COUNT_PARAMS: TypeAlias = struct_NV2080_CTRL_RC_GET_ERROR_COUNT_PARAMS +@c.record +class struct_NV2080_CTRL_RC_GET_ERROR_V2_PARAMS(c.Struct): + SIZE = 8200 + whichBuffer: int + outputRecordSize: int + recordBuffer: c.Array[ctypes.c_ubyte, Literal[8192]] +struct_NV2080_CTRL_RC_GET_ERROR_V2_PARAMS.register_fields([('whichBuffer', NvU32, 0), ('outputRecordSize', NvU32, 4), ('recordBuffer', c.Array[NvU8, Literal[8192]], 8)]) +NV2080_CTRL_RC_GET_ERROR_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_RC_GET_ERROR_V2_PARAMS +@c.record +class struct_NV2080_CTRL_RC_GET_WATCHDOG_INFO_PARAMS(c.Struct): + SIZE = 4 + watchdogStatusFlags: int +struct_NV2080_CTRL_RC_GET_WATCHDOG_INFO_PARAMS.register_fields([('watchdogStatusFlags', NvU32, 0)]) +NV2080_CTRL_RC_GET_WATCHDOG_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_RC_GET_WATCHDOG_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_RC_RECOVERY_PARAMS(c.Struct): + SIZE = 4 + rcEnable: int +struct_NV2080_CTRL_CMD_RC_RECOVERY_PARAMS.register_fields([('rcEnable', NvU32, 0)]) +NV2080_CTRL_CMD_RC_RECOVERY_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_RECOVERY_PARAMS +NV2080_CTRL_SET_RC_RECOVERY_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_RECOVERY_PARAMS +NV2080_CTRL_GET_RC_RECOVERY_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_RECOVERY_PARAMS +@c.record +class struct_NV2080_CTRL_TDR_SET_TIMEOUT_STATE_PARAMS(c.Struct): + SIZE = 8 + cmd: int + status: int +struct_NV2080_CTRL_TDR_SET_TIMEOUT_STATE_PARAMS.register_fields([('cmd', NvU32, 0), ('status', NvS32, 4)]) +NV2080_CTRL_TDR_SET_TIMEOUT_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_TDR_SET_TIMEOUT_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_RC_INFO_PARAMS(c.Struct): + SIZE = 8 + rcMode: int + rcBreak: int +struct_NV2080_CTRL_CMD_RC_INFO_PARAMS.register_fields([('rcMode', NvU32, 0), ('rcBreak', NvU32, 4)]) +NV2080_CTRL_CMD_RC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_INFO_PARAMS +NV2080_CTRL_SET_RC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_INFO_PARAMS +NV2080_CTRL_GET_RC_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_RC_WATCHDOG_INFO_PARAMS(c.Struct): + SIZE = 24 + channelTestInterval: int + thwapChannelMask: int + stompChannelMask: int + thwapRepeatMask: int + stompRepeatMask: int + allocFailMask: int +struct_NV2080_CTRL_CMD_RC_WATCHDOG_INFO_PARAMS.register_fields([('channelTestInterval', NvU32, 0), ('thwapChannelMask', NvU32, 4), ('stompChannelMask', NvU32, 8), ('thwapRepeatMask', NvU32, 12), ('stompRepeatMask', NvU32, 16), ('allocFailMask', NvU32, 20)]) +NV2080_CTRL_CMD_RC_WATCHDOG_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_WATCHDOG_INFO_PARAMS +NV2080_CTRL_SET_RC_WATCHDOG_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_WATCHDOG_INFO_PARAMS +NV2080_CTRL_GET_RC_WATCHDOG_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_RC_WATCHDOG_INFO_PARAMS +@c.record +class struct_RM_GSP_SPDM_CMD_CC_INIT(c.Struct): + SIZE = 1 + cmdType: int +struct_RM_GSP_SPDM_CMD_CC_INIT.register_fields([('cmdType', NvU8, 0)]) +RM_GSP_SPDM_CMD_CC_INIT: TypeAlias = struct_RM_GSP_SPDM_CMD_CC_INIT +PRM_GSP_SPDM_CMD_CC_INIT: TypeAlias = c.POINTER[struct_RM_GSP_SPDM_CMD_CC_INIT] +@c.record +class struct_RM_GSP_SPDM_CMD_CC_DEINIT(c.Struct): + SIZE = 1 + cmdType: int +struct_RM_GSP_SPDM_CMD_CC_DEINIT.register_fields([('cmdType', NvU8, 0)]) +RM_GSP_SPDM_CMD_CC_DEINIT: TypeAlias = struct_RM_GSP_SPDM_CMD_CC_DEINIT +PRM_GSP_SPDM_CMD_CC_DEINIT: TypeAlias = c.POINTER[struct_RM_GSP_SPDM_CMD_CC_DEINIT] +@c.record +class struct_RM_GSP_SPDM_CMD_CC_CTRL(c.Struct): + SIZE = 1 + cmdType: int +struct_RM_GSP_SPDM_CMD_CC_CTRL.register_fields([('cmdType', NvU8, 0)]) +RM_GSP_SPDM_CMD_CC_CTRL: TypeAlias = struct_RM_GSP_SPDM_CMD_CC_CTRL +PRM_GSP_SPDM_CMD_CC_CTRL: TypeAlias = c.POINTER[struct_RM_GSP_SPDM_CMD_CC_CTRL] +@c.record +class struct_RM_GSP_SPDM_CMD_CC_INIT_RM_DATA(c.Struct): + SIZE = 68 + cmdType: int + rsvd0: c.Array[ctypes.c_uint32, Literal[2]] + rsvd1: int + rsvd2: c.Array[ctypes.c_char, Literal[9]] + rsvd3: c.Array[ctypes.c_char, Literal[5]] + rsvd4: c.Array[ctypes.c_char, Literal[5]] + rsvd5: c.Array[ctypes.c_char, Literal[5]] + rsvd6: c.Array[ctypes.c_char, Literal[2]] + rsvd7: c.Array[ctypes.c_char, Literal[16]] + rsvd8: c.Array[ctypes.c_uint32, Literal[2]] +struct_RM_GSP_SPDM_CMD_CC_INIT_RM_DATA.register_fields([('cmdType', NvU8, 0), ('rsvd0', c.Array[NvU32, Literal[2]], 4), ('rsvd1', NvU32, 12), ('rsvd2', c.Array[ctypes.c_char, Literal[9]], 16), ('rsvd3', c.Array[ctypes.c_char, Literal[5]], 25), ('rsvd4', c.Array[ctypes.c_char, Literal[5]], 30), ('rsvd5', c.Array[ctypes.c_char, Literal[5]], 35), ('rsvd6', c.Array[ctypes.c_char, Literal[2]], 40), ('rsvd7', c.Array[ctypes.c_char, Literal[16]], 42), ('rsvd8', c.Array[NvU32, Literal[2]], 60)]) +RM_GSP_SPDM_CMD_CC_INIT_RM_DATA: TypeAlias = struct_RM_GSP_SPDM_CMD_CC_INIT_RM_DATA +PRM_GSP_SPDM_CMD_CC_INIT_RM_DATA: TypeAlias = c.POINTER[struct_RM_GSP_SPDM_CMD_CC_INIT_RM_DATA] +@c.record +class struct_RM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL(c.Struct): + SIZE = 2 + cmdType: int + bEnable: int +struct_RM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL.register_fields([('cmdType', NvU8, 0), ('bEnable', NvBool, 1)]) +RM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL: TypeAlias = struct_RM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL +PRM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL: TypeAlias = c.POINTER[struct_RM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL] +@c.record +class struct_RM_GSP_SPDM_CMD_FIPS_SELFTEST(c.Struct): + SIZE = 160 + cmdType: int + isEnc: int + kmb: struct_CC_KMB + text: c.Array[ctypes.c_ubyte, Literal[16]] + authTag: c.Array[ctypes.c_ubyte, Literal[16]] +@c.record +class struct_CC_KMB(c.Struct): + SIZE = 124 + encryptBundle: struct_CC_AES_CRYPTOBUNDLE + hmacBundle: struct_CC_HMAC_CRYPTOBUNDLE + decryptBundle: struct_CC_AES_CRYPTOBUNDLE + bIsWorkLaunch: int +CC_KMB: TypeAlias = struct_CC_KMB +@c.record +class struct_CC_AES_CRYPTOBUNDLE(c.Struct): + SIZE = 56 + iv: c.Array[ctypes.c_uint32, Literal[3]] + key: c.Array[ctypes.c_uint32, Literal[8]] + ivMask: c.Array[ctypes.c_uint32, Literal[3]] +CC_AES_CRYPTOBUNDLE: TypeAlias = struct_CC_AES_CRYPTOBUNDLE +struct_CC_AES_CRYPTOBUNDLE.register_fields([('iv', c.Array[NvU32, Literal[3]], 0), ('key', c.Array[NvU32, Literal[8]], 12), ('ivMask', c.Array[NvU32, Literal[3]], 44)]) +@c.record +class struct_CC_HMAC_CRYPTOBUNDLE(c.Struct): + SIZE = 64 + nonce: c.Array[ctypes.c_uint32, Literal[8]] + key: c.Array[ctypes.c_uint32, Literal[8]] +CC_HMAC_CRYPTOBUNDLE: TypeAlias = struct_CC_HMAC_CRYPTOBUNDLE +struct_CC_HMAC_CRYPTOBUNDLE.register_fields([('nonce', c.Array[NvU32, Literal[8]], 0), ('key', c.Array[NvU32, Literal[8]], 32)]) +struct_CC_KMB.register_fields([('encryptBundle', CC_AES_CRYPTOBUNDLE, 0), ('hmacBundle', CC_HMAC_CRYPTOBUNDLE, 56), ('decryptBundle', CC_AES_CRYPTOBUNDLE, 56), ('bIsWorkLaunch', NvBool, 120)]) +struct_RM_GSP_SPDM_CMD_FIPS_SELFTEST.register_fields([('cmdType', NvU8, 0), ('isEnc', NvU8, 1), ('kmb', CC_KMB, 4), ('text', c.Array[NvU8, Literal[16]], 128), ('authTag', c.Array[NvU8, Literal[16]], 144)]) +RM_GSP_SPDM_CMD_FIPS_SELFTEST: TypeAlias = struct_RM_GSP_SPDM_CMD_FIPS_SELFTEST +PRM_GSP_SPDM_CMD_FIPS_SELFTEST: TypeAlias = c.POINTER[struct_RM_GSP_SPDM_CMD_FIPS_SELFTEST] +@c.record +class union_RM_GSP_SPDM_CMD(c.Struct): + SIZE = 160 + cmdType: int + ccInit: struct_RM_GSP_SPDM_CMD_CC_INIT + ccDeinit: struct_RM_GSP_SPDM_CMD_CC_DEINIT + ccCtrl: struct_RM_GSP_SPDM_CMD_CC_CTRL + rmDataInitCmd: struct_RM_GSP_SPDM_CMD_CC_INIT_RM_DATA + ccHeartbeatCtrl: struct_RM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL + ccFipsTest: struct_RM_GSP_SPDM_CMD_FIPS_SELFTEST +union_RM_GSP_SPDM_CMD.register_fields([('cmdType', NvU8, 0), ('ccInit', RM_GSP_SPDM_CMD_CC_INIT, 0), ('ccDeinit', RM_GSP_SPDM_CMD_CC_DEINIT, 0), ('ccCtrl', RM_GSP_SPDM_CMD_CC_CTRL, 0), ('rmDataInitCmd', RM_GSP_SPDM_CMD_CC_INIT_RM_DATA, 0), ('ccHeartbeatCtrl', RM_GSP_SPDM_CMD_CC_HEARTBEAT_CTRL, 0), ('ccFipsTest', RM_GSP_SPDM_CMD_FIPS_SELFTEST, 0)]) +RM_GSP_SPDM_CMD: TypeAlias = union_RM_GSP_SPDM_CMD +PRM_GSP_SPDM_CMD: TypeAlias = c.POINTER[union_RM_GSP_SPDM_CMD] +@c.record +class struct_RM_GSP_SPDM_MSG(c.Struct): + SIZE = 28 + msgType: int + status: int + rsvd1: int + rsvd2: int + rsvd3: int + rsvd4: int + rsvd5: int +struct_RM_GSP_SPDM_MSG.register_fields([('msgType', NvU8, 0), ('status', NvU32, 4), ('rsvd1', NvU32, 8), ('rsvd2', NvU32, 12), ('rsvd3', NvU32, 16), ('rsvd4', NvU32, 20), ('rsvd5', NvU32, 24)]) +RM_GSP_SPDM_MSG: TypeAlias = struct_RM_GSP_SPDM_MSG +PRM_GSP_SPDM_MSG: TypeAlias = c.POINTER[struct_RM_GSP_SPDM_MSG] +@c.record +class struct_NV2080_CTRL_INTERNAL_SPDM_PARTITION_PARAMS(c.Struct): + SIZE = 192 + index: int + cmd: union_RM_GSP_SPDM_CMD + msg: struct_RM_GSP_SPDM_MSG +struct_NV2080_CTRL_INTERNAL_SPDM_PARTITION_PARAMS.register_fields([('index', NvU8, 0), ('cmd', RM_GSP_SPDM_CMD, 4), ('msg', RM_GSP_SPDM_MSG, 164)]) +NV2080_CTRL_INTERNAL_SPDM_PARTITION_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_SPDM_PARTITION_PARAMS +@c.record +class struct_NV2080_CTRL_INTERNAL_SPDM_RETRIEVE_TRANSCRIPT_PARAMS(c.Struct): + SIZE = 9220 + transcript: c.Array[ctypes.c_ubyte, Literal[9216]] + transcriptSize: int +struct_NV2080_CTRL_INTERNAL_SPDM_RETRIEVE_TRANSCRIPT_PARAMS.register_fields([('transcript', c.Array[NvU8, Literal[9216]], 0), ('transcriptSize', NvU32, 9216)]) +NV2080_CTRL_INTERNAL_SPDM_RETRIEVE_TRANSCRIPT_PARAMS: TypeAlias = struct_NV2080_CTRL_INTERNAL_SPDM_RETRIEVE_TRANSCRIPT_PARAMS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGETS_AVAILABLE_OPERANDS(c.Struct): + SIZE = 4 + availableTargets: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGETS_AVAILABLE_OPERANDS.register_fields([('availableTargets', NvU32, 0)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGETS_AVAILABLE_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGETS_AVAILABLE_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGET_TYPE_OPERANDS(c.Struct): + SIZE = 8 + targetIndex: int + type: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGET_TYPE_OPERANDS.register_fields([('targetIndex', NvU32, 0), ('type', NvU32, 4)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGET_TYPE_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGET_TYPE_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_PROVIDER_TYPE_OPERANDS(c.Struct): + SIZE = 8 + providerIndex: int + type: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_PROVIDER_TYPE_OPERANDS.register_fields([('providerIndex', NvU32, 0), ('type', NvU32, 4)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_PROVIDER_TYPE_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_PROVIDER_TYPE_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSORS_AVAILABLE_OPERANDS(c.Struct): + SIZE = 4 + availableSensors: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSORS_AVAILABLE_OPERANDS.register_fields([('availableSensors', NvU32, 0)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSORS_AVAILABLE_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSORS_AVAILABLE_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_PROVIDER_OPERANDS(c.Struct): + SIZE = 8 + sensorIndex: int + providerIndex: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_PROVIDER_OPERANDS.register_fields([('sensorIndex', NvU32, 0), ('providerIndex', NvU32, 4)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_PROVIDER_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_PROVIDER_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_STATUS_SENSOR_READING_OPERANDS(c.Struct): + SIZE = 8 + sensorIndex: int + value: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_STATUS_SENSOR_READING_OPERANDS.register_fields([('sensorIndex', NvU32, 0), ('value', NvS32, 4)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_STATUS_SENSOR_READING_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_STATUS_SENSOR_READING_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_TARGET_OPERANDS(c.Struct): + SIZE = 8 + sensorIndex: int + targetIndex: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_TARGET_OPERANDS.register_fields([('sensorIndex', NvU32, 0), ('targetIndex', NvU32, 4)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_TARGET_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_TARGET_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_READING_RANGE_OPERANDS(c.Struct): + SIZE = 12 + sensorIndex: int + minimum: int + maximum: int +struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_READING_RANGE_OPERANDS.register_fields([('sensorIndex', NvU32, 0), ('minimum', NvS32, 4), ('maximum', NvS32, 8)]) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_READING_RANGE_OPERANDS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_READING_RANGE_OPERANDS +@c.record +class union_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION_OPERANDS(c.Struct): + SIZE = 32 + getInfoTargetsAvailable: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGETS_AVAILABLE_OPERANDS + getInfoTargetType: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGET_TYPE_OPERANDS + getInfoProviderType: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_PROVIDER_TYPE_OPERANDS + getInfoSensorsAvailable: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSORS_AVAILABLE_OPERANDS + getInfoSensorProvider: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_PROVIDER_OPERANDS + getInfoSensorTarget: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_TARGET_OPERANDS + getInfoSensorReadingRange: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_READING_RANGE_OPERANDS + getStatusSensorReading: struct_NV2080_CTRL_THERMAL_SYSTEM_GET_STATUS_SENSOR_READING_OPERANDS + space: c.Array[ctypes.c_uint32, Literal[8]] +union_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION_OPERANDS.register_fields([('getInfoTargetsAvailable', NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGETS_AVAILABLE_OPERANDS, 0), ('getInfoTargetType', NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGET_TYPE_OPERANDS, 0), ('getInfoProviderType', NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_PROVIDER_TYPE_OPERANDS, 0), ('getInfoSensorsAvailable', NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSORS_AVAILABLE_OPERANDS, 0), ('getInfoSensorProvider', NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_PROVIDER_OPERANDS, 0), ('getInfoSensorTarget', NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_TARGET_OPERANDS, 0), ('getInfoSensorReadingRange', NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_READING_RANGE_OPERANDS, 0), ('getStatusSensorReading', NV2080_CTRL_THERMAL_SYSTEM_GET_STATUS_SENSOR_READING_OPERANDS, 0), ('space', c.Array[NvU32, Literal[8]], 0)]) +NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION_OPERANDS: TypeAlias = union_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION_OPERANDS +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION(c.Struct): + SIZE = 44 + result: int + executed: int + opcode: int + operands: union_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION_OPERANDS +struct_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION.register_fields([('result', NvU32, 0), ('executed', NvU32, 4), ('opcode', NvU32, 8), ('operands', NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION_OPERANDS, 12)]) +NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION +@c.record +class struct_NV2080_CTRL_THERMAL_SYSTEM_EXECUTE_V2_PARAMS(c.Struct): + SIZE = 1432 + clientAPIVersion: int + clientAPIRevision: int + clientInstructionSizeOf: int + executeFlags: int + successfulInstructions: int + instructionListSize: int + instructionList: c.Array[struct_NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION, Literal[32]] +struct_NV2080_CTRL_THERMAL_SYSTEM_EXECUTE_V2_PARAMS.register_fields([('clientAPIVersion', NvU32, 0), ('clientAPIRevision', NvU32, 4), ('clientInstructionSizeOf', NvU32, 8), ('executeFlags', NvU32, 12), ('successfulInstructions', NvU32, 16), ('instructionListSize', NvU32, 20), ('instructionList', c.Array[NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION, Literal[32]], 24)]) +NV2080_CTRL_THERMAL_SYSTEM_EXECUTE_V2_PARAMS: TypeAlias = struct_NV2080_CTRL_THERMAL_SYSTEM_EXECUTE_V2_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS(c.Struct): + SIZE = 16 + time_nsec: int + flags: int +struct_NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS.register_fields([('time_nsec', NvU64, 0), ('flags', NvU32, 8)]) +NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS +@c.record +class struct_NV2080_CTRL_TIMER_GET_TIME_PARAMS(c.Struct): + SIZE = 8 + time_nsec: int +struct_NV2080_CTRL_TIMER_GET_TIME_PARAMS.register_fields([('time_nsec', NvU64, 0)]) +NV2080_CTRL_TIMER_GET_TIME_PARAMS: TypeAlias = struct_NV2080_CTRL_TIMER_GET_TIME_PARAMS +@c.record +class struct_NV2080_CTRL_TIMER_GET_REGISTER_OFFSET_PARAMS(c.Struct): + SIZE = 4 + tmr_offset: int +struct_NV2080_CTRL_TIMER_GET_REGISTER_OFFSET_PARAMS.register_fields([('tmr_offset', NvU32, 0)]) +NV2080_CTRL_TIMER_GET_REGISTER_OFFSET_PARAMS: TypeAlias = struct_NV2080_CTRL_TIMER_GET_REGISTER_OFFSET_PARAMS +@c.record +class struct_NV2080_CTRL_TIMER_GPU_CPU_TIME_SAMPLE(c.Struct): + SIZE = 16 + cpuTime: int + gpuTime: int +struct_NV2080_CTRL_TIMER_GPU_CPU_TIME_SAMPLE.register_fields([('cpuTime', NvU64, 0), ('gpuTime', NvU64, 8)]) +NV2080_CTRL_TIMER_GPU_CPU_TIME_SAMPLE: TypeAlias = struct_NV2080_CTRL_TIMER_GPU_CPU_TIME_SAMPLE +@c.record +class struct_NV2080_CTRL_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO_PARAMS(c.Struct): + SIZE = 264 + cpuClkId: int + sampleCount: int + samples: c.Array[struct_NV2080_CTRL_TIMER_GPU_CPU_TIME_SAMPLE, Literal[16]] +struct_NV2080_CTRL_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO_PARAMS.register_fields([('cpuClkId', NvU8, 0), ('sampleCount', NvU8, 1), ('samples', c.Array[NV2080_CTRL_TIMER_GPU_CPU_TIME_SAMPLE, Literal[16]], 8)]) +NV2080_CTRL_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ_PARAMS(c.Struct): + SIZE = 1 + bSetMaxFreq: int +struct_NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ_PARAMS.register_fields([('bSetMaxFreq', NvBool, 0)]) +NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ_PARAMS +@c.record +class struct_NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS(c.Struct): + SIZE = 4 + action: int +struct_NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS.register_fields([('action', NvU32, 0)]) +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS: TypeAlias = struct_NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS +@c.record +class struct_NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS(c.Struct): + SIZE = 4 + action: int +struct_NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS.register_fields([('action', NvU32, 0)]) +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS: TypeAlias = struct_NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS +@c.record +class struct_NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS(c.Struct): + SIZE = 1 + bEnter: int +struct_NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS.register_fields([('bEnter', NvBool, 0)]) +NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS: TypeAlias = struct_NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS +@c.record +class struct_NV2080_CTRL_OS_UNIX_INSTALL_PROFILER_HOOKS_PARAMS(c.Struct): + SIZE = 24 + hNotifierResource: int + notifyDataSize: int + hNotifyDataMemory: int + perfmonIdCount: int + snapshotBufferSize: int + hSnapshotMemory: int +struct_NV2080_CTRL_OS_UNIX_INSTALL_PROFILER_HOOKS_PARAMS.register_fields([('hNotifierResource', NvHandle, 0), ('notifyDataSize', NvU32, 4), ('hNotifyDataMemory', NvHandle, 8), ('perfmonIdCount', NvU32, 12), ('snapshotBufferSize', NvU32, 16), ('hSnapshotMemory', NvHandle, 20)]) +NV2080_CTRL_OS_UNIX_INSTALL_PROFILER_HOOKS_PARAMS: TypeAlias = struct_NV2080_CTRL_OS_UNIX_INSTALL_PROFILER_HOOKS_PARAMS +@c.record +class struct_NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS(c.Struct): + SIZE = 1 + bVidmemPersistent: int +struct_NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS.register_fields([('bVidmemPersistent', NvBool, 0)]) +NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS(c.Struct): + SIZE = 1 + bUpdateTGP: int +struct_NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS.register_fields([('bUpdateTGP', NvBool, 0)]) +NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS(c.Struct): + SIZE = 6616 + dbdf: int + gfid: int + vgpuType: int + vmPid: int + swizzId: int + numChannels: int + numPluginChannels: int + chidOffset: c.Array[ctypes.c_uint32, Literal[84]] + bDisableDefaultSmcExecPartRestore: int + numGuestFbSegments: int + guestFbPhysAddrList: c.Array[ctypes.c_uint64, Literal[384]] + guestFbLengthList: c.Array[ctypes.c_uint64, Literal[384]] + pluginHeapMemoryPhysAddr: int + pluginHeapMemoryLength: int + ctrlBuffOffset: int + initTaskLogBuffOffset: int + initTaskLogBuffSize: int + vgpuTaskLogBuffOffset: int + vgpuTaskLogBuffSize: int + kernelLogBuffOffset: int + kernelLogBuffSize: int + migRmHeapMemoryPhysAddr: int + migRmHeapMemoryLength: int + bDeviceProfilingEnabled: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS.register_fields([('dbdf', NvU32, 0), ('gfid', NvU32, 4), ('vgpuType', NvU32, 8), ('vmPid', NvU32, 12), ('swizzId', NvU32, 16), ('numChannels', NvU32, 20), ('numPluginChannels', NvU32, 24), ('chidOffset', c.Array[NvU32, Literal[84]], 28), ('bDisableDefaultSmcExecPartRestore', NvBool, 364), ('numGuestFbSegments', NvU32, 368), ('guestFbPhysAddrList', c.Array[NvU64, Literal[384]], 376), ('guestFbLengthList', c.Array[NvU64, Literal[384]], 3448), ('pluginHeapMemoryPhysAddr', NvU64, 6520), ('pluginHeapMemoryLength', NvU64, 6528), ('ctrlBuffOffset', NvU64, 6536), ('initTaskLogBuffOffset', NvU64, 6544), ('initTaskLogBuffSize', NvU64, 6552), ('vgpuTaskLogBuffOffset', NvU64, 6560), ('vgpuTaskLogBuffSize', NvU64, 6568), ('kernelLogBuffOffset', NvU64, 6576), ('kernelLogBuffSize', NvU64, 6584), ('migRmHeapMemoryPhysAddr', NvU64, 6592), ('migRmHeapMemoryLength', NvU64, 6600), ('bDeviceProfilingEnabled', NvBool, 6608)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS(c.Struct): + SIZE = 4 + gfid: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS.register_fields([('gfid', NvU32, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS(c.Struct): + SIZE = 695432 + gspBuildVersion: c.Array[ctypes.c_ubyte, Literal[128]] + discardVgpuTypes: int + vgpuInfoCount: int + vgpuInfo: c.Array[struct_NVA081_CTRL_VGPU_INFO, Literal[128]] +@c.record +class struct_NVA081_CTRL_VGPU_INFO(c.Struct): + SIZE = 5432 + vgpuType: int + vgpuName: c.Array[ctypes.c_ubyte, Literal[64]] + vgpuClass: c.Array[ctypes.c_ubyte, Literal[64]] + vgpuSignature: c.Array[ctypes.c_ubyte, Literal[128]] + license: c.Array[ctypes.c_ubyte, Literal[128]] + maxInstance: int + numHeads: int + maxResolutionX: int + maxResolutionY: int + maxPixels: int + frlConfig: int + cudaEnabled: int + eccSupported: int + gpuInstanceSize: int + nominalFbRegionSize: int + multiVgpuSupported: int + vdevId: int + pdevId: int + profileSize: int + fbLength: int + gspHeapSize: int + fbReservation: int + mappableVideoSize: int + encoderCapacity: int + bar1Length: int + frlEnable: int + vgpuSsvid: int + adapterName: c.Array[ctypes.c_ubyte, Literal[64]] + adapterName_Unicode: c.Array[ctypes.c_uint16, Literal[64]] + shortGpuNameString: c.Array[ctypes.c_ubyte, Literal[64]] + licensedProductName: c.Array[ctypes.c_ubyte, Literal[128]] + vgpuExtraParams: c.Array[ctypes.c_uint32, Literal[1024]] + ftraceEnable: int + gpuDirectSupported: int + nvlinkP2PSupported: int + maxInstancePerGI: int + pvmrlSchedulingBaseWeightDivisor: int + pvmrlSchedulingCap: int + multiVgpuExclusive: int + exclusiveType: int + exclusiveSize: int + gpuInstanceProfileId: int + placementSize: int + homogeneousPlacementCount: int + homogeneousPlacementIds: c.Array[ctypes.c_uint32, Literal[48]] + heterogeneousPlacementCount: int + heterogeneousPlacementIds: c.Array[ctypes.c_uint32, Literal[48]] + isSuperProfile: int +NVA081_CTRL_VGPU_INFO: TypeAlias = struct_NVA081_CTRL_VGPU_INFO +struct_NVA081_CTRL_VGPU_INFO.register_fields([('vgpuType', NvU32, 0), ('vgpuName', c.Array[NvU8, Literal[64]], 4), ('vgpuClass', c.Array[NvU8, Literal[64]], 68), ('vgpuSignature', c.Array[NvU8, Literal[128]], 132), ('license', c.Array[NvU8, Literal[128]], 260), ('maxInstance', NvU32, 388), ('numHeads', NvU32, 392), ('maxResolutionX', NvU32, 396), ('maxResolutionY', NvU32, 400), ('maxPixels', NvU32, 404), ('frlConfig', NvU32, 408), ('cudaEnabled', NvU32, 412), ('eccSupported', NvU32, 416), ('gpuInstanceSize', NvU32, 420), ('nominalFbRegionSize', NvU32, 424), ('multiVgpuSupported', NvU32, 428), ('vdevId', NvU64, 432), ('pdevId', NvU64, 440), ('profileSize', NvU64, 448), ('fbLength', NvU64, 456), ('gspHeapSize', NvU64, 464), ('fbReservation', NvU64, 472), ('mappableVideoSize', NvU64, 480), ('encoderCapacity', NvU32, 488), ('bar1Length', NvU64, 496), ('frlEnable', NvU32, 504), ('vgpuSsvid', NvU16, 508), ('adapterName', c.Array[NvU8, Literal[64]], 510), ('adapterName_Unicode', c.Array[NvU16, Literal[64]], 574), ('shortGpuNameString', c.Array[NvU8, Literal[64]], 702), ('licensedProductName', c.Array[NvU8, Literal[128]], 766), ('vgpuExtraParams', c.Array[NvU32, Literal[1024]], 896), ('ftraceEnable', NvU32, 4992), ('gpuDirectSupported', NvU32, 4996), ('nvlinkP2PSupported', NvU32, 5000), ('maxInstancePerGI', NvU32, 5004), ('pvmrlSchedulingBaseWeightDivisor', NvU32, 5008), ('pvmrlSchedulingCap', NvU32, 5012), ('multiVgpuExclusive', NvU32, 5016), ('exclusiveType', NvU32, 5020), ('exclusiveSize', NvU32, 5024), ('gpuInstanceProfileId', NvU32, 5028), ('placementSize', NvU32, 5032), ('homogeneousPlacementCount', NvU32, 5036), ('homogeneousPlacementIds', c.Array[NvU32, Literal[48]], 5040), ('heterogeneousPlacementCount', NvU32, 5232), ('heterogeneousPlacementIds', c.Array[NvU32, Literal[48]], 5236), ('isSuperProfile', NvU32, 5428)]) +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS.register_fields([('gspBuildVersion', c.Array[NvU8, Literal[128]], 0), ('discardVgpuTypes', NvBool, 128), ('vgpuInfoCount', NvU32, 132), ('vgpuInfo', c.Array[NVA081_CTRL_VGPU_INFO, Literal[128]], 136)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS +@c.record +class struct_NV2080_GUEST_VM_INFO(c.Struct): + SIZE = 168 + vmPid: int + guestOs: int + migrationProhibited: int + guestNegotiatedVgpuVersion: int + frameRateLimit: int + licensed: int + licenseState: int + licenseExpiryTimestamp: int + licenseExpiryStatus: int + guestDriverVersion: c.Array[ctypes.c_ubyte, Literal[64]] + guestDriverBranch: c.Array[ctypes.c_ubyte, Literal[64]] + guestVmInfoState: int +enum_GUEST_VM_INFO_STATE: dict[int, str] = {(GUEST_VM_INFO_STATE_UNINITIALIZED:=0): 'GUEST_VM_INFO_STATE_UNINITIALIZED', (GUEST_VM_INFO_STATE_INITIALIZED:=1): 'GUEST_VM_INFO_STATE_INITIALIZED'} +GUEST_VM_INFO_STATE: TypeAlias = ctypes.c_uint32 +struct_NV2080_GUEST_VM_INFO.register_fields([('vmPid', NvU32, 0), ('guestOs', NvU32, 4), ('migrationProhibited', NvU32, 8), ('guestNegotiatedVgpuVersion', NvU32, 12), ('frameRateLimit', NvU32, 16), ('licensed', NvBool, 20), ('licenseState', NvU32, 24), ('licenseExpiryTimestamp', NvU32, 28), ('licenseExpiryStatus', NvU8, 32), ('guestDriverVersion', c.Array[NvU8, Literal[64]], 33), ('guestDriverBranch', c.Array[NvU8, Literal[64]], 97), ('guestVmInfoState', GUEST_VM_INFO_STATE, 164)]) +NV2080_GUEST_VM_INFO: TypeAlias = struct_NV2080_GUEST_VM_INFO +@c.record +class struct_NV2080_HOST_VGPU_DEVICE(c.Struct): + SIZE = 48 + gfid: int + vgpuPciId: int + vgpuDeviceInstanceId: int + accountingPid: int + fbUsed: int + encoderCapacity: int + eccState: int + bDriverLoaded: int +struct_NV2080_HOST_VGPU_DEVICE.register_fields([('gfid', NvU32, 0), ('vgpuPciId', NvU64, 8), ('vgpuDeviceInstanceId', NvU32, 16), ('accountingPid', NvU32, 20), ('fbUsed', NvU64, 24), ('encoderCapacity', NvU32, 32), ('eccState', NvU32, 36), ('bDriverLoaded', NvBool, 40)]) +NV2080_HOST_VGPU_DEVICE: TypeAlias = struct_NV2080_HOST_VGPU_DEVICE +@c.record +class struct_NV2080_VGPU_GUEST(c.Struct): + SIZE = 216 + guestVmInfo: struct_NV2080_GUEST_VM_INFO + vgpuDevice: struct_NV2080_HOST_VGPU_DEVICE +struct_NV2080_VGPU_GUEST.register_fields([('guestVmInfo', NV2080_GUEST_VM_INFO, 0), ('vgpuDevice', NV2080_HOST_VGPU_DEVICE, 168)]) +NV2080_VGPU_GUEST: TypeAlias = struct_NV2080_VGPU_GUEST +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS(c.Struct): + SIZE = 10376 + numVgpu: int + vgpuGuest: c.Array[struct_NV2080_VGPU_GUEST, Literal[48]] +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS.register_fields([('numVgpu', NvU32, 0), ('vgpuGuest', c.Array[NV2080_VGPU_GUEST, Literal[48]], 8)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS(c.Struct): + SIZE = 4 + gfid: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS.register_fields([('gfid', NvU32, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS +@c.record +class struct_NV2080_VGPU_FB_USAGE(c.Struct): + SIZE = 16 + gfid: int + fbUsed: int +struct_NV2080_VGPU_FB_USAGE.register_fields([('gfid', NvU32, 0), ('fbUsed', NvU64, 8)]) +NV2080_VGPU_FB_USAGE: TypeAlias = struct_NV2080_VGPU_FB_USAGE +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS(c.Struct): + SIZE = 776 + vgpuCount: int + vgpuFbUsage: c.Array[struct_NV2080_VGPU_FB_USAGE, Literal[48]] +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS.register_fields([('vgpuCount', NvU32, 0), ('vgpuFbUsage', c.Array[NV2080_VGPU_FB_USAGE, Literal[48]], 8)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS(c.Struct): + SIZE = 8 + gfid: int + encoderCapacity: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS.register_fields([('gfid', NvU32, 0), ('encoderCapacity', NvU32, 4)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS(c.Struct): + SIZE = 4 + gfid: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS.register_fields([('gfid', NvU32, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_FS_ENCODING_PARAMS(c.Struct): + SIZE = 260 + pgpuString: c.Array[ctypes.c_ubyte, Literal[256]] + pgpuStringSize: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_FS_ENCODING_PARAMS.register_fields([('pgpuString', c.Array[NvU8, Literal[256]], 0), ('pgpuStringSize', NvU32, 256)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_FS_ENCODING_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_FS_ENCODING_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_MIGRATION_SUPPORT_PARAMS(c.Struct): + SIZE = 1 + bIsMigrationSupported: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_MIGRATION_SUPPORT_PARAMS.register_fields([('bIsMigrationSupported', NvBool, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_MIGRATION_SUPPORT_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_MIGRATION_SUPPORT_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MGR_CONFIG_PARAMS(c.Struct): + SIZE = 1 + bSupportHeterogeneousTimeSlicedVgpuTypes: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MGR_CONFIG_PARAMS.register_fields([('bSupportHeterogeneousTimeSlicedVgpuTypes', NvBool, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MGR_CONFIG_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MGR_CONFIG_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_STATES_PARAMS(c.Struct): + SIZE = 8 + gfid: int + flags: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_STATES_PARAMS.register_fields([('gfid', NvU32, 0), ('flags', NvU32, 4)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_STATES_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_STATES_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_GET_FRAME_RATE_LIMITER_STATUS_PARAMS(c.Struct): + SIZE = 1 + bFlrDisabled: int +struct_NV2080_CTRL_VGPU_MGR_GET_FRAME_RATE_LIMITER_STATUS_PARAMS.register_fields([('bFlrDisabled', NvBool, 0)]) +NV2080_CTRL_VGPU_MGR_GET_FRAME_RATE_LIMITER_STATUS_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_GET_FRAME_RATE_LIMITER_STATUS_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_HETEROGENEOUS_MODE_PARAMS(c.Struct): + SIZE = 1 + bHeterogeneousMode: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_HETEROGENEOUS_MODE_PARAMS.register_fields([('bHeterogeneousMode', NvBool, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_HETEROGENEOUS_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_HETEROGENEOUS_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MIG_TIMESLICE_MODE_PARAMS(c.Struct): + SIZE = 1 + bMigTimeslicingModeEnabled: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MIG_TIMESLICE_MODE_PARAMS.register_fields([('bMigTimeslicingModeEnabled', NvBool, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MIG_TIMESLICE_MODE_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MIG_TIMESLICE_MODE_PARAMS +@c.record +class struct_NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_POWER_STATE_PARAMS(c.Struct): + SIZE = 4 + state: int +struct_NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_POWER_STATE_PARAMS.register_fields([('state', NvU32, 0)]) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_POWER_STATE_PARAMS: TypeAlias = struct_NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_POWER_STATE_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SCRUB_GUEST_FB_PARAMS(c.Struct): + SIZE = 32 + fbOffset: int + fbSize: int + submittedWorkId: int + gfid: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SCRUB_GUEST_FB_PARAMS.register_fields([('fbOffset', NvU64, 0), ('fbSize', NvU64, 8), ('submittedWorkId', NvU64, 16), ('gfid', NvU32, 24)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_SCRUB_GUEST_FB_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_SCRUB_GUEST_FB_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_CHECK_SCRUB_COMPLETE_PARAMS(c.Struct): + SIZE = 16 + submittedWorkId: int + gfid: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_CHECK_SCRUB_COMPLETE_PARAMS.register_fields([('submittedWorkId', NvU64, 0), ('gfid', NvU32, 8)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_CHECK_SCRUB_COMPLETE_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_CHECK_SCRUB_COMPLETE_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_ALLOC_GSP_CEUTILS_PARAMS(c.Struct): + SIZE = 32 + gfid: int + fixedChId: int + forceCeId: int + swizzId: int + semaPhysAddr: int + semaAperture: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_ALLOC_GSP_CEUTILS_PARAMS.register_fields([('gfid', NvU32, 0), ('fixedChId', NvU32, 4), ('forceCeId', NvU32, 8), ('swizzId', NvU32, 12), ('semaPhysAddr', NvU64, 16), ('semaAperture', NvU32, 24)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_ALLOC_GSP_CEUTILS_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_ALLOC_GSP_CEUTILS_PARAMS +@c.record +class struct_NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_GSP_CEUTILS_PARAMS(c.Struct): + SIZE = 4 + gfid: int +struct_NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_GSP_CEUTILS_PARAMS.register_fields([('gfid', NvU32, 0)]) +NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_GSP_CEUTILS_PARAMS: TypeAlias = struct_NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_GSP_CEUTILS_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_MODE_MMU_DEBUG_PARAMS(c.Struct): + SIZE = 4 + action: int +struct_NV83DE_CTRL_DEBUG_SET_MODE_MMU_DEBUG_PARAMS.register_fields([('action', NvU32, 0)]) +NV83DE_CTRL_DEBUG_SET_MODE_MMU_DEBUG_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_MODE_MMU_DEBUG_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_GET_MODE_MMU_DEBUG_PARAMS(c.Struct): + SIZE = 4 + value: int +struct_NV83DE_CTRL_DEBUG_GET_MODE_MMU_DEBUG_PARAMS.register_fields([('value', NvU32, 0)]) +NV83DE_CTRL_DEBUG_GET_MODE_MMU_DEBUG_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_GET_MODE_MMU_DEBUG_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PARAMS(c.Struct): + SIZE = 4 + exceptionMask: int +struct_NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PARAMS.register_fields([('exceptionMask', NvU32, 0)]) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PARAMS +@c.record +class struct_NV83DE_SM_ERROR_STATE_REGISTERS(c.Struct): + SIZE = 48 + hwwGlobalEsr: int + hwwWarpEsr: int + hwwWarpEsrPc: int + hwwGlobalEsrReportMask: int + hwwWarpEsrReportMask: int + hwwEsrAddr: int + hwwWarpEsrPc64: int + hwwCgaEsr: int + hwwCgaEsrReportMask: int +struct_NV83DE_SM_ERROR_STATE_REGISTERS.register_fields([('hwwGlobalEsr', NvU32, 0), ('hwwWarpEsr', NvU32, 4), ('hwwWarpEsrPc', NvU32, 8), ('hwwGlobalEsrReportMask', NvU32, 12), ('hwwWarpEsrReportMask', NvU32, 16), ('hwwEsrAddr', NvU64, 24), ('hwwWarpEsrPc64', NvU64, 32), ('hwwCgaEsr', NvU32, 40), ('hwwCgaEsrReportMask', NvU32, 44)]) +NV83DE_SM_ERROR_STATE_REGISTERS: TypeAlias = struct_NV83DE_SM_ERROR_STATE_REGISTERS +@c.record +class struct_NV83DE_CTRL_DEBUG_READ_SINGLE_SM_ERROR_STATE_PARAMS(c.Struct): + SIZE = 56 + hTargetChannel: int + smID: int + smErrorState: struct_NV83DE_SM_ERROR_STATE_REGISTERS +struct_NV83DE_CTRL_DEBUG_READ_SINGLE_SM_ERROR_STATE_PARAMS.register_fields([('hTargetChannel', NvHandle, 0), ('smID', NvU32, 4), ('smErrorState', NV83DE_SM_ERROR_STATE_REGISTERS, 8)]) +NV83DE_CTRL_DEBUG_READ_SINGLE_SM_ERROR_STATE_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_READ_SINGLE_SM_ERROR_STATE_PARAMS +@c.record +class struct_NV83DE_MMU_FAULT_INFO(c.Struct): + SIZE = 8 + valid: int + faultInfo: int +struct_NV83DE_MMU_FAULT_INFO.register_fields([('valid', NvBool, 0), ('faultInfo', NvU32, 4)]) +NV83DE_MMU_FAULT_INFO: TypeAlias = struct_NV83DE_MMU_FAULT_INFO +@c.record +class struct_NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS(c.Struct): + SIZE = 4824 + hTargetChannel: int + numSMsToRead: int + smErrorStateArray: c.Array[struct_NV83DE_SM_ERROR_STATE_REGISTERS, Literal[100]] + mmuFaultInfo: int + mmuFault: struct_NV83DE_MMU_FAULT_INFO + startingSM: int +struct_NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS.register_fields([('hTargetChannel', NvHandle, 0), ('numSMsToRead', NvU32, 4), ('smErrorStateArray', c.Array[NV83DE_SM_ERROR_STATE_REGISTERS, Literal[100]], 8), ('mmuFaultInfo', NvU32, 4808), ('mmuFault', NV83DE_MMU_FAULT_INFO, 4812), ('startingSM', NvU32, 4820)]) +NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_CLEAR_SINGLE_SM_ERROR_STATE_PARAMS(c.Struct): + SIZE = 8 + hTargetChannel: int + smID: int +struct_NV83DE_CTRL_DEBUG_CLEAR_SINGLE_SM_ERROR_STATE_PARAMS.register_fields([('hTargetChannel', NvHandle, 0), ('smID', NvU32, 4)]) +NV83DE_CTRL_DEBUG_CLEAR_SINGLE_SM_ERROR_STATE_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_CLEAR_SINGLE_SM_ERROR_STATE_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS(c.Struct): + SIZE = 8 + hTargetChannel: int + numSMsToClear: int +struct_NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS.register_fields([('hTargetChannel', NvHandle, 0), ('numSMsToClear', NvU32, 4)]) +NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS +@c.record +class struct_NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_PARAMS(c.Struct): + SIZE = 8 + waitForEvent: int + hResidentChannel: int +struct_NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_PARAMS.register_fields([('waitForEvent', NvU32, 0), ('hResidentChannel', NvHandle, 4)]) +NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_PARAMS: TypeAlias = struct_NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE_PARAMS(c.Struct): + SIZE = 4 + stopTriggerType: int +struct_NV83DE_CTRL_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE_PARAMS.register_fields([('stopTriggerType', NvU32, 0)]) +NV83DE_CTRL_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING_PARAMS(c.Struct): + SIZE = 4 + singleStepHandling: int +struct_NV83DE_CTRL_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING_PARAMS.register_fields([('singleStepHandling', NvU32, 0)]) +NV83DE_CTRL_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS(c.Struct): + SIZE = 24 + hMemory: int + length: int + offset: int + buffer: ctypes.c_void_p +struct_NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS.register_fields([('hMemory', NvU32, 0), ('length', NvU32, 4), ('offset', NvU64, 8), ('buffer', NvP64, 16)]) +NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_WRITE_MEMORY_PARAMS(c.Struct): + SIZE = 24 + hMemory: int + length: int + offset: int + buffer: ctypes.c_void_p +struct_NV83DE_CTRL_DEBUG_WRITE_MEMORY_PARAMS.register_fields([('hMemory', NvU32, 0), ('length', NvU32, 4), ('offset', NvU64, 8), ('buffer', NvP64, 16)]) +NV83DE_CTRL_DEBUG_WRITE_MEMORY_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_WRITE_MEMORY_PARAMS +NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT_PARAMS: TypeAlias = struct_NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_GET_HANDLES_PARAMS(c.Struct): + SIZE = 8 + hChannel: int + hSubdevice: int +struct_NV83DE_CTRL_DEBUG_GET_HANDLES_PARAMS.register_fields([('hChannel', NvHandle, 0), ('hSubdevice', NvHandle, 4)]) +NV83DE_CTRL_DEBUG_GET_HANDLES_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_GET_HANDLES_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_ACCESS_OP(c.Struct): + SIZE = 24 + gpuVA: int + pCpuVA: ctypes.c_void_p + size: int + valid: int +struct_NV83DE_CTRL_DEBUG_ACCESS_OP.register_fields([('gpuVA', NvU64, 0), ('pCpuVA', NvP64, 8), ('size', NvU32, 16), ('valid', NvU32, 20)]) +NV83DE_CTRL_DEBUG_ACCESS_OP: TypeAlias = struct_NV83DE_CTRL_DEBUG_ACCESS_OP +@c.record +class struct_NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS(c.Struct): + SIZE = 1544 + count: int + opsBuffer: c.Array[struct_NV83DE_CTRL_DEBUG_ACCESS_OP, Literal[64]] +struct_NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS.register_fields([('count', NvU32, 0), ('opsBuffer', c.Array[NV83DE_CTRL_DEBUG_ACCESS_OP, Literal[64]], 8)]) +NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS: TypeAlias = struct_NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS +@c.record +class struct_NV83DE_CTRL_DEBUG_GET_MAPPINGS_OP(c.Struct): + SIZE = 16 + gpuVA: int + size: int +struct_NV83DE_CTRL_DEBUG_GET_MAPPINGS_OP.register_fields([('gpuVA', NvU64, 0), ('size', NvU32, 8)]) +NV83DE_CTRL_DEBUG_GET_MAPPINGS_OP: TypeAlias = struct_NV83DE_CTRL_DEBUG_GET_MAPPINGS_OP +@c.record +class struct_NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS(c.Struct): + SIZE = 1048 + vaLo: int + vaHi: int + count: int + hasMore: int + opsBuffer: c.Array[struct_NV83DE_CTRL_DEBUG_GET_MAPPINGS_OP, Literal[64]] +struct_NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS.register_fields([('vaLo', NvU64, 0), ('vaHi', NvU64, 8), ('count', NvU32, 16), ('hasMore', NvU32, 20), ('opsBuffer', c.Array[NV83DE_CTRL_DEBUG_GET_MAPPINGS_OP, Literal[64]], 24)]) +NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS: TypeAlias = struct_NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS +@c.record +class struct_NV83DE_CTRL_DEBUG_EXEC_REG_OPS_PARAMS(c.Struct): + SIZE = 3208 + bNonTransactional: int + regOpCount: int + regOps: c.Array[struct_NV2080_CTRL_GPU_REG_OP, Literal[100]] +struct_NV83DE_CTRL_DEBUG_EXEC_REG_OPS_PARAMS.register_fields([('bNonTransactional', NvBool, 0), ('regOpCount', NvU32, 4), ('regOps', c.Array[NV2080_CTRL_GPU_REG_OP, Literal[100]], 8)]) +NV83DE_CTRL_DEBUG_EXEC_REG_OPS_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_EXEC_REG_OPS_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_MODE_ERRBAR_DEBUG_PARAMS(c.Struct): + SIZE = 4 + action: int +struct_NV83DE_CTRL_DEBUG_SET_MODE_ERRBAR_DEBUG_PARAMS.register_fields([('action', NvU32, 0)]) +NV83DE_CTRL_DEBUG_SET_MODE_ERRBAR_DEBUG_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_MODE_ERRBAR_DEBUG_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_GET_MODE_ERRBAR_DEBUG_PARAMS(c.Struct): + SIZE = 4 + value: int +struct_NV83DE_CTRL_DEBUG_GET_MODE_ERRBAR_DEBUG_PARAMS.register_fields([('value', NvU32, 0)]) +NV83DE_CTRL_DEBUG_GET_MODE_ERRBAR_DEBUG_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_GET_MODE_ERRBAR_DEBUG_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SINGLE_STEP_PARAMS(c.Struct): + SIZE = 8 + smID: int + bSingleStep: int +struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SINGLE_STEP_PARAMS.register_fields([('smID', NvU32, 0), ('bSingleStep', NvBool, 4)]) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SINGLE_STEP_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SINGLE_STEP_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_STOP_TRIGGER_PARAMS(c.Struct): + SIZE = 8 + smID: int + bStopTrigger: int +struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_STOP_TRIGGER_PARAMS.register_fields([('smID', NvU32, 0), ('bStopTrigger', NvBool, 4)]) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_STOP_TRIGGER_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_STOP_TRIGGER_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_RUN_TRIGGER_PARAMS(c.Struct): + SIZE = 8 + smID: int + bRunTrigger: int +struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_RUN_TRIGGER_PARAMS.register_fields([('smID', NvU32, 0), ('bRunTrigger', NvBool, 4)]) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_RUN_TRIGGER_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_RUN_TRIGGER_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SKIP_IDLE_WARP_DETECT_PARAMS(c.Struct): + SIZE = 8 + smID: int + bSkipIdleWarpDetect: int +struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SKIP_IDLE_WARP_DETECT_PARAMS.register_fields([('smID', NvU32, 0), ('bSkipIdleWarpDetect', NvBool, 4)]) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SKIP_IDLE_WARP_DETECT_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SKIP_IDLE_WARP_DETECT_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SINGLE_SM_DEBUGGER_STATUS(c.Struct): + SIZE = 5 + bInTrapMode: int + bCrsFlushDone: int + bRunTriggerInProgress: int + bComputeContext: int + bLockedDown: int +struct_NV83DE_CTRL_DEBUG_SINGLE_SM_DEBUGGER_STATUS.register_fields([('bInTrapMode', NvBool, 0), ('bCrsFlushDone', NvBool, 1), ('bRunTriggerInProgress', NvBool, 2), ('bComputeContext', NvBool, 3), ('bLockedDown', NvBool, 4)]) +NV83DE_CTRL_DEBUG_SINGLE_SM_DEBUGGER_STATUS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SINGLE_SM_DEBUGGER_STATUS +@c.record +class struct_NV83DE_CTRL_DEBUG_GET_SINGLE_SM_DEBUGGER_STATUS_PARAMS(c.Struct): + SIZE = 12 + smID: int + smDebuggerStatus: struct_NV83DE_CTRL_DEBUG_SINGLE_SM_DEBUGGER_STATUS +struct_NV83DE_CTRL_DEBUG_GET_SINGLE_SM_DEBUGGER_STATUS_PARAMS.register_fields([('smID', NvU32, 0), ('smDebuggerStatus', NV83DE_CTRL_DEBUG_SINGLE_SM_DEBUGGER_STATUS, 4)]) +NV83DE_CTRL_DEBUG_GET_SINGLE_SM_DEBUGGER_STATUS_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_GET_SINGLE_SM_DEBUGGER_STATUS_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_ENTRY(c.Struct): + SIZE = 24 + hMemory: int + length: int + memOffset: int + dataOffset: int + status: int +struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_ENTRY.register_fields([('hMemory', NvHandle, 0), ('length', NvU32, 4), ('memOffset', NvU64, 8), ('dataOffset', NvU32, 16), ('status', NV_STATUS, 20)]) +NV83DE_CTRL_DEBUG_ACCESS_MEMORY_ENTRY: TypeAlias = struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_ENTRY +@c.record +class struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS(c.Struct): + SIZE = 3616 + pData: ctypes.c_void_p + dataLength: int + count: int + entries: c.Array[struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_ENTRY, Literal[150]] +struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS.register_fields([('pData', NvP64, 0), ('dataLength', NvU32, 8), ('count', NvU32, 12), ('entries', c.Array[NV83DE_CTRL_DEBUG_ACCESS_MEMORY_ENTRY, Literal[150]], 16)]) +NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_ENTRY(c.Struct): + SIZE = 16 + faultAddress: int + faultType: int + accessType: int +struct_NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_ENTRY.register_fields([('faultAddress', NvU64, 0), ('faultType', NvU32, 8), ('accessType', NvU32, 12)]) +NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_ENTRY: TypeAlias = struct_NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_ENTRY +@c.record +class struct_NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS(c.Struct): + SIZE = 72 + mmuFaultInfoList: c.Array[struct_NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_ENTRY, Literal[4]] + count: int +struct_NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS.register_fields([('mmuFaultInfoList', c.Array[NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_ENTRY, Literal[4]], 0), ('count', NvU32, 64)]) +NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_DROP_DEFERRED_RC_PARAMS(c.Struct): + SIZE = 1 + bDropDeferredRc: int +struct_NV83DE_CTRL_DEBUG_SET_DROP_DEFERRED_RC_PARAMS.register_fields([('bDropDeferredRc', NvBool, 0)]) +NV83DE_CTRL_DEBUG_SET_DROP_DEFERRED_RC_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_DROP_DEFERRED_RC_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_MODE_MMU_GCC_DEBUG_PARAMS(c.Struct): + SIZE = 4 + action: int +struct_NV83DE_CTRL_DEBUG_SET_MODE_MMU_GCC_DEBUG_PARAMS.register_fields([('action', NvU32, 0)]) +NV83DE_CTRL_DEBUG_SET_MODE_MMU_GCC_DEBUG_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_MODE_MMU_GCC_DEBUG_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_GET_MODE_MMU_GCC_DEBUG_PARAMS(c.Struct): + SIZE = 4 + value: int +struct_NV83DE_CTRL_DEBUG_GET_MODE_MMU_GCC_DEBUG_PARAMS.register_fields([('value', NvU32, 0)]) +NV83DE_CTRL_DEBUG_GET_MODE_MMU_GCC_DEBUG_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_GET_MODE_MMU_GCC_DEBUG_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT_PARAMS(c.Struct): + SIZE = 1 + state: int +struct_NV83DE_CTRL_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT_PARAMS.register_fields([('state', NvBool, 0)]) +NV83DE_CTRL_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT_PARAMS +@c.record +class struct_NV83DE_CTRL_DEBUG_SET_RECOVERY_SUPPRESSION_TIMEOUT_PARAMS(c.Struct): + SIZE = 4 + timeoutSec: int +struct_NV83DE_CTRL_DEBUG_SET_RECOVERY_SUPPRESSION_TIMEOUT_PARAMS.register_fields([('timeoutSec', NvU32, 0)]) +NV83DE_CTRL_DEBUG_SET_RECOVERY_SUPPRESSION_TIMEOUT_PARAMS: TypeAlias = struct_NV83DE_CTRL_DEBUG_SET_RECOVERY_SUPPRESSION_TIMEOUT_PARAMS +@c.record +class struct_NVB0CC_CTRL_CMD_INTERNAL_ALLOC_PMA_STREAM_FINN_PARAMS(c.Struct): + SIZE = 56 + params: struct_NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS +@c.record +class struct_NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS(c.Struct): + SIZE = 56 + hMemPmaBuffer: int + pmaBufferOffset: int + pmaBufferSize: int + hMemPmaBytesAvailable: int + pmaBytesAvailableOffset: int + ctxsw: int + pmaChannelIdx: int + pmaBufferVA: int +NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS: TypeAlias = struct_NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS +struct_NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS.register_fields([('hMemPmaBuffer', NvHandle, 0), ('pmaBufferOffset', NvU64, 8), ('pmaBufferSize', NvU64, 16), ('hMemPmaBytesAvailable', NvHandle, 24), ('pmaBytesAvailableOffset', NvU64, 32), ('ctxsw', NvBool, 40), ('pmaChannelIdx', NvU32, 44), ('pmaBufferVA', NvU64, 48)]) +struct_NVB0CC_CTRL_CMD_INTERNAL_ALLOC_PMA_STREAM_FINN_PARAMS.register_fields([('params', NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS, 0)]) +NVB0CC_CTRL_CMD_INTERNAL_ALLOC_PMA_STREAM_FINN_PARAMS: TypeAlias = struct_NVB0CC_CTRL_CMD_INTERNAL_ALLOC_PMA_STREAM_FINN_PARAMS +@c.record +class struct_NVB0CC_CTRL_INTERNAL_QUIESCE_PMA_CHANNEL_PARAMS(c.Struct): + SIZE = 8 + pmaChannelIdx: int + bMembytesPollingRequired: int +struct_NVB0CC_CTRL_INTERNAL_QUIESCE_PMA_CHANNEL_PARAMS.register_fields([('pmaChannelIdx', NvU32, 0), ('bMembytesPollingRequired', NvBool, 4)]) +NVB0CC_CTRL_INTERNAL_QUIESCE_PMA_CHANNEL_PARAMS: TypeAlias = struct_NVB0CC_CTRL_INTERNAL_QUIESCE_PMA_CHANNEL_PARAMS +@c.record +class struct_NVB0CC_CTRL_INTERNAL_SRIOV_PROMOTE_PMA_STREAM_PARAMS(c.Struct): + SIZE = 48 + pmaChannelIdx: int + pmaBufferVA: int + pmaBufferSize: int + membytesVA: int + hwpmIBPA: int + hwpmIBAperture: int +struct_NVB0CC_CTRL_INTERNAL_SRIOV_PROMOTE_PMA_STREAM_PARAMS.register_fields([('pmaChannelIdx', NvU32, 0), ('pmaBufferVA', NvU64, 8), ('pmaBufferSize', NvU64, 16), ('membytesVA', NvU64, 24), ('hwpmIBPA', NvU64, 32), ('hwpmIBAperture', NvU8, 40)]) +NVB0CC_CTRL_INTERNAL_SRIOV_PROMOTE_PMA_STREAM_PARAMS: TypeAlias = struct_NVB0CC_CTRL_INTERNAL_SRIOV_PROMOTE_PMA_STREAM_PARAMS +@c.record +class struct_NVB0CC_CTRL_INTERNAL_PERMISSIONS_INIT_PARAMS(c.Struct): + SIZE = 6 + bDevProfilingPermitted: int + bDevTracePermitted: int + bCtxProfilingPermitted: int + bVideoMemoryProfilingPermitted: int + bSysMemoryProfilingPermitted: int + bAsyncCeProfilingPermitted: int +struct_NVB0CC_CTRL_INTERNAL_PERMISSIONS_INIT_PARAMS.register_fields([('bDevProfilingPermitted', NvBool, 0), ('bDevTracePermitted', NvBool, 1), ('bCtxProfilingPermitted', NvBool, 2), ('bVideoMemoryProfilingPermitted', NvBool, 3), ('bSysMemoryProfilingPermitted', NvBool, 4), ('bAsyncCeProfilingPermitted', NvBool, 5)]) +NVB0CC_CTRL_INTERNAL_PERMISSIONS_INIT_PARAMS: TypeAlias = struct_NVB0CC_CTRL_INTERNAL_PERMISSIONS_INIT_PARAMS +@c.record +class struct_NVB0CC_CTRL_INTERNAL_ALLOC_PMA_STREAM_PARAMS(c.Struct): + SIZE = 64 + hMemPmaBuffer: int + pmaBufferOffset: int + pmaBufferSize: int + hMemPmaBytesAvailable: int + pmaBytesAvailableOffset: int + ctxsw: int + pmaChannelIdx: int + pmaBufferVA: int + bInputPmaChIdx: int +struct_NVB0CC_CTRL_INTERNAL_ALLOC_PMA_STREAM_PARAMS.register_fields([('hMemPmaBuffer', NvHandle, 0), ('pmaBufferOffset', NvU64, 8), ('pmaBufferSize', NvU64, 16), ('hMemPmaBytesAvailable', NvHandle, 24), ('pmaBytesAvailableOffset', NvU64, 32), ('ctxsw', NvBool, 40), ('pmaChannelIdx', NvU32, 44), ('pmaBufferVA', NvU64, 48), ('bInputPmaChIdx', NvBool, 56)]) +NVB0CC_CTRL_INTERNAL_ALLOC_PMA_STREAM_PARAMS: TypeAlias = struct_NVB0CC_CTRL_INTERNAL_ALLOC_PMA_STREAM_PARAMS +@c.record +class struct_NVB0CC_CTRL_INTERNAL_FREE_PMA_STREAM_PARAMS(c.Struct): + SIZE = 4 + pmaChannelIdx: int +struct_NVB0CC_CTRL_INTERNAL_FREE_PMA_STREAM_PARAMS.register_fields([('pmaChannelIdx', NvU32, 0)]) +NVB0CC_CTRL_INTERNAL_FREE_PMA_STREAM_PARAMS: TypeAlias = struct_NVB0CC_CTRL_INTERNAL_FREE_PMA_STREAM_PARAMS +@c.record +class struct_NVB0CC_CTRL_INTERNAL_GET_MAX_PMAS_PARAMS(c.Struct): + SIZE = 4 + maxPmaChannels: int +struct_NVB0CC_CTRL_INTERNAL_GET_MAX_PMAS_PARAMS.register_fields([('maxPmaChannels', NvU32, 0)]) +NVB0CC_CTRL_INTERNAL_GET_MAX_PMAS_PARAMS: TypeAlias = struct_NVB0CC_CTRL_INTERNAL_GET_MAX_PMAS_PARAMS +@c.record +class struct_NVB0CC_CTRL_INTERNAL_RESERVE_HWPM_LEGACY_PARAMS(c.Struct): + SIZE = 1 + ctxsw: int +struct_NVB0CC_CTRL_INTERNAL_RESERVE_HWPM_LEGACY_PARAMS.register_fields([('ctxsw', NvBool, 0)]) +NVB0CC_CTRL_INTERNAL_RESERVE_HWPM_LEGACY_PARAMS: TypeAlias = struct_NVB0CC_CTRL_INTERNAL_RESERVE_HWPM_LEGACY_PARAMS +@c.record +class struct_NVB0CC_CTRL_POWER_REQUEST_FEATURES_PARAMS(c.Struct): + SIZE = 12 + globalStatus: int + controlMask: int + statusMask: int +struct_NVB0CC_CTRL_POWER_REQUEST_FEATURES_PARAMS.register_fields([('globalStatus', NvU32, 0), ('controlMask', NvU32, 4), ('statusMask', NvU32, 8)]) +NVB0CC_CTRL_POWER_REQUEST_FEATURES_PARAMS: TypeAlias = struct_NVB0CC_CTRL_POWER_REQUEST_FEATURES_PARAMS +@c.record +class struct_NVB0CC_CTRL_POWER_RELEASE_FEATURES_PARAMS(c.Struct): + SIZE = 4 + controlMask: int +struct_NVB0CC_CTRL_POWER_RELEASE_FEATURES_PARAMS.register_fields([('controlMask', NvU32, 0)]) +NVB0CC_CTRL_POWER_RELEASE_FEATURES_PARAMS: TypeAlias = struct_NVB0CC_CTRL_POWER_RELEASE_FEATURES_PARAMS +@c.record +class struct_NVB0CC_CTRL_RESERVE_HWPM_LEGACY_PARAMS(c.Struct): + SIZE = 1 + ctxsw: int +struct_NVB0CC_CTRL_RESERVE_HWPM_LEGACY_PARAMS.register_fields([('ctxsw', NvBool, 0)]) +NVB0CC_CTRL_RESERVE_HWPM_LEGACY_PARAMS: TypeAlias = struct_NVB0CC_CTRL_RESERVE_HWPM_LEGACY_PARAMS +@c.record +class struct_NVB0CC_CTRL_RESERVE_PM_AREA_SMPC_PARAMS(c.Struct): + SIZE = 1 + ctxsw: int +struct_NVB0CC_CTRL_RESERVE_PM_AREA_SMPC_PARAMS.register_fields([('ctxsw', NvBool, 0)]) +NVB0CC_CTRL_RESERVE_PM_AREA_SMPC_PARAMS: TypeAlias = struct_NVB0CC_CTRL_RESERVE_PM_AREA_SMPC_PARAMS +@c.record +class struct_NVB0CC_CTRL_FREE_PMA_STREAM_PARAMS(c.Struct): + SIZE = 4 + pmaChannelIdx: int +struct_NVB0CC_CTRL_FREE_PMA_STREAM_PARAMS.register_fields([('pmaChannelIdx', NvU32, 0)]) +NVB0CC_CTRL_FREE_PMA_STREAM_PARAMS: TypeAlias = struct_NVB0CC_CTRL_FREE_PMA_STREAM_PARAMS +@c.record +class struct_NVB0CC_CTRL_PMA_STREAM_UPDATE_GET_PUT_PARAMS(c.Struct): + SIZE = 48 + bytesConsumed: int + bUpdateAvailableBytes: int + bWait: int + bytesAvailable: int + bReturnPut: int + putPtr: int + pmaChannelIdx: int + bOverflowStatus: int +struct_NVB0CC_CTRL_PMA_STREAM_UPDATE_GET_PUT_PARAMS.register_fields([('bytesConsumed', NvU64, 0), ('bUpdateAvailableBytes', NvBool, 8), ('bWait', NvBool, 9), ('bytesAvailable', NvU64, 16), ('bReturnPut', NvBool, 24), ('putPtr', NvU64, 32), ('pmaChannelIdx', NvU32, 40), ('bOverflowStatus', NvBool, 44)]) +NVB0CC_CTRL_PMA_STREAM_UPDATE_GET_PUT_PARAMS: TypeAlias = struct_NVB0CC_CTRL_PMA_STREAM_UPDATE_GET_PUT_PARAMS +NVB0CC_GPU_REG_OP: TypeAlias = struct_NV2080_CTRL_GPU_REG_OP +enum_NVB0CC_REGOPS_MODE: dict[int, str] = {(NVB0CC_REGOPS_MODE_ALL_OR_NONE:=0): 'NVB0CC_REGOPS_MODE_ALL_OR_NONE', (NVB0CC_REGOPS_MODE_CONTINUE_ON_ERROR:=1): 'NVB0CC_REGOPS_MODE_CONTINUE_ON_ERROR'} +NVB0CC_REGOPS_MODE: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NVB0CC_CTRL_EXEC_REG_OPS_PARAMS(c.Struct): + SIZE = 3980 + regOpCount: int + mode: int + bPassed: int + bDirect: int + regOps: c.Array[struct_NV2080_CTRL_GPU_REG_OP, Literal[124]] +struct_NVB0CC_CTRL_EXEC_REG_OPS_PARAMS.register_fields([('regOpCount', NvU32, 0), ('mode', NVB0CC_REGOPS_MODE, 4), ('bPassed', NvBool, 8), ('bDirect', NvBool, 9), ('regOps', c.Array[NVB0CC_GPU_REG_OP, Literal[124]], 12)]) +NVB0CC_CTRL_EXEC_REG_OPS_PARAMS: TypeAlias = struct_NVB0CC_CTRL_EXEC_REG_OPS_PARAMS +@c.record +class struct_NVB0CC_CTRL_GET_TOTAL_HS_CREDITS_PARAMS(c.Struct): + SIZE = 4 + numCredits: int +struct_NVB0CC_CTRL_GET_TOTAL_HS_CREDITS_PARAMS.register_fields([('numCredits', NvU32, 0)]) +NVB0CC_CTRL_GET_TOTAL_HS_CREDITS_PARAMS: TypeAlias = struct_NVB0CC_CTRL_GET_TOTAL_HS_CREDITS_PARAMS +enum_NVB0CC_CHIPLET_TYPE: dict[int, str] = {(NVB0CC_CHIPLET_TYPE_INVALID:=0): 'NVB0CC_CHIPLET_TYPE_INVALID', (NVB0CC_CHIPLET_TYPE_FBP:=1): 'NVB0CC_CHIPLET_TYPE_FBP', (NVB0CC_CHIPLET_TYPE_GPC:=2): 'NVB0CC_CHIPLET_TYPE_GPC', (NVB0CC_CHIPLET_TYPE_SYS:=3): 'NVB0CC_CHIPLET_TYPE_SYS', (NVB0CC_CHIPLET_TYPE_IODN:=4): 'NVB0CC_CHIPLET_TYPE_IODN', (NVB0CC_CHIPLET_TYPE_IODS:=5): 'NVB0CC_CHIPLET_TYPE_IODS'} +NVB0CC_CHIPLET_TYPE: TypeAlias = ctypes.c_uint32 +enum_NVB0CC_HS_CREDITS_CMD_STATUS: dict[int, str] = {(NVB0CC_HS_CREDITS_CMD_STATUS_OK:=0): 'NVB0CC_HS_CREDITS_CMD_STATUS_OK', (NVB0CC_HS_CREDITS_CMD_STATUS_INVALID_CREDITS:=1): 'NVB0CC_HS_CREDITS_CMD_STATUS_INVALID_CREDITS', (NVB0CC_HS_CREDITS_CMD_STATUS_INVALID_CHIPLET:=2): 'NVB0CC_HS_CREDITS_CMD_STATUS_INVALID_CHIPLET'} +NVB0CC_HS_CREDITS_CMD_STATUS: TypeAlias = ctypes.c_uint32 +@c.record +class struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_INFO(c.Struct): + SIZE = 4 + chipletType: int + chipletIndex: int + numCredits: int +struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_INFO.register_fields([('chipletType', NvU8, 0), ('chipletIndex', NvU8, 1), ('numCredits', NvU16, 2)]) +NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_INFO: TypeAlias = struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_INFO +@c.record +class struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS(c.Struct): + SIZE = 2 + status: int + entryIndex: int +struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS.register_fields([('status', NvU8, 0), ('entryIndex', NvU8, 1)]) +NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS: TypeAlias = struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS +@c.record +class struct_NVB0CC_CTRL_HS_CREDITS_PARAMS(c.Struct): + SIZE = 256 + pmaChannelIdx: int + numEntries: int + statusInfo: struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS + creditInfo: c.Array[struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_INFO, Literal[63]] +struct_NVB0CC_CTRL_HS_CREDITS_PARAMS.register_fields([('pmaChannelIdx', NvU8, 0), ('numEntries', NvU8, 1), ('statusInfo', NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS, 2), ('creditInfo', c.Array[NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_INFO, Literal[63]], 4)]) +NVB0CC_CTRL_HS_CREDITS_PARAMS: TypeAlias = struct_NVB0CC_CTRL_HS_CREDITS_PARAMS +NVB0CC_CTRL_SET_HS_CREDITS_PARAMS: TypeAlias = struct_NVB0CC_CTRL_HS_CREDITS_PARAMS +NVB0CC_CTRL_GET_HS_CREDITS_PARAMS: TypeAlias = struct_NVB0CC_CTRL_HS_CREDITS_PARAMS +@c.record +class struct_NVB0CC_CTRL_RESERVE_HES_CWD_PARAMS(c.Struct): + SIZE = 1 + ctxsw: int +struct_NVB0CC_CTRL_RESERVE_HES_CWD_PARAMS.register_fields([('ctxsw', NvBool, 0)]) +NVB0CC_CTRL_RESERVE_HES_CWD_PARAMS: TypeAlias = struct_NVB0CC_CTRL_RESERVE_HES_CWD_PARAMS +@c.record +class struct_NVB0CC_CTRL_RESERVE_HES_TPC_PARAMS(c.Struct): + SIZE = 1 + ctxsw: int +struct_NVB0CC_CTRL_RESERVE_HES_TPC_PARAMS.register_fields([('ctxsw', NvBool, 0)]) +NVB0CC_CTRL_RESERVE_HES_TPC_PARAMS: TypeAlias = struct_NVB0CC_CTRL_RESERVE_HES_TPC_PARAMS +@c.record +class union_NVB0CC_CTRL_HES_RESERVATION_UNION(c.Struct): + SIZE = 1 + cwd: struct_NVB0CC_CTRL_RESERVE_HES_CWD_PARAMS + tpc: struct_NVB0CC_CTRL_RESERVE_HES_TPC_PARAMS +union_NVB0CC_CTRL_HES_RESERVATION_UNION.register_fields([('cwd', NVB0CC_CTRL_RESERVE_HES_CWD_PARAMS, 0), ('tpc', NVB0CC_CTRL_RESERVE_HES_TPC_PARAMS, 0)]) +NVB0CC_CTRL_HES_RESERVATION_UNION: TypeAlias = union_NVB0CC_CTRL_HES_RESERVATION_UNION +@c.record +class struct_NVB0CC_CTRL_RESERVE_HES_PARAMS(c.Struct): + SIZE = 8 + type: int + reserveParams: union_NVB0CC_CTRL_HES_RESERVATION_UNION +struct_NVB0CC_CTRL_RESERVE_HES_PARAMS.register_fields([('type', NvU32, 0), ('reserveParams', NVB0CC_CTRL_HES_RESERVATION_UNION, 4)]) +NVB0CC_CTRL_RESERVE_HES_PARAMS: TypeAlias = struct_NVB0CC_CTRL_RESERVE_HES_PARAMS +@c.record +class struct_NVB0CC_CTRL_RELEASE_HES_PARAMS(c.Struct): + SIZE = 4 + type: int +struct_NVB0CC_CTRL_RELEASE_HES_PARAMS.register_fields([('type', NvU32, 0)]) +NVB0CC_CTRL_RELEASE_HES_PARAMS: TypeAlias = struct_NVB0CC_CTRL_RELEASE_HES_PARAMS +@c.record +class struct_NVB0CC_CTRL_CREDIT_POOL_INFO(c.Struct): + SIZE = 4 + numCredits: int + poolIndex: int + chipletType: int +struct_NVB0CC_CTRL_CREDIT_POOL_INFO.register_fields([('numCredits', NvU16, 0), ('poolIndex', NvU8, 2), ('chipletType', NvU8, 3)]) +NVB0CC_CTRL_CREDIT_POOL_INFO: TypeAlias = struct_NVB0CC_CTRL_CREDIT_POOL_INFO +@c.record +class struct_NVB0CC_CTRL_GET_CHIPLET_HS_CREDIT_POOL(c.Struct): + SIZE = 124 + poolInfos: c.Array[struct_NVB0CC_CTRL_CREDIT_POOL_INFO, Literal[30]] + poolInfosCount: int +struct_NVB0CC_CTRL_GET_CHIPLET_HS_CREDIT_POOL.register_fields([('poolInfos', c.Array[NVB0CC_CTRL_CREDIT_POOL_INFO, Literal[30]], 0), ('poolInfosCount', NvU32, 120)]) +NVB0CC_CTRL_GET_CHIPLET_HS_CREDIT_POOL: TypeAlias = struct_NVB0CC_CTRL_GET_CHIPLET_HS_CREDIT_POOL +@c.record +class struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_MAPPING_INFO(c.Struct): + SIZE = 3 + chipletType: int + chipletIndex: int + poolIndex: int +struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_MAPPING_INFO.register_fields([('chipletType', NvU8, 0), ('chipletIndex', NvU8, 1), ('poolIndex', NvU8, 2)]) +NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_MAPPING_INFO: TypeAlias = struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_MAPPING_INFO +@c.record +class struct_NVB0CC_CTRL_GET_HS_CREDITS_POOL_MAPPING_PARAMS(c.Struct): + SIZE = 194 + numQueries: int + statusInfo: struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS + queries: c.Array[struct_NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_MAPPING_INFO, Literal[63]] +struct_NVB0CC_CTRL_GET_HS_CREDITS_POOL_MAPPING_PARAMS.register_fields([('numQueries', NvU16, 0), ('statusInfo', NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_STATUS, 2), ('queries', c.Array[NVB0CC_CTRL_PMA_STREAM_HS_CREDITS_MAPPING_INFO, Literal[63]], 4)]) +NVB0CC_CTRL_GET_HS_CREDITS_POOL_MAPPING_PARAMS: TypeAlias = struct_NVB0CC_CTRL_GET_HS_CREDITS_POOL_MAPPING_PARAMS +@c.record +class struct_NVB0CC_CTRL_DISABLE_DYNAMIC_MMA_BOOST_PARAMS(c.Struct): + SIZE = 1 + disable: int +struct_NVB0CC_CTRL_DISABLE_DYNAMIC_MMA_BOOST_PARAMS.register_fields([('disable', NvBool, 0)]) +NVB0CC_CTRL_DISABLE_DYNAMIC_MMA_BOOST_PARAMS: TypeAlias = struct_NVB0CC_CTRL_DISABLE_DYNAMIC_MMA_BOOST_PARAMS +@c.record +class struct_NVB0CC_CTRL_GET_DYNAMIC_MMA_BOOST_STATUS_PARAMS(c.Struct): + SIZE = 1 + enabled: int +struct_NVB0CC_CTRL_GET_DYNAMIC_MMA_BOOST_STATUS_PARAMS.register_fields([('enabled', NvBool, 0)]) +NVB0CC_CTRL_GET_DYNAMIC_MMA_BOOST_STATUS_PARAMS: TypeAlias = struct_NVB0CC_CTRL_GET_DYNAMIC_MMA_BOOST_STATUS_PARAMS +@c.record +class struct_NVB0CC_CTRL_RESERVE_CCUPROF_PARAMS(c.Struct): + SIZE = 1 + ctxsw: int +struct_NVB0CC_CTRL_RESERVE_CCUPROF_PARAMS.register_fields([('ctxsw', NvBool, 0)]) +NVB0CC_CTRL_RESERVE_CCUPROF_PARAMS: TypeAlias = struct_NVB0CC_CTRL_RESERVE_CCUPROF_PARAMS +nv_status_codes: dict[int, str] = {(NV_OK:=0): 'NV_OK', (NV_ERR_GENERIC:=65535): 'NV_ERR_GENERIC', (NV_ERR_BROKEN_FB:=1): 'NV_ERR_BROKEN_FB', (NV_ERR_BUFFER_TOO_SMALL:=2): 'NV_ERR_BUFFER_TOO_SMALL', (NV_ERR_BUSY_RETRY:=3): 'NV_ERR_BUSY_RETRY', (NV_ERR_CALLBACK_NOT_SCHEDULED:=4): 'NV_ERR_CALLBACK_NOT_SCHEDULED', (NV_ERR_CARD_NOT_PRESENT:=5): 'NV_ERR_CARD_NOT_PRESENT', (NV_ERR_CYCLE_DETECTED:=6): 'NV_ERR_CYCLE_DETECTED', (NV_ERR_DMA_IN_USE:=7): 'NV_ERR_DMA_IN_USE', (NV_ERR_DMA_MEM_NOT_LOCKED:=8): 'NV_ERR_DMA_MEM_NOT_LOCKED', (NV_ERR_DMA_MEM_NOT_UNLOCKED:=9): 'NV_ERR_DMA_MEM_NOT_UNLOCKED', (NV_ERR_DUAL_LINK_INUSE:=10): 'NV_ERR_DUAL_LINK_INUSE', (NV_ERR_ECC_ERROR:=11): 'NV_ERR_ECC_ERROR', (NV_ERR_FIFO_BAD_ACCESS:=12): 'NV_ERR_FIFO_BAD_ACCESS', (NV_ERR_FREQ_NOT_SUPPORTED:=13): 'NV_ERR_FREQ_NOT_SUPPORTED', (NV_ERR_GPU_DMA_NOT_INITIALIZED:=14): 'NV_ERR_GPU_DMA_NOT_INITIALIZED', (NV_ERR_GPU_IS_LOST:=15): 'NV_ERR_GPU_IS_LOST', (NV_ERR_GPU_IN_FULLCHIP_RESET:=16): 'NV_ERR_GPU_IN_FULLCHIP_RESET', (NV_ERR_GPU_NOT_FULL_POWER:=17): 'NV_ERR_GPU_NOT_FULL_POWER', (NV_ERR_GPU_UUID_NOT_FOUND:=18): 'NV_ERR_GPU_UUID_NOT_FOUND', (NV_ERR_HOT_SWITCH:=19): 'NV_ERR_HOT_SWITCH', (NV_ERR_I2C_ERROR:=20): 'NV_ERR_I2C_ERROR', (NV_ERR_I2C_SPEED_TOO_HIGH:=21): 'NV_ERR_I2C_SPEED_TOO_HIGH', (NV_ERR_ILLEGAL_ACTION:=22): 'NV_ERR_ILLEGAL_ACTION', (NV_ERR_IN_USE:=23): 'NV_ERR_IN_USE', (NV_ERR_INFLATE_COMPRESSED_DATA_FAILED:=24): 'NV_ERR_INFLATE_COMPRESSED_DATA_FAILED', (NV_ERR_INSERT_DUPLICATE_NAME:=25): 'NV_ERR_INSERT_DUPLICATE_NAME', (NV_ERR_INSUFFICIENT_RESOURCES:=26): 'NV_ERR_INSUFFICIENT_RESOURCES', (NV_ERR_INSUFFICIENT_PERMISSIONS:=27): 'NV_ERR_INSUFFICIENT_PERMISSIONS', (NV_ERR_INSUFFICIENT_POWER:=28): 'NV_ERR_INSUFFICIENT_POWER', (NV_ERR_INVALID_ACCESS_TYPE:=29): 'NV_ERR_INVALID_ACCESS_TYPE', (NV_ERR_INVALID_ADDRESS:=30): 'NV_ERR_INVALID_ADDRESS', (NV_ERR_INVALID_ARGUMENT:=31): 'NV_ERR_INVALID_ARGUMENT', (NV_ERR_INVALID_BASE:=32): 'NV_ERR_INVALID_BASE', (NV_ERR_INVALID_CHANNEL:=33): 'NV_ERR_INVALID_CHANNEL', (NV_ERR_INVALID_CLASS:=34): 'NV_ERR_INVALID_CLASS', (NV_ERR_INVALID_CLIENT:=35): 'NV_ERR_INVALID_CLIENT', (NV_ERR_INVALID_COMMAND:=36): 'NV_ERR_INVALID_COMMAND', (NV_ERR_INVALID_DATA:=37): 'NV_ERR_INVALID_DATA', (NV_ERR_INVALID_DEVICE:=38): 'NV_ERR_INVALID_DEVICE', (NV_ERR_INVALID_DMA_SPECIFIER:=39): 'NV_ERR_INVALID_DMA_SPECIFIER', (NV_ERR_INVALID_EVENT:=40): 'NV_ERR_INVALID_EVENT', (NV_ERR_INVALID_FLAGS:=41): 'NV_ERR_INVALID_FLAGS', (NV_ERR_INVALID_FUNCTION:=42): 'NV_ERR_INVALID_FUNCTION', (NV_ERR_INVALID_HEAP:=43): 'NV_ERR_INVALID_HEAP', (NV_ERR_INVALID_INDEX:=44): 'NV_ERR_INVALID_INDEX', (NV_ERR_INVALID_IRQ_LEVEL:=45): 'NV_ERR_INVALID_IRQ_LEVEL', (NV_ERR_INVALID_LIMIT:=46): 'NV_ERR_INVALID_LIMIT', (NV_ERR_INVALID_LOCK_STATE:=47): 'NV_ERR_INVALID_LOCK_STATE', (NV_ERR_INVALID_METHOD:=48): 'NV_ERR_INVALID_METHOD', (NV_ERR_INVALID_OBJECT:=49): 'NV_ERR_INVALID_OBJECT', (NV_ERR_INVALID_OBJECT_BUFFER:=50): 'NV_ERR_INVALID_OBJECT_BUFFER', (NV_ERR_INVALID_OBJECT_HANDLE:=51): 'NV_ERR_INVALID_OBJECT_HANDLE', (NV_ERR_INVALID_OBJECT_NEW:=52): 'NV_ERR_INVALID_OBJECT_NEW', (NV_ERR_INVALID_OBJECT_OLD:=53): 'NV_ERR_INVALID_OBJECT_OLD', (NV_ERR_INVALID_OBJECT_PARENT:=54): 'NV_ERR_INVALID_OBJECT_PARENT', (NV_ERR_INVALID_OFFSET:=55): 'NV_ERR_INVALID_OFFSET', (NV_ERR_INVALID_OPERATION:=56): 'NV_ERR_INVALID_OPERATION', (NV_ERR_INVALID_OWNER:=57): 'NV_ERR_INVALID_OWNER', (NV_ERR_INVALID_PARAM_STRUCT:=58): 'NV_ERR_INVALID_PARAM_STRUCT', (NV_ERR_INVALID_PARAMETER:=59): 'NV_ERR_INVALID_PARAMETER', (NV_ERR_INVALID_PATH:=60): 'NV_ERR_INVALID_PATH', (NV_ERR_INVALID_POINTER:=61): 'NV_ERR_INVALID_POINTER', (NV_ERR_INVALID_REGISTRY_KEY:=62): 'NV_ERR_INVALID_REGISTRY_KEY', (NV_ERR_INVALID_REQUEST:=63): 'NV_ERR_INVALID_REQUEST', (NV_ERR_INVALID_STATE:=64): 'NV_ERR_INVALID_STATE', (NV_ERR_INVALID_STRING_LENGTH:=65): 'NV_ERR_INVALID_STRING_LENGTH', (NV_ERR_INVALID_READ:=66): 'NV_ERR_INVALID_READ', (NV_ERR_INVALID_WRITE:=67): 'NV_ERR_INVALID_WRITE', (NV_ERR_INVALID_XLATE:=68): 'NV_ERR_INVALID_XLATE', (NV_ERR_IRQ_NOT_FIRING:=69): 'NV_ERR_IRQ_NOT_FIRING', (NV_ERR_IRQ_EDGE_TRIGGERED:=70): 'NV_ERR_IRQ_EDGE_TRIGGERED', (NV_ERR_MEMORY_TRAINING_FAILED:=71): 'NV_ERR_MEMORY_TRAINING_FAILED', (NV_ERR_MISMATCHED_SLAVE:=72): 'NV_ERR_MISMATCHED_SLAVE', (NV_ERR_MISMATCHED_TARGET:=73): 'NV_ERR_MISMATCHED_TARGET', (NV_ERR_MISSING_TABLE_ENTRY:=74): 'NV_ERR_MISSING_TABLE_ENTRY', (NV_ERR_MODULE_LOAD_FAILED:=75): 'NV_ERR_MODULE_LOAD_FAILED', (NV_ERR_MORE_DATA_AVAILABLE:=76): 'NV_ERR_MORE_DATA_AVAILABLE', (NV_ERR_MORE_PROCESSING_REQUIRED:=77): 'NV_ERR_MORE_PROCESSING_REQUIRED', (NV_ERR_MULTIPLE_MEMORY_TYPES:=78): 'NV_ERR_MULTIPLE_MEMORY_TYPES', (NV_ERR_NO_FREE_FIFOS:=79): 'NV_ERR_NO_FREE_FIFOS', (NV_ERR_NO_INTR_PENDING:=80): 'NV_ERR_NO_INTR_PENDING', (NV_ERR_NO_MEMORY:=81): 'NV_ERR_NO_MEMORY', (NV_ERR_NO_SUCH_DOMAIN:=82): 'NV_ERR_NO_SUCH_DOMAIN', (NV_ERR_NO_VALID_PATH:=83): 'NV_ERR_NO_VALID_PATH', (NV_ERR_NOT_COMPATIBLE:=84): 'NV_ERR_NOT_COMPATIBLE', (NV_ERR_NOT_READY:=85): 'NV_ERR_NOT_READY', (NV_ERR_NOT_SUPPORTED:=86): 'NV_ERR_NOT_SUPPORTED', (NV_ERR_OBJECT_NOT_FOUND:=87): 'NV_ERR_OBJECT_NOT_FOUND', (NV_ERR_OBJECT_TYPE_MISMATCH:=88): 'NV_ERR_OBJECT_TYPE_MISMATCH', (NV_ERR_OPERATING_SYSTEM:=89): 'NV_ERR_OPERATING_SYSTEM', (NV_ERR_OTHER_DEVICE_FOUND:=90): 'NV_ERR_OTHER_DEVICE_FOUND', (NV_ERR_OUT_OF_RANGE:=91): 'NV_ERR_OUT_OF_RANGE', (NV_ERR_OVERLAPPING_UVM_COMMIT:=92): 'NV_ERR_OVERLAPPING_UVM_COMMIT', (NV_ERR_PAGE_TABLE_NOT_AVAIL:=93): 'NV_ERR_PAGE_TABLE_NOT_AVAIL', (NV_ERR_PID_NOT_FOUND:=94): 'NV_ERR_PID_NOT_FOUND', (NV_ERR_PROTECTION_FAULT:=95): 'NV_ERR_PROTECTION_FAULT', (NV_ERR_RC_ERROR:=96): 'NV_ERR_RC_ERROR', (NV_ERR_REJECTED_VBIOS:=97): 'NV_ERR_REJECTED_VBIOS', (NV_ERR_RESET_REQUIRED:=98): 'NV_ERR_RESET_REQUIRED', (NV_ERR_STATE_IN_USE:=99): 'NV_ERR_STATE_IN_USE', (NV_ERR_SIGNAL_PENDING:=100): 'NV_ERR_SIGNAL_PENDING', (NV_ERR_TIMEOUT:=101): 'NV_ERR_TIMEOUT', (NV_ERR_TIMEOUT_RETRY:=102): 'NV_ERR_TIMEOUT_RETRY', (NV_ERR_TOO_MANY_PRIMARIES:=103): 'NV_ERR_TOO_MANY_PRIMARIES', (NV_ERR_UVM_ADDRESS_IN_USE:=104): 'NV_ERR_UVM_ADDRESS_IN_USE', (NV_ERR_MAX_SESSION_LIMIT_REACHED:=105): 'NV_ERR_MAX_SESSION_LIMIT_REACHED', (NV_ERR_LIB_RM_VERSION_MISMATCH:=106): 'NV_ERR_LIB_RM_VERSION_MISMATCH', (NV_ERR_PRIV_SEC_VIOLATION:=107): 'NV_ERR_PRIV_SEC_VIOLATION', (NV_ERR_GPU_IN_DEBUG_MODE:=108): 'NV_ERR_GPU_IN_DEBUG_MODE', (NV_ERR_FEATURE_NOT_ENABLED:=109): 'NV_ERR_FEATURE_NOT_ENABLED', (NV_ERR_RESOURCE_LOST:=110): 'NV_ERR_RESOURCE_LOST', (NV_ERR_PMU_NOT_READY:=111): 'NV_ERR_PMU_NOT_READY', (NV_ERR_FLCN_ERROR:=112): 'NV_ERR_FLCN_ERROR', (NV_ERR_FATAL_ERROR:=113): 'NV_ERR_FATAL_ERROR', (NV_ERR_MEMORY_ERROR:=114): 'NV_ERR_MEMORY_ERROR', (NV_ERR_INVALID_LICENSE:=115): 'NV_ERR_INVALID_LICENSE', (NV_ERR_NVLINK_INIT_ERROR:=116): 'NV_ERR_NVLINK_INIT_ERROR', (NV_ERR_NVLINK_MINION_ERROR:=117): 'NV_ERR_NVLINK_MINION_ERROR', (NV_ERR_NVLINK_CLOCK_ERROR:=118): 'NV_ERR_NVLINK_CLOCK_ERROR', (NV_ERR_NVLINK_TRAINING_ERROR:=119): 'NV_ERR_NVLINK_TRAINING_ERROR', (NV_ERR_NVLINK_CONFIGURATION_ERROR:=120): 'NV_ERR_NVLINK_CONFIGURATION_ERROR', (NV_ERR_RISCV_ERROR:=121): 'NV_ERR_RISCV_ERROR', (NV_ERR_FABRIC_MANAGER_NOT_PRESENT:=122): 'NV_ERR_FABRIC_MANAGER_NOT_PRESENT', (NV_ERR_ALREADY_SIGNALLED:=123): 'NV_ERR_ALREADY_SIGNALLED', (NV_ERR_QUEUE_TASK_SLOT_NOT_AVAILABLE:=124): 'NV_ERR_QUEUE_TASK_SLOT_NOT_AVAILABLE', (NV_ERR_KEY_ROTATION_IN_PROGRESS:=125): 'NV_ERR_KEY_ROTATION_IN_PROGRESS', (NV_ERR_TEST_ONLY_CODE_NOT_ENABLED:=126): 'NV_ERR_TEST_ONLY_CODE_NOT_ENABLED', (NV_ERR_SECURE_BOOT_FAILED:=127): 'NV_ERR_SECURE_BOOT_FAILED', (NV_ERR_INSUFFICIENT_ZBC_ENTRY:=128): 'NV_ERR_INSUFFICIENT_ZBC_ENTRY', (NV_ERR_NVLINK_FABRIC_NOT_READY:=129): 'NV_ERR_NVLINK_FABRIC_NOT_READY', (NV_ERR_NVLINK_FABRIC_FAILURE:=130): 'NV_ERR_NVLINK_FABRIC_FAILURE', (NV_ERR_GPU_MEMORY_ONLINING_FAILURE:=131): 'NV_ERR_GPU_MEMORY_ONLINING_FAILURE', (NV_ERR_REDUCTION_MANAGER_NOT_AVAILABLE:=132): 'NV_ERR_REDUCTION_MANAGER_NOT_AVAILABLE', (NV_ERR_THRESHOLD_CROSSED:=133): 'NV_ERR_THRESHOLD_CROSSED', (NV_ERR_RESOURCE_RETIREMENT_ERROR:=134): 'NV_ERR_RESOURCE_RETIREMENT_ERROR', (NV_ERR_FABRIC_STATE_OUT_OF_SYNC:=135): 'NV_ERR_FABRIC_STATE_OUT_OF_SYNC', (NV_ERR_BUFFER_FULL:=136): 'NV_ERR_BUFFER_FULL', (NV_ERR_BUFFER_EMPTY:=137): 'NV_ERR_BUFFER_EMPTY', (NV_ERR_MC_FLA_OFFSET_TABLE_FULL:=138): 'NV_ERR_MC_FLA_OFFSET_TABLE_FULL', (NV_ERR_OPERATION_ABORTED:=139): 'NV_ERR_OPERATION_ABORTED', (NV_ERR_DMA_XFER_FAILED:=140): 'NV_ERR_DMA_XFER_FAILED', (NV_ERR_RESOURCE_ACCOUNTING_HARD_LIMIT_EXCEEDED:=141): 'NV_ERR_RESOURCE_ACCOUNTING_HARD_LIMIT_EXCEEDED', (NV_ERR_NVLINK_FW_NOT_LOADED:=142): 'NV_ERR_NVLINK_FW_NOT_LOADED', (NV_ERR_INTR_STUCK:=143): 'NV_ERR_INTR_STUCK', (NV_ERR_GMCAPI_INVALID_MCTP:=144): 'NV_ERR_GMCAPI_INVALID_MCTP', (NV_ERR_GMCAPI_INVALID_NVDM_TYPE:=145): 'NV_ERR_GMCAPI_INVALID_NVDM_TYPE', (NV_ERR_GMCAPI_INVALID_COMMAND_ID:=146): 'NV_ERR_GMCAPI_INVALID_COMMAND_ID', (NV_ERR_GMCAPI_REQUEST_SIZE_TOO_LARGE:=147): 'NV_ERR_GMCAPI_REQUEST_SIZE_TOO_LARGE', (NV_ERR_GMCAPI_REQUEST_SIZE_TOO_SMALL:=148): 'NV_ERR_GMCAPI_REQUEST_SIZE_TOO_SMALL', (NV_ERR_GMCAPI_MAX_RESPONSE_SIZE_TOO_SMALL:=149): 'NV_ERR_GMCAPI_MAX_RESPONSE_SIZE_TOO_SMALL', (NV_ERR_GMCAPI_INTERNAL_RM_ERROR:=158): 'NV_ERR_GMCAPI_INTERNAL_RM_ERROR', (NV_ERR_GMCAPI_GENERIC_ERROR:=159): 'NV_ERR_GMCAPI_GENERIC_ERROR', (NV_WARN_HOT_SWITCH:=65537): 'NV_WARN_HOT_SWITCH', (NV_WARN_INCORRECT_PERFMON_DATA:=65538): 'NV_WARN_INCORRECT_PERFMON_DATA', (NV_WARN_MISMATCHED_SLAVE:=65539): 'NV_WARN_MISMATCHED_SLAVE', (NV_WARN_MISMATCHED_TARGET:=65540): 'NV_WARN_MISMATCHED_TARGET', (NV_WARN_MORE_PROCESSING_REQUIRED:=65541): 'NV_WARN_MORE_PROCESSING_REQUIRED', (NV_WARN_NOTHING_TO_DO:=65542): 'NV_WARN_NOTHING_TO_DO', (NV_WARN_NULL_OBJECT:=65543): 'NV_WARN_NULL_OBJECT', (NV_WARN_OUT_OF_RANGE:=65544): 'NV_WARN_OUT_OF_RANGE', (NV_WARN_THRESHOLD_CROSSED:=65545): 'NV_WARN_THRESHOLD_CROSSED', (NV_WARN_RESOURCE_ACCOUNTING_SOFT_LIMIT_EXCEEDED:=65546): 'NV_WARN_RESOURCE_ACCOUNTING_SOFT_LIMIT_EXCEEDED'} +NVC9B0_VIDEO_DECODER = (0x0000C9B0) +NVC9B0_NOP = (0x00000100) +NVC9B0_NOP_PARAMETER = (31, 0) +NVC9B0_PM_TRIGGER = (0x00000140) +NVC9B0_PM_TRIGGER_V = (31, 0) +NVC9B0_SET_APPLICATION_ID = (0x00000200) +NVC9B0_SET_APPLICATION_ID_ID = (31, 0) +NVC9B0_SET_APPLICATION_ID_ID_MPEG12 = (0x00000001) +NVC9B0_SET_APPLICATION_ID_ID_VC1 = (0x00000002) +NVC9B0_SET_APPLICATION_ID_ID_H264 = (0x00000003) +NVC9B0_SET_APPLICATION_ID_ID_MPEG4 = (0x00000004) +NVC9B0_SET_APPLICATION_ID_ID_VP8 = (0x00000005) +NVC9B0_SET_APPLICATION_ID_ID_CTR64 = (0x00000006) +NVC9B0_SET_APPLICATION_ID_ID_HEVC = (0x00000007) +NVC9B0_SET_APPLICATION_ID_ID_NEW_H264 = (0x00000008) +NVC9B0_SET_APPLICATION_ID_ID_VP9 = (0x00000009) +NVC9B0_SET_APPLICATION_ID_ID_PASS1 = (0x0000000A) +NVC9B0_SET_APPLICATION_ID_ID_HEVC_PARSER = (0x0000000C) +NVC9B0_SET_APPLICATION_ID_ID_UCODE_TEST = (0x0000000D) +NVC9B0_SET_APPLICATION_ID_ID_HWDRM_PR_DECRYPTAUDIO = (0x0000000E) +NVC9B0_SET_APPLICATION_ID_ID_HWDRM_PR_DECRYPTAUDIOMULTIPLE = (0x0000000F) +NVC9B0_SET_APPLICATION_ID_ID_HWDRM_PR_PREPROCESSENCRYPTEDDATA = (0x00000010) +NVC9B0_SET_APPLICATION_ID_ID_VP9_WITH_PARSER = (0x00000011) +NVC9B0_SET_APPLICATION_ID_ID_AVD = (0x00000012) +NVC9B0_SET_APPLICATION_ID_ID_HW_DRM_PR4_DECRYPTCONTENTMULTIPLE = (0x00000013) +NVC9B0_SET_APPLICATION_ID_ID_DHKE = (0x00000020) +NVC9B0_SET_WATCHDOG_TIMER = (0x00000204) +NVC9B0_SET_WATCHDOG_TIMER_TIMER = (31, 0) +NVC9B0_SEMAPHORE_A = (0x00000240) +NVC9B0_SEMAPHORE_A_UPPER = (7, 0) +NVC9B0_SEMAPHORE_B = (0x00000244) +NVC9B0_SEMAPHORE_B_LOWER = (31, 0) +NVC9B0_SEMAPHORE_C = (0x00000248) +NVC9B0_SEMAPHORE_C_PAYLOAD = (31, 0) +NVC9B0_CTX_SAVE_AREA = (0x0000024C) +NVC9B0_CTX_SAVE_AREA_OFFSET = (31, 0) +NVC9B0_CTX_SWITCH = (0x00000250) +NVC9B0_CTX_SWITCH_OP = (1, 0) +NVC9B0_CTX_SWITCH_OP_CTX_UPDATE = (0x00000000) +NVC9B0_CTX_SWITCH_OP_CTX_SAVE = (0x00000001) +NVC9B0_CTX_SWITCH_OP_CTX_RESTORE = (0x00000002) +NVC9B0_CTX_SWITCH_OP_CTX_FORCERESTORE = (0x00000003) +NVC9B0_CTX_SWITCH_CTXID_VALID = (2, 2) +NVC9B0_CTX_SWITCH_CTXID_VALID_FALSE = (0x00000000) +NVC9B0_CTX_SWITCH_CTXID_VALID_TRUE = (0x00000001) +NVC9B0_CTX_SWITCH_RESERVED0 = (7, 3) +NVC9B0_CTX_SWITCH_CTX_ID = (23, 8) +NVC9B0_CTX_SWITCH_RESERVED1 = (31, 24) +NVC9B0_SET_SEMAPHORE_PAYLOAD_LOWER = (0x00000254) +NVC9B0_SET_SEMAPHORE_PAYLOAD_LOWER_PAYLOAD_LOWER = (31, 0) +NVC9B0_SET_SEMAPHORE_PAYLOAD_UPPER = (0x00000258) +NVC9B0_SET_SEMAPHORE_PAYLOAD_UPPER_PAYLOAD_UPPER = (31, 0) +NVC9B0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_A = (0x0000025C) +NVC9B0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_A_LOWER = (31, 0) +NVC9B0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_B = (0x00000260) +NVC9B0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_B_UPPER = (31, 0) +NVC9B0_EXECUTE = (0x00000300) +NVC9B0_EXECUTE_NOTIFY = (0, 0) +NVC9B0_EXECUTE_NOTIFY_DISABLE = (0x00000000) +NVC9B0_EXECUTE_NOTIFY_ENABLE = (0x00000001) +NVC9B0_EXECUTE_NOTIFY_ON = (1, 1) +NVC9B0_EXECUTE_NOTIFY_ON_END = (0x00000000) +NVC9B0_EXECUTE_NOTIFY_ON_BEGIN = (0x00000001) +NVC9B0_EXECUTE_PREDICATION = (2, 2) +NVC9B0_EXECUTE_PREDICATION_DISABLE = (0x00000000) +NVC9B0_EXECUTE_PREDICATION_ENABLE = (0x00000001) +NVC9B0_EXECUTE_PREDICATION_OP = (3, 3) +NVC9B0_EXECUTE_PREDICATION_OP_EQUAL_ZERO = (0x00000000) +NVC9B0_EXECUTE_PREDICATION_OP_NOT_EQUAL_ZERO = (0x00000001) +NVC9B0_EXECUTE_AWAKEN = (8, 8) +NVC9B0_EXECUTE_AWAKEN_DISABLE = (0x00000000) +NVC9B0_EXECUTE_AWAKEN_ENABLE = (0x00000001) +NVC9B0_SEMAPHORE_D = (0x00000304) +NVC9B0_SEMAPHORE_D_STRUCTURE_SIZE = (1, 0) +NVC9B0_SEMAPHORE_D_STRUCTURE_SIZE_ONE = (0x00000000) +NVC9B0_SEMAPHORE_D_STRUCTURE_SIZE_FOUR = (0x00000001) +NVC9B0_SEMAPHORE_D_STRUCTURE_SIZE_TWO = (0x00000002) +NVC9B0_SEMAPHORE_D_AWAKEN_ENABLE = (8, 8) +NVC9B0_SEMAPHORE_D_AWAKEN_ENABLE_FALSE = (0x00000000) +NVC9B0_SEMAPHORE_D_AWAKEN_ENABLE_TRUE = (0x00000001) +NVC9B0_SEMAPHORE_D_OPERATION = (17, 16) +NVC9B0_SEMAPHORE_D_OPERATION_RELEASE = (0x00000000) +NVC9B0_SEMAPHORE_D_OPERATION_RESERVED_0 = (0x00000001) +NVC9B0_SEMAPHORE_D_OPERATION_RESERVED_1 = (0x00000002) +NVC9B0_SEMAPHORE_D_OPERATION_TRAP = (0x00000003) +NVC9B0_SEMAPHORE_D_FLUSH_DISABLE = (21, 21) +NVC9B0_SEMAPHORE_D_FLUSH_DISABLE_FALSE = (0x00000000) +NVC9B0_SEMAPHORE_D_FLUSH_DISABLE_TRUE = (0x00000001) +NVC9B0_SEMAPHORE_D_TRAP_TYPE = (23, 22) +NVC9B0_SEMAPHORE_D_TRAP_TYPE_UNCONDITIONAL = (0x00000000) +NVC9B0_SEMAPHORE_D_TRAP_TYPE_CONDITIONAL = (0x00000001) +NVC9B0_SEMAPHORE_D_TRAP_TYPE_CONDITIONAL_EXT = (0x00000002) +NVC9B0_SEMAPHORE_D_PAYLOAD_SIZE = (24, 24) +NVC9B0_SEMAPHORE_D_PAYLOAD_SIZE_32BIT = (0x00000000) +NVC9B0_SEMAPHORE_D_PAYLOAD_SIZE_64BIT = (0x00000001) +NVC9B0_SET_PREDICATION_OFFSET_UPPER = (0x00000308) +NVC9B0_SET_PREDICATION_OFFSET_UPPER_OFFSET = (7, 0) +NVC9B0_SET_PREDICATION_OFFSET_LOWER = (0x0000030C) +NVC9B0_SET_PREDICATION_OFFSET_LOWER_OFFSET = (31, 0) +NVC9B0_SET_AUXILIARY_DATA_BUFFER = (0x00000310) +NVC9B0_SET_AUXILIARY_DATA_BUFFER_OFFSET = (31, 0) +NVC9B0_SET_CONTROL_PARAMS = (0x00000400) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE = (3, 0) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_MPEG1 = (0x00000000) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_MPEG2 = (0x00000001) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_VC1 = (0x00000002) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_H264 = (0x00000003) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_MPEG4 = (0x00000004) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_DIVX3 = (0x00000004) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_VP8 = (0x00000005) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_HEVC = (0x00000007) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_VP9 = (0x00000009) +NVC9B0_SET_CONTROL_PARAMS_CODEC_TYPE_AV1 = (0x0000000A) +NVC9B0_SET_CONTROL_PARAMS_GPTIMER_ON = (4, 4) +NVC9B0_SET_CONTROL_PARAMS_RET_ERROR = (5, 5) +NVC9B0_SET_CONTROL_PARAMS_ERR_CONCEAL_ON = (6, 6) +NVC9B0_SET_CONTROL_PARAMS_ERROR_FRM_IDX = (12, 7) +NVC9B0_SET_CONTROL_PARAMS_MBTIMER_ON = (13, 13) +NVC9B0_SET_CONTROL_PARAMS_EC_INTRA_FRAME_USING_PSLC = (14, 14) +NVC9B0_SET_CONTROL_PARAMS_IGNORE_SOME_FIELDS_CRC_CHECK = (15, 15) +NVC9B0_SET_CONTROL_PARAMS_EVENT_TRACE_LOGGING_ON = (16, 16) +NVC9B0_SET_CONTROL_PARAMS_ALL_INTRA_FRAME = (17, 17) +NVC9B0_SET_CONTROL_PARAMS_TESTRUN_ENV = (19, 18) +NVC9B0_SET_CONTROL_PARAMS_TESTRUN_ENV_TRACE3D_RUN = (0x00000000) +NVC9B0_SET_CONTROL_PARAMS_TESTRUN_ENV_PROD_RUN = (0x00000001) +NVC9B0_SET_CONTROL_PARAMS_HINT_DUMP_EN = (20, 20) +NVC9B0_SET_CONTROL_PARAMS_RESERVED = (25, 21) +NVC9B0_SET_CONTROL_PARAMS_NVDECSIM_SKIP_SCP = (26, 26) +NVC9B0_SET_CONTROL_PARAMS_ENABLE_ENCRYPT = (27, 27) +NVC9B0_SET_CONTROL_PARAMS_ENCRYPTMODE = (31, 28) +NVC9B0_SET_DRV_PIC_SETUP_OFFSET = (0x00000404) +NVC9B0_SET_DRV_PIC_SETUP_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_IN_BUF_BASE_OFFSET = (0x00000408) +NVC9B0_SET_IN_BUF_BASE_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_INDEX = (0x0000040C) +NVC9B0_SET_PICTURE_INDEX_INDEX = (31, 0) +NVC9B0_SET_SLICE_OFFSETS_BUF_OFFSET = (0x00000410) +NVC9B0_SET_SLICE_OFFSETS_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_COLOC_DATA_OFFSET = (0x00000414) +NVC9B0_SET_COLOC_DATA_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_HISTORY_OFFSET = (0x00000418) +NVC9B0_SET_HISTORY_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_DISPLAY_BUF_SIZE = (0x0000041C) +NVC9B0_SET_DISPLAY_BUF_SIZE_SIZE = (31, 0) +NVC9B0_SET_HISTOGRAM_OFFSET = (0x00000420) +NVC9B0_SET_HISTOGRAM_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_NVDEC_STATUS_OFFSET = (0x00000424) +NVC9B0_SET_NVDEC_STATUS_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_DISPLAY_BUF_LUMA_OFFSET = (0x00000428) +NVC9B0_SET_DISPLAY_BUF_LUMA_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_DISPLAY_BUF_CHROMA_OFFSET = (0x0000042C) +NVC9B0_SET_DISPLAY_BUF_CHROMA_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET0 = (0x00000430) +NVC9B0_SET_PICTURE_LUMA_OFFSET0_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET1 = (0x00000434) +NVC9B0_SET_PICTURE_LUMA_OFFSET1_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET2 = (0x00000438) +NVC9B0_SET_PICTURE_LUMA_OFFSET2_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET3 = (0x0000043C) +NVC9B0_SET_PICTURE_LUMA_OFFSET3_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET4 = (0x00000440) +NVC9B0_SET_PICTURE_LUMA_OFFSET4_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET5 = (0x00000444) +NVC9B0_SET_PICTURE_LUMA_OFFSET5_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET6 = (0x00000448) +NVC9B0_SET_PICTURE_LUMA_OFFSET6_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET7 = (0x0000044C) +NVC9B0_SET_PICTURE_LUMA_OFFSET7_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET8 = (0x00000450) +NVC9B0_SET_PICTURE_LUMA_OFFSET8_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET9 = (0x00000454) +NVC9B0_SET_PICTURE_LUMA_OFFSET9_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET10 = (0x00000458) +NVC9B0_SET_PICTURE_LUMA_OFFSET10_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET11 = (0x0000045C) +NVC9B0_SET_PICTURE_LUMA_OFFSET11_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET12 = (0x00000460) +NVC9B0_SET_PICTURE_LUMA_OFFSET12_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET13 = (0x00000464) +NVC9B0_SET_PICTURE_LUMA_OFFSET13_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET14 = (0x00000468) +NVC9B0_SET_PICTURE_LUMA_OFFSET14_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET15 = (0x0000046C) +NVC9B0_SET_PICTURE_LUMA_OFFSET15_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_LUMA_OFFSET16 = (0x00000470) +NVC9B0_SET_PICTURE_LUMA_OFFSET16_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET0 = (0x00000474) +NVC9B0_SET_PICTURE_CHROMA_OFFSET0_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET1 = (0x00000478) +NVC9B0_SET_PICTURE_CHROMA_OFFSET1_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET2 = (0x0000047C) +NVC9B0_SET_PICTURE_CHROMA_OFFSET2_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET3 = (0x00000480) +NVC9B0_SET_PICTURE_CHROMA_OFFSET3_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET4 = (0x00000484) +NVC9B0_SET_PICTURE_CHROMA_OFFSET4_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET5 = (0x00000488) +NVC9B0_SET_PICTURE_CHROMA_OFFSET5_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET6 = (0x0000048C) +NVC9B0_SET_PICTURE_CHROMA_OFFSET6_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET7 = (0x00000490) +NVC9B0_SET_PICTURE_CHROMA_OFFSET7_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET8 = (0x00000494) +NVC9B0_SET_PICTURE_CHROMA_OFFSET8_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET9 = (0x00000498) +NVC9B0_SET_PICTURE_CHROMA_OFFSET9_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET10 = (0x0000049C) +NVC9B0_SET_PICTURE_CHROMA_OFFSET10_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET11 = (0x000004A0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET11_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET12 = (0x000004A4) +NVC9B0_SET_PICTURE_CHROMA_OFFSET12_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET13 = (0x000004A8) +NVC9B0_SET_PICTURE_CHROMA_OFFSET13_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET14 = (0x000004AC) +NVC9B0_SET_PICTURE_CHROMA_OFFSET14_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET15 = (0x000004B0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET15_OFFSET = (31, 0) +NVC9B0_SET_PICTURE_CHROMA_OFFSET16 = (0x000004B4) +NVC9B0_SET_PICTURE_CHROMA_OFFSET16_OFFSET = (31, 0) +NVC9B0_SET_PIC_SCRATCH_BUF_OFFSET = (0x000004B8) +NVC9B0_SET_PIC_SCRATCH_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_EXTERNAL_MVBUFFER_OFFSET = (0x000004BC) +NVC9B0_SET_EXTERNAL_MVBUFFER_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_SUB_SAMPLE_MAP_OFFSET = (0x000004C0) +NVC9B0_SET_SUB_SAMPLE_MAP_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_SUB_SAMPLE_MAP_IV_OFFSET = (0x000004C4) +NVC9B0_SET_SUB_SAMPLE_MAP_IV_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_INTRA_TOP_BUF_OFFSET = (0x000004C8) +NVC9B0_SET_INTRA_TOP_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_TILE_SIZE_BUF_OFFSET = (0x000004CC) +NVC9B0_SET_TILE_SIZE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_FILTER_BUFFER_OFFSET = (0x000004D0) +NVC9B0_SET_FILTER_BUFFER_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_CRC_STRUCT_OFFSET = (0x000004D4) +NVC9B0_SET_CRC_STRUCT_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_PR_SSM_CONTENT_INFO_BUF_OFFSET = (0x000004D8) +NVC9B0_SET_PR_SSM_CONTENT_INFO_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_H264_SET_MBHIST_BUF_OFFSET = (0x00000500) +NVC9B0_H264_SET_MBHIST_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP8_SET_PROB_DATA_OFFSET = (0x00000540) +NVC9B0_VP8_SET_PROB_DATA_OFFSET_OFFSET = (31, 0) +NVC9B0_VP8_SET_HEADER_PARTITION_BUF_BASE_OFFSET = (0x00000544) +NVC9B0_VP8_SET_HEADER_PARTITION_BUF_BASE_OFFSET_OFFSET = (31, 0) +NVC9B0_HEVC_SET_SCALING_LIST_OFFSET = (0x00000580) +NVC9B0_HEVC_SET_SCALING_LIST_OFFSET_OFFSET = (31, 0) +NVC9B0_HEVC_SET_TILE_SIZES_OFFSET = (0x00000584) +NVC9B0_HEVC_SET_TILE_SIZES_OFFSET_OFFSET = (31, 0) +NVC9B0_HEVC_SET_FILTER_BUFFER_OFFSET = (0x00000588) +NVC9B0_HEVC_SET_FILTER_BUFFER_OFFSET_OFFSET = (31, 0) +NVC9B0_HEVC_SET_SAO_BUFFER_OFFSET = (0x0000058C) +NVC9B0_HEVC_SET_SAO_BUFFER_OFFSET_OFFSET = (31, 0) +NVC9B0_HEVC_SET_SLICE_INFO_BUFFER_OFFSET = (0x00000590) +NVC9B0_HEVC_SET_SLICE_INFO_BUFFER_OFFSET_OFFSET = (31, 0) +NVC9B0_HEVC_SET_SLICE_GROUP_INDEX = (0x00000594) +NVC9B0_HEVC_SET_SLICE_GROUP_INDEX_OFFSET = (31, 0) +NVC9B0_VP9_SET_PROB_TAB_BUF_OFFSET = (0x000005C0) +NVC9B0_VP9_SET_PROB_TAB_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_CTX_COUNTER_BUF_OFFSET = (0x000005C4) +NVC9B0_VP9_SET_CTX_COUNTER_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_SEGMENT_READ_BUF_OFFSET = (0x000005C8) +NVC9B0_VP9_SET_SEGMENT_READ_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_SEGMENT_WRITE_BUF_OFFSET = (0x000005CC) +NVC9B0_VP9_SET_SEGMENT_WRITE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_TILE_SIZE_BUF_OFFSET = (0x000005D0) +NVC9B0_VP9_SET_TILE_SIZE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_COL_MVWRITE_BUF_OFFSET = (0x000005D4) +NVC9B0_VP9_SET_COL_MVWRITE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_COL_MVREAD_BUF_OFFSET = (0x000005D8) +NVC9B0_VP9_SET_COL_MVREAD_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_FILTER_BUFFER_OFFSET = (0x000005DC) +NVC9B0_VP9_SET_FILTER_BUFFER_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_PARSER_SET_PIC_SETUP_OFFSET = (0x000005E0) +NVC9B0_VP9_PARSER_SET_PIC_SETUP_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_PARSER_SET_PREV_PIC_SETUP_OFFSET = (0x000005E4) +NVC9B0_VP9_PARSER_SET_PREV_PIC_SETUP_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_PARSER_SET_PROB_TAB_BUF_OFFSET = (0x000005E8) +NVC9B0_VP9_PARSER_SET_PROB_TAB_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_VP9_SET_HINT_DUMP_BUF_OFFSET = (0x000005EC) +NVC9B0_VP9_SET_HINT_DUMP_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_PASS1_SET_CLEAR_HEADER_OFFSET = (0x00000600) +NVC9B0_PASS1_SET_CLEAR_HEADER_OFFSET_OFFSET = (31, 0) +NVC9B0_PASS1_SET_RE_ENCRYPT_OFFSET = (0x00000604) +NVC9B0_PASS1_SET_RE_ENCRYPT_OFFSET_OFFSET = (31, 0) +NVC9B0_PASS1_SET_VP8_TOKEN_OFFSET = (0x00000608) +NVC9B0_PASS1_SET_VP8_TOKEN_OFFSET_OFFSET = (31, 0) +NVC9B0_PASS1_SET_INPUT_DATA_OFFSET = (0x0000060C) +NVC9B0_PASS1_SET_INPUT_DATA_OFFSET_OFFSET = (31, 0) +NVC9B0_PASS1_SET_OUTPUT_DATA_SIZE_OFFSET = (0x00000610) +NVC9B0_PASS1_SET_OUTPUT_DATA_SIZE_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_PROB_TAB_READ_BUF_OFFSET = (0x00000640) +NVC9B0_AV1_SET_PROB_TAB_READ_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_PROB_TAB_WRITE_BUF_OFFSET = (0x00000644) +NVC9B0_AV1_SET_PROB_TAB_WRITE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_SEGMENT_READ_BUF_OFFSET = (0x00000648) +NVC9B0_AV1_SET_SEGMENT_READ_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_SEGMENT_WRITE_BUF_OFFSET = (0x0000064C) +NVC9B0_AV1_SET_SEGMENT_WRITE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_COL_MV0_READ_BUF_OFFSET = (0x00000650) +NVC9B0_AV1_SET_COL_MV0_READ_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_COL_MV1_READ_BUF_OFFSET = (0x00000654) +NVC9B0_AV1_SET_COL_MV1_READ_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_COL_MV2_READ_BUF_OFFSET = (0x00000658) +NVC9B0_AV1_SET_COL_MV2_READ_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_COL_MVWRITE_BUF_OFFSET = (0x0000065C) +NVC9B0_AV1_SET_COL_MVWRITE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_GLOBAL_MODEL_BUF_OFFSET = (0x00000660) +NVC9B0_AV1_SET_GLOBAL_MODEL_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_FILM_GRAIN_BUF_OFFSET = (0x00000664) +NVC9B0_AV1_SET_FILM_GRAIN_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_TILE_STREAM_INFO_BUF_OFFSET = (0x00000668) +NVC9B0_AV1_SET_TILE_STREAM_INFO_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_SUB_STREAM_ENTRY_BUF_OFFSET = (0x0000066C) +NVC9B0_AV1_SET_SUB_STREAM_ENTRY_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_AV1_SET_HINT_DUMP_BUF_OFFSET = (0x00000670) +NVC9B0_AV1_SET_HINT_DUMP_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_H264_SET_SCALING_LIST_OFFSET = (0x00000680) +NVC9B0_H264_SET_SCALING_LIST_OFFSET_OFFSET = (31, 0) +NVC9B0_H264_SET_VLDHIST_BUF_OFFSET = (0x00000684) +NVC9B0_H264_SET_VLDHIST_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_H264_SET_EDOBOFFSET0 = (0x00000688) +NVC9B0_H264_SET_EDOBOFFSET0_OFFSET = (31, 0) +NVC9B0_H264_SET_EDOBOFFSET1 = (0x0000068C) +NVC9B0_H264_SET_EDOBOFFSET1_OFFSET = (31, 0) +NVC9B0_H264_SET_EDOBOFFSET2 = (0x00000690) +NVC9B0_H264_SET_EDOBOFFSET2_OFFSET = (31, 0) +NVC9B0_H264_SET_EDOBOFFSET3 = (0x00000694) +NVC9B0_H264_SET_EDOBOFFSET3_OFFSET = (31, 0) +NVC9B0_SET_CONTENT_INITIAL_VECTOR = lambda b: (0x00000C00 + (b)*0x00000004) # type: ignore +NVC9B0_SET_CONTENT_INITIAL_VECTOR_VALUE = (31, 0) +NVC9B0_SET_CTL_COUNT = (0x00000C10) +NVC9B0_SET_CTL_COUNT_VALUE = (31, 0) +NVC9B0_SET_UPPER_SRC = (0x00000C14) +NVC9B0_SET_UPPER_SRC_OFFSET = (7, 0) +NVC9B0_SET_LOWER_SRC = (0x00000C18) +NVC9B0_SET_LOWER_SRC_OFFSET = (31, 0) +NVC9B0_SET_UPPER_DST = (0x00000C1C) +NVC9B0_SET_UPPER_DST_OFFSET = (7, 0) +NVC9B0_SET_LOWER_DST = (0x00000C20) +NVC9B0_SET_LOWER_DST_OFFSET = (31, 0) +NVC9B0_SET_BLOCK_COUNT = (0x00000C24) +NVC9B0_SET_BLOCK_COUNT_VALUE = (31, 0) +NVC9B0_PR_SET_REQUEST_BUF_OFFSET = (0x00000D00) +NVC9B0_PR_SET_REQUEST_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_PR_SET_REQUEST_BUF_SIZE = (0x00000D04) +NVC9B0_PR_SET_REQUEST_BUF_SIZE_SIZE = (31, 0) +NVC9B0_PR_SET_RESPONSE_BUF_OFFSET = (0x00000D08) +NVC9B0_PR_SET_RESPONSE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_PR_SET_RESPONSE_BUF_SIZE = (0x00000D0C) +NVC9B0_PR_SET_RESPONSE_BUF_SIZE_SIZE = (31, 0) +NVC9B0_PR_SET_REQUEST_MESSAGE_BUF_OFFSET = (0x00000D10) +NVC9B0_PR_SET_REQUEST_MESSAGE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_PR_SET_RESPONSE_MESSAGE_BUF_OFFSET = (0x00000D14) +NVC9B0_PR_SET_RESPONSE_MESSAGE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_PR_SET_LOCAL_DECRYPT_BUF_OFFSET = (0x00000D18) +NVC9B0_PR_SET_LOCAL_DECRYPT_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_PR_SET_LOCAL_DECRYPT_BUF_SIZE = (0x00000D1C) +NVC9B0_PR_SET_LOCAL_DECRYPT_BUF_SIZE_SIZE = (31, 0) +NVC9B0_PR_SET_CONTENT_DECRYPT_INFO_BUF_OFFSET = (0x00000D20) +NVC9B0_PR_SET_CONTENT_DECRYPT_INFO_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_PR_SET_REENCRYPTED_BITSTREAM_BUF_OFFSET = (0x00000D24) +NVC9B0_PR_SET_REENCRYPTED_BITSTREAM_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_DH_KE_SET_CHALLENGE_BUF_OFFSET = (0x00000E00) +NVC9B0_DH_KE_SET_CHALLENGE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_DH_KE_SET_RESPONSE_BUF_OFFSET = (0x00000E04) +NVC9B0_DH_KE_SET_RESPONSE_BUF_OFFSET_OFFSET = (31, 0) +NVC9B0_SET_SESSION_KEY = lambda b: (0x00000F00 + (b)*0x00000004) # type: ignore +NVC9B0_SET_SESSION_KEY_VALUE = (31, 0) +NVC9B0_SET_CONTENT_KEY = lambda b: (0x00000F10 + (b)*0x00000004) # type: ignore +NVC9B0_SET_CONTENT_KEY_VALUE = (31, 0) +NVC9B0_PM_TRIGGER_END = (0x00001114) +NVC9B0_PM_TRIGGER_END_V = (31, 0) +NVC9B0_ERROR_NONE = (0x00000000) +NVC9B0_OS_ERROR_EXECUTE_INSUFFICIENT_DATA = (0x00000001) +NVC9B0_OS_ERROR_SEMAPHORE_INSUFFICIENT_DATA = (0x00000002) +NVC9B0_OS_ERROR_INVALID_METHOD = (0x00000003) +NVC9B0_OS_ERROR_INVALID_DMA_PAGE = (0x00000004) +NVC9B0_OS_ERROR_UNHANDLED_INTERRUPT = (0x00000005) +NVC9B0_OS_ERROR_EXCEPTION = (0x00000006) +NVC9B0_OS_ERROR_INVALID_CTXSW_REQUEST = (0x00000007) +NVC9B0_OS_ERROR_APPLICATION = (0x00000008) +NVC9B0_OS_ERROR_SW_BREAKPT = (0x00000009) +NVC9B0_OS_INTERRUPT_EXECUTE_AWAKEN = (0x00000100) +NVC9B0_OS_INTERRUPT_BACKEND_SEMAPHORE_AWAKEN = (0x00000200) +NVC9B0_OS_INTERRUPT_CTX_ERROR_FBIF = (0x00000300) +NVC9B0_OS_INTERRUPT_LIMIT_VIOLATION = (0x00000400) +NVC9B0_OS_INTERRUPT_LIMIT_AND_FBIF_CTX_ERROR = (0x00000500) +NVC9B0_OS_INTERRUPT_HALT_ENGINE = (0x00000600) +NVC9B0_OS_INTERRUPT_TRAP_NONSTALL = (0x00000700) +NVC9B0_H264_VLD_ERR_SEQ_DATA_INCONSISTENT = (0x00004001) +NVC9B0_H264_VLD_ERR_PIC_DATA_INCONSISTENT = (0x00004002) +NVC9B0_H264_VLD_ERR_SLC_DATA_BUF_ADDR_OUT_OF_BOUNDS = (0x00004100) +NVC9B0_H264_VLD_ERR_BITSTREAM_ERROR = (0x00004101) +NVC9B0_H264_VLD_ERR_CTX_DMA_ID_CTRL_IN_INVALID = (0x000041F8) +NVC9B0_H264_VLD_ERR_SLC_HDR_OUT_SIZE_NOT_MULT256 = (0x00004200) +NVC9B0_H264_VLD_ERR_SLC_DATA_OUT_SIZE_NOT_MULT256 = (0x00004201) +NVC9B0_H264_VLD_ERR_CTX_DMA_ID_FLOW_CTRL_INVALID = (0x00004203) +NVC9B0_H264_VLD_ERR_CTX_DMA_ID_SLC_HDR_OUT_INVALID = (0x00004204) +NVC9B0_H264_VLD_ERR_SLC_HDR_OUT_BUF_TOO_SMALL = (0x00004205) +NVC9B0_H264_VLD_ERR_SLC_HDR_OUT_BUF_ALREADY_VALID = (0x00004206) +NVC9B0_H264_VLD_ERR_SLC_DATA_OUT_BUF_TOO_SMALL = (0x00004207) +NVC9B0_H264_VLD_ERR_DATA_BUF_CNT_TOO_SMALL = (0x00004208) +NVC9B0_H264_VLD_ERR_BITSTREAM_EMPTY = (0x00004209) +NVC9B0_H264_VLD_ERR_FRAME_WIDTH_TOO_LARGE = (0x0000420A) +NVC9B0_H264_VLD_ERR_FRAME_HEIGHT_TOO_LARGE = (0x0000420B) +NVC9B0_H264_VLD_ERR_HIST_BUF_TOO_SMALL = (0x00004300) +NVC9B0_VC1_VLD_ERR_PIC_DATA_BUF_ADDR_OUT_OF_BOUND = (0x00005100) +NVC9B0_VC1_VLD_ERR_BITSTREAM_ERROR = (0x00005101) +NVC9B0_VC1_VLD_ERR_PIC_HDR_OUT_SIZE_NOT_MULT256 = (0x00005200) +NVC9B0_VC1_VLD_ERR_PIC_DATA_OUT_SIZE_NOT_MULT256 = (0x00005201) +NVC9B0_VC1_VLD_ERR_CTX_DMA_ID_CTRL_IN_INVALID = (0x00005202) +NVC9B0_VC1_VLD_ERR_CTX_DMA_ID_FLOW_CTRL_INVALID = (0x00005203) +NVC9B0_VC1_VLD_ERR_CTX_DMA_ID_PIC_HDR_OUT_INVALID = (0x00005204) +NVC9B0_VC1_VLD_ERR_SLC_HDR_OUT_BUF_TOO_SMALL = (0x00005205) +NVC9B0_VC1_VLD_ERR_PIC_HDR_OUT_BUF_ALREADY_VALID = (0x00005206) +NVC9B0_VC1_VLD_ERR_PIC_DATA_OUT_BUF_TOO_SMALL = (0x00005207) +NVC9B0_VC1_VLD_ERR_DATA_INFO_IN_BUF_TOO_SMALL = (0x00005208) +NVC9B0_VC1_VLD_ERR_BITSTREAM_EMPTY = (0x00005209) +NVC9B0_VC1_VLD_ERR_FRAME_WIDTH_TOO_LARGE = (0x0000520A) +NVC9B0_VC1_VLD_ERR_FRAME_HEIGHT_TOO_LARGE = (0x0000520B) +NVC9B0_VC1_VLD_ERR_PIC_DATA_OUT_BUF_FULL_TIME_OUT = (0x00005300) +NVC9B0_MPEG12_VLD_ERR_SLC_DATA_BUF_ADDR_OUT_OF_BOUNDS = (0x00006100) +NVC9B0_MPEG12_VLD_ERR_BITSTREAM_ERROR = (0x00006101) +NVC9B0_MPEG12_VLD_ERR_SLC_DATA_OUT_SIZE_NOT_MULT256 = (0x00006200) +NVC9B0_MPEG12_VLD_ERR_CTX_DMA_ID_CTRL_IN_INVALID = (0x00006201) +NVC9B0_MPEG12_VLD_ERR_CTX_DMA_ID_FLOW_CTRL_INVALID = (0x00006202) +NVC9B0_MPEG12_VLD_ERR_SLC_DATA_OUT_BUF_TOO_SMALL = (0x00006203) +NVC9B0_MPEG12_VLD_ERR_DATA_INFO_IN_BUF_TOO_SMALL = (0x00006204) +NVC9B0_MPEG12_VLD_ERR_BITSTREAM_EMPTY = (0x00006205) +NVC9B0_MPEG12_VLD_ERR_INVALID_PIC_STRUCTURE = (0x00006206) +NVC9B0_MPEG12_VLD_ERR_INVALID_PIC_CODING_TYPE = (0x00006207) +NVC9B0_MPEG12_VLD_ERR_FRAME_WIDTH_TOO_LARGE = (0x00006208) +NVC9B0_MPEG12_VLD_ERR_FRAME_HEIGHT_TOO_LARGE = (0x00006209) +NVC9B0_MPEG12_VLD_ERR_SLC_DATA_OUT_BUF_FULL_TIME_OUT = (0x00006300) +NVC9B0_CMN_VLD_ERR_PDEC_RETURNED_ERROR = (0x00007101) +NVC9B0_CMN_VLD_ERR_EDOB_FLUSH_TIME_OUT = (0x00007102) +NVC9B0_CMN_VLD_ERR_EDOB_REWIND_TIME_OUT = (0x00007103) +NVC9B0_CMN_VLD_ERR_VLD_WD_TIME_OUT = (0x00007104) +NVC9B0_CMN_VLD_ERR_NUM_SLICES_ZERO = (0x00007105) +NVC9B0_MPEG4_VLD_ERR_PIC_DATA_BUF_ADDR_OUT_OF_BOUND = (0x00008100) +NVC9B0_MPEG4_VLD_ERR_BITSTREAM_ERROR = (0x00008101) +NVC9B0_MPEG4_VLD_ERR_PIC_HDR_OUT_SIZE_NOT_MULT256 = (0x00008200) +NVC9B0_MPEG4_VLD_ERR_PIC_DATA_OUT_SIZE_NOT_MULT256 = (0x00008201) +NVC9B0_MPEG4_VLD_ERR_CTX_DMA_ID_CTRL_IN_INVALID = (0x00008202) +NVC9B0_MPEG4_VLD_ERR_CTX_DMA_ID_FLOW_CTRL_INVALID = (0x00008203) +NVC9B0_MPEG4_VLD_ERR_CTX_DMA_ID_PIC_HDR_OUT_INVALID = (0x00008204) +NVC9B0_MPEG4_VLD_ERR_SLC_HDR_OUT_BUF_TOO_SMALL = (0x00008205) +NVC9B0_MPEG4_VLD_ERR_PIC_HDR_OUT_BUF_ALREADY_VALID = (0x00008206) +NVC9B0_MPEG4_VLD_ERR_PIC_DATA_OUT_BUF_TOO_SMALL = (0x00008207) +NVC9B0_MPEG4_VLD_ERR_DATA_INFO_IN_BUF_TOO_SMALL = (0x00008208) +NVC9B0_MPEG4_VLD_ERR_BITSTREAM_EMPTY = (0x00008209) +NVC9B0_MPEG4_VLD_ERR_FRAME_WIDTH_TOO_LARGE = (0x0000820A) +NVC9B0_MPEG4_VLD_ERR_FRAME_HEIGHT_TOO_LARGE = (0x0000820B) +NVC9B0_MPEG4_VLD_ERR_PIC_DATA_OUT_BUF_FULL_TIME_OUT = (0x00051E01) +NVC9B0_DEC_ERROR_MPEG12_APPTIMER_EXPIRED = (0xDEC10001) +NVC9B0_DEC_ERROR_MPEG12_MVTIMER_EXPIRED = (0xDEC10002) +NVC9B0_DEC_ERROR_MPEG12_INVALID_TOKEN = (0xDEC10003) +NVC9B0_DEC_ERROR_MPEG12_SLICEDATA_MISSING = (0xDEC10004) +NVC9B0_DEC_ERROR_MPEG12_HWERR_INTERRUPT = (0xDEC10005) +NVC9B0_DEC_ERROR_MPEG12_DETECTED_VLD_FAILURE = (0xDEC10006) +NVC9B0_DEC_ERROR_MPEG12_PICTURE_INIT = (0xDEC10100) +NVC9B0_DEC_ERROR_MPEG12_STATEMACHINE_FAILURE = (0xDEC10101) +NVC9B0_DEC_ERROR_MPEG12_INVALID_CTXID_PIC = (0xDEC10901) +NVC9B0_DEC_ERROR_MPEG12_INVALID_CTXID_UCODE = (0xDEC10902) +NVC9B0_DEC_ERROR_MPEG12_INVALID_CTXID_FC = (0xDEC10903) +NVC9B0_DEC_ERROR_MPEG12_INVALID_CTXID_SLH = (0xDEC10904) +NVC9B0_DEC_ERROR_MPEG12_INVALID_UCODE_SIZE = (0xDEC10905) +NVC9B0_DEC_ERROR_MPEG12_INVALID_SLICE_COUNT = (0xDEC10906) +NVC9B0_DEC_ERROR_VC1_APPTIMER_EXPIRED = (0xDEC20001) +NVC9B0_DEC_ERROR_VC1_MVTIMER_EXPIRED = (0xDEC20002) +NVC9B0_DEC_ERROR_VC1_INVALID_TOKEN = (0xDEC20003) +NVC9B0_DEC_ERROR_VC1_SLICEDATA_MISSING = (0xDEC20004) +NVC9B0_DEC_ERROR_VC1_HWERR_INTERRUPT = (0xDEC20005) +NVC9B0_DEC_ERROR_VC1_DETECTED_VLD_FAILURE = (0xDEC20006) +NVC9B0_DEC_ERROR_VC1_TIMEOUT_POLLING_FOR_DATA = (0xDEC20007) +NVC9B0_DEC_ERROR_VC1_PDEC_PIC_END_UNALIGNED = (0xDEC20008) +NVC9B0_DEC_ERROR_VC1_WDTIMER_EXPIRED = (0xDEC20009) +NVC9B0_DEC_ERROR_VC1_ERRINTSTART = (0xDEC20010) +NVC9B0_DEC_ERROR_VC1_IQT_ERRINT = (0xDEC20011) +NVC9B0_DEC_ERROR_VC1_MC_ERRINT = (0xDEC20012) +NVC9B0_DEC_ERROR_VC1_MC_IQT_ERRINT = (0xDEC20013) +NVC9B0_DEC_ERROR_VC1_REC_ERRINT = (0xDEC20014) +NVC9B0_DEC_ERROR_VC1_REC_IQT_ERRINT = (0xDEC20015) +NVC9B0_DEC_ERROR_VC1_REC_MC_ERRINT = (0xDEC20016) +NVC9B0_DEC_ERROR_VC1_REC_MC_IQT_ERRINT = (0xDEC20017) +NVC9B0_DEC_ERROR_VC1_DBF_ERRINT = (0xDEC20018) +NVC9B0_DEC_ERROR_VC1_DBF_IQT_ERRINT = (0xDEC20019) +NVC9B0_DEC_ERROR_VC1_DBF_MC_ERRINT = (0xDEC2001A) +NVC9B0_DEC_ERROR_VC1_DBF_MC_IQT_ERRINT = (0xDEC2001B) +NVC9B0_DEC_ERROR_VC1_DBF_REC_ERRINT = (0xDEC2001C) +NVC9B0_DEC_ERROR_VC1_DBF_REC_IQT_ERRINT = (0xDEC2001D) +NVC9B0_DEC_ERROR_VC1_DBF_REC_MC_ERRINT = (0xDEC2001E) +NVC9B0_DEC_ERROR_VC1_DBF_REC_MC_IQT_ERRINT = (0xDEC2001F) +NVC9B0_DEC_ERROR_VC1_PICTURE_INIT = (0xDEC20100) +NVC9B0_DEC_ERROR_VC1_STATEMACHINE_FAILURE = (0xDEC20101) +NVC9B0_DEC_ERROR_VC1_INVALID_CTXID_PIC = (0xDEC20901) +NVC9B0_DEC_ERROR_VC1_INVALID_CTXID_UCODE = (0xDEC20902) +NVC9B0_DEC_ERROR_VC1_INVALID_CTXID_FC = (0xDEC20903) +NVC9B0_DEC_ERROR_VC1_INVAILD_CTXID_SLH = (0xDEC20904) +NVC9B0_DEC_ERROR_VC1_INVALID_UCODE_SIZE = (0xDEC20905) +NVC9B0_DEC_ERROR_VC1_INVALID_SLICE_COUNT = (0xDEC20906) +NVC9B0_DEC_ERROR_H264_APPTIMER_EXPIRED = (0xDEC30001) +NVC9B0_DEC_ERROR_H264_MVTIMER_EXPIRED = (0xDEC30002) +NVC9B0_DEC_ERROR_H264_INVALID_TOKEN = (0xDEC30003) +NVC9B0_DEC_ERROR_H264_SLICEDATA_MISSING = (0xDEC30004) +NVC9B0_DEC_ERROR_H264_HWERR_INTERRUPT = (0xDEC30005) +NVC9B0_DEC_ERROR_H264_DETECTED_VLD_FAILURE = (0xDEC30006) +NVC9B0_DEC_ERROR_H264_ERRINTSTART = (0xDEC30010) +NVC9B0_DEC_ERROR_H264_IQT_ERRINT = (0xDEC30011) +NVC9B0_DEC_ERROR_H264_MC_ERRINT = (0xDEC30012) +NVC9B0_DEC_ERROR_H264_MC_IQT_ERRINT = (0xDEC30013) +NVC9B0_DEC_ERROR_H264_REC_ERRINT = (0xDEC30014) +NVC9B0_DEC_ERROR_H264_REC_IQT_ERRINT = (0xDEC30015) +NVC9B0_DEC_ERROR_H264_REC_MC_ERRINT = (0xDEC30016) +NVC9B0_DEC_ERROR_H264_REC_MC_IQT_ERRINT = (0xDEC30017) +NVC9B0_DEC_ERROR_H264_DBF_ERRINT = (0xDEC30018) +NVC9B0_DEC_ERROR_H264_DBF_IQT_ERRINT = (0xDEC30019) +NVC9B0_DEC_ERROR_H264_DBF_MC_ERRINT = (0xDEC3001A) +NVC9B0_DEC_ERROR_H264_DBF_MC_IQT_ERRINT = (0xDEC3001B) +NVC9B0_DEC_ERROR_H264_DBF_REC_ERRINT = (0xDEC3001C) +NVC9B0_DEC_ERROR_H264_DBF_REC_IQT_ERRINT = (0xDEC3001D) +NVC9B0_DEC_ERROR_H264_DBF_REC_MC_ERRINT = (0xDEC3001E) +NVC9B0_DEC_ERROR_H264_DBF_REC_MC_IQT_ERRINT = (0xDEC3001F) +NVC9B0_DEC_ERROR_H264_PICTURE_INIT = (0xDEC30100) +NVC9B0_DEC_ERROR_H264_STATEMACHINE_FAILURE = (0xDEC30101) +NVC9B0_DEC_ERROR_H264_INVALID_CTXID_PIC = (0xDEC30901) +NVC9B0_DEC_ERROR_H264_INVALID_CTXID_UCODE = (0xDEC30902) +NVC9B0_DEC_ERROR_H264_INVALID_CTXID_FC = (0xDEC30903) +NVC9B0_DEC_ERROR_H264_INVALID_CTXID_SLH = (0xDEC30904) +NVC9B0_DEC_ERROR_H264_INVALID_UCODE_SIZE = (0xDEC30905) +NVC9B0_DEC_ERROR_H264_INVALID_SLICE_COUNT = (0xDEC30906) +NVC9B0_DEC_ERROR_MPEG4_APPTIMER_EXPIRED = (0xDEC40001) +NVC9B0_DEC_ERROR_MPEG4_MVTIMER_EXPIRED = (0xDEC40002) +NVC9B0_DEC_ERROR_MPEG4_INVALID_TOKEN = (0xDEC40003) +NVC9B0_DEC_ERROR_MPEG4_SLICEDATA_MISSING = (0xDEC40004) +NVC9B0_DEC_ERROR_MPEG4_HWERR_INTERRUPT = (0xDEC40005) +NVC9B0_DEC_ERROR_MPEG4_DETECTED_VLD_FAILURE = (0xDEC40006) +NVC9B0_DEC_ERROR_MPEG4_TIMEOUT_POLLING_FOR_DATA = (0xDEC40007) +NVC9B0_DEC_ERROR_MPEG4_PDEC_PIC_END_UNALIGNED = (0xDEC40008) +NVC9B0_DEC_ERROR_MPEG4_WDTIMER_EXPIRED = (0xDEC40009) +NVC9B0_DEC_ERROR_MPEG4_ERRINTSTART = (0xDEC40010) +NVC9B0_DEC_ERROR_MPEG4_IQT_ERRINT = (0xDEC40011) +NVC9B0_DEC_ERROR_MPEG4_MC_ERRINT = (0xDEC40012) +NVC9B0_DEC_ERROR_MPEG4_MC_IQT_ERRINT = (0xDEC40013) +NVC9B0_DEC_ERROR_MPEG4_REC_ERRINT = (0xDEC40014) +NVC9B0_DEC_ERROR_MPEG4_REC_IQT_ERRINT = (0xDEC40015) +NVC9B0_DEC_ERROR_MPEG4_REC_MC_ERRINT = (0xDEC40016) +NVC9B0_DEC_ERROR_MPEG4_REC_MC_IQT_ERRINT = (0xDEC40017) +NVC9B0_DEC_ERROR_MPEG4_DBF_ERRINT = (0xDEC40018) +NVC9B0_DEC_ERROR_MPEG4_DBF_IQT_ERRINT = (0xDEC40019) +NVC9B0_DEC_ERROR_MPEG4_DBF_MC_ERRINT = (0xDEC4001A) +NVC9B0_DEC_ERROR_MPEG4_DBF_MC_IQT_ERRINT = (0xDEC4001B) +NVC9B0_DEC_ERROR_MPEG4_DBF_REC_ERRINT = (0xDEC4001C) +NVC9B0_DEC_ERROR_MPEG4_DBF_REC_IQT_ERRINT = (0xDEC4001D) +NVC9B0_DEC_ERROR_MPEG4_DBF_REC_MC_ERRINT = (0xDEC4001E) +NVC9B0_DEC_ERROR_MPEG4_DBF_REC_MC_IQT_ERRINT = (0xDEC4001F) +NVC9B0_DEC_ERROR_MPEG4_PICTURE_INIT = (0xDEC40100) +NVC9B0_DEC_ERROR_MPEG4_STATEMACHINE_FAILURE = (0xDEC40101) +NVC9B0_DEC_ERROR_MPEG4_INVALID_CTXID_PIC = (0xDEC40901) +NVC9B0_DEC_ERROR_MPEG4_INVALID_CTXID_UCODE = (0xDEC40902) +NVC9B0_DEC_ERROR_MPEG4_INVALID_CTXID_FC = (0xDEC40903) +NVC9B0_DEC_ERROR_MPEG4_INVALID_CTXID_SLH = (0xDEC40904) +NVC9B0_DEC_ERROR_MPEG4_INVALID_UCODE_SIZE = (0xDEC40905) +NVC9B0_DEC_ERROR_MPEG4_INVALID_SLICE_COUNT = (0xDEC40906) +NVC6C0_QMDV02_03_OUTER_PUT = (30, 0) +NVC6C0_QMDV02_03_OUTER_OVERFLOW = (31, 31) +NVC6C0_QMDV02_03_OUTER_GET = (62, 32) +NVC6C0_QMDV02_03_OUTER_STICKY_OVERFLOW = (63, 63) +NVC6C0_QMDV02_03_INNER_GET = (94, 64) +NVC6C0_QMDV02_03_INNER_OVERFLOW = (95, 95) +NVC6C0_QMDV02_03_INNER_PUT = (126, 96) +NVC6C0_QMDV02_03_INNER_STICKY_OVERFLOW = (127, 127) +NVC6C0_QMDV02_03_QMD_GROUP_ID = (133, 128) +NVC6C0_QMDV02_03_SM_GLOBAL_CACHING_ENABLE = (134, 134) +NVC6C0_QMDV02_03_RUN_CTA_IN_ONE_SM_PARTITION = (135, 135) +NVC6C0_QMDV02_03_RUN_CTA_IN_ONE_SM_PARTITION_FALSE = 0x00000000 +NVC6C0_QMDV02_03_RUN_CTA_IN_ONE_SM_PARTITION_TRUE = 0x00000001 +NVC6C0_QMDV02_03_IS_QUEUE = (136, 136) +NVC6C0_QMDV02_03_IS_QUEUE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_IS_QUEUE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST = (137, 137) +NVC6C0_QMDV02_03_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_FALSE = 0x00000000 +NVC6C0_QMDV02_03_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_TRUE = 0x00000001 +NVC6C0_QMDV02_03_SEMAPHORE_RELEASE_ENABLE0 = (138, 138) +NVC6C0_QMDV02_03_SEMAPHORE_RELEASE_ENABLE0_FALSE = 0x00000000 +NVC6C0_QMDV02_03_SEMAPHORE_RELEASE_ENABLE0_TRUE = 0x00000001 +NVC6C0_QMDV02_03_SEMAPHORE_RELEASE_ENABLE1 = (139, 139) +NVC6C0_QMDV02_03_SEMAPHORE_RELEASE_ENABLE1_FALSE = 0x00000000 +NVC6C0_QMDV02_03_SEMAPHORE_RELEASE_ENABLE1_TRUE = 0x00000001 +NVC6C0_QMDV02_03_REQUIRE_SCHEDULING_PCAS = (140, 140) +NVC6C0_QMDV02_03_REQUIRE_SCHEDULING_PCAS_FALSE = 0x00000000 +NVC6C0_QMDV02_03_REQUIRE_SCHEDULING_PCAS_TRUE = 0x00000001 +NVC6C0_QMDV02_03_DEPENDENT_QMD_SCHEDULE_ENABLE = (141, 141) +NVC6C0_QMDV02_03_DEPENDENT_QMD_SCHEDULE_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_DEPENDENT_QMD_SCHEDULE_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_DEPENDENT_QMD_TYPE = (142, 142) +NVC6C0_QMDV02_03_DEPENDENT_QMD_TYPE_QUEUE = 0x00000000 +NVC6C0_QMDV02_03_DEPENDENT_QMD_TYPE_GRID = 0x00000001 +NVC6C0_QMDV02_03_DEPENDENT_QMD_FIELD_COPY = (143, 143) +NVC6C0_QMDV02_03_DEPENDENT_QMD_FIELD_COPY_FALSE = 0x00000000 +NVC6C0_QMDV02_03_DEPENDENT_QMD_FIELD_COPY_TRUE = 0x00000001 +NVC6C0_QMDV02_03_QMD_RESERVED_B = (159, 144) +NVC6C0_QMDV02_03_CIRCULAR_QUEUE_SIZE = (184, 160) +NVC6C0_QMDV02_03_QMD_RESERVED_C = (185, 185) +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_HEADER_CACHE = (186, 186) +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_HEADER_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_HEADER_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_SAMPLER_CACHE = (187, 187) +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_SAMPLER_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_SAMPLER_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_DATA_CACHE = (188, 188) +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_INVALIDATE_SHADER_DATA_CACHE = (189, 189) +NVC6C0_QMDV02_03_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_INVALIDATE_INSTRUCTION_CACHE = (190, 190) +NVC6C0_QMDV02_03_INVALIDATE_INSTRUCTION_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_INVALIDATE_INSTRUCTION_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_INVALIDATE_SHADER_CONSTANT_CACHE = (191, 191) +NVC6C0_QMDV02_03_INVALIDATE_SHADER_CONSTANT_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_INVALIDATE_SHADER_CONSTANT_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_CTA_RASTER_WIDTH_RESUME = (223, 192) +NVC6C0_QMDV02_03_CTA_RASTER_HEIGHT_RESUME = (239, 224) +NVC6C0_QMDV02_03_CTA_RASTER_DEPTH_RESUME = (255, 240) +NVC6C0_QMDV02_03_PROGRAM_PREFETCH_ADDR_LOWER_SHIFTED = (287, 256) +NVC6C0_QMDV02_03_CIRCULAR_QUEUE_ADDR_LOWER = (319, 288) +NVC6C0_QMDV02_03_CIRCULAR_QUEUE_ADDR_UPPER = (327, 320) +NVC6C0_QMDV02_03_QMD_RESERVED_D = (335, 328) +NVC6C0_QMDV02_03_CIRCULAR_QUEUE_ENTRY_SIZE = (351, 336) +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_ID = (357, 352) +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_DELTA_MINUS_ONE = (365, 358) +NVC6C0_QMDV02_03_RELEASE_MEMBAR_TYPE = (366, 366) +NVC6C0_QMDV02_03_RELEASE_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVC6C0_QMDV02_03_RELEASE_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_INCR_ENABLE = (367, 367) +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_INCR_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_INCR_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_CWD_MEMBAR_TYPE = (369, 368) +NVC6C0_QMDV02_03_CWD_MEMBAR_TYPE_L1_NONE = 0x00000000 +NVC6C0_QMDV02_03_CWD_MEMBAR_TYPE_L1_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV02_03_CWD_MEMBAR_TYPE_L1_MEMBAR = 0x00000003 +NVC6C0_QMDV02_03_SEQUENTIALLY_RUN_CTAS = (370, 370) +NVC6C0_QMDV02_03_SEQUENTIALLY_RUN_CTAS_FALSE = 0x00000000 +NVC6C0_QMDV02_03_SEQUENTIALLY_RUN_CTAS_TRUE = 0x00000001 +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_DECR_ENABLE = (371, 371) +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_DECR_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_CWD_REFERENCE_COUNT_DECR_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_API_VISIBLE_CALL_LIMIT = (378, 378) +NVC6C0_QMDV02_03_API_VISIBLE_CALL_LIMIT__32 = 0x00000000 +NVC6C0_QMDV02_03_API_VISIBLE_CALL_LIMIT_NO_CHECK = 0x00000001 +NVC6C0_QMDV02_03_SAMPLER_INDEX = (382, 382) +NVC6C0_QMDV02_03_SAMPLER_INDEX_INDEPENDENTLY = 0x00000000 +NVC6C0_QMDV02_03_SAMPLER_INDEX_VIA_HEADER_INDEX = 0x00000001 +NVC6C0_QMDV02_03_CTA_RASTER_WIDTH = (415, 384) +NVC6C0_QMDV02_03_CTA_RASTER_HEIGHT = (431, 416) +NVC6C0_QMDV02_03_QMD_RESERVED13A = (447, 432) +NVC6C0_QMDV02_03_CTA_RASTER_DEPTH = (463, 448) +NVC6C0_QMDV02_03_QMD_RESERVED14A = (479, 464) +NVC6C0_QMDV02_03_DEPENDENT_QMD_POINTER = (511, 480) +NVC6C0_QMDV02_03_COALESCE_WAITING_PERIOD = (529, 522) +NVC6C0_QMDV02_03_QUEUE_ENTRIES_PER_CTA_LOG2 = (534, 530) +NVC6C0_QMDV02_03_SHARED_MEMORY_SIZE = (561, 544) +NVC6C0_QMDV02_03_MIN_SM_CONFIG_SHARED_MEM_SIZE = (568, 562) +NVC6C0_QMDV02_03_MAX_SM_CONFIG_SHARED_MEM_SIZE = (575, 569) +NVC6C0_QMDV02_03_QMD_VERSION = (579, 576) +NVC6C0_QMDV02_03_QMD_MAJOR_VERSION = (583, 580) +NVC6C0_QMDV02_03_QMD_RESERVED_H = (591, 584) +NVC6C0_QMDV02_03_CTA_THREAD_DIMENSION0 = (607, 592) +NVC6C0_QMDV02_03_CTA_THREAD_DIMENSION1 = (623, 608) +NVC6C0_QMDV02_03_CTA_THREAD_DIMENSION2 = (639, 624) +NVC6C0_QMDV02_03_CONSTANT_BUFFER_VALID = lambda i: ((640+(i)*1), (640+(i)*1)) # type: ignore +NVC6C0_QMDV02_03_CONSTANT_BUFFER_VALID_FALSE = 0x00000000 +NVC6C0_QMDV02_03_CONSTANT_BUFFER_VALID_TRUE = 0x00000001 +NVC6C0_QMDV02_03_REGISTER_COUNT_V = (656, 648) +NVC6C0_QMDV02_03_TARGET_SM_CONFIG_SHARED_MEM_SIZE = (663, 657) +NVC6C0_QMDV02_03_FREE_CTA_SLOTS_EMPTY_SM = (671, 664) +NVC6C0_QMDV02_03_SM_DISABLE_MASK_LOWER = (703, 672) +NVC6C0_QMDV02_03_SM_DISABLE_MASK_UPPER = (735, 704) +NVC6C0_QMDV02_03_RELEASE0_ADDRESS_LOWER = (767, 736) +NVC6C0_QMDV02_03_RELEASE0_ADDRESS_UPPER = (775, 768) +NVC6C0_QMDV02_03_QMD_RESERVED_J = (783, 776) +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP = (790, 788) +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_QMDV02_03_QMD_RESERVED_K = (791, 791) +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_FORMAT = (793, 792) +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_ENABLE = (794, 794) +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_RELEASE0_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_RELEASE0_STRUCTURE_SIZE = (799, 799) +NVC6C0_QMDV02_03_RELEASE0_STRUCTURE_SIZE_FOUR_WORDS = 0x00000000 +NVC6C0_QMDV02_03_RELEASE0_STRUCTURE_SIZE_ONE_WORD = 0x00000001 +NVC6C0_QMDV02_03_RELEASE0_PAYLOAD = (831, 800) +NVC6C0_QMDV02_03_RELEASE1_ADDRESS_LOWER = (863, 832) +NVC6C0_QMDV02_03_RELEASE1_ADDRESS_UPPER = (871, 864) +NVC6C0_QMDV02_03_QMD_RESERVED_L = (879, 872) +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP = (886, 884) +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_QMDV02_03_QMD_RESERVED_M = (887, 887) +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_FORMAT = (889, 888) +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_ENABLE = (890, 890) +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_RELEASE1_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_RELEASE1_STRUCTURE_SIZE = (895, 895) +NVC6C0_QMDV02_03_RELEASE1_STRUCTURE_SIZE_FOUR_WORDS = 0x00000000 +NVC6C0_QMDV02_03_RELEASE1_STRUCTURE_SIZE_ONE_WORD = 0x00000001 +NVC6C0_QMDV02_03_RELEASE1_PAYLOAD = (927, 896) +NVC6C0_QMDV02_03_SHADER_LOCAL_MEMORY_LOW_SIZE = (951, 928) +NVC6C0_QMDV02_03_QMD_RESERVED_N = (954, 952) +NVC6C0_QMDV02_03_BARRIER_COUNT = (959, 955) +NVC6C0_QMDV02_03_SHADER_LOCAL_MEMORY_HIGH_SIZE = (983, 960) +NVC6C0_QMDV02_03_REGISTER_COUNT = (991, 984) +NVC6C0_QMDV02_03_PROGRAM_PREFETCH_ADDR_UPPER_SHIFTED = (1000, 992) +NVC6C0_QMDV02_03_PROGRAM_PREFETCH_SIZE = (1009, 1001) +NVC6C0_QMDV02_03_QMD_RESERVED_A = (1015, 1010) +NVC6C0_QMDV02_03_SASS_VERSION = (1023, 1016) +NVC6C0_QMDV02_03_CONSTANT_BUFFER_ADDR_LOWER = lambda i: ((1055+(i)*64), (1024+(i)*64)) # type: ignore +NVC6C0_QMDV02_03_CONSTANT_BUFFER_ADDR_UPPER = lambda i: ((1072+(i)*64), (1056+(i)*64)) # type: ignore +NVC6C0_QMDV02_03_CONSTANT_BUFFER_PREFETCH_POST = lambda i: ((1073+(i)*64), (1073+(i)*64)) # type: ignore +NVC6C0_QMDV02_03_CONSTANT_BUFFER_PREFETCH_POST_FALSE = 0x00000000 +NVC6C0_QMDV02_03_CONSTANT_BUFFER_PREFETCH_POST_TRUE = 0x00000001 +NVC6C0_QMDV02_03_CONSTANT_BUFFER_INVALIDATE = lambda i: ((1074+(i)*64), (1074+(i)*64)) # type: ignore +NVC6C0_QMDV02_03_CONSTANT_BUFFER_INVALIDATE_FALSE = 0x00000000 +NVC6C0_QMDV02_03_CONSTANT_BUFFER_INVALIDATE_TRUE = 0x00000001 +NVC6C0_QMDV02_03_CONSTANT_BUFFER_SIZE_SHIFTED4 = lambda i: ((1087+(i)*64), (1075+(i)*64)) # type: ignore +NVC6C0_QMDV02_03_PROGRAM_ADDRESS_LOWER = (1567, 1536) +NVC6C0_QMDV02_03_PROGRAM_ADDRESS_UPPER = (1584, 1568) +NVC6C0_QMDV02_03_QMD_RESERVED_S = (1599, 1585) +NVC6C0_QMDV02_03_HW_ONLY_INNER_GET = (1630, 1600) +NVC6C0_QMDV02_03_HW_ONLY_REQUIRE_SCHEDULING_PCAS = (1631, 1631) +NVC6C0_QMDV02_03_HW_ONLY_INNER_PUT = (1662, 1632) +NVC6C0_QMDV02_03_HW_ONLY_SCG_TYPE = (1663, 1663) +NVC6C0_QMDV02_03_HW_ONLY_SPAN_LIST_HEAD_INDEX = (1693, 1664) +NVC6C0_QMDV02_03_QMD_RESERVED_Q = (1694, 1694) +NVC6C0_QMDV02_03_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID = (1695, 1695) +NVC6C0_QMDV02_03_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_FALSE = 0x00000000 +NVC6C0_QMDV02_03_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_TRUE = 0x00000001 +NVC6C0_QMDV02_03_HW_ONLY_SKED_NEXT_QMD_POINTER = (1727, 1696) +NVC6C0_QMDV02_03_QMD_SPARE_G = (1759, 1728) +NVC6C0_QMDV02_03_QMD_SPARE_H = (1791, 1760) +NVC6C0_QMDV02_03_QMD_SPARE_I = (1823, 1792) +NVC6C0_QMDV02_03_QMD_SPARE_J = (1855, 1824) +NVC6C0_QMDV02_03_QMD_SPARE_K = (1887, 1856) +NVC6C0_QMDV02_03_QMD_SPARE_L = (1919, 1888) +NVC6C0_QMDV02_03_QMD_SPARE_M = (1951, 1920) +NVC6C0_QMDV02_03_QMD_SPARE_N = (1983, 1952) +NVC6C0_QMDV02_03_DEBUG_ID_UPPER = (2015, 1984) +NVC6C0_QMDV02_03_DEBUG_ID_LOWER = (2047, 2016) +NVC6C0_QMDV02_04_OUTER_PUT = (30, 0) +NVC6C0_QMDV02_04_OUTER_OVERFLOW = (31, 31) +NVC6C0_QMDV02_04_OUTER_GET = (62, 32) +NVC6C0_QMDV02_04_OUTER_STICKY_OVERFLOW = (63, 63) +NVC6C0_QMDV02_04_INNER_GET = (94, 64) +NVC6C0_QMDV02_04_INNER_OVERFLOW = (95, 95) +NVC6C0_QMDV02_04_INNER_PUT = (126, 96) +NVC6C0_QMDV02_04_INNER_STICKY_OVERFLOW = (127, 127) +NVC6C0_QMDV02_04_QMD_GROUP_ID = (133, 128) +NVC6C0_QMDV02_04_SM_GLOBAL_CACHING_ENABLE = (134, 134) +NVC6C0_QMDV02_04_RUN_CTA_IN_ONE_SM_PARTITION = (135, 135) +NVC6C0_QMDV02_04_RUN_CTA_IN_ONE_SM_PARTITION_FALSE = 0x00000000 +NVC6C0_QMDV02_04_RUN_CTA_IN_ONE_SM_PARTITION_TRUE = 0x00000001 +NVC6C0_QMDV02_04_IS_QUEUE = (136, 136) +NVC6C0_QMDV02_04_IS_QUEUE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_IS_QUEUE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST = (137, 137) +NVC6C0_QMDV02_04_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_FALSE = 0x00000000 +NVC6C0_QMDV02_04_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_TRUE = 0x00000001 +NVC6C0_QMDV02_04_SEMAPHORE_RELEASE_ENABLE0 = (138, 138) +NVC6C0_QMDV02_04_SEMAPHORE_RELEASE_ENABLE0_FALSE = 0x00000000 +NVC6C0_QMDV02_04_SEMAPHORE_RELEASE_ENABLE0_TRUE = 0x00000001 +NVC6C0_QMDV02_04_SEMAPHORE_RELEASE_ENABLE1 = (139, 139) +NVC6C0_QMDV02_04_SEMAPHORE_RELEASE_ENABLE1_FALSE = 0x00000000 +NVC6C0_QMDV02_04_SEMAPHORE_RELEASE_ENABLE1_TRUE = 0x00000001 +NVC6C0_QMDV02_04_REQUIRE_SCHEDULING_PCAS = (140, 140) +NVC6C0_QMDV02_04_REQUIRE_SCHEDULING_PCAS_FALSE = 0x00000000 +NVC6C0_QMDV02_04_REQUIRE_SCHEDULING_PCAS_TRUE = 0x00000001 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ENABLE = (141, 141) +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ACTION = (144, 142) +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ACTION_QMD_SCHEDULE = 0x00000001 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_PREFETCH = (145, 145) +NVC6C0_QMDV02_04_DEPENDENT_QMD0_PREFETCH_FALSE = 0x00000000 +NVC6C0_QMDV02_04_DEPENDENT_QMD0_PREFETCH_TRUE = 0x00000001 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ENABLE = (146, 146) +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ACTION = (149, 147) +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ACTION_QMD_SCHEDULE = 0x00000001 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_PREFETCH = (150, 150) +NVC6C0_QMDV02_04_DEPENDENT_QMD1_PREFETCH_FALSE = 0x00000000 +NVC6C0_QMDV02_04_DEPENDENT_QMD1_PREFETCH_TRUE = 0x00000001 +NVC6C0_QMDV02_04_DEPENDENCE_COUNTER = (157, 151) +NVC6C0_QMDV02_04_SELF_COPY_ON_COMPLETION = (158, 158) +NVC6C0_QMDV02_04_SELF_COPY_ON_COMPLETION_FALSE = 0x00000000 +NVC6C0_QMDV02_04_SELF_COPY_ON_COMPLETION_TRUE = 0x00000001 +NVC6C0_QMDV02_04_QMD_RESERVED_B = (159, 159) +NVC6C0_QMDV02_04_CIRCULAR_QUEUE_SIZE = (184, 160) +NVC6C0_QMDV02_04_DEMOTE_L2_EVICT_LAST = (185, 185) +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_HEADER_CACHE = (186, 186) +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_HEADER_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_HEADER_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_SAMPLER_CACHE = (187, 187) +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_SAMPLER_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_SAMPLER_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_DATA_CACHE = (188, 188) +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_INVALIDATE_SHADER_DATA_CACHE = (189, 189) +NVC6C0_QMDV02_04_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_INVALIDATE_INSTRUCTION_CACHE = (190, 190) +NVC6C0_QMDV02_04_INVALIDATE_INSTRUCTION_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_INVALIDATE_INSTRUCTION_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_INVALIDATE_SHADER_CONSTANT_CACHE = (191, 191) +NVC6C0_QMDV02_04_INVALIDATE_SHADER_CONSTANT_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_INVALIDATE_SHADER_CONSTANT_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_CTA_RASTER_WIDTH_RESUME = (223, 192) +NVC6C0_QMDV02_04_CTA_RASTER_HEIGHT_RESUME = (239, 224) +NVC6C0_QMDV02_04_CTA_RASTER_DEPTH_RESUME = (255, 240) +NVC6C0_QMDV02_04_PROGRAM_PREFETCH_ADDR_LOWER_SHIFTED = (287, 256) +NVC6C0_QMDV02_04_CIRCULAR_QUEUE_ADDR_LOWER = (319, 288) +NVC6C0_QMDV02_04_CIRCULAR_QUEUE_ADDR_UPPER = (327, 320) +NVC6C0_QMDV02_04_QMD_RESERVED_D = (335, 328) +NVC6C0_QMDV02_04_CIRCULAR_QUEUE_ENTRY_SIZE = (351, 336) +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_ID = (357, 352) +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_DELTA_MINUS_ONE = (365, 358) +NVC6C0_QMDV02_04_RELEASE_MEMBAR_TYPE = (366, 366) +NVC6C0_QMDV02_04_RELEASE_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVC6C0_QMDV02_04_RELEASE_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_INCR_ENABLE = (367, 367) +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_INCR_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_INCR_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_CWD_MEMBAR_TYPE = (369, 368) +NVC6C0_QMDV02_04_CWD_MEMBAR_TYPE_L1_NONE = 0x00000000 +NVC6C0_QMDV02_04_CWD_MEMBAR_TYPE_L1_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV02_04_CWD_MEMBAR_TYPE_L1_MEMBAR = 0x00000003 +NVC6C0_QMDV02_04_SEQUENTIALLY_RUN_CTAS = (370, 370) +NVC6C0_QMDV02_04_SEQUENTIALLY_RUN_CTAS_FALSE = 0x00000000 +NVC6C0_QMDV02_04_SEQUENTIALLY_RUN_CTAS_TRUE = 0x00000001 +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_DECR_ENABLE = (371, 371) +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_DECR_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_CWD_REFERENCE_COUNT_DECR_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_API_VISIBLE_CALL_LIMIT = (378, 378) +NVC6C0_QMDV02_04_API_VISIBLE_CALL_LIMIT__32 = 0x00000000 +NVC6C0_QMDV02_04_API_VISIBLE_CALL_LIMIT_NO_CHECK = 0x00000001 +NVC6C0_QMDV02_04_SAMPLER_INDEX = (382, 382) +NVC6C0_QMDV02_04_SAMPLER_INDEX_INDEPENDENTLY = 0x00000000 +NVC6C0_QMDV02_04_SAMPLER_INDEX_VIA_HEADER_INDEX = 0x00000001 +NVC6C0_QMDV02_04_DISABLE_AUTO_INVALIDATE = (383, 383) +NVC6C0_QMDV02_04_DISABLE_AUTO_INVALIDATE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_DISABLE_AUTO_INVALIDATE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_CTA_RASTER_WIDTH = (415, 384) +NVC6C0_QMDV02_04_CTA_RASTER_HEIGHT = (431, 416) +NVC6C0_QMDV02_04_QMD_RESERVED13A = (447, 432) +NVC6C0_QMDV02_04_CTA_RASTER_DEPTH = (463, 448) +NVC6C0_QMDV02_04_QMD_RESERVED14A = (479, 464) +NVC6C0_QMDV02_04_DEPENDENT_QMD0_POINTER = (511, 480) +NVC6C0_QMDV02_04_COALESCE_WAITING_PERIOD = (529, 522) +NVC6C0_QMDV02_04_QUEUE_ENTRIES_PER_CTA_LOG2 = (534, 530) +NVC6C0_QMDV02_04_SHARED_MEMORY_SIZE = (561, 544) +NVC6C0_QMDV02_04_MIN_SM_CONFIG_SHARED_MEM_SIZE = (568, 562) +NVC6C0_QMDV02_04_MAX_SM_CONFIG_SHARED_MEM_SIZE = (575, 569) +NVC6C0_QMDV02_04_QMD_VERSION = (579, 576) +NVC6C0_QMDV02_04_QMD_MAJOR_VERSION = (583, 580) +NVC6C0_QMDV02_04_QMD_RESERVED_H = (591, 584) +NVC6C0_QMDV02_04_CTA_THREAD_DIMENSION0 = (607, 592) +NVC6C0_QMDV02_04_CTA_THREAD_DIMENSION1 = (623, 608) +NVC6C0_QMDV02_04_CTA_THREAD_DIMENSION2 = (639, 624) +NVC6C0_QMDV02_04_CONSTANT_BUFFER_VALID = lambda i: ((640+(i)*1), (640+(i)*1)) # type: ignore +NVC6C0_QMDV02_04_CONSTANT_BUFFER_VALID_FALSE = 0x00000000 +NVC6C0_QMDV02_04_CONSTANT_BUFFER_VALID_TRUE = 0x00000001 +NVC6C0_QMDV02_04_REGISTER_COUNT_V = (656, 648) +NVC6C0_QMDV02_04_TARGET_SM_CONFIG_SHARED_MEM_SIZE = (663, 657) +NVC6C0_QMDV02_04_FREE_CTA_SLOTS_EMPTY_SM = (671, 664) +NVC6C0_QMDV02_04_SM_DISABLE_MASK_LOWER = (703, 672) +NVC6C0_QMDV02_04_SM_DISABLE_MASK_UPPER = (735, 704) +NVC6C0_QMDV02_04_RELEASE0_ADDRESS_LOWER = (767, 736) +NVC6C0_QMDV02_04_RELEASE0_ADDRESS_UPPER = (775, 768) +NVC6C0_QMDV02_04_QMD_RESERVED_J = (783, 776) +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP = (790, 788) +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_QMDV02_04_QMD_RESERVED_K = (791, 791) +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_FORMAT = (793, 792) +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_ENABLE = (794, 794) +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_RELEASE0_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_RELEASE0_STRUCTURE_SIZE = (799, 799) +NVC6C0_QMDV02_04_RELEASE0_STRUCTURE_SIZE_FOUR_WORDS = 0x00000000 +NVC6C0_QMDV02_04_RELEASE0_STRUCTURE_SIZE_ONE_WORD = 0x00000001 +NVC6C0_QMDV02_04_RELEASE0_PAYLOAD = (831, 800) +NVC6C0_QMDV02_04_RELEASE1_ADDRESS_LOWER = (863, 832) +NVC6C0_QMDV02_04_RELEASE1_ADDRESS_UPPER = (871, 864) +NVC6C0_QMDV02_04_QMD_RESERVED_L = (879, 872) +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP = (886, 884) +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_QMDV02_04_QMD_RESERVED_M = (887, 887) +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_FORMAT = (889, 888) +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_ENABLE = (890, 890) +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_RELEASE1_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_RELEASE1_STRUCTURE_SIZE = (895, 895) +NVC6C0_QMDV02_04_RELEASE1_STRUCTURE_SIZE_FOUR_WORDS = 0x00000000 +NVC6C0_QMDV02_04_RELEASE1_STRUCTURE_SIZE_ONE_WORD = 0x00000001 +NVC6C0_QMDV02_04_RELEASE1_PAYLOAD = (927, 896) +NVC6C0_QMDV02_04_SHADER_LOCAL_MEMORY_LOW_SIZE = (951, 928) +NVC6C0_QMDV02_04_QMD_RESERVED_N = (954, 952) +NVC6C0_QMDV02_04_BARRIER_COUNT = (959, 955) +NVC6C0_QMDV02_04_SHADER_LOCAL_MEMORY_HIGH_SIZE = (983, 960) +NVC6C0_QMDV02_04_QMD_RESERVED_G = (991, 984) +NVC6C0_QMDV02_04_PROGRAM_PREFETCH_ADDR_UPPER_SHIFTED = (1000, 992) +NVC6C0_QMDV02_04_PROGRAM_PREFETCH_SIZE = (1009, 1001) +NVC6C0_QMDV02_04_PROGRAM_PREFETCH_TYPE = (1011, 1010) +NVC6C0_QMDV02_04_PROGRAM_PREFETCH_TYPE_PREFETCH_LAUNCH = 0x00000000 +NVC6C0_QMDV02_04_PROGRAM_PREFETCH_TYPE_PREFTECH_POST = 0x00000001 +NVC6C0_QMDV02_04_QMD_RESERVED_A = (1015, 1012) +NVC6C0_QMDV02_04_SASS_VERSION = (1023, 1016) +NVC6C0_QMDV02_04_CONSTANT_BUFFER_ADDR_LOWER = lambda i: ((1055+(i)*64), (1024+(i)*64)) # type: ignore +NVC6C0_QMDV02_04_CONSTANT_BUFFER_ADDR_UPPER = lambda i: ((1072+(i)*64), (1056+(i)*64)) # type: ignore +NVC6C0_QMDV02_04_CONSTANT_BUFFER_PREFETCH_POST = lambda i: ((1073+(i)*64), (1073+(i)*64)) # type: ignore +NVC6C0_QMDV02_04_CONSTANT_BUFFER_PREFETCH_POST_FALSE = 0x00000000 +NVC6C0_QMDV02_04_CONSTANT_BUFFER_PREFETCH_POST_TRUE = 0x00000001 +NVC6C0_QMDV02_04_CONSTANT_BUFFER_INVALIDATE = lambda i: ((1074+(i)*64), (1074+(i)*64)) # type: ignore +NVC6C0_QMDV02_04_CONSTANT_BUFFER_INVALIDATE_FALSE = 0x00000000 +NVC6C0_QMDV02_04_CONSTANT_BUFFER_INVALIDATE_TRUE = 0x00000001 +NVC6C0_QMDV02_04_CONSTANT_BUFFER_SIZE_SHIFTED4 = lambda i: ((1087+(i)*64), (1075+(i)*64)) # type: ignore +NVC6C0_QMDV02_04_PROGRAM_ADDRESS_LOWER = (1567, 1536) +NVC6C0_QMDV02_04_PROGRAM_ADDRESS_UPPER = (1584, 1568) +NVC6C0_QMDV02_04_QMD_RESERVED_S = (1599, 1585) +NVC6C0_QMDV02_04_HW_ONLY_INNER_GET = (1630, 1600) +NVC6C0_QMDV02_04_HW_ONLY_REQUIRE_SCHEDULING_PCAS = (1631, 1631) +NVC6C0_QMDV02_04_HW_ONLY_INNER_PUT = (1662, 1632) +NVC6C0_QMDV02_04_HW_ONLY_SCG_TYPE = (1663, 1663) +NVC6C0_QMDV02_04_HW_ONLY_SPAN_LIST_HEAD_INDEX = (1693, 1664) +NVC6C0_QMDV02_04_QMD_RESERVED_Q = (1694, 1694) +NVC6C0_QMDV02_04_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID = (1695, 1695) +NVC6C0_QMDV02_04_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_FALSE = 0x00000000 +NVC6C0_QMDV02_04_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_TRUE = 0x00000001 +NVC6C0_QMDV02_04_HW_ONLY_SKED_NEXT_QMD_POINTER = (1727, 1696) +NVC6C0_QMDV02_04_HW_ONLY_DEPENDENCE_COUNTER = (1734, 1728) +NVC6C0_QMDV02_04_QMD_RESERVED_I = (1759, 1735) +NVC6C0_QMDV02_04_QMD_SPARE_H = (1791, 1760) +NVC6C0_QMDV02_04_QMD_SPARE_I = (1823, 1792) +NVC6C0_QMDV02_04_QMD_SPARE_J = (1855, 1824) +NVC6C0_QMDV02_04_QMD_SPARE_K = (1887, 1856) +NVC6C0_QMDV02_04_QMD_SPARE_L = (1919, 1888) +NVC6C0_QMDV02_04_QMD_SPARE_M = (1951, 1920) +NVC6C0_QMDV02_04_QMD_SPARE_N = (1983, 1952) +NVC6C0_QMDV02_04_DEBUG_ID_UPPER = (2015, 1984) +NVC6C0_QMDV02_04_DEBUG_ID_LOWER = (2047, 2016) +NVC6C0_QMDV03_00_OUTER_PUT = (30, 0) +NVC6C0_QMDV03_00_OUTER_OVERFLOW = (31, 31) +NVC6C0_QMDV03_00_OUTER_GET = (62, 32) +NVC6C0_QMDV03_00_OUTER_STICKY_OVERFLOW = (63, 63) +NVC6C0_QMDV03_00_INNER_GET = (94, 64) +NVC6C0_QMDV03_00_INNER_OVERFLOW = (95, 95) +NVC6C0_QMDV03_00_INNER_PUT = (126, 96) +NVC6C0_QMDV03_00_INNER_STICKY_OVERFLOW = (127, 127) +NVC6C0_QMDV03_00_QMD_GROUP_ID = (133, 128) +NVC6C0_QMDV03_00_SM_GLOBAL_CACHING_ENABLE = (134, 134) +NVC6C0_QMDV03_00_RUN_CTA_IN_ONE_SM_PARTITION = (135, 135) +NVC6C0_QMDV03_00_RUN_CTA_IN_ONE_SM_PARTITION_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RUN_CTA_IN_ONE_SM_PARTITION_TRUE = 0x00000001 +NVC6C0_QMDV03_00_IS_QUEUE = (136, 136) +NVC6C0_QMDV03_00_IS_QUEUE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_IS_QUEUE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST = (137, 137) +NVC6C0_QMDV03_00_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_FALSE = 0x00000000 +NVC6C0_QMDV03_00_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_TRUE = 0x00000001 +NVC6C0_QMDV03_00_QMD_RESERVED04A = (139, 138) +NVC6C0_QMDV03_00_REQUIRE_SCHEDULING_PCAS = (140, 140) +NVC6C0_QMDV03_00_REQUIRE_SCHEDULING_PCAS_FALSE = 0x00000000 +NVC6C0_QMDV03_00_REQUIRE_SCHEDULING_PCAS_TRUE = 0x00000001 +NVC6C0_QMDV03_00_QMD_RESERVED04B = (141, 141) +NVC6C0_QMDV03_00_DEPENDENCE_COUNTER = (157, 142) +NVC6C0_QMDV03_00_SELF_COPY_ON_COMPLETION = (158, 158) +NVC6C0_QMDV03_00_SELF_COPY_ON_COMPLETION_FALSE = 0x00000000 +NVC6C0_QMDV03_00_SELF_COPY_ON_COMPLETION_TRUE = 0x00000001 +NVC6C0_QMDV03_00_QMD_RESERVED04C = (159, 159) +NVC6C0_QMDV03_00_CIRCULAR_QUEUE_SIZE = (184, 160) +NVC6C0_QMDV03_00_DEMOTE_L2_EVICT_LAST = (185, 185) +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_HEADER_CACHE = (186, 186) +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_HEADER_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_HEADER_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_SAMPLER_CACHE = (187, 187) +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_SAMPLER_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_SAMPLER_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_DATA_CACHE = (188, 188) +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_INVALIDATE_SHADER_DATA_CACHE = (189, 189) +NVC6C0_QMDV03_00_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_INVALIDATE_INSTRUCTION_CACHE = (190, 190) +NVC6C0_QMDV03_00_INVALIDATE_INSTRUCTION_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_INVALIDATE_INSTRUCTION_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_INVALIDATE_SHADER_CONSTANT_CACHE = (191, 191) +NVC6C0_QMDV03_00_INVALIDATE_SHADER_CONSTANT_CACHE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_INVALIDATE_SHADER_CONSTANT_CACHE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_CTA_RASTER_WIDTH_RESUME = (223, 192) +NVC6C0_QMDV03_00_CTA_RASTER_HEIGHT_RESUME = (239, 224) +NVC6C0_QMDV03_00_CTA_RASTER_DEPTH_RESUME = (255, 240) +NVC6C0_QMDV03_00_PROGRAM_PREFETCH_ADDR_LOWER_SHIFTED = (287, 256) +NVC6C0_QMDV03_00_CIRCULAR_QUEUE_ADDR_LOWER = (319, 288) +NVC6C0_QMDV03_00_CIRCULAR_QUEUE_ADDR_UPPER = (327, 320) +NVC6C0_QMDV03_00_QMD_RESERVED_D = (335, 328) +NVC6C0_QMDV03_00_CIRCULAR_QUEUE_ENTRY_SIZE = (351, 336) +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_ID = (357, 352) +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_DELTA_MINUS_ONE = (365, 358) +NVC6C0_QMDV03_00_QMD_RESERVED11A = (366, 366) +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_INCR_ENABLE = (367, 367) +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_INCR_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_INCR_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_CWD_MEMBAR_TYPE = (369, 368) +NVC6C0_QMDV03_00_CWD_MEMBAR_TYPE_L1_NONE = 0x00000000 +NVC6C0_QMDV03_00_CWD_MEMBAR_TYPE_L1_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV03_00_CWD_MEMBAR_TYPE_L1_MEMBAR = 0x00000003 +NVC6C0_QMDV03_00_SEQUENTIALLY_RUN_CTAS = (370, 370) +NVC6C0_QMDV03_00_SEQUENTIALLY_RUN_CTAS_FALSE = 0x00000000 +NVC6C0_QMDV03_00_SEQUENTIALLY_RUN_CTAS_TRUE = 0x00000001 +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_DECR_ENABLE = (371, 371) +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_DECR_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_CWD_REFERENCE_COUNT_DECR_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_QMD_RESERVED11B = (377, 372) +NVC6C0_QMDV03_00_API_VISIBLE_CALL_LIMIT = (378, 378) +NVC6C0_QMDV03_00_API_VISIBLE_CALL_LIMIT__32 = 0x00000000 +NVC6C0_QMDV03_00_API_VISIBLE_CALL_LIMIT_NO_CHECK = 0x00000001 +NVC6C0_QMDV03_00_QMD_RESERVED11C = (381, 379) +NVC6C0_QMDV03_00_SAMPLER_INDEX = (382, 382) +NVC6C0_QMDV03_00_SAMPLER_INDEX_INDEPENDENTLY = 0x00000000 +NVC6C0_QMDV03_00_SAMPLER_INDEX_VIA_HEADER_INDEX = 0x00000001 +NVC6C0_QMDV03_00_DISABLE_AUTO_INVALIDATE = (383, 383) +NVC6C0_QMDV03_00_DISABLE_AUTO_INVALIDATE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_DISABLE_AUTO_INVALIDATE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_CTA_RASTER_WIDTH = (415, 384) +NVC6C0_QMDV03_00_CTA_RASTER_HEIGHT = (431, 416) +NVC6C0_QMDV03_00_CTA_RASTER_DEPTH = (463, 448) +NVC6C0_QMDV03_00_DEPENDENT_QMD0_POINTER = (511, 480) +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ENABLE = (512, 512) +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ACTION = (515, 513) +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ACTION_QMD_SCHEDULE = 0x00000001 +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVC6C0_QMDV03_00_DEPENDENT_QMD0_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVC6C0_QMDV03_00_DEPENDENT_QMD0_PREFETCH = (516, 516) +NVC6C0_QMDV03_00_DEPENDENT_QMD0_PREFETCH_FALSE = 0x00000000 +NVC6C0_QMDV03_00_DEPENDENT_QMD0_PREFETCH_TRUE = 0x00000001 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ENABLE = (517, 517) +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ACTION = (520, 518) +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ACTION_QMD_SCHEDULE = 0x00000001 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_PREFETCH = (521, 521) +NVC6C0_QMDV03_00_DEPENDENT_QMD1_PREFETCH_FALSE = 0x00000000 +NVC6C0_QMDV03_00_DEPENDENT_QMD1_PREFETCH_TRUE = 0x00000001 +NVC6C0_QMDV03_00_COALESCE_WAITING_PERIOD = (529, 522) +NVC6C0_QMDV03_00_QUEUE_ENTRIES_PER_CTA_LOG2 = (534, 530) +NVC6C0_QMDV03_00_SHARED_MEMORY_SIZE = (561, 544) +NVC6C0_QMDV03_00_MIN_SM_CONFIG_SHARED_MEM_SIZE = (567, 562) +NVC6C0_QMDV03_00_QMD_RESERVED17A = (568, 568) +NVC6C0_QMDV03_00_MAX_SM_CONFIG_SHARED_MEM_SIZE = (574, 569) +NVC6C0_QMDV03_00_QMD_RESERVED17B = (575, 575) +NVC6C0_QMDV03_00_QMD_VERSION = (579, 576) +NVC6C0_QMDV03_00_QMD_MAJOR_VERSION = (583, 580) +NVC6C0_QMDV03_00_CTA_THREAD_DIMENSION0 = (607, 592) +NVC6C0_QMDV03_00_CTA_THREAD_DIMENSION1 = (623, 608) +NVC6C0_QMDV03_00_CTA_THREAD_DIMENSION2 = (639, 624) +NVC6C0_QMDV03_00_CONSTANT_BUFFER_VALID = lambda i: ((640+(i)*1), (640+(i)*1)) # type: ignore +NVC6C0_QMDV03_00_CONSTANT_BUFFER_VALID_FALSE = 0x00000000 +NVC6C0_QMDV03_00_CONSTANT_BUFFER_VALID_TRUE = 0x00000001 +NVC6C0_QMDV03_00_REGISTER_COUNT_V = (656, 648) +NVC6C0_QMDV03_00_TARGET_SM_CONFIG_SHARED_MEM_SIZE = (662, 657) +NVC6C0_QMDV03_00_SHARED_ALLOCATION_ENABLE = (663, 663) +NVC6C0_QMDV03_00_SHARED_ALLOCATION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_SHARED_ALLOCATION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_FREE_CTA_SLOTS_EMPTY_SM = (671, 664) +NVC6C0_QMDV03_00_SM_DISABLE_MASK_LOWER = (703, 672) +NVC6C0_QMDV03_00_SM_DISABLE_MASK_UPPER = (735, 704) +NVC6C0_QMDV03_00_SHADER_LOCAL_MEMORY_LOW_SIZE = (759, 736) +NVC6C0_QMDV03_00_BARRIER_COUNT = (767, 763) +NVC6C0_QMDV03_00_RELEASE0_ADDRESS_LOWER = (799, 768) +NVC6C0_QMDV03_00_RELEASE0_ADDRESS_UPPER = (807, 800) +NVC6C0_QMDV03_00_SEMAPHORE_RESERVED25A = (818, 808) +NVC6C0_QMDV03_00_RELEASE0_MEMBAR_TYPE = (819, 819) +NVC6C0_QMDV03_00_RELEASE0_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP = (822, 820) +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_QMDV03_00_RELEASE0_ENABLE = (823, 823) +NVC6C0_QMDV03_00_RELEASE0_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_FORMAT = (825, 824) +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_ENABLE = (826, 826) +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_NON_BLOCKING_INTR_TYPE = (828, 827) +NVC6C0_QMDV03_00_RELEASE0_NON_BLOCKING_INTR_TYPE_NONE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_NON_BLOCKING_INTR_TYPE_TRAP = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_NON_BLOCKING_INTR_TYPE_CONDITIONAL_TRAP = 0x00000002 +NVC6C0_QMDV03_00_RELEASE0_PAYLOAD64B = (829, 829) +NVC6C0_QMDV03_00_RELEASE0_PAYLOAD64B_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_PAYLOAD64B_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_STRUCTURE_SIZE = (831, 830) +NVC6C0_QMDV03_00_RELEASE0_STRUCTURE_SIZE_SEMAPHORE_FOUR_WORDS = 0x00000000 +NVC6C0_QMDV03_00_RELEASE0_STRUCTURE_SIZE_SEMAPHORE_ONE_WORD = 0x00000001 +NVC6C0_QMDV03_00_RELEASE0_STRUCTURE_SIZE_SEMAPHORE_TWO_WORDS = 0x00000002 +NVC6C0_QMDV03_00_RELEASE0_PAYLOAD_LOWER = (863, 832) +NVC6C0_QMDV03_00_RELEASE0_PAYLOAD_UPPER = (895, 864) +NVC6C0_QMDV03_00_RELEASE1_ADDRESS_LOWER = (927, 896) +NVC6C0_QMDV03_00_RELEASE1_ADDRESS_UPPER = (935, 928) +NVC6C0_QMDV03_00_SEMAPHORE_RESERVED29A = (946, 936) +NVC6C0_QMDV03_00_RELEASE1_MEMBAR_TYPE = (947, 947) +NVC6C0_QMDV03_00_RELEASE1_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP = (950, 948) +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_QMDV03_00_RELEASE1_ENABLE = (951, 951) +NVC6C0_QMDV03_00_RELEASE1_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_FORMAT = (953, 952) +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_ENABLE = (954, 954) +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_NON_BLOCKING_INTR_TYPE = (956, 955) +NVC6C0_QMDV03_00_RELEASE1_NON_BLOCKING_INTR_TYPE_NONE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_NON_BLOCKING_INTR_TYPE_TRAP = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_NON_BLOCKING_INTR_TYPE_CONDITIONAL_TRAP = 0x00000002 +NVC6C0_QMDV03_00_RELEASE1_PAYLOAD64B = (957, 957) +NVC6C0_QMDV03_00_RELEASE1_PAYLOAD64B_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_PAYLOAD64B_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_STRUCTURE_SIZE = (959, 958) +NVC6C0_QMDV03_00_RELEASE1_STRUCTURE_SIZE_SEMAPHORE_FOUR_WORDS = 0x00000000 +NVC6C0_QMDV03_00_RELEASE1_STRUCTURE_SIZE_SEMAPHORE_ONE_WORD = 0x00000001 +NVC6C0_QMDV03_00_RELEASE1_STRUCTURE_SIZE_SEMAPHORE_TWO_WORDS = 0x00000002 +NVC6C0_QMDV03_00_RELEASE1_PAYLOAD_LOWER = (991, 960) +NVC6C0_QMDV03_00_RELEASE1_PAYLOAD_UPPER = (1023, 992) +NVC6C0_QMDV03_00_CONSTANT_BUFFER_ADDR_LOWER = lambda i: ((1055+(i)*64), (1024+(i)*64)) # type: ignore +NVC6C0_QMDV03_00_CONSTANT_BUFFER_ADDR_UPPER = lambda i: ((1072+(i)*64), (1056+(i)*64)) # type: ignore +NVC6C0_QMDV03_00_CONSTANT_BUFFER_PREFETCH_POST = lambda i: ((1073+(i)*64), (1073+(i)*64)) # type: ignore +NVC6C0_QMDV03_00_CONSTANT_BUFFER_PREFETCH_POST_FALSE = 0x00000000 +NVC6C0_QMDV03_00_CONSTANT_BUFFER_PREFETCH_POST_TRUE = 0x00000001 +NVC6C0_QMDV03_00_CONSTANT_BUFFER_INVALIDATE = lambda i: ((1074+(i)*64), (1074+(i)*64)) # type: ignore +NVC6C0_QMDV03_00_CONSTANT_BUFFER_INVALIDATE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_CONSTANT_BUFFER_INVALIDATE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_CONSTANT_BUFFER_SIZE_SHIFTED4 = lambda i: ((1087+(i)*64), (1075+(i)*64)) # type: ignore +NVC6C0_QMDV03_00_PROGRAM_ADDRESS_LOWER = (1567, 1536) +NVC6C0_QMDV03_00_PROGRAM_ADDRESS_UPPER = (1584, 1568) +NVC6C0_QMDV03_00_SHADER_LOCAL_MEMORY_HIGH_SIZE = (1623, 1600) +NVC6C0_QMDV03_00_PROGRAM_PREFETCH_ADDR_UPPER_SHIFTED = (1640, 1632) +NVC6C0_QMDV03_00_PROGRAM_PREFETCH_SIZE = (1649, 1641) +NVC6C0_QMDV03_00_PROGRAM_PREFETCH_TYPE = (1651, 1650) +NVC6C0_QMDV03_00_PROGRAM_PREFETCH_TYPE_PREFETCH_LAUNCH = 0x00000000 +NVC6C0_QMDV03_00_PROGRAM_PREFETCH_TYPE_PREFTECH_POST = 0x00000001 +NVC6C0_QMDV03_00_SASS_VERSION = (1663, 1656) +NVC6C0_QMDV03_00_RELEASE2_ADDRESS_LOWER = (1695, 1664) +NVC6C0_QMDV03_00_RELEASE2_ADDRESS_UPPER = (1703, 1696) +NVC6C0_QMDV03_00_SEMAPHORE_RESERVED53A = (1714, 1704) +NVC6C0_QMDV03_00_RELEASE2_MEMBAR_TYPE = (1715, 1715) +NVC6C0_QMDV03_00_RELEASE2_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP = (1718, 1716) +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_QMDV03_00_RELEASE2_ENABLE = (1719, 1719) +NVC6C0_QMDV03_00_RELEASE2_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_FORMAT = (1721, 1720) +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_ENABLE = (1722, 1722) +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_NON_BLOCKING_INTR_TYPE = (1724, 1723) +NVC6C0_QMDV03_00_RELEASE2_NON_BLOCKING_INTR_TYPE_NONE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_NON_BLOCKING_INTR_TYPE_TRAP = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_NON_BLOCKING_INTR_TYPE_CONDITIONAL_TRAP = 0x00000002 +NVC6C0_QMDV03_00_RELEASE2_PAYLOAD64B = (1725, 1725) +NVC6C0_QMDV03_00_RELEASE2_PAYLOAD64B_FALSE = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_PAYLOAD64B_TRUE = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_STRUCTURE_SIZE = (1727, 1726) +NVC6C0_QMDV03_00_RELEASE2_STRUCTURE_SIZE_SEMAPHORE_FOUR_WORDS = 0x00000000 +NVC6C0_QMDV03_00_RELEASE2_STRUCTURE_SIZE_SEMAPHORE_ONE_WORD = 0x00000001 +NVC6C0_QMDV03_00_RELEASE2_STRUCTURE_SIZE_SEMAPHORE_TWO_WORDS = 0x00000002 +NVC6C0_QMDV03_00_RELEASE2_PAYLOAD_LOWER = (1759, 1728) +NVC6C0_QMDV03_00_RELEASE2_PAYLOAD_UPPER = (1791, 1760) +NVC6C0_QMDV03_00_QMD_SPARE_I = (1823, 1792) +NVC6C0_QMDV03_00_HW_ONLY_INNER_GET = (1854, 1824) +NVC6C0_QMDV03_00_HW_ONLY_REQUIRE_SCHEDULING_PCAS = (1855, 1855) +NVC6C0_QMDV03_00_HW_ONLY_INNER_PUT = (1886, 1856) +NVC6C0_QMDV03_00_HW_ONLY_SPAN_LIST_HEAD_INDEX = (1917, 1888) +NVC6C0_QMDV03_00_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID = (1919, 1919) +NVC6C0_QMDV03_00_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_FALSE = 0x00000000 +NVC6C0_QMDV03_00_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_TRUE = 0x00000001 +NVC6C0_QMDV03_00_HW_ONLY_SKED_NEXT_QMD_POINTER = (1951, 1920) +NVC6C0_QMDV03_00_HW_ONLY_DEPENDENCE_COUNTER = (1958, 1952) +NVC6C0_QMDV03_00_DEBUG_ID_UPPER = (2015, 1984) +NVC6C0_QMDV03_00_DEBUG_ID_LOWER = (2047, 2016) +NVCEC0_QMDV05_00_HW_ONLY_SPAN_LIST_HEAD_INDEX = (29, 0) +NVCEC0_QMDV05_00_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID = (30, 30) +NVCEC0_QMDV05_00_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_00_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_00_HW_ONLY_REQUIRE_SCHEDULING_PCAS = (31, 31) +NVCEC0_QMDV05_00_HW_ONLY_SKED_NEXT_QMD_POINTER = (63, 32) +NVCEC0_QMDV05_00_INNER_GET = (94, 64) +NVCEC0_QMDV05_00_INNER_OVERFLOW = (95, 95) +NVCEC0_QMDV05_00_INNER_PUT = (126, 96) +NVCEC0_QMDV05_00_INNER_STICKY_OVERFLOW = (127, 127) +NVCEC0_QMDV05_00_DEPENDENCE_COUNTER = (143, 128) +NVCEC0_QMDV05_00_QMD_GROUP_ID = (149, 144) +NVCEC0_QMDV05_00_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST = (150, 150) +NVCEC0_QMDV05_00_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_FALSE = 0x00000000 +NVCEC0_QMDV05_00_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_TRUE = 0x00000001 +NVCEC0_QMDV05_00_QMD_TYPE = (153, 151) +NVCEC0_QMDV05_00_QMD_TYPE_QUEUE = 0x00000000 +NVCEC0_QMDV05_00_QMD_TYPE_GRID_NULL = 0x00000001 +NVCEC0_QMDV05_00_QMD_TYPE_GRID_CTA = 0x00000002 +NVCEC0_QMDV05_00_QMD_TYPE_GRID_GPC_CGA = 0x00000003 +NVCEC0_QMDV05_00_QMD_TYPE_GRID_GPU_CGA = 0x00000004 +NVCEC0_QMDV05_00_QMD_TYPE_GRID_GPU_GPC_CGA = 0x00000005 +NVCEC0_QMDV05_00_NUM_SUB_TASKS_PER_TASK = (157, 154) +NVCEC0_QMDV05_00_REQUIRE_SCHEDULING_PCAS = (158, 158) +NVCEC0_QMDV05_00_REQUIRE_SCHEDULING_PCAS_FALSE = 0x00000000 +NVCEC0_QMDV05_00_REQUIRE_SCHEDULING_PCAS_TRUE = 0x00000001 +NVCEC0_QMDV05_00_TPC_DISABLE_MASK_VALID = (159, 159) +NVCEC0_QMDV05_00_TPC_DISABLE_MASK_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_00_TPC_DISABLE_MASK_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CIRCULAR_QUEUE_SIZE = (184, 160) +NVCEC0_QMDV05_00_HW_ONLY_DEPENDENCE_COUNTER = (207, 192) +NVCEC0_QMDV05_00_RESUME_SUB_TASK_ID = (210, 208) +NVCEC0_QMDV05_00_COMPLETED_SUB_TASK_MASK = (218, 211) +NVCEC0_QMDV05_00_GRID_WIDTH_RESUME = (255, 224) +NVCEC0_QMDV05_00_GRID_HEIGHT_RESUME = (271, 256) +NVCEC0_QMDV05_00_GRID_DEPTH_RESUME = (287, 272) +NVCEC0_QMDV05_00_RELEASE_ENABLE = lambda i: ((288+(i)*16), (288+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE0_ENABLE = NVCEC0_QMDV05_00_RELEASE_ENABLE(0) +NVCEC0_QMDV05_00_RELEASE1_ENABLE = NVCEC0_QMDV05_00_RELEASE_ENABLE(1) +NVCEC0_QMDV05_00_RELEASE_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_STRUCTURE_SIZE = lambda i: ((290+(i)*16), (289+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE_STRUCTURE_SIZE_SEMAPHORE_FOUR_WORDS = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_STRUCTURE_SIZE_SEMAPHORE_ONE_WORD = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_STRUCTURE_SIZE_SEMAPHORE_TWO_WORDS = 0x00000002 +NVCEC0_QMDV05_00_RELEASE_MEMBAR_TYPE = lambda i: ((291+(i)*16), (291+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_ENABLE = lambda i: ((292+(i)*16), (292+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE_REDUCTION_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP = lambda i: ((295+(i)*16), (293+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_ADD = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_MIN = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_MAX = 0x00000002 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_INC = 0x00000003 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_DEC = 0x00000004 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_AND = 0x00000005 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_OR = 0x00000006 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_OP_RED_XOR = 0x00000007 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_FORMAT = lambda i: ((297+(i)*16), (296+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE_REDUCTION_FORMAT_UNSIGNED = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_REDUCTION_FORMAT_SIGNED = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_TRAP_TYPE = lambda i: ((299+(i)*16), (298+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE_TRAP_TYPE_TRAP_NONE = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_TRAP_TYPE_TRAP_UNCONDITIONAL = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_TRAP_TYPE_TRAP_CONDITIONAL = 0x00000002 +NVCEC0_QMDV05_00_RELEASE_TRAP_TYPE_TRAP_CONDITIONAL_EXT = 0x00000003 +NVCEC0_QMDV05_00_RELEASE_PAYLOAD64B = lambda i: ((300+(i)*16), (300+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_RELEASE_PAYLOAD64B_FALSE = 0x00000000 +NVCEC0_QMDV05_00_RELEASE_PAYLOAD64B_TRUE = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_RESERVED_INFO = lambda i: ((303+(i)*16), (301+(i)*16)) # type: ignore +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ENABLE = (336, 336) +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ACTION = (339, 337) +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ACTION_QMD_SCHEDULE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCEC0_QMDV05_00_DEPENDENT_QMD0_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVCEC0_QMDV05_00_DEPENDENT_QMD0_PREFETCH = (340, 340) +NVCEC0_QMDV05_00_DEPENDENT_QMD0_PREFETCH_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD0_PREFETCH_TRUE = 0x00000001 +NVCEC0_QMDV05_00_SELF_COPY_ON_COMPLETION = (341, 341) +NVCEC0_QMDV05_00_SELF_COPY_ON_COMPLETION_FALSE = 0x00000000 +NVCEC0_QMDV05_00_SELF_COPY_ON_COMPLETION_TRUE = 0x00000001 +NVCEC0_QMDV05_00_DEMOTE_L2_EVICT_LAST = (342, 342) +NVCEC0_QMDV05_00_DEMOTE_L2_EVICT_LAST_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DEMOTE_L2_EVICT_LAST_TRUE = 0x00000001 +NVCEC0_QMDV05_00_DISABLE_AUTO_INVALIDATE = (343, 343) +NVCEC0_QMDV05_00_DISABLE_AUTO_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DISABLE_AUTO_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ENABLE = (344, 344) +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ACTION = (347, 345) +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ACTION_QMD_SCHEDULE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_PREFETCH = (348, 348) +NVCEC0_QMDV05_00_DEPENDENT_QMD1_PREFETCH_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD1_PREFETCH_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CORRELATION_ID_INTERNAL = (349, 349) +NVCEC0_QMDV05_00_CORRELATION_ID_INTERNAL_FALSE = 0x00000000 +NVCEC0_QMDV05_00_CORRELATION_ID_INTERNAL_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CWD_MEMBAR_TASK_CHASING_ENABLE = (350, 350) +NVCEC0_QMDV05_00_CWD_MEMBAR_TASK_CHASING_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_CWD_MEMBAR_TASK_CHASING_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_SHARED_ALLOCATION_ENABLE = (351, 351) +NVCEC0_QMDV05_00_SHARED_ALLOCATION_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_SHARED_ALLOCATION_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CORRELATION_ID = (383, 352) +NVCEC0_QMDV05_00_DEPENDENT_QMD0_POINTER = (415, 384) +NVCEC0_QMDV05_00_DEPENDENT_QMD1_POINTER = (447, 416) +NVCEC0_QMDV05_00_SASS_VERSION = (455, 448) +NVCEC0_QMDV05_00_API_VISIBLE_CALL_LIMIT = (456, 456) +NVCEC0_QMDV05_00_API_VISIBLE_CALL_LIMIT__32 = 0x00000000 +NVCEC0_QMDV05_00_API_VISIBLE_CALL_LIMIT_NO_CHECK = 0x00000001 +NVCEC0_QMDV05_00_SAMPLER_INDEX = (457, 457) +NVCEC0_QMDV05_00_SAMPLER_INDEX_INDEPENDENTLY = 0x00000000 +NVCEC0_QMDV05_00_SAMPLER_INDEX_VIA_HEADER_INDEX = 0x00000001 +NVCEC0_QMDV05_00_CONSTANT_BANK_PREFETCH_PRE_MAX_SIZE_SHIFTED8 = (463, 458) +NVCEC0_QMDV05_00_QMD_MINOR_VERSION = (467, 464) +NVCEC0_QMDV05_00_QMD_MAJOR_VERSION = (471, 468) +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_HEADER_CACHE = (472, 472) +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_HEADER_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_HEADER_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_SAMPLER_CACHE = (473, 473) +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_SAMPLER_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_SAMPLER_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_DATA_CACHE = (474, 474) +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_INVALIDATE_SHADER_DATA_CACHE = (475, 475) +NVCEC0_QMDV05_00_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_INVALIDATE_INSTRUCTION_CACHE = (476, 476) +NVCEC0_QMDV05_00_INVALIDATE_INSTRUCTION_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_INVALIDATE_INSTRUCTION_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_INVALIDATE_SHADER_CONSTANT_CACHE = (477, 477) +NVCEC0_QMDV05_00_INVALIDATE_SHADER_CONSTANT_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_INVALIDATE_SHADER_CONSTANT_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE = (478, 478) +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE = (479, 479) +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE0_ADDR_LOWER = (511, 480) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE0_ADDR_UPPER = (536, 512) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE0_PAYLOAD_LOWER = (575, 544) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE0_PAYLOAD_UPPER = (607, 576) +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_DELTA_MINUS_ONE = (615, 608) +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_ID = (621, 616) +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_INCR_ENABLE = (622, 622) +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_INCR_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_INCR_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_DECR_ENABLE = (623, 623) +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_DECR_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_CWD_REFERENCE_COUNT_DECR_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CWD_MEMBAR_TYPE = (625, 624) +NVCEC0_QMDV05_00_CWD_MEMBAR_TYPE_L1_NONE = 0x00000000 +NVCEC0_QMDV05_00_CWD_MEMBAR_TYPE_L1_SYSMEMBAR = 0x00000001 +NVCEC0_QMDV05_00_CWD_MEMBAR_TYPE_L1_MEMBAR = 0x00000003 +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_CONSTANT_CACHE = (626, 626) +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_CONSTANT_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_LATCH_ACQUIRE_INVALIDATE_CONSTANT_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CTA_LAUNCH_QUEUE = (627, 627) +NVCEC0_QMDV05_00_FREE_CTA_SLOTS_EMPTY_SM = (635, 628) +NVCEC0_QMDV05_00_SYNC_DOMAIN_ID = (637, 636) +NVCEC0_QMDV05_00_PRE_EXIT_AT_LAST_CTA_LAUNCH = (638, 638) +NVCEC0_QMDV05_00_PRE_EXIT_AT_LAST_CTA_LAUNCH_FALSE = 0x00000000 +NVCEC0_QMDV05_00_PRE_EXIT_AT_LAST_CTA_LAUNCH_TRUE = 0x00000001 +NVCEC0_QMDV05_00_ENABLE_PROGRAM_PRE_EXIT = (639, 639) +NVCEC0_QMDV05_00_ENABLE_PROGRAM_PRE_EXIT_FALSE = 0x00000000 +NVCEC0_QMDV05_00_ENABLE_PROGRAM_PRE_EXIT_TRUE = 0x00000001 +NVCEC0_QMDV05_00_ARRIVE_AT_LATCH_ID = (671, 640) +NVCEC0_QMDV05_00_WAIT_ON_LATCH_ID = (703, 672) +NVCEC0_QMDV05_00_OCCUPANCY_THRESHOLD_SHARED_MEM = (721, 714) +NVCEC0_QMDV05_00_OCCUPANCY_MAX_SHARED_MEM = (729, 722) +NVCEC0_QMDV05_00_ARRIVE_AT_LATCH_VALID = (730, 730) +NVCEC0_QMDV05_00_ARRIVE_AT_LATCH_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_00_ARRIVE_AT_LATCH_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_00_WAIT_ON_LATCH_VALID = (731, 731) +NVCEC0_QMDV05_00_WAIT_ON_LATCH_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_00_WAIT_ON_LATCH_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_00_LATCH_RELEASE_INVALIDATE_ENABLE = (732, 732) +NVCEC0_QMDV05_00_LATCH_RELEASE_INVALIDATE_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_LATCH_RELEASE_INVALIDATE_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_HOLD_CTA_LAUNCH_UNTIL_PARENT_LATCH_ACQUIRE_AND_CTA_COMPLETE = (733, 733) +NVCEC0_QMDV05_00_HOLD_CTA_LAUNCH_UNTIL_PARENT_LATCH_ACQUIRE_AND_CTA_COMPLETE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_HOLD_CTA_LAUNCH_UNTIL_PARENT_LATCH_ACQUIRE_AND_CTA_COMPLETE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_HOLD_MEMBAR_UNTIL_LATCH_ACQUIRE = (734, 734) +NVCEC0_QMDV05_00_HOLD_MEMBAR_UNTIL_LATCH_ACQUIRE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_HOLD_MEMBAR_UNTIL_LATCH_ACQUIRE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_PRIORITY_DEMOTE_UNTIL_LATCH_ACQUIRE = (735, 735) +NVCEC0_QMDV05_00_PRIORITY_DEMOTE_UNTIL_LATCH_ACQUIRE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_PRIORITY_DEMOTE_UNTIL_LATCH_ACQUIRE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_OCCUPANCY_THRESHOLD_WARP = (743, 736) +NVCEC0_QMDV05_00_OCCUPANCY_MAX_WARP = (751, 744) +NVCEC0_QMDV05_00_OCCUPANCY_THRESHOLD_REGISTER = (759, 752) +NVCEC0_QMDV05_00_OCCUPANCY_MAX_REGISTER = (767, 760) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE1_ADDR_LOWER = (799, 768) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE1_ADDR_UPPER = (824, 800) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE1_PAYLOAD_LOWER = (863, 832) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE1_PAYLOAD_UPPER = (895, 864) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE2_ADDR_LOWER = (927, 896) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE2_ADDR_UPPER = (952, 928) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE2_PAYLOAD_LOWER = (991, 960) +NVCEC0_QMDV05_00_RELEASE_SEMAPHORE2_PAYLOAD_UPPER = (1023, 992) +NVCEC0_QMDV05_00_PROGRAM_ADDRESS_LOWER_SHIFTED4 = (1055, 1024) +NVCEC0_QMDV05_00_PROGRAM_ADDRESS_UPPER_SHIFTED4 = (1076, 1056) +NVCEC0_QMDV05_00_PROGRAM_PREFETCH_SIZE = (1085, 1077) +NVCEC0_QMDV05_00_PROGRAM_PREFETCH_TYPE = (1087, 1086) +NVCEC0_QMDV05_00_PROGRAM_PREFETCH_TYPE_PREFETCH_LAUNCH = 0x00000000 +NVCEC0_QMDV05_00_PROGRAM_PREFETCH_TYPE_PREFTECH_POST = 0x00000001 +NVCEC0_QMDV05_00_CTA_THREAD_DIMENSION0 = (1103, 1088) +NVCEC0_QMDV05_00_CTA_THREAD_DIMENSION1 = (1119, 1104) +NVCEC0_QMDV05_00_CTA_THREAD_DIMENSION2 = (1127, 1120) +NVCEC0_QMDV05_00_REGISTER_COUNT = (1136, 1128) +NVCEC0_QMDV05_00_BARRIER_COUNT = (1141, 1137) +NVCEC0_QMDV05_00_ICC_PREFETCH_SIZE = (1147, 1142) +NVCEC0_QMDV05_00_SHARED_MEMORY_SIZE_SHIFTED7 = (1162, 1152) +NVCEC0_QMDV05_00_MIN_SM_CONFIG_SHARED_MEM_SIZE = (1168, 1163) +NVCEC0_QMDV05_00_MAX_SM_CONFIG_SHARED_MEM_SIZE = (1174, 1169) +NVCEC0_QMDV05_00_TARGET_SM_CONFIG_SHARED_MEM_SIZE = (1180, 1175) +NVCEC0_QMDV05_00_SHARED_MEM_BARRIER_INIT_ENABLE = (1181, 1181) +NVCEC0_QMDV05_00_SHADER_LOCAL_MEMORY_LOW_SIZE_SHIFTED4 = (1199, 1184) +NVCEC0_QMDV05_00_SHADER_LOCAL_MEMORY_HIGH_SIZE_SHIFTED4 = (1215, 1200) +NVCEC0_QMDV05_00_VIRTUAL_RESOURCE_COUNT = (1223, 1216) +NVCEC0_QMDV05_00_GRID_WIDTH = (1279, 1248) +NVCEC0_QMDV05_00_GRID_HEIGHT = (1295, 1280) +NVCEC0_QMDV05_00_GRID_DEPTH = (1327, 1312) +NVCEC0_QMDV05_00_CONSTANT_BUFFER_ADDR_LOWER_SHIFTED6 = lambda i: ((1375+(i)*64), (1344+(i)*64)) # type: ignore +NVCEC0_QMDV05_00_CONSTANT_BUFFER_ADDR_UPPER_SHIFTED6 = lambda i: ((1394+(i)*64), (1376+(i)*64)) # type: ignore +NVCEC0_QMDV05_00_CONSTANT_BUFFER_SIZE_SHIFTED4 = lambda i: ((1407+(i)*64), (1395+(i)*64)) # type: ignore +NVCEC0_QMDV05_00_CONSTANT_BUFFER_VALID = lambda i: ((1856+(i)*4), (1856+(i)*4)) # type: ignore +NVCEC0_QMDV05_00_CONSTANT_BUFFER_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_00_CONSTANT_BUFFER_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_00_CONSTANT_BUFFER_PREFETCH = lambda i: ((1858+(i)*4), (1857+(i)*4)) # type: ignore +NVCEC0_QMDV05_00_CONSTANT_BUFFER_PREFETCH_PREFETCH_NONE = 0x00000000 +NVCEC0_QMDV05_00_CONSTANT_BUFFER_PREFETCH_PREFETCH_PRE = 0x00000001 +NVCEC0_QMDV05_00_CONSTANT_BUFFER_PREFETCH_PREFETCH_POST = 0x00000002 +NVCEC0_QMDV05_00_CONSTANT_BUFFER_INVALIDATE = lambda i: ((1859+(i)*4), (1859+(i)*4)) # type: ignore +NVCEC0_QMDV05_00_CONSTANT_BUFFER_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_CONSTANT_BUFFER_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_PROGRAM_PREFETCH_ADDR_LOWER_SHIFTED = (1919, 1888) +NVCEC0_QMDV05_00_PROGRAM_PREFETCH_ADDR_UPPER_SHIFTED = (1936, 1920) +NVCEC0_QMDV05_00_GPC_CGA_WIDTH = (2053, 2048) +NVCEC0_QMDV05_00_GPC_CGA_HEIGHT = (2061, 2056) +NVCEC0_QMDV05_00_GPC_CGA_DEPTH = (2069, 2064) +NVCEC0_QMDV05_00_LARGE_GPC_CGA_WIDTH_MINUS_ONE = (2075, 2072) +NVCEC0_QMDV05_00_LARGE_GPC_CGA_HEIGHT_MINUS_ONE = (2079, 2076) +NVCEC0_QMDV05_00_CGA_CTA_DISTRIBUTION_MODE = (2111, 2111) +NVCEC0_QMDV05_00_CGA_CTA_DISTRIBUTION_MODE_LOAD_BALANCING = 0x00000000 +NVCEC0_QMDV05_00_CGA_CTA_DISTRIBUTION_MODE_MULTI_CAST = 0x00000001 +NVCEC0_QMDV05_00_GPU_CGA_WIDTH = (2127, 2112) +NVCEC0_QMDV05_00_GPU_CGA_HEIGHT = (2143, 2128) +NVCEC0_QMDV05_00_GPU_CGA_DEPTH = (2159, 2144) +NVCEC0_QMDV05_00_DEBUG_ID_LOWER = (2207, 2176) +NVCEC0_QMDV05_00_DEBUG_ID_UPPER = (2239, 2208) +NVCEC0_QMDV05_00_TPC_DISABLE_MASK = lambda i: ((2271+(i)*32), (2240+(i)*32)) # type: ignore +NVCEC0_QMDV05_00_INCOMPLETE_BOX_BASE_WIDTH_RESUME = (2527, 2496) +NVCEC0_QMDV05_00_INCOMPLETE_BOX_BASE_HEIGHT_RESUME = (2543, 2528) +NVCEC0_QMDV05_00_INCOMPLETE_BOX_BASE_DEPTH_RESUME = (2559, 2544) +NVCEC0_QMDV05_00_INCOMPLETE_BOX_OFFSET_WIDTH_RESUME = (2563, 2560) +NVCEC0_QMDV05_00_INCOMPLETE_BOX_OFFSET_HEIGHT_RESUME = (2567, 2564) +NVCEC0_QMDV05_00_QUEUE_ENTRIES_PER_CTA_LOG2 = (2596, 2592) +NVCEC0_QMDV05_00_HW_ONLY_INNER_GET = (2654, 2624) +NVCEC0_QMDV05_00_HW_ONLY_INNER_PUT = (2686, 2656) +NVCEC0_QMDV05_00_OUTER_PUT = (3038, 3008) +NVCEC0_QMDV05_00_OUTER_OVERFLOW = (3039, 3039) +NVCEC0_QMDV05_00_OUTER_GET = (3070, 3040) +NVCEC0_QMDV05_00_OUTER_STICKY_OVERFLOW = (3071, 3071) +NVCEC0_QMDV05_00_DEPENDENT_QMD_ENABLE = lambda i: ((336+(i)*8), (336+(i)*8)) # type: ignore +NVCEC0_QMDV05_00_DEPENDENT_QMD_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD_ACTION = lambda i: ((339+(i)*8), (337+(i)*8)) # type: ignore +NVCEC0_QMDV05_00_DEPENDENT_QMD_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD_ACTION_QMD_SCHEDULE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCEC0_QMDV05_00_DEPENDENT_QMD_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVCEC0_QMDV05_00_DEPENDENT_QMD_PREFETCH = lambda i: ((340+(i)*8), (340+(i)*8)) # type: ignore +NVCEC0_QMDV05_00_DEPENDENT_QMD_PREFETCH_FALSE = 0x00000000 +NVCEC0_QMDV05_00_DEPENDENT_QMD_PREFETCH_TRUE = 0x00000001 +NVCEC0_QMDV05_00_DEPENDENT_QMD_POINTER = lambda i: ((415+(i)*32), (384+(i)*32)) # type: ignore +NVCEC0_QMDV05_01_HW_ONLY_SPAN_LIST_HEAD_INDEX = (29, 0) +NVCEC0_QMDV05_01_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID = (30, 30) +NVCEC0_QMDV05_01_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_01_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_01_HW_ONLY_REQUIRE_SCHEDULING_PCAS = (31, 31) +NVCEC0_QMDV05_01_HW_ONLY_SKED_NEXT_QMD_POINTER = (63, 32) +NVCEC0_QMDV05_01_INNER_GET = (94, 64) +NVCEC0_QMDV05_01_INNER_OVERFLOW = (95, 95) +NVCEC0_QMDV05_01_INNER_PUT = (126, 96) +NVCEC0_QMDV05_01_INNER_STICKY_OVERFLOW = (127, 127) +NVCEC0_QMDV05_01_DEPENDENCE_COUNTER = (143, 128) +NVCEC0_QMDV05_01_QMD_GROUP_ID = (149, 144) +NVCEC0_QMDV05_01_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST = (150, 150) +NVCEC0_QMDV05_01_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_FALSE = 0x00000000 +NVCEC0_QMDV05_01_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_TRUE = 0x00000001 +NVCEC0_QMDV05_01_QMD_TYPE = (153, 151) +NVCEC0_QMDV05_01_QMD_TYPE_QUEUE = 0x00000000 +NVCEC0_QMDV05_01_QMD_TYPE_GRID_NULL = 0x00000001 +NVCEC0_QMDV05_01_QMD_TYPE_GRID_CTA = 0x00000002 +NVCEC0_QMDV05_01_QMD_TYPE_GRID_GPC_CGA = 0x00000003 +NVCEC0_QMDV05_01_QMD_TYPE_GRID_GPU_CGA = 0x00000004 +NVCEC0_QMDV05_01_QMD_TYPE_GRID_GPU_GPC_CGA = 0x00000005 +NVCEC0_QMDV05_01_NUM_SUB_TASKS_PER_TASK = (157, 154) +NVCEC0_QMDV05_01_REQUIRE_SCHEDULING_PCAS = (158, 158) +NVCEC0_QMDV05_01_REQUIRE_SCHEDULING_PCAS_FALSE = 0x00000000 +NVCEC0_QMDV05_01_REQUIRE_SCHEDULING_PCAS_TRUE = 0x00000001 +NVCEC0_QMDV05_01_TPC_DISABLE_MASK_VALID = (159, 159) +NVCEC0_QMDV05_01_TPC_DISABLE_MASK_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_01_TPC_DISABLE_MASK_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_01_CIRCULAR_QUEUE_SIZE = (184, 160) +NVCEC0_QMDV05_01_HW_ONLY_DEPENDENCE_COUNTER = (207, 192) +NVCEC0_QMDV05_01_RESUME_SUB_TASK_ID = (210, 208) +NVCEC0_QMDV05_01_COMPLETED_SUB_TASK_MASK = (218, 211) +NVCEC0_QMDV05_01_GRID_WIDTH_RESUME = (255, 224) +NVCEC0_QMDV05_01_GRID_HEIGHT_RESUME = (271, 256) +NVCEC0_QMDV05_01_GRID_DEPTH_RESUME = (287, 272) +NVCEC0_QMDV05_01_RELEASE_ENABLE = lambda i: ((288+(i)*16), (288+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_STRUCTURE_SIZE = lambda i: ((290+(i)*16), (289+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_STRUCTURE_SIZE_SEMAPHORE_FOUR_WORDS = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_STRUCTURE_SIZE_SEMAPHORE_ONE_WORD = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_STRUCTURE_SIZE_SEMAPHORE_TWO_WORDS = 0x00000002 +NVCEC0_QMDV05_01_RELEASE_MEMBAR_TYPE = lambda i: ((291+(i)*16), (291+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_ENABLE = lambda i: ((292+(i)*16), (292+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_REDUCTION_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP = lambda i: ((295+(i)*16), (293+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_ADD = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_MIN = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_MAX = 0x00000002 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_INC = 0x00000003 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_DEC = 0x00000004 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_AND = 0x00000005 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_OR = 0x00000006 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_OP_RED_XOR = 0x00000007 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_FORMAT = lambda i: ((297+(i)*16), (296+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_REDUCTION_FORMAT_UNSIGNED = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_REDUCTION_FORMAT_SIGNED = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_TRAP_TYPE = lambda i: ((299+(i)*16), (298+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_TRAP_TYPE_TRAP_NONE = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_TRAP_TYPE_TRAP_UNCONDITIONAL = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_TRAP_TYPE_TRAP_CONDITIONAL = 0x00000002 +NVCEC0_QMDV05_01_RELEASE_TRAP_TYPE_TRAP_CONDITIONAL_EXT = 0x00000003 +NVCEC0_QMDV05_01_RELEASE_PAYLOAD64B = lambda i: ((300+(i)*16), (300+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_RELEASE_PAYLOAD64B_FALSE = 0x00000000 +NVCEC0_QMDV05_01_RELEASE_PAYLOAD64B_TRUE = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_RESERVED_INFO = lambda i: ((303+(i)*16), (301+(i)*16)) # type: ignore +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ENABLE = (336, 336) +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ACTION = (339, 337) +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ACTION_QMD_SCHEDULE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCEC0_QMDV05_01_DEPENDENT_QMD0_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVCEC0_QMDV05_01_DEPENDENT_QMD0_PREFETCH = (340, 340) +NVCEC0_QMDV05_01_DEPENDENT_QMD0_PREFETCH_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD0_PREFETCH_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SELF_COPY_ON_COMPLETION = (341, 341) +NVCEC0_QMDV05_01_SELF_COPY_ON_COMPLETION_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SELF_COPY_ON_COMPLETION_TRUE = 0x00000001 +NVCEC0_QMDV05_01_DEMOTE_L2_EVICT_LAST = (342, 342) +NVCEC0_QMDV05_01_DEMOTE_L2_EVICT_LAST_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DEMOTE_L2_EVICT_LAST_TRUE = 0x00000001 +NVCEC0_QMDV05_01_DISABLE_AUTO_INVALIDATE = (343, 343) +NVCEC0_QMDV05_01_DISABLE_AUTO_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DISABLE_AUTO_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ENABLE = (344, 344) +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ACTION = (347, 345) +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ACTION_QMD_SCHEDULE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_PREFETCH = (348, 348) +NVCEC0_QMDV05_01_DEPENDENT_QMD1_PREFETCH_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD1_PREFETCH_TRUE = 0x00000001 +NVCEC0_QMDV05_01_CORRELATION_ID_INTERNAL = (349, 349) +NVCEC0_QMDV05_01_CORRELATION_ID_INTERNAL_FALSE = 0x00000000 +NVCEC0_QMDV05_01_CORRELATION_ID_INTERNAL_TRUE = 0x00000001 +NVCEC0_QMDV05_01_CWD_MEMBAR_TASK_CHASING_ENABLE = (350, 350) +NVCEC0_QMDV05_01_CWD_MEMBAR_TASK_CHASING_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_CWD_MEMBAR_TASK_CHASING_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SHARED_ALLOCATION_ENABLE = (351, 351) +NVCEC0_QMDV05_01_SHARED_ALLOCATION_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SHARED_ALLOCATION_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_CORRELATION_ID = (383, 352) +NVCEC0_QMDV05_01_DEPENDENT_QMD0_POINTER = (415, 384) +NVCEC0_QMDV05_01_DEPENDENT_QMD1_POINTER = (447, 416) +NVCEC0_QMDV05_01_SASS_VERSION = (455, 448) +NVCEC0_QMDV05_01_API_VISIBLE_CALL_LIMIT = (456, 456) +NVCEC0_QMDV05_01_API_VISIBLE_CALL_LIMIT__32 = 0x00000000 +NVCEC0_QMDV05_01_API_VISIBLE_CALL_LIMIT_NO_CHECK = 0x00000001 +NVCEC0_QMDV05_01_SAMPLER_INDEX = (457, 457) +NVCEC0_QMDV05_01_SAMPLER_INDEX_INDEPENDENTLY = 0x00000000 +NVCEC0_QMDV05_01_SAMPLER_INDEX_VIA_HEADER_INDEX = 0x00000001 +NVCEC0_QMDV05_01_CONSTANT_BANK_PREFETCH_PRE_MAX_SIZE_SHIFTED8 = (463, 458) +NVCEC0_QMDV05_01_QMD_MINOR_VERSION = (467, 464) +NVCEC0_QMDV05_01_QMD_MAJOR_VERSION = (471, 468) +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_HEADER_CACHE = (472, 472) +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_HEADER_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_HEADER_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_SAMPLER_CACHE = (473, 473) +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_SAMPLER_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_SAMPLER_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_DATA_CACHE = (474, 474) +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_INVALIDATE_SHADER_DATA_CACHE = (475, 475) +NVCEC0_QMDV05_01_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_INVALIDATE_INSTRUCTION_CACHE = (476, 476) +NVCEC0_QMDV05_01_INVALIDATE_INSTRUCTION_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_INVALIDATE_INSTRUCTION_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_INVALIDATE_SHADER_CONSTANT_CACHE = (477, 477) +NVCEC0_QMDV05_01_INVALIDATE_SHADER_CONSTANT_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_INVALIDATE_SHADER_CONSTANT_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE = (478, 478) +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE = (479, 479) +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_RELEASE_SEMAPHORE0_ADDR_LOWER = (511, 480) +NVCEC0_QMDV05_01_RELEASE_SEMAPHORE0_ADDR_UPPER = (536, 512) +NVCEC0_QMDV05_01_RELEASE_SEMAPHORE0_PAYLOAD_LOWER = (575, 544) +NVCEC0_QMDV05_01_RELEASE_SEMAPHORE0_PAYLOAD_UPPER = (607, 576) +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_DELTA_MINUS_ONE = (615, 608) +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_ID = (621, 616) +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_INCR_ENABLE = (622, 622) +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_INCR_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_INCR_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_DECR_ENABLE = (623, 623) +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_DECR_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_CWD_REFERENCE_COUNT_DECR_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_CWD_MEMBAR_TYPE = (625, 624) +NVCEC0_QMDV05_01_CWD_MEMBAR_TYPE_L1_NONE = 0x00000000 +NVCEC0_QMDV05_01_CWD_MEMBAR_TYPE_L1_SYSMEMBAR = 0x00000001 +NVCEC0_QMDV05_01_CWD_MEMBAR_TYPE_L1_MEMBAR = 0x00000003 +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_CONSTANT_CACHE = (626, 626) +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_CONSTANT_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_LATCH_ACQUIRE_INVALIDATE_CONSTANT_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_CTA_LAUNCH_QUEUE = (627, 627) +NVCEC0_QMDV05_01_FREE_CTA_SLOTS_EMPTY_SM = (635, 628) +NVCEC0_QMDV05_01_SYNC_DOMAIN_ID = (637, 636) +NVCEC0_QMDV05_01_PRE_EXIT_AT_LAST_CTA_LAUNCH = (638, 638) +NVCEC0_QMDV05_01_PRE_EXIT_AT_LAST_CTA_LAUNCH_FALSE = 0x00000000 +NVCEC0_QMDV05_01_PRE_EXIT_AT_LAST_CTA_LAUNCH_TRUE = 0x00000001 +NVCEC0_QMDV05_01_ENABLE_PROGRAM_PRE_EXIT = (639, 639) +NVCEC0_QMDV05_01_ENABLE_PROGRAM_PRE_EXIT_FALSE = 0x00000000 +NVCEC0_QMDV05_01_ENABLE_PROGRAM_PRE_EXIT_TRUE = 0x00000001 +NVCEC0_QMDV05_01_ARRIVE_AT_LATCH_ID = (671, 640) +NVCEC0_QMDV05_01_WAIT_ON_LATCH_ID = (703, 672) +NVCEC0_QMDV05_01_OCCUPANCY_THRESHOLD_SHARED_MEM = (721, 714) +NVCEC0_QMDV05_01_OCCUPANCY_MAX_SHARED_MEM = (729, 722) +NVCEC0_QMDV05_01_ARRIVE_AT_LATCH_VALID = (730, 730) +NVCEC0_QMDV05_01_ARRIVE_AT_LATCH_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_01_ARRIVE_AT_LATCH_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_01_WAIT_ON_LATCH_VALID = (731, 731) +NVCEC0_QMDV05_01_WAIT_ON_LATCH_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_01_WAIT_ON_LATCH_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_01_LATCH_RELEASE_INVALIDATE_ENABLE = (732, 732) +NVCEC0_QMDV05_01_LATCH_RELEASE_INVALIDATE_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_LATCH_RELEASE_INVALIDATE_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_HOLD_CTA_LAUNCH_UNTIL_PARENT_LATCH_ACQUIRE_AND_CTA_COMPLETE = (733, 733) +NVCEC0_QMDV05_01_HOLD_CTA_LAUNCH_UNTIL_PARENT_LATCH_ACQUIRE_AND_CTA_COMPLETE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_HOLD_CTA_LAUNCH_UNTIL_PARENT_LATCH_ACQUIRE_AND_CTA_COMPLETE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_HOLD_MEMBAR_UNTIL_LATCH_ACQUIRE = (734, 734) +NVCEC0_QMDV05_01_HOLD_MEMBAR_UNTIL_LATCH_ACQUIRE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_HOLD_MEMBAR_UNTIL_LATCH_ACQUIRE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_PRIORITY_DEMOTE_UNTIL_LATCH_ACQUIRE = (735, 735) +NVCEC0_QMDV05_01_PRIORITY_DEMOTE_UNTIL_LATCH_ACQUIRE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_PRIORITY_DEMOTE_UNTIL_LATCH_ACQUIRE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_OCCUPANCY_THRESHOLD_WARP = (743, 736) +NVCEC0_QMDV05_01_OCCUPANCY_MAX_WARP = (751, 744) +NVCEC0_QMDV05_01_OCCUPANCY_THRESHOLD_REGISTER = (759, 752) +NVCEC0_QMDV05_01_OCCUPANCY_MAX_REGISTER = (767, 760) +NVCEC0_QMDV05_01_SUB_TASK_PROGRAM_ADDRESS_LOWER_SHIFTED4 = lambda i: ((799+(i)*416), (768+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_PROGRAM_ADDRESS_UPPER_SHIFTED4 = lambda i: ((820+(i)*416), (800+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_PROGRAM_PREFETCH_SIZE = lambda i: ((829+(i)*416), (821+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_PROGRAM_PREFETCH_TYPE = lambda i: ((831+(i)*416), (830+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_PROGRAM_PREFETCH_TYPE_PREFETCH_LAUNCH = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_PROGRAM_PREFETCH_TYPE_PREFTECH_POST = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CTA_THREAD_DIMENSION0 = lambda i: ((847+(i)*416), (832+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CTA_THREAD_DIMENSION1 = lambda i: ((863+(i)*416), (848+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CTA_THREAD_DIMENSION2 = lambda i: ((871+(i)*416), (864+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_REGISTER_COUNT = lambda i: ((880+(i)*416), (872+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_BARRIER_COUNT = lambda i: ((885+(i)*416), (881+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_VALID = lambda i: ((886+(i)*416), (886+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_PREFETCH = lambda i: ((888+(i)*416), (887+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_PREFETCH_PREFETCH_NONE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_PREFETCH_PREFETCH_PRE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_PREFETCH_PREFETCH_POST = 0x00000002 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_INVALIDATE = lambda i: ((889+(i)*416), (889+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_ICC_PREFETCH_SIZE = lambda i: ((895+(i)*416), (890+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_SHARED_MEMORY_SIZE_SHIFTED7 = lambda i: ((906+(i)*416), (896+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_MIN_SM_CONFIG_SHARED_MEM_SIZE = lambda i: ((912+(i)*416), (907+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_MAX_SM_CONFIG_SHARED_MEM_SIZE = lambda i: ((918+(i)*416), (913+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_TARGET_SM_CONFIG_SHARED_MEM_SIZE = lambda i: ((924+(i)*416), (919+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_SHARED_MEM_BARRIER_INIT_ENABLE = lambda i: ((925+(i)*416), (925+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_SHADER_LOCAL_MEMORY_LOW_SIZE_SHIFTED4 = lambda i: ((943+(i)*416), (928+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_SHADER_LOCAL_MEMORY_HIGH_SIZE_SHIFTED4 = lambda i: ((959+(i)*416), (944+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_ADDR_LOWER_SHIFTED6 = lambda i: ((991+(i)*416), (960+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_ADDR_UPPER_SHIFTED6 = lambda i: ((1010+(i)*416), (992+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK0_SIZE_SHIFTED4 = lambda i: ((1023+(i)*416), (1011+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_ADDR_LOWER_SHIFTED6 = lambda i: ((1055+(i)*416), (1024+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_ADDR_UPPER_SHIFTED6 = lambda i: ((1074+(i)*416), (1056+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_SIZE_SHIFTED4 = lambda i: ((1087+(i)*416), (1075+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_VALID = lambda i: ((1088+(i)*416), (1088+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_PREFETCH = lambda i: ((1090+(i)*416), (1089+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_PREFETCH_PREFETCH_NONE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_PREFETCH_PREFETCH_PRE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_PREFETCH_PREFETCH_POST = 0x00000002 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_INVALIDATE = lambda i: ((1091+(i)*416), (1091+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK1_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_VIRTUAL_RESOURCE_COUNT = lambda i: ((1099+(i)*416), (1092+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_GRID_WIDTH = lambda i: ((1151+(i)*416), (1120+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_GRID_HEIGHT = lambda i: ((1167+(i)*416), (1152+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_GRID_DEPTH = lambda i: ((1183+(i)*416), (1168+(i)*416)) # type: ignore +NVCEC0_QMDV05_01_DEPENDENT_QMD_ENABLE = lambda i: ((336+(i)*8), (336+(i)*8)) # type: ignore +NVCEC0_QMDV05_01_DEPENDENT_QMD_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD_ACTION = lambda i: ((339+(i)*8), (337+(i)*8)) # type: ignore +NVCEC0_QMDV05_01_DEPENDENT_QMD_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD_ACTION_QMD_SCHEDULE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCEC0_QMDV05_01_DEPENDENT_QMD_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVCEC0_QMDV05_01_DEPENDENT_QMD_PREFETCH = lambda i: ((340+(i)*8), (340+(i)*8)) # type: ignore +NVCEC0_QMDV05_01_DEPENDENT_QMD_PREFETCH_FALSE = 0x00000000 +NVCEC0_QMDV05_01_DEPENDENT_QMD_PREFETCH_TRUE = 0x00000001 +NVCEC0_QMDV05_01_DEPENDENT_QMD_POINTER = lambda i: ((415+(i)*32), (384+(i)*32)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_VALID = lambda i,j: ((886+(i)*416+(j)*202), (886+(i)*416+(j)*202)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_VALID_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_VALID_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_PREFETCH = lambda i,j: ((888+(i)*416+(j)*202), (887+(i)*416+(j)*202)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_PREFETCH_PREFETCH_NONE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_PREFETCH_PREFETCH_PRE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_PREFETCH_PREFETCH_POST = 0x00000002 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_INVALIDATE = lambda i,j: ((889+(i)*416+(j)*202), (889+(i)*416+(j)*202)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_ADDR_LOWER_SHIFTED6 = lambda i,j: ((991+(i)*416+(j)*64), (960+(i)*416+(j)*64)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_ADDR_UPPER_SHIFTED6 = lambda i,j: ((1010+(i)*416+(j)*64), (992+(i)*416+(j)*64)) # type: ignore +NVCEC0_QMDV05_01_SUB_TASK_CONSTANT_BANK_SIZE_SHIFTED4 = lambda i,j: ((1023+(i)*416+(j)*64), (1011+(i)*416+(j)*64)) # type: ignore +NVCEC0_QMDV04_01_DEPENDENCE_COUNTER = (15, 0) +NVCEC0_QMDV04_01_QMD_GROUP_ID = (21, 16) +NVCEC0_QMDV04_01_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST = (22, 22) +NVCEC0_QMDV04_01_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_FALSE = 0x00000000 +NVCEC0_QMDV04_01_ADD_TO_HEAD_OF_QMD_GROUP_LINKED_LIST_TRUE = 0x00000001 +NVCEC0_QMDV04_01_QMD_TYPE = (25, 23) +NVCEC0_QMDV04_01_QMD_TYPE_QUEUE = 0x00000000 +NVCEC0_QMDV04_01_QMD_TYPE_GRID_NULL = 0x00000001 +NVCEC0_QMDV04_01_QMD_TYPE_GRID_CTA = 0x00000002 +NVCEC0_QMDV04_01_QMD_TYPE_GRID_GPC_CGA = 0x00000003 +NVCEC0_QMDV04_01_QMD_TYPE_GRID_GPU_CGA = 0x00000004 +NVCEC0_QMDV04_01_QMD_TYPE_GRID_GPU_GPC_CGA = 0x00000005 +NVCEC0_QMDV04_01_ARRIVE_AT_LATCH_VALID = (28, 28) +NVCEC0_QMDV04_01_WAIT_ON_LATCH_VALID = (29, 29) +NVCEC0_QMDV04_01_REQUIRE_SCHEDULING_PCAS = (30, 30) +NVCEC0_QMDV04_01_REQUIRE_SCHEDULING_PCAS_FALSE = 0x00000000 +NVCEC0_QMDV04_01_REQUIRE_SCHEDULING_PCAS_TRUE = 0x00000001 +NVCEC0_QMDV04_01_TPC_DISABLE_MASK_VALID = (31, 31) +NVCEC0_QMDV04_01_TPC_DISABLE_MASK_VALID_FALSE = 0x00000000 +NVCEC0_QMDV04_01_TPC_DISABLE_MASK_VALID_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CIRCULAR_QUEUE_SIZE = (56, 32) +NVCEC0_QMDV04_01_INNER_GET = (94, 64) +NVCEC0_QMDV04_01_INNER_OVERFLOW = (95, 95) +NVCEC0_QMDV04_01_INNER_PUT = (126, 96) +NVCEC0_QMDV04_01_INNER_STICKY_OVERFLOW = (127, 127) +NVCEC0_QMDV04_01_HW_ONLY_INNER_GET = (190, 160) +NVCEC0_QMDV04_01_HW_ONLY_INNER_PUT = (222, 192) +NVCEC0_QMDV04_01_HW_ONLY_SPAN_LIST_HEAD_INDEX = (253, 224) +NVCEC0_QMDV04_01_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID = (254, 254) +NVCEC0_QMDV04_01_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_FALSE = 0x00000000 +NVCEC0_QMDV04_01_HW_ONLY_SPAN_LIST_HEAD_INDEX_VALID_TRUE = 0x00000001 +NVCEC0_QMDV04_01_HW_ONLY_SKED_NEXT_QMD_POINTER = (287, 256) +NVCEC0_QMDV04_01_HW_ONLY_DEPENDENCE_COUNTER = (303, 288) +NVCEC0_QMDV04_01_HW_ONLY_REQUIRE_SCHEDULING_PCAS = (304, 304) +NVCEC0_QMDV04_01_RELEASE_ENABLE = lambda i: ((320+(i)*16), (320+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_STRUCTURE_SIZE = lambda i: ((322+(i)*16), (321+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_STRUCTURE_SIZE_SEMAPHORE_FOUR_WORDS = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_STRUCTURE_SIZE_SEMAPHORE_ONE_WORD = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_STRUCTURE_SIZE_SEMAPHORE_TWO_WORDS = 0x00000002 +NVCEC0_QMDV04_01_RELEASE_MEMBAR_TYPE = lambda i: ((323+(i)*16), (323+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_MEMBAR_TYPE_FE_NONE = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_MEMBAR_TYPE_FE_SYSMEMBAR = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_ENABLE = lambda i: ((324+(i)*16), (324+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_REDUCTION_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP = lambda i: ((327+(i)*16), (325+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_ADD = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_MIN = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_MAX = 0x00000002 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_INC = 0x00000003 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_DEC = 0x00000004 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_AND = 0x00000005 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_OR = 0x00000006 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_OP_RED_XOR = 0x00000007 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_FORMAT = lambda i: ((329+(i)*16), (328+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_REDUCTION_FORMAT_UNSIGNED = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_REDUCTION_FORMAT_SIGNED = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_TRAP_TYPE = lambda i: ((331+(i)*16), (330+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_TRAP_TYPE_TRAP_NONE = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_TRAP_TYPE_TRAP_UNCONDITIONAL = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_TRAP_TYPE_TRAP_CONDITIONAL = 0x00000002 +NVCEC0_QMDV04_01_RELEASE_TRAP_TYPE_TRAP_CONDITIONAL_EXT = 0x00000003 +NVCEC0_QMDV04_01_RELEASE_PAYLOAD64B = lambda i: ((332+(i)*16), (332+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_RELEASE_PAYLOAD64B_FALSE = 0x00000000 +NVCEC0_QMDV04_01_RELEASE_PAYLOAD64B_TRUE = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_RESERVED_INFO = lambda i: ((335+(i)*16), (333+(i)*16)) # type: ignore +NVCEC0_QMDV04_01_DEPENDENT_QMD_ENABLE = lambda i: ((368+(i)*5), (368+(i)*5)) # type: ignore +NVCEC0_QMDV04_01_DEPENDENT_QMD_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_DEPENDENT_QMD_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_DEPENDENT_QMD_ACTION = lambda i: ((371+(i)*5), (369+(i)*5)) # type: ignore +NVCEC0_QMDV04_01_DEPENDENT_QMD_ACTION_QMD_INCREMENT_PUT = 0x00000000 +NVCEC0_QMDV04_01_DEPENDENT_QMD_ACTION_QMD_SCHEDULE = 0x00000001 +NVCEC0_QMDV04_01_DEPENDENT_QMD_ACTION_QMD_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCEC0_QMDV04_01_DEPENDENT_QMD_ACTION_QMD_DECREMENT_DEPENDENCE = 0x00000004 +NVCEC0_QMDV04_01_DEPENDENT_QMD_PREFETCH = lambda i: ((372+(i)*5), (372+(i)*5)) # type: ignore +NVCEC0_QMDV04_01_DEPENDENT_QMD_PREFETCH_FALSE = 0x00000000 +NVCEC0_QMDV04_01_DEPENDENT_QMD_PREFETCH_TRUE = 0x00000001 +NVCEC0_QMDV04_01_SELF_COPY_ON_COMPLETION = (378, 378) +NVCEC0_QMDV04_01_SELF_COPY_ON_COMPLETION_FALSE = 0x00000000 +NVCEC0_QMDV04_01_SELF_COPY_ON_COMPLETION_TRUE = 0x00000001 +NVCEC0_QMDV04_01_DEMOTE_L2_EVICT_LAST = (379, 379) +NVCEC0_QMDV04_01_DEMOTE_L2_EVICT_LAST_FALSE = 0x00000000 +NVCEC0_QMDV04_01_DEMOTE_L2_EVICT_LAST_TRUE = 0x00000001 +NVCEC0_QMDV04_01_DISABLE_AUTO_INVALIDATE = (380, 380) +NVCEC0_QMDV04_01_DISABLE_AUTO_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_DISABLE_AUTO_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CORRELATION_ID_INTERNAL = (381, 381) +NVCEC0_QMDV04_01_CORRELATION_ID_INTERNAL_FALSE = 0x00000000 +NVCEC0_QMDV04_01_CORRELATION_ID_INTERNAL_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CWD_MEMBAR_TASK_CHASING_ENABLE = (382, 382) +NVCEC0_QMDV04_01_CWD_MEMBAR_TASK_CHASING_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_CWD_MEMBAR_TASK_CHASING_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CORRELATION_ID = (415, 384) +NVCEC0_QMDV04_01_CONSTANT_BUFFER_VALID = lambda i: ((416+(i)*4), (416+(i)*4)) # type: ignore +NVCEC0_QMDV04_01_CONSTANT_BUFFER_VALID_FALSE = 0x00000000 +NVCEC0_QMDV04_01_CONSTANT_BUFFER_VALID_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CONSTANT_BUFFER_PREFETCH = lambda i: ((418+(i)*4), (417+(i)*4)) # type: ignore +NVCEC0_QMDV04_01_CONSTANT_BUFFER_PREFETCH_PREFETCH_NONE = 0x00000000 +NVCEC0_QMDV04_01_CONSTANT_BUFFER_PREFETCH_PREFETCH_PRE = 0x00000001 +NVCEC0_QMDV04_01_CONSTANT_BUFFER_PREFETCH_PREFETCH_POST = 0x00000002 +NVCEC0_QMDV04_01_CONSTANT_BUFFER_INVALIDATE = lambda i: ((419+(i)*4), (419+(i)*4)) # type: ignore +NVCEC0_QMDV04_01_CONSTANT_BUFFER_INVALIDATE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_CONSTANT_BUFFER_INVALIDATE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_DEPENDENT_QMD0_POINTER = (479, 448) +NVCEC0_QMDV04_01_DEPENDENT_QMD1_POINTER = (511, 480) +NVCEC0_QMDV04_01_SHADER_LOCAL_MEMORY_LOW_SIZE = (535, 512) +NVCEC0_QMDV04_01_SASS_VERSION = (543, 536) +NVCEC0_QMDV04_01_SHADER_LOCAL_MEMORY_HIGH_SIZE = (567, 544) +NVCEC0_QMDV04_01_API_VISIBLE_CALL_LIMIT = (568, 568) +NVCEC0_QMDV04_01_API_VISIBLE_CALL_LIMIT__32 = 0x00000000 +NVCEC0_QMDV04_01_API_VISIBLE_CALL_LIMIT_NO_CHECK = 0x00000001 +NVCEC0_QMDV04_01_SAMPLER_INDEX = (569, 569) +NVCEC0_QMDV04_01_SAMPLER_INDEX_INDEPENDENTLY = 0x00000000 +NVCEC0_QMDV04_01_SAMPLER_INDEX_VIA_HEADER_INDEX = 0x00000001 +NVCEC0_QMDV04_01_CONSTANT_BUFFER_PREFETCH_PRE_MAX_SIZE_SHIFTED8 = (575, 570) +NVCEC0_QMDV04_01_QMD_MINOR_VERSION = (579, 576) +NVCEC0_QMDV04_01_QMD_MAJOR_VERSION = (583, 580) +NVCEC0_QMDV04_01_SHARED_MEMORY_SIZE = (601, 584) +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_HEADER_CACHE = (602, 602) +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_HEADER_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_HEADER_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_SAMPLER_CACHE = (603, 603) +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_SAMPLER_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_SAMPLER_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_DATA_CACHE = (604, 604) +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_INVALIDATE_SHADER_DATA_CACHE = (605, 605) +NVCEC0_QMDV04_01_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_INVALIDATE_INSTRUCTION_CACHE = (606, 606) +NVCEC0_QMDV04_01_INVALIDATE_INSTRUCTION_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_INVALIDATE_INSTRUCTION_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_INVALIDATE_SHADER_CONSTANT_CACHE = (607, 607) +NVCEC0_QMDV04_01_INVALIDATE_SHADER_CONSTANT_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_INVALIDATE_SHADER_CONSTANT_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_MIN_SM_CONFIG_SHARED_MEM_SIZE = (613, 608) +NVCEC0_QMDV04_01_MAX_SM_CONFIG_SHARED_MEM_SIZE = (619, 614) +NVCEC0_QMDV04_01_TARGET_SM_CONFIG_SHARED_MEM_SIZE = (625, 620) +NVCEC0_QMDV04_01_SHARED_ALLOCATION_ENABLE = (626, 626) +NVCEC0_QMDV04_01_SHARED_ALLOCATION_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_SHARED_ALLOCATION_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE0_ADDR_LOWER = (671, 640) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE0_ADDR_UPPER = (696, 672) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE0_PAYLOAD_LOWER = (735, 704) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE0_PAYLOAD_UPPER = (767, 736) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE1_ADDR_LOWER = (799, 768) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE1_ADDR_UPPER = (824, 800) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE1_PAYLOAD_LOWER = (863, 832) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE1_PAYLOAD_UPPER = (895, 864) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE2_ADDR_LOWER = (927, 896) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE2_ADDR_UPPER = (952, 928) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE2_PAYLOAD_LOWER = (991, 960) +NVCEC0_QMDV04_01_RELEASE_SEMAPHORE2_PAYLOAD_UPPER = (1023, 992) +NVCEC0_QMDV04_01_GRID_WIDTH = (1055, 1024) +NVCEC0_QMDV04_01_GRID_HEIGHT = (1071, 1056) +NVCEC0_QMDV04_01_GRID_DEPTH = (1103, 1088) +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_DELTA_MINUS_ONE = (1127, 1120) +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_ID = (1133, 1128) +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_INCR_ENABLE = (1134, 1134) +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_INCR_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_INCR_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_DECR_ENABLE = (1135, 1135) +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_DECR_ENABLE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_CWD_REFERENCE_COUNT_DECR_ENABLE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CWD_MEMBAR_TYPE = (1137, 1136) +NVCEC0_QMDV04_01_CWD_MEMBAR_TYPE_L1_NONE = 0x00000000 +NVCEC0_QMDV04_01_CWD_MEMBAR_TYPE_L1_SYSMEMBAR = 0x00000001 +NVCEC0_QMDV04_01_CWD_MEMBAR_TYPE_L1_MEMBAR = 0x00000003 +NVCEC0_QMDV04_01_SEQUENTIALLY_RUN_CTAS = (1138, 1138) +NVCEC0_QMDV04_01_SEQUENTIALLY_RUN_CTAS_FALSE = 0x00000000 +NVCEC0_QMDV04_01_SEQUENTIALLY_RUN_CTAS_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CTA_LAUNCH_QUEUE = (1139, 1139) +NVCEC0_QMDV04_01_FREE_CTA_SLOTS_EMPTY_SM = (1147, 1140) +NVCEC0_QMDV04_01_SYNC_DOMAIN_ID = (1149, 1148) +NVCEC0_QMDV04_01_PRE_EXIT_AT_LAST_CTA_LAUNCH = (1150, 1150) +NVCEC0_QMDV04_01_PRE_EXIT_AT_LAST_CTA_LAUNCH_FALSE = 0x00000000 +NVCEC0_QMDV04_01_PRE_EXIT_AT_LAST_CTA_LAUNCH_TRUE = 0x00000001 +NVCEC0_QMDV04_01_ENABLE_PROGRAM_PRE_EXIT = (1151, 1151) +NVCEC0_QMDV04_01_ENABLE_PROGRAM_PRE_EXIT_FALSE = 0x00000000 +NVCEC0_QMDV04_01_ENABLE_PROGRAM_PRE_EXIT_TRUE = 0x00000001 +NVCEC0_QMDV04_01_CTA_THREAD_DIMENSION0 = (1167, 1152) +NVCEC0_QMDV04_01_CTA_THREAD_DIMENSION1 = (1183, 1168) +NVCEC0_QMDV04_01_CTA_THREAD_DIMENSION2 = (1191, 1184) +NVCEC0_QMDV04_01_VIRTUAL_RESOURCE_COUNT = (1199, 1192) +NVCEC0_QMDV04_01_REGISTER_COUNT = (1208, 1200) +NVCEC0_QMDV04_01_SHARED_MEM_BARRIER_INIT_ENABLE = (1210, 1210) +NVCEC0_QMDV04_01_BARRIER_COUNT = (1215, 1211) +NVCEC0_QMDV04_01_PROGRAM_ADDRESS_LOWER = (1247, 1216) +NVCEC0_QMDV04_01_PROGRAM_ADDRESS_UPPER = (1272, 1248) +NVCEC0_QMDV04_01_OCCUPANCY_THRESHOLD_WARP = (1287, 1280) +NVCEC0_QMDV04_01_OCCUPANCY_MAX_WARP = (1295, 1288) +NVCEC0_QMDV04_01_OCCUPANCY_THRESHOLD_REGISTER = (1303, 1296) +NVCEC0_QMDV04_01_OCCUPANCY_MAX_REGISTER = (1311, 1304) +NVCEC0_QMDV04_01_OCCUPANCY_THRESHOLD_SHARED_MEM = (1319, 1312) +NVCEC0_QMDV04_01_OCCUPANCY_MAX_SHARED_MEM = (1327, 1320) +NVCEC0_QMDV04_01_ICC_PREFETCH_SIZE = (1333, 1328) +NVCEC0_QMDV04_01_PROGRAM_PREFETCH_ADDR_LOWER_SHIFTED = (1375, 1344) +NVCEC0_QMDV04_01_PROGRAM_PREFETCH_ADDR_UPPER_SHIFTED = (1392, 1376) +NVCEC0_QMDV04_01_PROGRAM_PREFETCH_SIZE = (1401, 1393) +NVCEC0_QMDV04_01_PROGRAM_PREFETCH_TYPE = (1403, 1402) +NVCEC0_QMDV04_01_PROGRAM_PREFETCH_TYPE_PREFETCH_LAUNCH = 0x00000000 +NVCEC0_QMDV04_01_PROGRAM_PREFETCH_TYPE_PREFTECH_POST = 0x00000001 +NVCEC0_QMDV04_01_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE = (1406, 1406) +NVCEC0_QMDV04_01_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_LATCH_ACQUIRE_INVALIDATE_SHADER_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE = (1407, 1407) +NVCEC0_QMDV04_01_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE_FALSE = 0x00000000 +NVCEC0_QMDV04_01_LATCH_ACQUIRE_INVALIDATE_TEXTURE_DATA_CACHE_TRUE = 0x00000001 +NVCEC0_QMDV04_01_GRID_WIDTH_RESUME = (1439, 1408) +NVCEC0_QMDV04_01_GRID_HEIGHT_RESUME = (1455, 1440) +NVCEC0_QMDV04_01_GRID_DEPTH_RESUME = (1471, 1456) +NVCEC0_QMDV04_01_ARRIVE_AT_LATCH_ID = (1503, 1472) +NVCEC0_QMDV04_01_WAIT_ON_LATCH_ID = (1535, 1504) +NVCEC0_QMDV04_01_CONSTANT_BUFFER_ADDR_LOWER_SHIFTED6 = lambda i: ((1567+(i)*64), (1536+(i)*64)) # type: ignore +NVCEC0_QMDV04_01_CONSTANT_BUFFER_ADDR_UPPER_SHIFTED6 = lambda i: ((1586+(i)*64), (1568+(i)*64)) # type: ignore +NVCEC0_QMDV04_01_CONSTANT_BUFFER_SIZE_SHIFTED4 = lambda i: ((1599+(i)*64), (1587+(i)*64)) # type: ignore +NVCEC0_QMDV04_01_COALESCE_WAITING_PERIOD = (2135, 2128) +NVCEC0_QMDV04_01_QUEUE_ENTRIES_PER_CTA_LOG2 = (2140, 2136) +NVCEC0_QMDV04_01_GPC_CGA_WIDTH = (2149, 2144) +NVCEC0_QMDV04_01_GPC_CGA_HEIGHT = (2157, 2152) +NVCEC0_QMDV04_01_GPC_CGA_DEPTH = (2165, 2160) +NVCEC0_QMDV04_01_LARGE_GPC_CGA_WIDTH_MINUS_ONE = (2171, 2168) +NVCEC0_QMDV04_01_LARGE_GPC_CGA_HEIGHT_MINUS_ONE = (2175, 2172) +NVCEC0_QMDV04_01_CGA_CTA_DISTRIBUTION_MODE = (2207, 2207) +NVCEC0_QMDV04_01_CGA_CTA_DISTRIBUTION_MODE_LOAD_BALANCING = 0x00000000 +NVCEC0_QMDV04_01_CGA_CTA_DISTRIBUTION_MODE_MULTI_CAST = 0x00000001 +NVCEC0_QMDV04_01_GPU_CGA_WIDTH = (2223, 2208) +NVCEC0_QMDV04_01_GPU_CGA_HEIGHT = (2239, 2224) +NVCEC0_QMDV04_01_GPU_CGA_DEPTH = (2255, 2240) +NVCEC0_QMDV04_01_DEBUG_ID_LOWER = (2399, 2368) +NVCEC0_QMDV04_01_DEBUG_ID_UPPER = (2431, 2400) +NVCEC0_QMDV04_01_TPC_DISABLE_MASK = lambda i: ((2463+(i)*32), (2432+(i)*32)) # type: ignore +NVCEC0_QMDV04_01_INCOMPLETE_BOX_BASE_WIDTH_RESUME = (2591, 2560) +NVCEC0_QMDV04_01_INCOMPLETE_BOX_BASE_HEIGHT_RESUME = (2607, 2592) +NVCEC0_QMDV04_01_INCOMPLETE_BOX_BASE_DEPTH_RESUME = (2623, 2608) +NVCEC0_QMDV04_01_INCOMPLETE_BOX_OFFSET_WIDTH_RESUME = (2627, 2624) +NVCEC0_QMDV04_01_INCOMPLETE_BOX_OFFSET_HEIGHT_RESUME = (2631, 2628) +NVCEC0_QMDV04_01_TPC_DISABLE_MASK_UPPER = lambda i: ((2719+(i)*32), (2688+(i)*32)) # type: ignore +NVCEC0_QMDV04_01_OUTER_PUT = (3038, 3008) +NVCEC0_QMDV04_01_OUTER_OVERFLOW = (3039, 3039) +NVCEC0_QMDV04_01_OUTER_GET = (3070, 3040) +NVCEC0_QMDV04_01_OUTER_STICKY_OVERFLOW = (3071, 3071) +ALIGN_UP = lambda v,n: (((v) + ((n)-1)) &~ ((n)-1)) # type: ignore +NVDEC_ALIGN = lambda value: ALIGN_UP(value,256) # type: ignore +NVDEC_MAX_MPEG2_SLICE = 65536 +NVDEC_CODEC_MPEG1 = 0 +NVDEC_CODEC_MPEG2 = 1 +NVDEC_CODEC_VC1 = 2 +NVDEC_CODEC_H264 = 3 +NVDEC_CODEC_MPEG4 = 4 +NVDEC_CODEC_DIVX = NVDEC_CODEC_MPEG4 +NVDEC_CODEC_VP8 = 5 +NVDEC_CODEC_HEVC = 7 +NVDEC_CODEC_VP9 = 9 +NVDEC_CODEC_HEVC_PARSER = 12 +NVDEC_CODEC_AV1 = 10 +AES_MODE_MASK = 0x7 +AES_CTS_MASK = 0x1 +AES_PADDING_TYPE_MASK = 0x7 +AES_UNWRAP_KEY_MASK = 0x1 +AES_MODE_SHIFT = 0 +AES_CTS_SHIFT = 3 +AES_PADDING_TYPE_SHIFT = 4 +AES_UNWRAP_KEY_SHIFT = 7 +AES_SET_FLAG = lambda M,C,P: ((M & AES_MODE_MASK) << AES_MODE_SHIFT) | ((C & AES_CTS_MASK) << AES_CTS_SHIFT) | ((P & AES_PADDING_TYPE_MASK) << AES_PADDING_TYPE_SHIFT) # type: ignore +DRM_MODE_MASK = 0x7f +AES_GET_DRM_MODE = lambda V: (V & DRM_MODE_MASK) # type: ignore +GIP_ASIC_VERT_FILTER_RAM_SIZE = 16 +GIP_ASIC_BSD_CTRL_RAM_SIZE = 4 +GIP_ASIC_SCALING_LIST_SIZE = (16*64) +GIP_ASIC_VERT_SAO_RAM_SIZE = 16 +GIP_ASIC_TILE_SIZE = ((20*22*2*2+16+15) & ~0xF) +GIP_ASIC_VP9_CTB_SEG_SIZE = 32 +HEVC_DBLK_TOP_SIZE_IN_SB16 = ALIGN_UP(632, 128) +HEVC_DBLK_TOP_BUF_SIZE = lambda w: NVDEC_ALIGN( (ALIGN_UP(w,16)/16 + 2) * HEVC_DBLK_TOP_SIZE_IN_SB16) # type: ignore +HEVC_DBLK_LEFT_SIZE_IN_SB16 = ALIGN_UP(506, 128) +HEVC_DBLK_LEFT_BUF_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,16)/16 + 2) * HEVC_DBLK_LEFT_SIZE_IN_SB16) # type: ignore +HEVC_SAO_LEFT_SIZE_IN_SB16 = ALIGN_UP(713, 128) +HEVC_SAO_LEFT_BUF_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,16)/16 + 2) * HEVC_SAO_LEFT_SIZE_IN_SB16) # type: ignore +VP9_DBLK_TOP_SIZE_IN_SB64 = ALIGN_UP(2000, 128) +VP9_DBLK_TOP_BUF_SIZE = lambda w: NVDEC_ALIGN( (ALIGN_UP(w,64)/64 + 2) * VP9_DBLK_TOP_SIZE_IN_SB64) # type: ignore +VP9_DBLK_LEFT_SIZE_IN_SB64 = ALIGN_UP(1600, 128) +VP9_DBLK_LEFT_BUF_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,64)/64 + 2) * VP9_DBLK_LEFT_SIZE_IN_SB64) # type: ignore +VP9_HINT_DUMP_SIZE_IN_SB64 = ((64*64)/(4*4)*8) +VP9_HINT_DUMP_SIZE = lambda w,h: NVDEC_ALIGN(VP9_HINT_DUMP_SIZE_IN_SB64*((w+63)/64)*((h+63)/64)) # type: ignore +NUM_SUBSAMPLES = 32 +VP8_MAX_TOKEN_PARTITIONS = 8 +VP9_MAX_FRAMES_IN_SUPERFRAME = 8 +AV1_MAX_TILES = 256 +MAX_SUBSTREAM_ENTRY_SIZE = 32 +AV1_TEMPORAL_MV_SIZE_IN_64x64 = 256 +AV1_TEMPORAL_MV_BUF_SIZE = lambda w,h: ALIGN_UP( ALIGN_UP(w,128) * ALIGN_UP(h,128) / (64*64) * AV1_TEMPORAL_MV_SIZE_IN_64x64, 4096) # type: ignore +AV1_SEGMENT_ID_SIZE_IN_64x64 = 128 +AV1_SEGMENT_ID_BUF_SIZE = lambda w,h: ALIGN_UP( ALIGN_UP(w,128) * ALIGN_UP(h,128) / (64*64) * AV1_SEGMENT_ID_SIZE_IN_64x64, 4096) # type: ignore +AV1_GLOBAL_MOTION_BUF_SIZE = NVDEC_ALIGN(7*32) +AV1_INTRA_TOP_BUF_SIZE = NVDEC_ALIGN(8*8192) +AV1_HISTOGRAM_BUF_SIZE = NVDEC_ALIGN(1024) +AV1_DBLK_TOP_SIZE_IN_SB64 = ALIGN_UP(1920, 128) +AV1_DBLK_TOP_BUF_SIZE = lambda w: NVDEC_ALIGN( (ALIGN_UP(w,64)/64 + 2) * AV1_DBLK_TOP_SIZE_IN_SB64) # type: ignore +AV1_DBLK_LEFT_SIZE_IN_SB64 = ALIGN_UP(1536, 128) +AV1_DBLK_LEFT_BUF_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,64)/64 + 2) * AV1_DBLK_LEFT_SIZE_IN_SB64) # type: ignore +AV1_CDEF_LEFT_SIZE_IN_SB64 = ALIGN_UP(1792, 128) +AV1_CDEF_LEFT_BUF_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,64)/64 + 2) * AV1_CDEF_LEFT_SIZE_IN_SB64) # type: ignore +AV1_HUS_LEFT_SIZE_IN_SB64 = ALIGN_UP(12544, 128) +AV1_ASIC_HUS_LEFT_BUFFER_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,64)/64 + 2) * AV1_HUS_LEFT_SIZE_IN_SB64) # type: ignore +AV1_HUS_LEFT_BUF_SIZE = lambda h: 2*AV1_ASIC_HUS_LEFT_BUFFER_SIZE(h) # type: ignore +AV1_LR_LEFT_SIZE_IN_SB64 = ALIGN_UP(1920, 128) +AV1_LR_LEFT_BUF_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,64)/64 + 2) * AV1_LR_LEFT_SIZE_IN_SB64) # type: ignore +AV1_FGS_LEFT_SIZE_IN_SB64 = ALIGN_UP(320, 128) +AV1_FGS_LEFT_BUF_SIZE = lambda h: NVDEC_ALIGN( (ALIGN_UP(h,64)/64 + 2) * AV1_FGS_LEFT_SIZE_IN_SB64) # type: ignore +AV1_HINT_DUMP_SIZE_IN_SB64 = ((64*64)/(4*4)*8) +AV1_HINT_DUMP_SIZE_IN_SB128 = ((128*128)/(4*4)*8) +AV1_HINT_DUMP_SIZE = lambda w,h: NVDEC_ALIGN(AV1_HINT_DUMP_SIZE_IN_SB128*((w+127)/128)*((h+127)/128)) # type: ignore +NVBIT = lambda b: (1<<(b)) # type: ignore +NVBIT32 = lambda b: NVBIT_TYPE(b, NvU32) # type: ignore +NVBIT64 = lambda b: NVBIT_TYPE(b, NvU64) # type: ignore +NV_BITMASK32_IDX = lambda chId: (((chId) & ~(0x1F)) >> 5) # type: ignore +NV_BITMASK32_OFFSET = lambda chId: ((chId) & (0x1F)) # type: ignore +NV_BITMASK32_GET = lambda pChannelMask,chId: ((pChannelMask)[NV_BITMASK32_IDX(chId)] & NVBIT(NV_BITMASK32_OFFSET(chId))) # type: ignore +DRF_SHIFT = lambda drf: ((DRF_ISBIT(0,drf)) % 32) # type: ignore +DRF_SHIFT_RT = lambda drf: ((DRF_ISBIT(1,drf)) % 32) # type: ignore +DRF_SIZE = lambda drf: (DRF_EXTENT(drf)-DRF_BASE(drf)+1) # type: ignore +DRF_MASK = lambda drf: (0xFFFFFFFF>>(31-((DRF_ISBIT(1,drf)) % 32)+((DRF_ISBIT(0,drf)) % 32))) # type: ignore +DRF_SHIFTMASK = lambda drf: (DRF_MASK(drf)<<(DRF_SHIFT(drf))) # type: ignore +DRF_SHIFT64 = lambda drf: ((DRF_ISBIT(0,drf)) % 64) # type: ignore +DRF_MASK64 = lambda drf: (NV_U64_MAX>>(63-((DRF_ISBIT(1,drf)) % 64)+((DRF_ISBIT(0,drf)) % 64))) # type: ignore +DRF_SHIFTMASK64 = lambda drf: (DRF_MASK64(drf)<<(DRF_SHIFT64(drf))) # type: ignore +FLD_TEST_DRF_NUM64 = lambda d,r,f,n,v: (DRF_VAL64(d, r, f, (v)) == (n)) # type: ignore +REF_VAL64 = lambda drf,v: (((NvU64)(v)>>DRF_SHIFT64(drf))&DRF_MASK64(drf)) # type: ignore +REF_NUM64 = lambda drf,n: (((NvU64)(n)&DRF_MASK64(drf))<>DRF_SHIFT(drf))&DRF_MASK(drf)) # type: ignore +REF_NUM = lambda drf,n: (((n)&DRF_MASK(drf))<>((31-(DRF_EXTENT_MW(drf))+(DRF_BASE_MW(drf)))%32)) # type: ignore +DRF_SHIFTMASK_MW = lambda drf: ((DRF_MASK_MW(drf))<<(DRF_SHIFT_MW(drf))) # type: ignore +DRF_SIZE_MW = lambda drf: (DRF_EXTENT_MW(drf)-DRF_BASE_MW(drf)+1) # type: ignore +DRF_WORD_MW_LOW = lambda drf: (DRF_PICK_MW(drf,0)/32) # type: ignore +DRF_WORD_MW_HIGH = lambda drf: (DRF_PICK_MW(drf,1)/32) # type: ignore +DRF_MASK_MW_LOW = lambda drf: (0xFFFFFFFF) # type: ignore +DRF_MASK_MW_HIGH = lambda drf: (0xFFFFFFFF>>(31-(DRF_EXTENT_MW(drf)))) # type: ignore +DRF_SHIFT_MW_LOW = lambda drf: (DRF_PICK_MW(drf,0)%32) # type: ignore +DRF_SHIFT_MW_HIGH = lambda drf: (0) # type: ignore +DRF_MERGE_SHIFT = lambda drf: ((32-((DRF_PICK_MW(drf,0)%32)))%32) # type: ignore +FLD_MERGE_MW = lambda drf,n,v: (((v)[DRF_WORD_MW(drf)] & ~DRF_SHIFTMASK_MW(drf)) | n) # type: ignore +FLD_IDX_MERGE_MW = lambda drf,i,n,v: (((v)[DRF_WORD_MW(drf(i))] & ~DRF_SHIFTMASK_MW(drf(i))) | n) # type: ignore +FLD_TEST_DRF_NUM_MW = lambda d,r,f,n,v: ((DRF_VAL_MW(d, r, f, (v)) == n)) # type: ignore +NV_TWO_N_MINUS_ONE = lambda n: (((1<<(n/2))<<((n+1)/2))-1) # type: ignore +LOWESTBIT = lambda x: ( (x) & (((x) - 1) ^ (x)) ) # type: ignore +NV_ANYSIZE_ARRAY = 1 +NV_CEIL = lambda a,b: (((a)+(b)-1)/(b)) # type: ignore +NV_DIV_AND_CEIL = lambda a,b: NV_CEIL(a,b) # type: ignore +NV_SIGN = lambda s: ((NvS8)(((s) > 0) - ((s) < 0))) # type: ignore +NV_ZERO_SIGN = lambda s: ((NvS8)((((s) >= 0) * 2) - 1)) # type: ignore +NV_UNSIGNED_ROUNDED_DIV = lambda a,b: (((a) + ((b) / 2)) / (b)) # type: ignore +NV_UNSIGNED_DIV_CEIL = lambda a,b: (((a) + (b - 1)) / (b)) # type: ignore +NV_ALIGN_DOWN = lambda v,gran: ((v) & ~((v) - (v) + (gran) - 1)) # type: ignore +NV_ALIGN_UP = lambda v,gran: (((v) + ((gran) - 1)) & ~((v) - (v) + (gran) - 1)) # type: ignore +NV_IS_ALIGNED = lambda v,gran: (0 == ((v) & ((gran) - 1))) # type: ignore +NV_BIT_GET = lambda k,x: (((x) >> (k)) & 1) # type: ignore +NV_ARRAY_ELEMENTS = lambda x: ((sizeof(x)/sizeof((x)[0]))) # type: ignore +BIT = lambda b: (1<<(b)) # type: ignore +NV01_NULL_OBJECT = (0x0) +NV1_NULL_OBJECT = NV01_NULL_OBJECT +NV01_ROOT = (0x0) +NV0000_ALLOC_PARAMETERS_MESSAGE_ID = (0x0000) +NV01_MEMORY_VIRTUAL = (0x70) +NV01_MEMORY_SYSTEM_DYNAMIC = (0x70) +NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS_MESSAGE_ID = (0x0070) +NV_MEMORY_VIRTUAL_SYSMEM_DYNAMIC_HVASPACE = 0xffffffff +NV01_DEVICE_0 = (0x80) +NV0080_ALLOC_PARAMETERS_MESSAGE_ID = (0x0080) +NV20_SUBDEVICE_0 = (0x2080) +NV2080_ALLOC_PARAMETERS_MESSAGE_ID = (0x2080) +NV2080_NOTIFIERS_SW = (0) +NV2080_NOTIFIERS_HOTPLUG = (1) +NV2080_NOTIFIERS_POWER_CONNECTOR = (2) +NV2080_NOTIFIERS_THERMAL_SW = (3) +NV2080_NOTIFIERS_THERMAL_HW = (4) +NV2080_NOTIFIERS_FULL_SCREEN_CHANGE = (5) +NV2080_NOTIFIERS_EVENTBUFFER = (6) +NV2080_NOTIFIERS_DP_IRQ = (7) +NV2080_NOTIFIERS_GR_DEBUG_INTR = (8) +NV2080_NOTIFIERS_PMU_EVENT = (9) +NV2080_NOTIFIERS_PMU_COMMAND = (10) +NV2080_NOTIFIERS_TIMER = (11) +NV2080_NOTIFIERS_GRAPHICS = (12) +NV2080_NOTIFIERS_PPP = (13) +NV2080_NOTIFIERS_VLD = (14) +NV2080_NOTIFIERS_NVDEC0 = NV2080_NOTIFIERS_VLD +NV2080_NOTIFIERS_NVDEC1 = (15) +NV2080_NOTIFIERS_NVDEC2 = (16) +NV2080_NOTIFIERS_NVDEC3 = (17) +NV2080_NOTIFIERS_NVDEC4 = (18) +NV2080_NOTIFIERS_NVDEC5 = (19) +NV2080_NOTIFIERS_NVDEC6 = (20) +NV2080_NOTIFIERS_NVDEC7 = (21) +NV2080_NOTIFIERS_PDEC = (22) +NV2080_NOTIFIERS_CE0 = (23) +NV2080_NOTIFIERS_CE1 = (24) +NV2080_NOTIFIERS_CE2 = (25) +NV2080_NOTIFIERS_CE3 = (26) +NV2080_NOTIFIERS_CE4 = (27) +NV2080_NOTIFIERS_CE5 = (28) +NV2080_NOTIFIERS_CE6 = (29) +NV2080_NOTIFIERS_CE7 = (30) +NV2080_NOTIFIERS_CE8 = (31) +NV2080_NOTIFIERS_CE9 = (32) +NV2080_NOTIFIERS_PSTATE_CHANGE = (33) +NV2080_NOTIFIERS_HDCP_STATUS_CHANGE = (34) +NV2080_NOTIFIERS_FIFO_EVENT_MTHD = (35) +NV2080_NOTIFIERS_PRIV_RING_HANG = (36) +NV2080_NOTIFIERS_RC_ERROR = (37) +NV2080_NOTIFIERS_MSENC = (38) +NV2080_NOTIFIERS_NVENC0 = NV2080_NOTIFIERS_MSENC +NV2080_NOTIFIERS_NVENC1 = (39) +NV2080_NOTIFIERS_NVENC2 = (40) +NV2080_NOTIFIERS_UNUSED_0 = (41) +NV2080_NOTIFIERS_ACPI_NOTIFY = (42) +NV2080_NOTIFIERS_COOLER_DIAG_ZONE = (43) +NV2080_NOTIFIERS_THERMAL_DIAG_ZONE = (44) +NV2080_NOTIFIERS_AUDIO_HDCP_REQUEST = (45) +NV2080_NOTIFIERS_WORKLOAD_MODULATION_CHANGE = (46) +NV2080_NOTIFIERS_GPIO_0_RISING_INTERRUPT = (47) +NV2080_NOTIFIERS_GPIO_1_RISING_INTERRUPT = (48) +NV2080_NOTIFIERS_GPIO_2_RISING_INTERRUPT = (49) +NV2080_NOTIFIERS_GPIO_3_RISING_INTERRUPT = (50) +NV2080_NOTIFIERS_GPIO_4_RISING_INTERRUPT = (51) +NV2080_NOTIFIERS_GPIO_5_RISING_INTERRUPT = (52) +NV2080_NOTIFIERS_GPIO_6_RISING_INTERRUPT = (53) +NV2080_NOTIFIERS_GPIO_7_RISING_INTERRUPT = (54) +NV2080_NOTIFIERS_GPIO_8_RISING_INTERRUPT = (55) +NV2080_NOTIFIERS_GPIO_9_RISING_INTERRUPT = (56) +NV2080_NOTIFIERS_GPIO_10_RISING_INTERRUPT = (57) +NV2080_NOTIFIERS_GPIO_11_RISING_INTERRUPT = (58) +NV2080_NOTIFIERS_GPIO_12_RISING_INTERRUPT = (59) +NV2080_NOTIFIERS_GPIO_13_RISING_INTERRUPT = (60) +NV2080_NOTIFIERS_GPIO_14_RISING_INTERRUPT = (61) +NV2080_NOTIFIERS_GPIO_15_RISING_INTERRUPT = (62) +NV2080_NOTIFIERS_GPIO_16_RISING_INTERRUPT = (63) +NV2080_NOTIFIERS_GPIO_17_RISING_INTERRUPT = (64) +NV2080_NOTIFIERS_GPIO_18_RISING_INTERRUPT = (65) +NV2080_NOTIFIERS_GPIO_19_RISING_INTERRUPT = (66) +NV2080_NOTIFIERS_GPIO_20_RISING_INTERRUPT = (67) +NV2080_NOTIFIERS_GPIO_21_RISING_INTERRUPT = (68) +NV2080_NOTIFIERS_GPIO_22_RISING_INTERRUPT = (69) +NV2080_NOTIFIERS_GPIO_23_RISING_INTERRUPT = (70) +NV2080_NOTIFIERS_GPIO_24_RISING_INTERRUPT = (71) +NV2080_NOTIFIERS_GPIO_25_RISING_INTERRUPT = (72) +NV2080_NOTIFIERS_GPIO_26_RISING_INTERRUPT = (73) +NV2080_NOTIFIERS_GPIO_27_RISING_INTERRUPT = (74) +NV2080_NOTIFIERS_GPIO_28_RISING_INTERRUPT = (75) +NV2080_NOTIFIERS_GPIO_29_RISING_INTERRUPT = (76) +NV2080_NOTIFIERS_GPIO_30_RISING_INTERRUPT = (77) +NV2080_NOTIFIERS_GPIO_31_RISING_INTERRUPT = (78) +NV2080_NOTIFIERS_GPIO_0_FALLING_INTERRUPT = (79) +NV2080_NOTIFIERS_GPIO_1_FALLING_INTERRUPT = (80) +NV2080_NOTIFIERS_GPIO_2_FALLING_INTERRUPT = (81) +NV2080_NOTIFIERS_GPIO_3_FALLING_INTERRUPT = (82) +NV2080_NOTIFIERS_GPIO_4_FALLING_INTERRUPT = (83) +NV2080_NOTIFIERS_GPIO_5_FALLING_INTERRUPT = (84) +NV2080_NOTIFIERS_GPIO_6_FALLING_INTERRUPT = (85) +NV2080_NOTIFIERS_GPIO_7_FALLING_INTERRUPT = (86) +NV2080_NOTIFIERS_GPIO_8_FALLING_INTERRUPT = (87) +NV2080_NOTIFIERS_GPIO_9_FALLING_INTERRUPT = (88) +NV2080_NOTIFIERS_GPIO_10_FALLING_INTERRUPT = (89) +NV2080_NOTIFIERS_GPIO_11_FALLING_INTERRUPT = (90) +NV2080_NOTIFIERS_GPIO_12_FALLING_INTERRUPT = (91) +NV2080_NOTIFIERS_GPIO_13_FALLING_INTERRUPT = (92) +NV2080_NOTIFIERS_GPIO_14_FALLING_INTERRUPT = (93) +NV2080_NOTIFIERS_GPIO_15_FALLING_INTERRUPT = (94) +NV2080_NOTIFIERS_GPIO_16_FALLING_INTERRUPT = (95) +NV2080_NOTIFIERS_GPIO_17_FALLING_INTERRUPT = (96) +NV2080_NOTIFIERS_GPIO_18_FALLING_INTERRUPT = (97) +NV2080_NOTIFIERS_GPIO_19_FALLING_INTERRUPT = (98) +NV2080_NOTIFIERS_GPIO_20_FALLING_INTERRUPT = (99) +NV2080_NOTIFIERS_GPIO_21_FALLING_INTERRUPT = (100) +NV2080_NOTIFIERS_GPIO_22_FALLING_INTERRUPT = (101) +NV2080_NOTIFIERS_GPIO_23_FALLING_INTERRUPT = (102) +NV2080_NOTIFIERS_GPIO_24_FALLING_INTERRUPT = (103) +NV2080_NOTIFIERS_GPIO_25_FALLING_INTERRUPT = (104) +NV2080_NOTIFIERS_GPIO_26_FALLING_INTERRUPT = (105) +NV2080_NOTIFIERS_GPIO_27_FALLING_INTERRUPT = (106) +NV2080_NOTIFIERS_GPIO_28_FALLING_INTERRUPT = (107) +NV2080_NOTIFIERS_GPIO_29_FALLING_INTERRUPT = (108) +NV2080_NOTIFIERS_GPIO_30_FALLING_INTERRUPT = (109) +NV2080_NOTIFIERS_GPIO_31_FALLING_INTERRUPT = (110) +NV2080_NOTIFIERS_ECC_SBE = (111) +NV2080_NOTIFIERS_ECC_DBE = (112) +NV2080_NOTIFIERS_STEREO_EMITTER_DETECTION = (113) +NV2080_NOTIFIERS_GC5_GPU_READY = (114) +NV2080_NOTIFIERS_SEC2 = (115) +NV2080_NOTIFIERS_GC6_REFCOUNT_INC = (116) +NV2080_NOTIFIERS_GC6_REFCOUNT_DEC = (117) +NV2080_NOTIFIERS_POWER_EVENT = (118) +NV2080_NOTIFIERS_CLOCKS_CHANGE = (119) +NV2080_NOTIFIERS_HOTPLUG_PROCESSING_COMPLETE = (120) +NV2080_NOTIFIERS_PHYSICAL_PAGE_FAULT = (121) +NV2080_NOTIFIERS_RESERVED122 = (122) +NV2080_NOTIFIERS_NVLINK_ERROR_FATAL = (123) +NV2080_NOTIFIERS_PRIV_REG_ACCESS_FAULT = (124) +NV2080_NOTIFIERS_NVLINK_ERROR_RECOVERY_REQUIRED = (125) +NV2080_NOTIFIERS_NVJPG = (126) +NV2080_NOTIFIERS_NVJPEG0 = NV2080_NOTIFIERS_NVJPG +NV2080_NOTIFIERS_NVJPEG1 = (127) +NV2080_NOTIFIERS_NVJPEG2 = (128) +NV2080_NOTIFIERS_NVJPEG3 = (129) +NV2080_NOTIFIERS_NVJPEG4 = (130) +NV2080_NOTIFIERS_NVJPEG5 = (131) +NV2080_NOTIFIERS_NVJPEG6 = (132) +NV2080_NOTIFIERS_NVJPEG7 = (133) +NV2080_NOTIFIERS_RUNLIST_AND_ENG_IDLE = (134) +NV2080_NOTIFIERS_RUNLIST_ACQUIRE = (135) +NV2080_NOTIFIERS_RUNLIST_ACQUIRE_AND_ENG_IDLE = (136) +NV2080_NOTIFIERS_RUNLIST_IDLE = (137) +NV2080_NOTIFIERS_TSG_PREEMPT_COMPLETE = (138) +NV2080_NOTIFIERS_RUNLIST_PREEMPT_COMPLETE = (139) +NV2080_NOTIFIERS_CTXSW_TIMEOUT = (140) +NV2080_NOTIFIERS_INFOROM_ECC_OBJECT_UPDATED = (141) +NV2080_NOTIFIERS_NVTELEMETRY_REPORT_EVENT = (142) +NV2080_NOTIFIERS_DSTATE_XUSB_PPC = (143) +NV2080_NOTIFIERS_FECS_CTX_SWITCH = (144) +NV2080_NOTIFIERS_XUSB_PPC_CONNECTED = (145) +NV2080_NOTIFIERS_GR0 = NV2080_NOTIFIERS_GRAPHICS +NV2080_NOTIFIERS_GR1 = (146) +NV2080_NOTIFIERS_GR2 = (147) +NV2080_NOTIFIERS_GR3 = (148) +NV2080_NOTIFIERS_GR4 = (149) +NV2080_NOTIFIERS_GR5 = (150) +NV2080_NOTIFIERS_GR6 = (151) +NV2080_NOTIFIERS_GR7 = (152) +NV2080_NOTIFIERS_OFA = (153) +NV2080_NOTIFIERS_OFA0 = NV2080_NOTIFIERS_OFA +NV2080_NOTIFIERS_DSTATE_HDA = (154) +NV2080_NOTIFIERS_POISON_ERROR_NON_FATAL = (155) +NV2080_NOTIFIERS_POISON_ERROR_FATAL = (156) +NV2080_NOTIFIERS_UCODE_RESET = (157) +NV2080_NOTIFIERS_PLATFORM_POWER_MODE_CHANGE = (158) +NV2080_NOTIFIERS_SMC_CONFIG_UPDATE = (159) +NV2080_NOTIFIERS_INFOROM_RRL_OBJECT_UPDATED = (160) +NV2080_NOTIFIERS_INFOROM_PBL_OBJECT_UPDATED = (161) +NV2080_NOTIFIERS_LPWR_DIFR_PREFETCH_REQUEST = (162) +NV2080_NOTIFIERS_SEC_FAULT_ERROR = (163) +NV2080_NOTIFIERS_UNUSED_1 = (164) +NV2080_NOTIFIERS_NVLINK_INFO_LINK_UP = (165) +NV2080_NOTIFIERS_CE10 = (166) +NV2080_NOTIFIERS_CE11 = (167) +NV2080_NOTIFIERS_CE12 = (168) +NV2080_NOTIFIERS_CE13 = (169) +NV2080_NOTIFIERS_CE14 = (170) +NV2080_NOTIFIERS_CE15 = (171) +NV2080_NOTIFIERS_CE16 = (172) +NV2080_NOTIFIERS_CE17 = (173) +NV2080_NOTIFIERS_CE18 = (174) +NV2080_NOTIFIERS_CE19 = (175) +NV2080_NOTIFIERS_NVLINK_INFO_LINK_DOWN = (176) +NV2080_NOTIFIERS_NVPCF_EVENTS = (177) +NV2080_NOTIFIERS_HDMI_FRL_RETRAINING_REQUEST = (178) +NV2080_NOTIFIERS_OFA1 = (180) +NV2080_NOTIFIERS_AUX_POWER_EVENT = (181) +NV2080_NOTIFIERS_AUX_POWER_STATE_CHANGE = (182) +NV2080_NOTIFIERS_NVENC3 = (183) +NV2080_NOTIFIERS_GSP_PERF_TRACE = (184) +NV2080_NOTIFIERS_INBAND_RESPONSE = (185) +NV2080_NOTIFIERS_RESERVED_186 = (186) +NV2080_NOTIFIERS_ECC_SBE_STORM = (187) +NV2080_NOTIFIERS_DRAM_RETIREMENT_EVENT = (188) +NV2080_NOTIFIERS_DRAM_RETIREMENT_FAILURE = (189) +NV2080_NOTIFIERS_NVLINK_UNCONTAINED_ERROR = (190) +NV2080_NOTIFIERS_GPU_UNAVAILABLE = (191) +NV2080_NOTIFIERS_GPU_RECOVERY_ACTION = (192) +NV2080_NOTIFIERS_POWER_SUSPEND = (193) +NV2080_NOTIFIERS_POWER_RESUME = (194) +NV2080_NOTIFIERS_CTXSW_UCODE_ERROR = (195) +NV2080_NOTIFIERS_USE_GC6_REDUCED_THRESHOLD = (196) +NV2080_NOTIFIERS_GPU_RC_RESET = (197) +NV2080_NOTIFIERS_RESERVED_198 = (198) +NV2080_NOTIFIERS_INFOROM_BRL_OBJECT_UPDATED = (199) +NV2080_NOTIFIERS_MEMACCT_SOFT_LIMIT_EXCEEDED = (200) +NV2080_NOTIFIERS_MEMACCT_RETURNED_BELOW_SOFT_LIMIT = (201) +NV2080_NOTIFIERS_MAXCOUNT = (202) +NV2080_NOTIFIERS_GR_IDX = lambda x: ((x) - NV2080_NOTIFIERS_GR0) # type: ignore +NV2080_NOTIFIER_TYPE_IS_GR = lambda x: (((x) == NV2080_NOTIFIERS_GR0) or (((x) >= NV2080_NOTIFIERS_GR1) and ((x) <= NV2080_NOTIFIERS_GR7))) # type: ignore +NV2080_NOTIFIER_TYPE_IS_CE = lambda x: ((((x) >= NV2080_NOTIFIERS_CE0) and ((x) <= NV2080_NOTIFIERS_CE9)) or (((x) >= NV2080_NOTIFIERS_CE10) and ((x) <= NV2080_NOTIFIERS_CE19))) # type: ignore +NV2080_NOTIFIER_TYPE_IS_NVENC = lambda x: ((((x) >= NV2080_NOTIFIERS_NVENC0) and ((x) <= NV2080_NOTIFIERS_NVENC2)) or (((x) == NV2080_NOTIFIERS_NVENC3))) # type: ignore +NV2080_NOTIFIERS_NVDEC = lambda x: (NV2080_NOTIFIERS_NVDEC0 + (x)) # type: ignore +NV2080_NOTIFIERS_NVDEC_IDX = lambda x: ((x) - NV2080_NOTIFIERS_NVDEC0) # type: ignore +NV2080_NOTIFIER_TYPE_IS_NVDEC = lambda x: (((x) >= NV2080_NOTIFIERS_NVDEC0) and ((x) <= NV2080_NOTIFIERS_NVDEC7)) # type: ignore +NV2080_NOTIFIERS_NVJPEG = lambda x: (NV2080_NOTIFIERS_NVJPEG0 + (x)) # type: ignore +NV2080_NOTIFIERS_NVJPEG_IDX = lambda x: ((x) - NV2080_NOTIFIERS_NVJPEG0) # type: ignore +NV2080_NOTIFIER_TYPE_IS_NVJPEG = lambda x: (((x) >= NV2080_NOTIFIERS_NVJPEG0) and ((x) <= NV2080_NOTIFIERS_NVJPEG7)) # type: ignore +NV2080_NOTIFIER_TYPE_IS_OFA = lambda x: (((x) == NV2080_NOTIFIERS_OFA0) or ((x) == NV2080_NOTIFIERS_OFA1)) # type: ignore +NV2080_NOTIFIERS_GPIO_RISING_INTERRUPT = lambda pin: (NV2080_NOTIFIERS_GPIO_0_RISING_INTERRUPT + (pin)) # type: ignore +NV2080_NOTIFIERS_GPIO_FALLING_INTERRUPT = lambda pin: (NV2080_NOTIFIERS_GPIO_0_FALLING_INTERRUPT + (pin)) # type: ignore +NV2080_SUBDEVICE_NOTIFICATION_STATUS_IN_PROGRESS = (0x8000) +NV2080_SUBDEVICE_NOTIFICATION_STATUS_BAD_ARGUMENT = (0x4000) +NV2080_SUBDEVICE_NOTIFICATION_STATUS_ERROR_INVALID_STATE = (0x2000) +NV2080_SUBDEVICE_NOTIFICATION_STATUS_ERROR_STATE_IN_USE = (0x1000) +NV2080_SUBDEVICE_NOTIFICATION_STATUS_DONE_SUCCESS = (0x0000) +NV2080_ENGINE_TYPE_NULL = (0x00000000) +NV2080_ENGINE_TYPE_GRAPHICS = (0x00000001) +NV2080_ENGINE_TYPE_GR0 = NV2080_ENGINE_TYPE_GRAPHICS +NV2080_ENGINE_TYPE_GR1 = (0x00000002) +NV2080_ENGINE_TYPE_GR2 = (0x00000003) +NV2080_ENGINE_TYPE_GR3 = (0x00000004) +NV2080_ENGINE_TYPE_GR4 = (0x00000005) +NV2080_ENGINE_TYPE_GR5 = (0x00000006) +NV2080_ENGINE_TYPE_GR6 = (0x00000007) +NV2080_ENGINE_TYPE_GR7 = (0x00000008) +NV2080_ENGINE_TYPE_COPY0 = (0x00000009) +NV2080_ENGINE_TYPE_COPY1 = (0x0000000a) +NV2080_ENGINE_TYPE_COPY2 = (0x0000000b) +NV2080_ENGINE_TYPE_COPY3 = (0x0000000c) +NV2080_ENGINE_TYPE_COPY4 = (0x0000000d) +NV2080_ENGINE_TYPE_COPY5 = (0x0000000e) +NV2080_ENGINE_TYPE_COPY6 = (0x0000000f) +NV2080_ENGINE_TYPE_COPY7 = (0x00000010) +NV2080_ENGINE_TYPE_COPY8 = (0x00000011) +NV2080_ENGINE_TYPE_COPY9 = (0x00000012) +NV2080_ENGINE_TYPE_BSP = (0x00000013) +NV2080_ENGINE_TYPE_NVDEC0 = NV2080_ENGINE_TYPE_BSP +NV2080_ENGINE_TYPE_NVDEC1 = (0x00000014) +NV2080_ENGINE_TYPE_NVDEC2 = (0x00000015) +NV2080_ENGINE_TYPE_NVDEC3 = (0x00000016) +NV2080_ENGINE_TYPE_NVDEC4 = (0x00000017) +NV2080_ENGINE_TYPE_NVDEC5 = (0x00000018) +NV2080_ENGINE_TYPE_NVDEC6 = (0x00000019) +NV2080_ENGINE_TYPE_NVDEC7 = (0x0000001a) +NV2080_ENGINE_TYPE_MSENC = (0x0000001b) +NV2080_ENGINE_TYPE_NVENC0 = NV2080_ENGINE_TYPE_MSENC +NV2080_ENGINE_TYPE_NVENC1 = (0x0000001c) +NV2080_ENGINE_TYPE_NVENC2 = (0x0000001d) +NV2080_ENGINE_TYPE_VP = (0x0000001e) +NV2080_ENGINE_TYPE_ME = (0x0000001f) +NV2080_ENGINE_TYPE_PPP = (0x00000020) +NV2080_ENGINE_TYPE_MPEG = (0x00000021) +NV2080_ENGINE_TYPE_SW = (0x00000022) +NV2080_ENGINE_TYPE_CIPHER = (0x00000023) +NV2080_ENGINE_TYPE_TSEC = NV2080_ENGINE_TYPE_CIPHER +NV2080_ENGINE_TYPE_VIC = (0x00000024) +NV2080_ENGINE_TYPE_MP = (0x00000025) +NV2080_ENGINE_TYPE_SEC2 = (0x00000026) +NV2080_ENGINE_TYPE_HOST = (0x00000027) +NV2080_ENGINE_TYPE_DPU = (0x00000028) +NV2080_ENGINE_TYPE_PMU = (0x00000029) +NV2080_ENGINE_TYPE_FBFLCN = (0x0000002a) +NV2080_ENGINE_TYPE_NVJPG = (0x0000002b) +NV2080_ENGINE_TYPE_NVJPEG0 = NV2080_ENGINE_TYPE_NVJPG +NV2080_ENGINE_TYPE_NVJPEG1 = (0x0000002c) +NV2080_ENGINE_TYPE_NVJPEG2 = (0x0000002d) +NV2080_ENGINE_TYPE_NVJPEG3 = (0x0000002e) +NV2080_ENGINE_TYPE_NVJPEG4 = (0x0000002f) +NV2080_ENGINE_TYPE_NVJPEG5 = (0x00000030) +NV2080_ENGINE_TYPE_NVJPEG6 = (0x00000031) +NV2080_ENGINE_TYPE_NVJPEG7 = (0x00000032) +NV2080_ENGINE_TYPE_OFA = (0x00000033) +NV2080_ENGINE_TYPE_OFA0 = NV2080_ENGINE_TYPE_OFA +NV2080_ENGINE_TYPE_COPY10 = (0x00000034) +NV2080_ENGINE_TYPE_COPY11 = (0x00000035) +NV2080_ENGINE_TYPE_COPY12 = (0x00000036) +NV2080_ENGINE_TYPE_COPY13 = (0x00000037) +NV2080_ENGINE_TYPE_COPY14 = (0x00000038) +NV2080_ENGINE_TYPE_COPY15 = (0x00000039) +NV2080_ENGINE_TYPE_COPY16 = (0x0000003a) +NV2080_ENGINE_TYPE_COPY17 = (0x0000003b) +NV2080_ENGINE_TYPE_COPY18 = (0x0000003c) +NV2080_ENGINE_TYPE_COPY19 = (0x0000003d) +NV2080_ENGINE_TYPE_OFA1 = (0x0000003e) +NV2080_ENGINE_TYPE_NVENC3 = (0x0000003f) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY0 = (0x00000040) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY1 = (0x00000041) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY2 = (0x00000042) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY3 = (0x00000043) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY4 = (0x00000044) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY5 = (0x00000045) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY6 = (0x00000046) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY7 = (0x00000047) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY8 = (0x00000048) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY9 = (0x00000049) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY10 = (0x0000004a) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY11 = (0x0000004b) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY12 = (0x0000004c) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY13 = (0x0000004d) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY14 = (0x0000004e) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY15 = (0x0000004f) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY16 = (0x00000050) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY17 = (0x00000051) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY18 = (0x00000052) +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY19 = (0x00000053) +NV2080_ENGINE_TYPE_LAST = (0x00000054) +NV2080_ENGINE_TYPE_ALLENGINES = (0xffffffff) +NV2080_ENGINE_TYPE_COPY_SIZE = 64 +NV2080_ENGINE_TYPE_NVENC_SIZE = 4 +NV2080_ENGINE_TYPE_NVJPEG_SIZE = 8 +NV2080_ENGINE_TYPE_NVDEC_SIZE = 8 +NV2080_ENGINE_TYPE_GR_SIZE = 8 +NV2080_ENGINE_TYPE_OFA_SIZE = 2 +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY = lambda i: (NV2080_ENGINE_TYPE_COMP_DECOMP_COPY0 + (i)) # type: ignore +NV2080_ENGINE_TYPE_IS_COMP_DECOMP_COPY = lambda i: (((i) >= NV2080_ENGINE_TYPE_COMP_DECOMP_COPY0) and ((i) <= NV2080_ENGINE_TYPE_COMP_DECOMP_COPY19)) # type: ignore +NV2080_ENGINE_TYPE_COMP_DECOMP_COPY_IDX = lambda i: ((i) - NV2080_ENGINE_TYPE_COMP_DECOMP_COPY0) # type: ignore +NV2080_ENGINE_TYPE_IS_COPY = lambda i: ((((i) >= NV2080_ENGINE_TYPE_COPY0) and ((i) <= NV2080_ENGINE_TYPE_COPY9)) or (((i) >= NV2080_ENGINE_TYPE_COPY10) and ((i) <= NV2080_ENGINE_TYPE_COPY19))) # type: ignore +NV2080_ENGINE_TYPE_IS_NVENC = lambda i: ((((i) >= NV2080_ENGINE_TYPE_NVENC0) and ((i) <= NV2080_ENGINE_TYPE_NVENC2)) or (((i) == NV2080_ENGINE_TYPE_NVENC3))) # type: ignore +NV2080_ENGINE_TYPE_NVDEC = lambda i: (NV2080_ENGINE_TYPE_NVDEC0+(i)) # type: ignore +NV2080_ENGINE_TYPE_IS_NVDEC = lambda i: (((i) >= NV2080_ENGINE_TYPE_NVDEC0) and ((i) < NV2080_ENGINE_TYPE_NVDEC(NV2080_ENGINE_TYPE_NVDEC_SIZE))) # type: ignore +NV2080_ENGINE_TYPE_NVDEC_IDX = lambda i: ((i) - NV2080_ENGINE_TYPE_NVDEC0) # type: ignore +NV2080_ENGINE_TYPE_NVJPEG = lambda i: (NV2080_ENGINE_TYPE_NVJPEG0+(i)) # type: ignore +NV2080_ENGINE_TYPE_IS_NVJPEG = lambda i: (((i) >= NV2080_ENGINE_TYPE_NVJPEG0) and ((i) < NV2080_ENGINE_TYPE_NVJPEG(NV2080_ENGINE_TYPE_NVJPEG_SIZE))) # type: ignore +NV2080_ENGINE_TYPE_NVJPEG_IDX = lambda i: ((i) - NV2080_ENGINE_TYPE_NVJPEG0) # type: ignore +NV2080_ENGINE_TYPE_GR = lambda i: (NV2080_ENGINE_TYPE_GR0 + (i)) # type: ignore +NV2080_ENGINE_TYPE_IS_GR = lambda i: (((i) >= NV2080_ENGINE_TYPE_GR0) and ((i) < NV2080_ENGINE_TYPE_GR(NV2080_ENGINE_TYPE_GR_SIZE))) # type: ignore +NV2080_ENGINE_TYPE_GR_IDX = lambda i: ((i) - NV2080_ENGINE_TYPE_GR0) # type: ignore +NV2080_ENGINE_TYPE_IS_OFA = lambda i: (((i) == NV2080_ENGINE_TYPE_OFA0) or ((i) == NV2080_ENGINE_TYPE_OFA1)) # type: ignore +NV2080_ENGINE_TYPE_IS_VALID = lambda i: (((i) > (NV2080_ENGINE_TYPE_NULL)) and ((i) < (NV2080_ENGINE_TYPE_LAST))) # type: ignore +NV2080_CLIENT_TYPE_TEX = (0x00000001) +NV2080_CLIENT_TYPE_COLOR = (0x00000002) +NV2080_CLIENT_TYPE_DEPTH = (0x00000003) +NV2080_CLIENT_TYPE_DA = (0x00000004) +NV2080_CLIENT_TYPE_FE = (0x00000005) +NV2080_CLIENT_TYPE_SCC = (0x00000006) +NV2080_CLIENT_TYPE_WID = (0x00000007) +NV2080_CLIENT_TYPE_MSVLD = (0x00000008) +NV2080_CLIENT_TYPE_MSPDEC = (0x00000009) +NV2080_CLIENT_TYPE_MSPPP = (0x0000000a) +NV2080_CLIENT_TYPE_VIC = (0x0000000b) +NV2080_CLIENT_TYPE_ALLCLIENTS = (0xffffffff) +NV2080_GC5_EXIT_COMPLETE = (0x00000001) +NV2080_GC5_ENTRY_ABORTED = (0x00000002) +NV2080_PLATFORM_POWER_MODE_CHANGE_COMPLETION = (0x00000000) +NV2080_PLATFORM_POWER_MODE_CHANGE_ACPI_NOTIFICATION = (0x00000001) +NV2080_NOTIFICATION_STATUS_ERROR_PROTECTION_FAULT = (0x4000) +NV2080_TYPEDEF = Nv20Subdevice0 +NV2080_PLATFORM_POWER_MODE_CHANGE_INFO_INDEX = (7, 0) +NV2080_PLATFORM_POWER_MODE_CHANGE_INFO_MASK = (15, 8) +NV2080_PLATFORM_POWER_MODE_CHANGE_INFO_REASON = (23, 16) +AMPERE_CHANNEL_GPFIFO_A = (0x0000C56F) +NVC56F_NUMBER_OF_SUBCHANNELS = (8) +NVC56F_SET_OBJECT = (0x00000000) +NVC56F_SET_OBJECT_NVCLASS = (15, 0) +NVC56F_SET_OBJECT_ENGINE = (20, 16) +NVC56F_SET_OBJECT_ENGINE_SW = 0x0000001f +NVC56F_ILLEGAL = (0x00000004) +NVC56F_ILLEGAL_HANDLE = (31, 0) +NVC56F_NOP = (0x00000008) +NVC56F_NOP_HANDLE = (31, 0) +NVC56F_SEMAPHOREA = (0x00000010) +NVC56F_SEMAPHOREA_OFFSET_UPPER = (7, 0) +NVC56F_SEMAPHOREB = (0x00000014) +NVC56F_SEMAPHOREB_OFFSET_LOWER = (31, 2) +NVC56F_SEMAPHOREC = (0x00000018) +NVC56F_SEMAPHOREC_PAYLOAD = (31, 0) +NVC56F_SEMAPHORED = (0x0000001C) +NVC56F_SEMAPHORED_OPERATION = (4, 0) +NVC56F_SEMAPHORED_OPERATION_ACQUIRE = 0x00000001 +NVC56F_SEMAPHORED_OPERATION_RELEASE = 0x00000002 +NVC56F_SEMAPHORED_OPERATION_ACQ_GEQ = 0x00000004 +NVC56F_SEMAPHORED_OPERATION_ACQ_AND = 0x00000008 +NVC56F_SEMAPHORED_OPERATION_REDUCTION = 0x00000010 +NVC56F_SEMAPHORED_ACQUIRE_SWITCH = (12, 12) +NVC56F_SEMAPHORED_ACQUIRE_SWITCH_DISABLED = 0x00000000 +NVC56F_SEMAPHORED_ACQUIRE_SWITCH_ENABLED = 0x00000001 +NVC56F_SEMAPHORED_RELEASE_WFI = (20, 20) +NVC56F_SEMAPHORED_RELEASE_WFI_EN = 0x00000000 +NVC56F_SEMAPHORED_RELEASE_WFI_DIS = 0x00000001 +NVC56F_SEMAPHORED_RELEASE_SIZE = (24, 24) +NVC56F_SEMAPHORED_RELEASE_SIZE_16BYTE = 0x00000000 +NVC56F_SEMAPHORED_RELEASE_SIZE_4BYTE = 0x00000001 +NVC56F_SEMAPHORED_REDUCTION = (30, 27) +NVC56F_SEMAPHORED_REDUCTION_MIN = 0x00000000 +NVC56F_SEMAPHORED_REDUCTION_MAX = 0x00000001 +NVC56F_SEMAPHORED_REDUCTION_XOR = 0x00000002 +NVC56F_SEMAPHORED_REDUCTION_AND = 0x00000003 +NVC56F_SEMAPHORED_REDUCTION_OR = 0x00000004 +NVC56F_SEMAPHORED_REDUCTION_ADD = 0x00000005 +NVC56F_SEMAPHORED_REDUCTION_INC = 0x00000006 +NVC56F_SEMAPHORED_REDUCTION_DEC = 0x00000007 +NVC56F_SEMAPHORED_FORMAT = (31, 31) +NVC56F_SEMAPHORED_FORMAT_SIGNED = 0x00000000 +NVC56F_SEMAPHORED_FORMAT_UNSIGNED = 0x00000001 +NVC56F_NON_STALL_INTERRUPT = (0x00000020) +NVC56F_NON_STALL_INTERRUPT_HANDLE = (31, 0) +NVC56F_FB_FLUSH = (0x00000024) +NVC56F_FB_FLUSH_HANDLE = (31, 0) +NVC56F_MEM_OP_A = (0x00000028) +NVC56F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_CLIENT_UNIT_ID = (5, 0) +NVC56F_MEM_OP_A_TLB_INVALIDATE_INVALIDATION_SIZE = (5, 0) +NVC56F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_GPC_ID = (10, 6) +NVC56F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE = (7, 6) +NVC56F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_ALL_TLBS = 0 +NVC56F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_LINK_TLBS = 1 +NVC56F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_NON_LINK_TLBS = 2 +NVC56F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_RSVRVD = 3 +NVC56F_MEM_OP_A_TLB_INVALIDATE_CANCEL_MMU_ENGINE_ID = (6, 0) +NVC56F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR = (11, 11) +NVC56F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_EN = 0x00000001 +NVC56F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_DIS = 0x00000000 +NVC56F_MEM_OP_A_TLB_INVALIDATE_TARGET_ADDR_LO = (31, 12) +NVC56F_MEM_OP_B = (0x0000002c) +NVC56F_MEM_OP_B_TLB_INVALIDATE_TARGET_ADDR_HI = (31, 0) +NVC56F_MEM_OP_C = (0x00000030) +NVC56F_MEM_OP_C_MEMBAR_TYPE = (2, 0) +NVC56F_MEM_OP_C_MEMBAR_TYPE_SYS_MEMBAR = 0x00000000 +NVC56F_MEM_OP_C_MEMBAR_TYPE_MEMBAR = 0x00000001 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB = (0, 0) +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE = 0x00000000 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL = 0x00000001 +NVC56F_MEM_OP_C_TLB_INVALIDATE_GPC = (1, 1) +NVC56F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE = 0x00000000 +NVC56F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE = 0x00000001 +NVC56F_MEM_OP_C_TLB_INVALIDATE_REPLAY = (4, 2) +NVC56F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE = 0x00000000 +NVC56F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START = 0x00000001 +NVC56F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START_ACK_ALL = 0x00000002 +NVC56F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_TARGETED = 0x00000003 +NVC56F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_GLOBAL = 0x00000004 +NVC56F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_VA_GLOBAL = 0x00000005 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE = (6, 5) +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_NONE = 0x00000000 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_GLOBALLY = 0x00000001 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_INTRANODE = 0x00000002 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE = (9, 7) +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_READ = 0 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE = 1 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_STRONG = 2 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_RSVRVD = 3 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_WEAK = 4 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_ALL = 5 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE_AND_ATOMIC = 6 +NVC56F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ALL = 7 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL = (9, 7) +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_ALL = 0x00000000 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_PTE_ONLY = 0x00000001 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE0 = 0x00000002 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE1 = 0x00000003 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE2 = 0x00000004 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 = 0x00000005 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 = 0x00000006 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE5 = 0x00000007 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE = (11, 10) +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_VID_MEM = 0x00000000 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_COHERENT = 0x00000002 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_NONCOHERENT = 0x00000003 +NVC56F_MEM_OP_C_TLB_INVALIDATE_PDB_ADDR_LO = (31, 12) +NVC56F_MEM_OP_C_ACCESS_COUNTER_CLR_TARGETED_NOTIFY_TAG = (19, 0) +NVC56F_MEM_OP_D = (0x00000034) +NVC56F_MEM_OP_D_TLB_INVALIDATE_PDB_ADDR_HI = (26, 0) +NVC56F_MEM_OP_D_OPERATION = (31, 27) +NVC56F_MEM_OP_D_OPERATION_MEMBAR = 0x00000005 +NVC56F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE = 0x00000009 +NVC56F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE_TARGETED = 0x0000000a +NVC56F_MEM_OP_D_OPERATION_L2_PEERMEM_INVALIDATE = 0x0000000d +NVC56F_MEM_OP_D_OPERATION_L2_SYSMEM_INVALIDATE = 0x0000000e +NVC56F_MEM_OP_B_OPERATION_L2_INVALIDATE_CLEAN_LINES = 0x0000000e +NVC56F_MEM_OP_D_OPERATION_L2_CLEAN_COMPTAGS = 0x0000000f +NVC56F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY = 0x00000010 +NVC56F_MEM_OP_D_OPERATION_L2_WAIT_FOR_SYS_PENDING_READS = 0x00000015 +NVC56F_MEM_OP_D_OPERATION_ACCESS_COUNTER_CLR = 0x00000016 +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE = (1, 0) +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MIMC = 0x00000000 +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MOMC = 0x00000001 +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_ALL = 0x00000002 +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_TARGETED = 0x00000003 +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE = (2, 2) +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MIMC = 0x00000000 +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MOMC = 0x00000001 +NVC56F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_BANK = (6, 3) +NVC56F_SET_REFERENCE = (0x00000050) +NVC56F_SET_REFERENCE_COUNT = (31, 0) +NVC56F_SEM_ADDR_LO = (0x0000005c) +NVC56F_SEM_ADDR_LO_OFFSET = (31, 2) +NVC56F_SEM_ADDR_HI = (0x00000060) +NVC56F_SEM_ADDR_HI_OFFSET = (7, 0) +NVC56F_SEM_PAYLOAD_LO = (0x00000064) +NVC56F_SEM_PAYLOAD_LO_PAYLOAD = (31, 0) +NVC56F_SEM_PAYLOAD_HI = (0x00000068) +NVC56F_SEM_PAYLOAD_HI_PAYLOAD = (31, 0) +NVC56F_SEM_EXECUTE = (0x0000006c) +NVC56F_SEM_EXECUTE_OPERATION = (2, 0) +NVC56F_SEM_EXECUTE_OPERATION_ACQUIRE = 0x00000000 +NVC56F_SEM_EXECUTE_OPERATION_RELEASE = 0x00000001 +NVC56F_SEM_EXECUTE_OPERATION_ACQ_STRICT_GEQ = 0x00000002 +NVC56F_SEM_EXECUTE_OPERATION_ACQ_CIRC_GEQ = 0x00000003 +NVC56F_SEM_EXECUTE_OPERATION_ACQ_AND = 0x00000004 +NVC56F_SEM_EXECUTE_OPERATION_ACQ_NOR = 0x00000005 +NVC56F_SEM_EXECUTE_OPERATION_REDUCTION = 0x00000006 +NVC56F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG = (12, 12) +NVC56F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_DIS = 0x00000000 +NVC56F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_EN = 0x00000001 +NVC56F_SEM_EXECUTE_RELEASE_WFI = (20, 20) +NVC56F_SEM_EXECUTE_RELEASE_WFI_DIS = 0x00000000 +NVC56F_SEM_EXECUTE_RELEASE_WFI_EN = 0x00000001 +NVC56F_SEM_EXECUTE_PAYLOAD_SIZE = (24, 24) +NVC56F_SEM_EXECUTE_PAYLOAD_SIZE_32BIT = 0x00000000 +NVC56F_SEM_EXECUTE_PAYLOAD_SIZE_64BIT = 0x00000001 +NVC56F_SEM_EXECUTE_RELEASE_TIMESTAMP = (25, 25) +NVC56F_SEM_EXECUTE_RELEASE_TIMESTAMP_DIS = 0x00000000 +NVC56F_SEM_EXECUTE_RELEASE_TIMESTAMP_EN = 0x00000001 +NVC56F_SEM_EXECUTE_REDUCTION = (30, 27) +NVC56F_SEM_EXECUTE_REDUCTION_IMIN = 0x00000000 +NVC56F_SEM_EXECUTE_REDUCTION_IMAX = 0x00000001 +NVC56F_SEM_EXECUTE_REDUCTION_IXOR = 0x00000002 +NVC56F_SEM_EXECUTE_REDUCTION_IAND = 0x00000003 +NVC56F_SEM_EXECUTE_REDUCTION_IOR = 0x00000004 +NVC56F_SEM_EXECUTE_REDUCTION_IADD = 0x00000005 +NVC56F_SEM_EXECUTE_REDUCTION_INC = 0x00000006 +NVC56F_SEM_EXECUTE_REDUCTION_DEC = 0x00000007 +NVC56F_SEM_EXECUTE_REDUCTION_FORMAT = (31, 31) +NVC56F_SEM_EXECUTE_REDUCTION_FORMAT_SIGNED = 0x00000000 +NVC56F_SEM_EXECUTE_REDUCTION_FORMAT_UNSIGNED = 0x00000001 +NVC56F_WFI = (0x00000078) +NVC56F_WFI_SCOPE = (0, 0) +NVC56F_WFI_SCOPE_CURRENT_SCG_TYPE = 0x00000000 +NVC56F_WFI_SCOPE_CURRENT_VEID = 0x00000000 +NVC56F_WFI_SCOPE_ALL = 0x00000001 +NVC56F_YIELD = (0x00000080) +NVC56F_YIELD_OP = (1, 0) +NVC56F_YIELD_OP_NOP = 0x00000000 +NVC56F_YIELD_OP_TSG = 0x00000003 +NVC56F_CLEAR_FAULTED = (0x00000084) +NVC56F_CLEAR_FAULTED_HANDLE = (30, 0) +NVC56F_CLEAR_FAULTED_TYPE = (31, 31) +NVC56F_CLEAR_FAULTED_TYPE_PBDMA_FAULTED = 0x00000000 +NVC56F_CLEAR_FAULTED_TYPE_ENG_FAULTED = 0x00000001 +NVC56F_GP_ENTRY__SIZE = 8 +NVC56F_GP_ENTRY0_FETCH = (0, 0) +NVC56F_GP_ENTRY0_FETCH_UNCONDITIONAL = 0x00000000 +NVC56F_GP_ENTRY0_FETCH_CONDITIONAL = 0x00000001 +NVC56F_GP_ENTRY0_GET = (31, 2) +NVC56F_GP_ENTRY0_OPERAND = (31, 0) +NVC56F_GP_ENTRY1_GET_HI = (7, 0) +NVC56F_GP_ENTRY1_LEVEL = (9, 9) +NVC56F_GP_ENTRY1_LEVEL_MAIN = 0x00000000 +NVC56F_GP_ENTRY1_LEVEL_SUBROUTINE = 0x00000001 +NVC56F_GP_ENTRY1_LENGTH = (30, 10) +NVC56F_GP_ENTRY1_SYNC = (31, 31) +NVC56F_GP_ENTRY1_SYNC_PROCEED = 0x00000000 +NVC56F_GP_ENTRY1_SYNC_WAIT = 0x00000001 +NVC56F_GP_ENTRY1_OPCODE = (7, 0) +NVC56F_GP_ENTRY1_OPCODE_NOP = 0x00000000 +NVC56F_GP_ENTRY1_OPCODE_ILLEGAL = 0x00000001 +NVC56F_GP_ENTRY1_OPCODE_GP_CRC = 0x00000002 +NVC56F_GP_ENTRY1_OPCODE_PB_CRC = 0x00000003 +NVC56F_DMA_METHOD_ADDRESS_OLD = (12, 2) +NVC56F_DMA_METHOD_ADDRESS = (11, 0) +NVC56F_DMA_SUBDEVICE_MASK = (15, 4) +NVC56F_DMA_METHOD_SUBCHANNEL = (15, 13) +NVC56F_DMA_TERT_OP = (17, 16) +NVC56F_DMA_TERT_OP_GRP0_INC_METHOD = (0x00000000) +NVC56F_DMA_TERT_OP_GRP0_SET_SUB_DEV_MASK = (0x00000001) +NVC56F_DMA_TERT_OP_GRP0_STORE_SUB_DEV_MASK = (0x00000002) +NVC56F_DMA_TERT_OP_GRP0_USE_SUB_DEV_MASK = (0x00000003) +NVC56F_DMA_TERT_OP_GRP2_NON_INC_METHOD = (0x00000000) +NVC56F_DMA_METHOD_COUNT_OLD = (28, 18) +NVC56F_DMA_METHOD_COUNT = (28, 16) +NVC56F_DMA_IMMD_DATA = (28, 16) +NVC56F_DMA_SEC_OP = (31, 29) +NVC56F_DMA_SEC_OP_GRP0_USE_TERT = (0x00000000) +NVC56F_DMA_SEC_OP_INC_METHOD = (0x00000001) +NVC56F_DMA_SEC_OP_GRP2_USE_TERT = (0x00000002) +NVC56F_DMA_SEC_OP_NON_INC_METHOD = (0x00000003) +NVC56F_DMA_SEC_OP_IMMD_DATA_METHOD = (0x00000004) +NVC56F_DMA_SEC_OP_ONE_INC = (0x00000005) +NVC56F_DMA_SEC_OP_RESERVED6 = (0x00000006) +NVC56F_DMA_SEC_OP_END_PB_SEGMENT = (0x00000007) +NVC56F_DMA_INCR_ADDRESS = (11, 0) +NVC56F_DMA_INCR_SUBCHANNEL = (15, 13) +NVC56F_DMA_INCR_COUNT = (28, 16) +NVC56F_DMA_INCR_OPCODE = (31, 29) +NVC56F_DMA_INCR_OPCODE_VALUE = (0x00000001) +NVC56F_DMA_INCR_DATA = (31, 0) +NVC56F_DMA_NONINCR_ADDRESS = (11, 0) +NVC56F_DMA_NONINCR_SUBCHANNEL = (15, 13) +NVC56F_DMA_NONINCR_COUNT = (28, 16) +NVC56F_DMA_NONINCR_OPCODE = (31, 29) +NVC56F_DMA_NONINCR_OPCODE_VALUE = (0x00000003) +NVC56F_DMA_NONINCR_DATA = (31, 0) +NVC56F_DMA_ONEINCR_ADDRESS = (11, 0) +NVC56F_DMA_ONEINCR_SUBCHANNEL = (15, 13) +NVC56F_DMA_ONEINCR_COUNT = (28, 16) +NVC56F_DMA_ONEINCR_OPCODE = (31, 29) +NVC56F_DMA_ONEINCR_OPCODE_VALUE = (0x00000005) +NVC56F_DMA_ONEINCR_DATA = (31, 0) +NVC56F_DMA_NOP = (0x00000000) +NVC56F_DMA_IMMD_ADDRESS = (11, 0) +NVC56F_DMA_IMMD_SUBCHANNEL = (15, 13) +NVC56F_DMA_IMMD_DATA = (28, 16) +NVC56F_DMA_IMMD_OPCODE = (31, 29) +NVC56F_DMA_IMMD_OPCODE_VALUE = (0x00000004) +NVC56F_DMA_SET_SUBDEVICE_MASK_VALUE = (15, 4) +NVC56F_DMA_SET_SUBDEVICE_MASK_OPCODE = (31, 16) +NVC56F_DMA_SET_SUBDEVICE_MASK_OPCODE_VALUE = (0x00000001) +NVC56F_DMA_STORE_SUBDEVICE_MASK_VALUE = (15, 4) +NVC56F_DMA_STORE_SUBDEVICE_MASK_OPCODE = (31, 16) +NVC56F_DMA_STORE_SUBDEVICE_MASK_OPCODE_VALUE = (0x00000002) +NVC56F_DMA_USE_SUBDEVICE_MASK_OPCODE = (31, 16) +NVC56F_DMA_USE_SUBDEVICE_MASK_OPCODE_VALUE = (0x00000003) +NVC56F_DMA_ENDSEG_OPCODE = (31, 29) +NVC56F_DMA_ENDSEG_OPCODE_VALUE = (0x00000007) +NVC56F_DMA_ADDRESS = (12, 2) +NVC56F_DMA_SUBCH = (15, 13) +NVC56F_DMA_OPCODE3 = (17, 16) +NVC56F_DMA_OPCODE3_NONE = (0x00000000) +NVC56F_DMA_COUNT = (28, 18) +NVC56F_DMA_OPCODE = (31, 29) +NVC56F_DMA_OPCODE_METHOD = (0x00000000) +NVC56F_DMA_OPCODE_NONINC_METHOD = (0x00000002) +NVC56F_DMA_DATA = (31, 0) +HOPPER_CHANNEL_GPFIFO_A = (0x0000C86F) +NVC86F_SET_OBJECT = (0x00000000) +NVC86F_MEM_OP_A = (0x00000028) +NVC86F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_CLIENT_UNIT_ID = (5, 0) +NVC86F_MEM_OP_A_TLB_INVALIDATE_INVALIDATION_SIZE = (5, 0) +NVC86F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_GPC_ID = (10, 6) +NVC86F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE = (7, 6) +NVC86F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_ALL_TLBS = 0 +NVC86F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_LINK_TLBS = 1 +NVC86F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_NON_LINK_TLBS = 2 +NVC86F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_RSVRVD = 3 +NVC86F_MEM_OP_A_TLB_INVALIDATE_CANCEL_MMU_ENGINE_ID = (8, 0) +NVC86F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR = (11, 11) +NVC86F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_EN = 0x00000001 +NVC86F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_DIS = 0x00000000 +NVC86F_MEM_OP_A_TLB_INVALIDATE_TARGET_ADDR_LO = (31, 12) +NVC86F_MEM_OP_B = (0x0000002c) +NVC86F_MEM_OP_B_TLB_INVALIDATE_TARGET_ADDR_HI = (31, 0) +NVC86F_MEM_OP_C = (0x00000030) +NVC86F_MEM_OP_C_MEMBAR_TYPE = (2, 0) +NVC86F_MEM_OP_C_MEMBAR_TYPE_SYS_MEMBAR = 0x00000000 +NVC86F_MEM_OP_C_MEMBAR_TYPE_MEMBAR = 0x00000001 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB = (0, 0) +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE = 0x00000000 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL = 0x00000001 +NVC86F_MEM_OP_C_TLB_INVALIDATE_GPC = (1, 1) +NVC86F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE = 0x00000000 +NVC86F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE = 0x00000001 +NVC86F_MEM_OP_C_TLB_INVALIDATE_REPLAY = (4, 2) +NVC86F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE = 0x00000000 +NVC86F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START = 0x00000001 +NVC86F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START_ACK_ALL = 0x00000002 +NVC86F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_TARGETED = 0x00000003 +NVC86F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_GLOBAL = 0x00000004 +NVC86F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_VA_GLOBAL = 0x00000005 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE = (6, 5) +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_NONE = 0x00000000 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_GLOBALLY = 0x00000001 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_INTRANODE = 0x00000002 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE = (9, 7) +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_READ = 0 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE = 1 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_STRONG = 2 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_RSVRVD = 3 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_WEAK = 4 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_ALL = 5 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE_AND_ATOMIC = 6 +NVC86F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ALL = 7 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL = (9, 7) +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_ALL = 0x00000000 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_PTE_ONLY = 0x00000001 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE0 = 0x00000002 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE1 = 0x00000003 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE2 = 0x00000004 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 = 0x00000005 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 = 0x00000006 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE5 = 0x00000007 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE = (11, 10) +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_VID_MEM = 0x00000000 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_COHERENT = 0x00000002 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_NONCOHERENT = 0x00000003 +NVC86F_MEM_OP_C_TLB_INVALIDATE_PDB_ADDR_LO = (31, 12) +NVC86F_MEM_OP_C_ACCESS_COUNTER_CLR_TARGETED_NOTIFY_TAG = (19, 0) +NVC86F_MEM_OP_D = (0x00000034) +NVC86F_MEM_OP_D_TLB_INVALIDATE_PDB_ADDR_HI = (26, 0) +NVC86F_MEM_OP_D_OPERATION = (31, 27) +NVC86F_MEM_OP_D_OPERATION_MEMBAR = 0x00000005 +NVC86F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE = 0x00000009 +NVC86F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE_TARGETED = 0x0000000a +NVC86F_MEM_OP_D_OPERATION_MMU_OPERATION = 0x0000000b +NVC86F_MEM_OP_D_OPERATION_L2_PEERMEM_INVALIDATE = 0x0000000d +NVC86F_MEM_OP_D_OPERATION_L2_SYSMEM_INVALIDATE = 0x0000000e +NVC86F_MEM_OP_B_OPERATION_L2_INVALIDATE_CLEAN_LINES = 0x0000000e +NVC86F_MEM_OP_D_OPERATION_L2_CLEAN_COMPTAGS = 0x0000000f +NVC86F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY = 0x00000010 +NVC86F_MEM_OP_D_OPERATION_L2_WAIT_FOR_SYS_PENDING_READS = 0x00000015 +NVC86F_MEM_OP_D_OPERATION_ACCESS_COUNTER_CLR = 0x00000016 +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE = (1, 0) +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MIMC = 0x00000000 +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MOMC = 0x00000001 +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_ALL = 0x00000002 +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_TARGETED = 0x00000003 +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE = (2, 2) +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MIMC = 0x00000000 +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MOMC = 0x00000001 +NVC86F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_BANK = (6, 3) +NVC86F_MEM_OP_D_MMU_OPERATION_TYPE = (23, 20) +NVC86F_MEM_OP_D_MMU_OPERATION_TYPE_RESERVED = 0x00000000 +NVC86F_MEM_OP_D_MMU_OPERATION_TYPE_VIDMEM_ACCESS_BIT_DUMP = 0x00000001 +NVC86F_SEM_ADDR_LO = (0x0000005c) +NVC86F_SEM_ADDR_LO_OFFSET = (31, 2) +NVC86F_SEM_ADDR_HI = (0x00000060) +NVC86F_SEM_ADDR_HI_OFFSET = (24, 0) +NVC86F_SEM_PAYLOAD_LO = (0x00000064) +NVC86F_SEM_PAYLOAD_HI = (0x00000068) +NVC86F_SEM_EXECUTE = (0x0000006c) +NVC86F_SEM_EXECUTE_OPERATION = (2, 0) +NVC86F_SEM_EXECUTE_OPERATION_ACQUIRE = 0x00000000 +NVC86F_SEM_EXECUTE_OPERATION_RELEASE = 0x00000001 +NVC86F_SEM_EXECUTE_OPERATION_ACQ_CIRC_GEQ = 0x00000003 +NVC86F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG = (12, 12) +NVC86F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_EN = 0x00000001 +NVC86F_SEM_EXECUTE_RELEASE_WFI = (20, 20) +NVC86F_SEM_EXECUTE_RELEASE_WFI_DIS = 0x00000000 +NVC86F_SEM_EXECUTE_PAYLOAD_SIZE = (24, 24) +NVC86F_SEM_EXECUTE_PAYLOAD_SIZE_32BIT = 0x00000000 +NVC86F_SEM_EXECUTE_RELEASE_TIMESTAMP = (25, 25) +NVC86F_SEM_EXECUTE_RELEASE_TIMESTAMP_DIS = 0x00000000 +NVC86F_SEM_EXECUTE_RELEASE_TIMESTAMP_EN = 0x00000001 +NVC86F_WFI = (0x00000078) +NVC86F_WFI_SCOPE = (0, 0) +NVC86F_WFI_SCOPE_CURRENT_SCG_TYPE = 0x00000000 +NVC86F_WFI_SCOPE_CURRENT_VEID = 0x00000000 +NVC86F_WFI_SCOPE_ALL = 0x00000001 +NVC86F_GP_ENTRY__SIZE = 8 +NVC86F_GP_ENTRY0_FETCH = (0, 0) +NVC86F_GP_ENTRY0_FETCH_UNCONDITIONAL = 0x00000000 +NVC86F_GP_ENTRY0_FETCH_CONDITIONAL = 0x00000001 +NVC86F_GP_ENTRY0_GET = (31, 2) +NVC86F_GP_ENTRY0_OPERAND = (31, 0) +NVC86F_GP_ENTRY0_PB_EXTENDED_BASE_OPERAND = (24, 8) +NVC86F_GP_ENTRY1_GET_HI = (7, 0) +NVC86F_GP_ENTRY1_LEVEL = (9, 9) +NVC86F_GP_ENTRY1_LEVEL_MAIN = 0x00000000 +NVC86F_GP_ENTRY1_LEVEL_SUBROUTINE = 0x00000001 +NVC86F_GP_ENTRY1_LENGTH = (30, 10) +NVC86F_GP_ENTRY1_SYNC = (31, 31) +NVC86F_GP_ENTRY1_SYNC_PROCEED = 0x00000000 +NVC86F_GP_ENTRY1_SYNC_WAIT = 0x00000001 +NVC86F_GP_ENTRY1_OPCODE = (7, 0) +NVC86F_GP_ENTRY1_OPCODE_NOP = 0x00000000 +NVC86F_GP_ENTRY1_OPCODE_ILLEGAL = 0x00000001 +NVC86F_GP_ENTRY1_OPCODE_GP_CRC = 0x00000002 +NVC86F_GP_ENTRY1_OPCODE_PB_CRC = 0x00000003 +NVC86F_GP_ENTRY1_OPCODE_SET_PB_SEGMENT_EXTENDED_BASE = 0x00000004 +BLACKWELL_CHANNEL_GPFIFO_A = (0x0000C96F) +NVC96F_SET_OBJECT = (0x00000000) +NVC96F_MEM_OP_A = (0x00000028) +NVC96F_MEM_OP_A_TLB_INVALIDATE_INVALIDATION_SIZE = (5, 0) +NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE = (7, 6) +NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_NON_LINK_TLBS = 2 +NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR = (11, 11) +NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_EN = 0x00000001 +NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_DIS = 0x00000000 +NVC96F_MEM_OP_A_TLB_INVALIDATE_TARGET_ADDR_LO = (31, 12) +NVC96F_MEM_OP_B = (0x0000002c) +NVC96F_MEM_OP_B_TLB_INVALIDATE_TARGET_ADDR_HI = (31, 0) +NVC96F_MEM_OP_C = (0x00000030) +NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB = (0, 0) +NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE = 0x00000000 +NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL = 0x00000001 +NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC = (1, 1) +NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE = 0x00000000 +NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE = 0x00000001 +NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY = (4, 2) +NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE = 0x00000000 +NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE = (6, 5) +NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_NONE = 0x00000000 +NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_GLOBALLY = 0x00000001 +NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL = (9, 7) +NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_ALL = 0x00000000 +NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_PTE_ONLY = 0x00000001 +NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 = 0x00000006 +NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE = (11, 10) +NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_VID_MEM = 0x00000000 +NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_COHERENT = 0x00000002 +NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ADDR_LO = (31, 12) +NVC96F_MEM_OP_D = (0x00000034) +NVC96F_MEM_OP_D_TLB_INVALIDATE_PDB_ADDR_HI = (26, 0) +NVC96F_MEM_OP_D_OPERATION = (31, 27) +NVC96F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE = 0x00000009 +NVC96F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE_TARGETED = 0x0000000a +NVC96F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY = 0x00000010 +NVC96F_MEM_OP_D_OPERATION_L2_SYSMEM_NCOH_INVALIDATE = 0x00000011 +NVC96F_SEM_ADDR_LO = (0x0000005c) +NVC96F_SEM_ADDR_LO_OFFSET = (31, 2) +NVC96F_SEM_ADDR_HI = (0x00000060) +NVC96F_SEM_ADDR_HI_OFFSET = (24, 0) +NVC96F_SEM_PAYLOAD_LO = (0x00000064) +NVC96F_SEM_PAYLOAD_HI = (0x00000068) +NVC96F_SEM_EXECUTE = (0x0000006c) +NVC96F_SEM_EXECUTE_OPERATION = (2, 0) +NVC96F_SEM_EXECUTE_OPERATION_ACQUIRE = 0x00000000 +NVC96F_SEM_EXECUTE_OPERATION_RELEASE = 0x00000001 +NVC96F_SEM_EXECUTE_RELEASE_WFI = (20, 20) +NVC96F_SEM_EXECUTE_RELEASE_WFI_DIS = 0x00000000 +NVC96F_SEM_EXECUTE_PAYLOAD_SIZE = (24, 24) +NVC96F_SEM_EXECUTE_PAYLOAD_SIZE_32BIT = 0x00000000 +NVC96F_GP_ENTRY__SIZE = 8 +NVC96F_GP_ENTRY0_FETCH = (0, 0) +NVC96F_GP_ENTRY0_FETCH_UNCONDITIONAL = 0x00000000 +NVC96F_GP_ENTRY0_FETCH_CONDITIONAL = 0x00000001 +NVC96F_GP_ENTRY0_GET = (31, 2) +NVC96F_GP_ENTRY0_OPERAND = (31, 0) +NVC96F_GP_ENTRY0_PB_EXTENDED_BASE_OPERAND = (24, 8) +NVC96F_GP_ENTRY1_GET_HI = (7, 0) +NVC96F_GP_ENTRY1_LEVEL = (9, 9) +NVC96F_GP_ENTRY1_LEVEL_MAIN = 0x00000000 +NVC96F_GP_ENTRY1_LEVEL_SUBROUTINE = 0x00000001 +NVC96F_GP_ENTRY1_LENGTH = (30, 10) +NVC96F_GP_ENTRY1_SYNC = (31, 31) +NVC96F_GP_ENTRY1_SYNC_PROCEED = 0x00000000 +NVC96F_GP_ENTRY1_SYNC_WAIT = 0x00000001 +NVC96F_GP_ENTRY1_OPCODE = (7, 0) +NVC96F_GP_ENTRY1_OPCODE_NOP = 0x00000000 +NVC96F_GP_ENTRY1_OPCODE_ILLEGAL = 0x00000001 +NVC96F_GP_ENTRY1_OPCODE_GP_CRC = 0x00000002 +NVC96F_GP_ENTRY1_OPCODE_PB_CRC = 0x00000003 +NVC96F_GP_ENTRY1_OPCODE_SET_PB_SEGMENT_EXTENDED_BASE = 0x00000004 +BLACKWELL_USERMODE_A = (0xc761) +GT200_DEBUGGER = (0x83de) +NV83DE_ALLOC_PARAMETERS_MESSAGE_ID = (0x83de) +MAXWELL_PROFILER_DEVICE = (0xb2cc) +NVB2CC_ALLOC_PARAMETERS_MESSAGE_ID = (0xb2cc) +AMPERE_COMPUTE_A = 0xC6C0 +NVC6C0_SET_OBJECT = 0x0000 +NVC6C0_SET_OBJECT_CLASS_ID = (15, 0) +NVC6C0_SET_OBJECT_ENGINE_ID = (20, 16) +NVC6C0_NO_OPERATION = 0x0100 +NVC6C0_NO_OPERATION_V = (31, 0) +NVC6C0_SET_NOTIFY_A = 0x0104 +NVC6C0_SET_NOTIFY_A_ADDRESS_UPPER = (7, 0) +NVC6C0_SET_NOTIFY_B = 0x0108 +NVC6C0_SET_NOTIFY_B_ADDRESS_LOWER = (31, 0) +NVC6C0_NOTIFY = 0x010c +NVC6C0_NOTIFY_TYPE = (31, 0) +NVC6C0_NOTIFY_TYPE_WRITE_ONLY = 0x00000000 +NVC6C0_NOTIFY_TYPE_WRITE_THEN_AWAKEN = 0x00000001 +NVC6C0_WAIT_FOR_IDLE = 0x0110 +NVC6C0_WAIT_FOR_IDLE_V = (31, 0) +NVC6C0_SET_GLOBAL_RENDER_ENABLE_A = 0x0130 +NVC6C0_SET_GLOBAL_RENDER_ENABLE_A_OFFSET_UPPER = (7, 0) +NVC6C0_SET_GLOBAL_RENDER_ENABLE_B = 0x0134 +NVC6C0_SET_GLOBAL_RENDER_ENABLE_B_OFFSET_LOWER = (31, 0) +NVC6C0_SET_GLOBAL_RENDER_ENABLE_C = 0x0138 +NVC6C0_SET_GLOBAL_RENDER_ENABLE_C_MODE = (2, 0) +NVC6C0_SET_GLOBAL_RENDER_ENABLE_C_MODE_FALSE = 0x00000000 +NVC6C0_SET_GLOBAL_RENDER_ENABLE_C_MODE_TRUE = 0x00000001 +NVC6C0_SET_GLOBAL_RENDER_ENABLE_C_MODE_CONDITIONAL = 0x00000002 +NVC6C0_SET_GLOBAL_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL = 0x00000003 +NVC6C0_SET_GLOBAL_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL = 0x00000004 +NVC6C0_SEND_GO_IDLE = 0x013c +NVC6C0_SEND_GO_IDLE_V = (31, 0) +NVC6C0_PM_TRIGGER = 0x0140 +NVC6C0_PM_TRIGGER_V = (31, 0) +NVC6C0_PM_TRIGGER_WFI = 0x0144 +NVC6C0_PM_TRIGGER_WFI_V = (31, 0) +NVC6C0_FE_ATOMIC_SEQUENCE_BEGIN = 0x0148 +NVC6C0_FE_ATOMIC_SEQUENCE_BEGIN_V = (31, 0) +NVC6C0_FE_ATOMIC_SEQUENCE_END = 0x014c +NVC6C0_FE_ATOMIC_SEQUENCE_END_V = (31, 0) +NVC6C0_SET_INSTRUMENTATION_METHOD_HEADER = 0x0150 +NVC6C0_SET_INSTRUMENTATION_METHOD_HEADER_V = (31, 0) +NVC6C0_SET_INSTRUMENTATION_METHOD_DATA = 0x0154 +NVC6C0_SET_INSTRUMENTATION_METHOD_DATA_V = (31, 0) +NVC6C0_LINE_LENGTH_IN = 0x0180 +NVC6C0_LINE_LENGTH_IN_VALUE = (31, 0) +NVC6C0_LINE_COUNT = 0x0184 +NVC6C0_LINE_COUNT_VALUE = (31, 0) +NVC6C0_OFFSET_OUT_UPPER = 0x0188 +NVC6C0_OFFSET_OUT_UPPER_VALUE = (16, 0) +NVC6C0_OFFSET_OUT = 0x018c +NVC6C0_OFFSET_OUT_VALUE = (31, 0) +NVC6C0_PITCH_OUT = 0x0190 +NVC6C0_PITCH_OUT_VALUE = (31, 0) +NVC6C0_SET_DST_BLOCK_SIZE = 0x0194 +NVC6C0_SET_DST_BLOCK_SIZE_WIDTH = (3, 0) +NVC6C0_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB = 0x00000000 +NVC6C0_SET_DST_BLOCK_SIZE_HEIGHT = (7, 4) +NVC6C0_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB = 0x00000000 +NVC6C0_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS = 0x00000001 +NVC6C0_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS = 0x00000002 +NVC6C0_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS = 0x00000003 +NVC6C0_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS = 0x00000004 +NVC6C0_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS = 0x00000005 +NVC6C0_SET_DST_BLOCK_SIZE_DEPTH = (11, 8) +NVC6C0_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB = 0x00000000 +NVC6C0_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS = 0x00000001 +NVC6C0_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS = 0x00000002 +NVC6C0_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS = 0x00000003 +NVC6C0_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS = 0x00000004 +NVC6C0_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS = 0x00000005 +NVC6C0_SET_DST_WIDTH = 0x0198 +NVC6C0_SET_DST_WIDTH_V = (31, 0) +NVC6C0_SET_DST_HEIGHT = 0x019c +NVC6C0_SET_DST_HEIGHT_V = (31, 0) +NVC6C0_SET_DST_DEPTH = 0x01a0 +NVC6C0_SET_DST_DEPTH_V = (31, 0) +NVC6C0_SET_DST_LAYER = 0x01a4 +NVC6C0_SET_DST_LAYER_V = (31, 0) +NVC6C0_SET_DST_ORIGIN_BYTES_X = 0x01a8 +NVC6C0_SET_DST_ORIGIN_BYTES_X_V = (20, 0) +NVC6C0_SET_DST_ORIGIN_SAMPLES_Y = 0x01ac +NVC6C0_SET_DST_ORIGIN_SAMPLES_Y_V = (16, 0) +NVC6C0_LAUNCH_DMA = 0x01b0 +NVC6C0_LAUNCH_DMA_DST_MEMORY_LAYOUT = (0, 0) +NVC6C0_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR = 0x00000000 +NVC6C0_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH = 0x00000001 +NVC6C0_LAUNCH_DMA_COMPLETION_TYPE = (5, 4) +NVC6C0_LAUNCH_DMA_COMPLETION_TYPE_FLUSH_DISABLE = 0x00000000 +NVC6C0_LAUNCH_DMA_COMPLETION_TYPE_FLUSH_ONLY = 0x00000001 +NVC6C0_LAUNCH_DMA_COMPLETION_TYPE_RELEASE_SEMAPHORE = 0x00000002 +NVC6C0_LAUNCH_DMA_INTERRUPT_TYPE = (9, 8) +NVC6C0_LAUNCH_DMA_INTERRUPT_TYPE_NONE = 0x00000000 +NVC6C0_LAUNCH_DMA_INTERRUPT_TYPE_INTERRUPT = 0x00000001 +NVC6C0_LAUNCH_DMA_SEMAPHORE_STRUCT_SIZE = (12, 12) +NVC6C0_LAUNCH_DMA_SEMAPHORE_STRUCT_SIZE_FOUR_WORDS = 0x00000000 +NVC6C0_LAUNCH_DMA_SEMAPHORE_STRUCT_SIZE_ONE_WORD = 0x00000001 +NVC6C0_LAUNCH_DMA_REDUCTION_ENABLE = (1, 1) +NVC6C0_LAUNCH_DMA_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_LAUNCH_DMA_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_LAUNCH_DMA_REDUCTION_OP = (15, 13) +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_LAUNCH_DMA_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_LAUNCH_DMA_REDUCTION_FORMAT = (3, 2) +NVC6C0_LAUNCH_DMA_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_LAUNCH_DMA_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_LAUNCH_DMA_SYSMEMBAR_DISABLE = (6, 6) +NVC6C0_LAUNCH_DMA_SYSMEMBAR_DISABLE_FALSE = 0x00000000 +NVC6C0_LAUNCH_DMA_SYSMEMBAR_DISABLE_TRUE = 0x00000001 +NVC6C0_LOAD_INLINE_DATA = 0x01b4 +NVC6C0_LOAD_INLINE_DATA_V = (31, 0) +NVC6C0_SET_I2M_SEMAPHORE_A = 0x01dc +NVC6C0_SET_I2M_SEMAPHORE_A_OFFSET_UPPER = (7, 0) +NVC6C0_SET_I2M_SEMAPHORE_B = 0x01e0 +NVC6C0_SET_I2M_SEMAPHORE_B_OFFSET_LOWER = (31, 0) +NVC6C0_SET_I2M_SEMAPHORE_C = 0x01e4 +NVC6C0_SET_I2M_SEMAPHORE_C_PAYLOAD = (31, 0) +NVC6C0_SET_SM_SCG_CONTROL = 0x01e8 +NVC6C0_SET_SM_SCG_CONTROL_COMPUTE_IN_GRAPHICS = (0, 0) +NVC6C0_SET_SM_SCG_CONTROL_COMPUTE_IN_GRAPHICS_FALSE = 0x00000000 +NVC6C0_SET_SM_SCG_CONTROL_COMPUTE_IN_GRAPHICS_TRUE = 0x00000001 +NVC6C0_SET_I2M_SPARE_NOOP00 = 0x01f0 +NVC6C0_SET_I2M_SPARE_NOOP00_V = (31, 0) +NVC6C0_SET_I2M_SPARE_NOOP01 = 0x01f4 +NVC6C0_SET_I2M_SPARE_NOOP01_V = (31, 0) +NVC6C0_SET_I2M_SPARE_NOOP02 = 0x01f8 +NVC6C0_SET_I2M_SPARE_NOOP02_V = (31, 0) +NVC6C0_SET_I2M_SPARE_NOOP03 = 0x01fc +NVC6C0_SET_I2M_SPARE_NOOP03_V = (31, 0) +NVC6C0_SET_VALID_SPAN_OVERFLOW_AREA_A = 0x0200 +NVC6C0_SET_VALID_SPAN_OVERFLOW_AREA_A_ADDRESS_UPPER = (7, 0) +NVC6C0_SET_VALID_SPAN_OVERFLOW_AREA_B = 0x0204 +NVC6C0_SET_VALID_SPAN_OVERFLOW_AREA_B_ADDRESS_LOWER = (31, 0) +NVC6C0_SET_VALID_SPAN_OVERFLOW_AREA_C = 0x0208 +NVC6C0_SET_VALID_SPAN_OVERFLOW_AREA_C_SIZE = (31, 0) +NVC6C0_PERFMON_TRANSFER = 0x0210 +NVC6C0_PERFMON_TRANSFER_V = (31, 0) +NVC6C0_SET_QMD_VIRTUALIZATION_BASE_A = 0x0214 +NVC6C0_SET_QMD_VIRTUALIZATION_BASE_A_ADDRESS_UPPER = (7, 0) +NVC6C0_SET_QMD_VIRTUALIZATION_BASE_B = 0x0218 +NVC6C0_SET_QMD_VIRTUALIZATION_BASE_B_ADDRESS_LOWER = (31, 0) +NVC6C0_INVALIDATE_SHADER_CACHES = 0x021c +NVC6C0_INVALIDATE_SHADER_CACHES_INSTRUCTION = (0, 0) +NVC6C0_INVALIDATE_SHADER_CACHES_INSTRUCTION_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_INSTRUCTION_TRUE = 0x00000001 +NVC6C0_INVALIDATE_SHADER_CACHES_DATA = (4, 4) +NVC6C0_INVALIDATE_SHADER_CACHES_DATA_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_DATA_TRUE = 0x00000001 +NVC6C0_INVALIDATE_SHADER_CACHES_CONSTANT = (12, 12) +NVC6C0_INVALIDATE_SHADER_CACHES_CONSTANT_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_CONSTANT_TRUE = 0x00000001 +NVC6C0_INVALIDATE_SHADER_CACHES_LOCKS = (1, 1) +NVC6C0_INVALIDATE_SHADER_CACHES_LOCKS_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_LOCKS_TRUE = 0x00000001 +NVC6C0_INVALIDATE_SHADER_CACHES_FLUSH_DATA = (2, 2) +NVC6C0_INVALIDATE_SHADER_CACHES_FLUSH_DATA_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_FLUSH_DATA_TRUE = 0x00000001 +NVC6C0_SET_RESERVED_SW_METHOD00 = 0x0220 +NVC6C0_SET_RESERVED_SW_METHOD00_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD01 = 0x0224 +NVC6C0_SET_RESERVED_SW_METHOD01_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD02 = 0x0228 +NVC6C0_SET_RESERVED_SW_METHOD02_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD03 = 0x022c +NVC6C0_SET_RESERVED_SW_METHOD03_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD04 = 0x0230 +NVC6C0_SET_RESERVED_SW_METHOD04_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD05 = 0x0234 +NVC6C0_SET_RESERVED_SW_METHOD05_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD06 = 0x0238 +NVC6C0_SET_RESERVED_SW_METHOD06_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD07 = 0x023c +NVC6C0_SET_RESERVED_SW_METHOD07_V = (31, 0) +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI = 0x0244 +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_LINES = (0, 0) +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_LINES_ALL = 0x00000000 +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_LINES_ONE = 0x00000001 +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_TAG = (25, 4) +NVC6C0_SET_CWD_REF_COUNTER = 0x0248 +NVC6C0_SET_CWD_REF_COUNTER_SELECT = (5, 0) +NVC6C0_SET_CWD_REF_COUNTER_VALUE = (23, 8) +NVC6C0_SET_RESERVED_SW_METHOD08 = 0x024c +NVC6C0_SET_RESERVED_SW_METHOD08_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD09 = 0x0250 +NVC6C0_SET_RESERVED_SW_METHOD09_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD10 = 0x0254 +NVC6C0_SET_RESERVED_SW_METHOD10_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD11 = 0x0258 +NVC6C0_SET_RESERVED_SW_METHOD11_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD12 = 0x025c +NVC6C0_SET_RESERVED_SW_METHOD12_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD13 = 0x0260 +NVC6C0_SET_RESERVED_SW_METHOD13_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD14 = 0x0264 +NVC6C0_SET_RESERVED_SW_METHOD14_V = (31, 0) +NVC6C0_SET_RESERVED_SW_METHOD15 = 0x0268 +NVC6C0_SET_RESERVED_SW_METHOD15_V = (31, 0) +NVC6C0_SET_SCG_CONTROL = 0x0270 +NVC6C0_SET_SCG_CONTROL_COMPUTE1_MAX_SM_COUNT = (8, 0) +NVC6C0_SET_SCG_CONTROL_COMPUTE1_MIN_SM_COUNT = (20, 12) +NVC6C0_SET_SCG_CONTROL_DISABLE_COMPUTE1_LIMIT_IN_ALL_COMPUTE = (24, 24) +NVC6C0_SET_SCG_CONTROL_DISABLE_COMPUTE1_LIMIT_IN_ALL_COMPUTE_FALSE = 0x00000000 +NVC6C0_SET_SCG_CONTROL_DISABLE_COMPUTE1_LIMIT_IN_ALL_COMPUTE_TRUE = 0x00000001 +NVC6C0_SET_COMPUTE_CLASS_VERSION = 0x0280 +NVC6C0_SET_COMPUTE_CLASS_VERSION_CURRENT = (15, 0) +NVC6C0_SET_COMPUTE_CLASS_VERSION_OLDEST_SUPPORTED = (31, 16) +NVC6C0_CHECK_COMPUTE_CLASS_VERSION = 0x0284 +NVC6C0_CHECK_COMPUTE_CLASS_VERSION_CURRENT = (15, 0) +NVC6C0_CHECK_COMPUTE_CLASS_VERSION_OLDEST_SUPPORTED = (31, 16) +NVC6C0_SET_QMD_VERSION = 0x0288 +NVC6C0_SET_QMD_VERSION_CURRENT = (15, 0) +NVC6C0_SET_QMD_VERSION_OLDEST_SUPPORTED = (31, 16) +NVC6C0_CHECK_QMD_VERSION = 0x0290 +NVC6C0_CHECK_QMD_VERSION_CURRENT = (15, 0) +NVC6C0_CHECK_QMD_VERSION_OLDEST_SUPPORTED = (31, 16) +NVC6C0_INVALIDATE_SKED_CACHES = 0x0298 +NVC6C0_INVALIDATE_SKED_CACHES_V = (0, 0) +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL = 0x029c +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_CONSTANT_BUFFER_MASK = (7, 0) +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_I2M_ADDR_ENABLE = (8, 8) +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_I2M_ADDR_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_I2M_ADDR_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_I2M_CONSTANT_BUFFER_ENABLE = (12, 12) +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_I2M_CONSTANT_BUFFER_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_I2M_CONSTANT_BUFFER_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_IQ2M_ADDR_ENABLE = (16, 16) +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_IQ2M_ADDR_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_IQ2M_ADDR_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_IQ2M_CONSTANT_BUFFER_ENABLE = (20, 20) +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_IQ2M_CONSTANT_BUFFER_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_IQ2M_CONSTANT_BUFFER_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_SEND_PCAS_ENABLE = (24, 24) +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_SEND_PCAS_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_QMD_VIRTUALIZATION_CONTROL_SEND_PCAS_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_SHADER_SHARED_MEMORY_WINDOW_A = 0x02a0 +NVC6C0_SET_SHADER_SHARED_MEMORY_WINDOW_A_BASE_ADDRESS_UPPER = (16, 0) +NVC6C0_SET_SHADER_SHARED_MEMORY_WINDOW_B = 0x02a4 +NVC6C0_SET_SHADER_SHARED_MEMORY_WINDOW_B_BASE_ADDRESS = (31, 0) +NVC6C0_SCG_HYSTERESIS_CONTROL = 0x02a8 +NVC6C0_SCG_HYSTERESIS_CONTROL_USE_TIMEOUT_ONCE = (0, 0) +NVC6C0_SCG_HYSTERESIS_CONTROL_USE_TIMEOUT_ONCE_FALSE = 0x00000000 +NVC6C0_SCG_HYSTERESIS_CONTROL_USE_TIMEOUT_ONCE_TRUE = 0x00000001 +NVC6C0_SCG_HYSTERESIS_CONTROL_USE_NULL_TIMEOUT_ONCE = (1, 1) +NVC6C0_SCG_HYSTERESIS_CONTROL_USE_NULL_TIMEOUT_ONCE_FALSE = 0x00000000 +NVC6C0_SCG_HYSTERESIS_CONTROL_USE_NULL_TIMEOUT_ONCE_TRUE = 0x00000001 +NVC6C0_SET_CWD_SLOT_COUNT = 0x02b0 +NVC6C0_SET_CWD_SLOT_COUNT_V = (7, 0) +NVC6C0_SEND_PCAS_A = 0x02b4 +NVC6C0_SEND_PCAS_A_QMD_ADDRESS_SHIFTED8 = (31, 0) +NVC6C0_SEND_PCAS_B = 0x02b8 +NVC6C0_SEND_PCAS_B_FROM = (23, 0) +NVC6C0_SEND_PCAS_B_DELTA = (31, 24) +NVC6C0_SEND_SIGNALING_PCAS_B = 0x02bc +NVC6C0_SEND_SIGNALING_PCAS_B_INVALIDATE = (0, 0) +NVC6C0_SEND_SIGNALING_PCAS_B_INVALIDATE_FALSE = 0x00000000 +NVC6C0_SEND_SIGNALING_PCAS_B_INVALIDATE_TRUE = 0x00000001 +NVC6C0_SEND_SIGNALING_PCAS_B_SCHEDULE = (1, 1) +NVC6C0_SEND_SIGNALING_PCAS_B_SCHEDULE_FALSE = 0x00000000 +NVC6C0_SEND_SIGNALING_PCAS_B_SCHEDULE_TRUE = 0x00000001 +NVC6C0_SEND_SIGNALING_PCAS2_B = 0x02c0 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION = (3, 0) +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_NOP = 0x00000000 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE = 0x00000001 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_SCHEDULE = 0x00000002 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INCREMENT_PUT = 0x00000006 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_DECREMENT_DEPENDENCE = 0x00000007 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_PREFETCH = 0x00000008 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_PREFETCH_SCHEDULE = 0x00000009 +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE_PREFETCH_COPY_SCHEDULE = 0x0000000A +NVC6C0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE_PREFETCH_COPY_FORCE_REQUIRE_SCHEDULING = 0x0000000B +NVC6C0_SET_SKED_CACHE_CONTROL = 0x02cc +NVC6C0_SET_SKED_CACHE_CONTROL_IGNORE_VEID = (0, 0) +NVC6C0_SET_SKED_CACHE_CONTROL_IGNORE_VEID_FALSE = 0x00000000 +NVC6C0_SET_SKED_CACHE_CONTROL_IGNORE_VEID_TRUE = 0x00000001 +NVC6C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A = 0x02e4 +NVC6C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A_SIZE_UPPER = (7, 0) +NVC6C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_B = 0x02e8 +NVC6C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_B_SIZE_LOWER = (31, 0) +NVC6C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C = 0x02ec +NVC6C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C_MAX_SM_COUNT = (8, 0) +NVC6C0_SET_SPA_VERSION = 0x0310 +NVC6C0_SET_SPA_VERSION_MINOR = (7, 0) +NVC6C0_SET_SPA_VERSION_MAJOR = (15, 8) +NVC6C0_SET_INLINE_QMD_ADDRESS_A = 0x0318 +NVC6C0_SET_INLINE_QMD_ADDRESS_A_QMD_ADDRESS_SHIFTED8_UPPER = (31, 0) +NVC6C0_SET_INLINE_QMD_ADDRESS_B = 0x031c +NVC6C0_SET_INLINE_QMD_ADDRESS_B_QMD_ADDRESS_SHIFTED8_LOWER = (31, 0) +NVC6C0_LOAD_INLINE_QMD_DATA = lambda i: (0x0320+(i)*4) # type: ignore +NVC6C0_LOAD_INLINE_QMD_DATA_V = (31, 0) +NVC6C0_SET_FALCON00 = 0x0500 +NVC6C0_SET_FALCON00_V = (31, 0) +NVC6C0_SET_FALCON01 = 0x0504 +NVC6C0_SET_FALCON01_V = (31, 0) +NVC6C0_SET_FALCON02 = 0x0508 +NVC6C0_SET_FALCON02_V = (31, 0) +NVC6C0_SET_FALCON03 = 0x050c +NVC6C0_SET_FALCON03_V = (31, 0) +NVC6C0_SET_FALCON04 = 0x0510 +NVC6C0_SET_FALCON04_V = (31, 0) +NVC6C0_SET_FALCON05 = 0x0514 +NVC6C0_SET_FALCON05_V = (31, 0) +NVC6C0_SET_FALCON06 = 0x0518 +NVC6C0_SET_FALCON06_V = (31, 0) +NVC6C0_SET_FALCON07 = 0x051c +NVC6C0_SET_FALCON07_V = (31, 0) +NVC6C0_SET_FALCON08 = 0x0520 +NVC6C0_SET_FALCON08_V = (31, 0) +NVC6C0_SET_FALCON09 = 0x0524 +NVC6C0_SET_FALCON09_V = (31, 0) +NVC6C0_SET_FALCON10 = 0x0528 +NVC6C0_SET_FALCON10_V = (31, 0) +NVC6C0_SET_FALCON11 = 0x052c +NVC6C0_SET_FALCON11_V = (31, 0) +NVC6C0_SET_FALCON12 = 0x0530 +NVC6C0_SET_FALCON12_V = (31, 0) +NVC6C0_SET_FALCON13 = 0x0534 +NVC6C0_SET_FALCON13_V = (31, 0) +NVC6C0_SET_FALCON14 = 0x0538 +NVC6C0_SET_FALCON14_V = (31, 0) +NVC6C0_SET_FALCON15 = 0x053c +NVC6C0_SET_FALCON15_V = (31, 0) +NVC6C0_SET_SHADER_LOCAL_MEMORY_A = 0x0790 +NVC6C0_SET_SHADER_LOCAL_MEMORY_A_ADDRESS_UPPER = (16, 0) +NVC6C0_SET_SHADER_LOCAL_MEMORY_B = 0x0794 +NVC6C0_SET_SHADER_LOCAL_MEMORY_B_ADDRESS_LOWER = (31, 0) +NVC6C0_SET_SHADER_LOCAL_MEMORY_WINDOW_A = 0x07b0 +NVC6C0_SET_SHADER_LOCAL_MEMORY_WINDOW_A_BASE_ADDRESS_UPPER = (16, 0) +NVC6C0_SET_SHADER_LOCAL_MEMORY_WINDOW_B = 0x07b4 +NVC6C0_SET_SHADER_LOCAL_MEMORY_WINDOW_B_BASE_ADDRESS = (31, 0) +NVC6C0_SET_SHADER_CACHE_CONTROL = 0x0d94 +NVC6C0_SET_SHADER_CACHE_CONTROL_ICACHE_PREFETCH_ENABLE = (0, 0) +NVC6C0_SET_SHADER_CACHE_CONTROL_ICACHE_PREFETCH_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_SHADER_CACHE_CONTROL_ICACHE_PREFETCH_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_SCG_COMPUTE_SCHEDULING_PARAMETERS = lambda i: (0x0da0+(i)*4) # type: ignore +NVC6C0_SET_SCG_COMPUTE_SCHEDULING_PARAMETERS_V = (31, 0) +NVC6C0_SET_SM_TIMEOUT_INTERVAL = 0x0de4 +NVC6C0_SET_SM_TIMEOUT_INTERVAL_COUNTER_BIT = (5, 0) +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI = 0x1288 +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_LINES = (0, 0) +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_LINES_ALL = 0x00000000 +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_LINES_ONE = 0x00000001 +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_TAG = (25, 4) +NVC6C0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT = 0x12a8 +NVC6C0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT_ALL = (0, 0) +NVC6C0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT_ALL_FALSE = 0x00000000 +NVC6C0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT_ALL_TRUE = 0x00000001 +NVC6C0_INVALIDATE_SAMPLER_CACHE = 0x1330 +NVC6C0_INVALIDATE_SAMPLER_CACHE_LINES = (0, 0) +NVC6C0_INVALIDATE_SAMPLER_CACHE_LINES_ALL = 0x00000000 +NVC6C0_INVALIDATE_SAMPLER_CACHE_LINES_ONE = 0x00000001 +NVC6C0_INVALIDATE_SAMPLER_CACHE_TAG = (25, 4) +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE = 0x1334 +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_LINES = (0, 0) +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_LINES_ALL = 0x00000000 +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_LINES_ONE = 0x00000001 +NVC6C0_INVALIDATE_TEXTURE_HEADER_CACHE_TAG = (25, 4) +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE = 0x1338 +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_LINES = (0, 0) +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_LINES_ALL = 0x00000000 +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_LINES_ONE = 0x00000001 +NVC6C0_INVALIDATE_TEXTURE_DATA_CACHE_TAG = (25, 4) +NVC6C0_INVALIDATE_SAMPLER_CACHE_NO_WFI = 0x1424 +NVC6C0_INVALIDATE_SAMPLER_CACHE_NO_WFI_LINES = (0, 0) +NVC6C0_INVALIDATE_SAMPLER_CACHE_NO_WFI_LINES_ALL = 0x00000000 +NVC6C0_INVALIDATE_SAMPLER_CACHE_NO_WFI_LINES_ONE = 0x00000001 +NVC6C0_INVALIDATE_SAMPLER_CACHE_NO_WFI_TAG = (25, 4) +NVC6C0_SET_SHADER_EXCEPTIONS = 0x1528 +NVC6C0_SET_SHADER_EXCEPTIONS_ENABLE = (0, 0) +NVC6C0_SET_SHADER_EXCEPTIONS_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_SHADER_EXCEPTIONS_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_RENDER_ENABLE_A = 0x1550 +NVC6C0_SET_RENDER_ENABLE_A_OFFSET_UPPER = (7, 0) +NVC6C0_SET_RENDER_ENABLE_B = 0x1554 +NVC6C0_SET_RENDER_ENABLE_B_OFFSET_LOWER = (31, 0) +NVC6C0_SET_RENDER_ENABLE_C = 0x1558 +NVC6C0_SET_RENDER_ENABLE_C_MODE = (2, 0) +NVC6C0_SET_RENDER_ENABLE_C_MODE_FALSE = 0x00000000 +NVC6C0_SET_RENDER_ENABLE_C_MODE_TRUE = 0x00000001 +NVC6C0_SET_RENDER_ENABLE_C_MODE_CONDITIONAL = 0x00000002 +NVC6C0_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL = 0x00000003 +NVC6C0_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL = 0x00000004 +NVC6C0_SET_TEX_SAMPLER_POOL_A = 0x155c +NVC6C0_SET_TEX_SAMPLER_POOL_A_OFFSET_UPPER = (16, 0) +NVC6C0_SET_TEX_SAMPLER_POOL_B = 0x1560 +NVC6C0_SET_TEX_SAMPLER_POOL_B_OFFSET_LOWER = (31, 0) +NVC6C0_SET_TEX_SAMPLER_POOL_C = 0x1564 +NVC6C0_SET_TEX_SAMPLER_POOL_C_MAXIMUM_INDEX = (19, 0) +NVC6C0_SET_TEX_HEADER_POOL_A = 0x1574 +NVC6C0_SET_TEX_HEADER_POOL_A_OFFSET_UPPER = (16, 0) +NVC6C0_SET_TEX_HEADER_POOL_B = 0x1578 +NVC6C0_SET_TEX_HEADER_POOL_B_OFFSET_LOWER = (31, 0) +NVC6C0_SET_TEX_HEADER_POOL_C = 0x157c +NVC6C0_SET_TEX_HEADER_POOL_C_MAXIMUM_INDEX = (21, 0) +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI = 0x1698 +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_INSTRUCTION = (0, 0) +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_INSTRUCTION_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_INSTRUCTION_TRUE = 0x00000001 +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_GLOBAL_DATA = (4, 4) +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_GLOBAL_DATA_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_GLOBAL_DATA_TRUE = 0x00000001 +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_CONSTANT = (12, 12) +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_CONSTANT_FALSE = 0x00000000 +NVC6C0_INVALIDATE_SHADER_CACHES_NO_WFI_CONSTANT_TRUE = 0x00000001 +NVC6C0_SET_RENDER_ENABLE_OVERRIDE = 0x1944 +NVC6C0_SET_RENDER_ENABLE_OVERRIDE_MODE = (1, 0) +NVC6C0_SET_RENDER_ENABLE_OVERRIDE_MODE_USE_RENDER_ENABLE = 0x00000000 +NVC6C0_SET_RENDER_ENABLE_OVERRIDE_MODE_ALWAYS_RENDER = 0x00000001 +NVC6C0_SET_RENDER_ENABLE_OVERRIDE_MODE_NEVER_RENDER = 0x00000002 +NVC6C0_PIPE_NOP = 0x1a2c +NVC6C0_PIPE_NOP_V = (31, 0) +NVC6C0_SET_SPARE00 = 0x1a30 +NVC6C0_SET_SPARE00_V = (31, 0) +NVC6C0_SET_SPARE01 = 0x1a34 +NVC6C0_SET_SPARE01_V = (31, 0) +NVC6C0_SET_SPARE02 = 0x1a38 +NVC6C0_SET_SPARE02_V = (31, 0) +NVC6C0_SET_SPARE03 = 0x1a3c +NVC6C0_SET_SPARE03_V = (31, 0) +NVC6C0_SET_REPORT_SEMAPHORE_A = 0x1b00 +NVC6C0_SET_REPORT_SEMAPHORE_A_OFFSET_UPPER = (7, 0) +NVC6C0_SET_REPORT_SEMAPHORE_B = 0x1b04 +NVC6C0_SET_REPORT_SEMAPHORE_B_OFFSET_LOWER = (31, 0) +NVC6C0_SET_REPORT_SEMAPHORE_C = 0x1b08 +NVC6C0_SET_REPORT_SEMAPHORE_C_PAYLOAD = (31, 0) +NVC6C0_SET_REPORT_SEMAPHORE_D = 0x1b0c +NVC6C0_SET_REPORT_SEMAPHORE_D_OPERATION = (1, 0) +NVC6C0_SET_REPORT_SEMAPHORE_D_OPERATION_RELEASE = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_OPERATION_TRAP = 0x00000003 +NVC6C0_SET_REPORT_SEMAPHORE_D_AWAKEN_ENABLE = (20, 20) +NVC6C0_SET_REPORT_SEMAPHORE_D_AWAKEN_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_AWAKEN_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_REPORT_SEMAPHORE_D_STRUCTURE_SIZE = (28, 28) +NVC6C0_SET_REPORT_SEMAPHORE_D_STRUCTURE_SIZE_FOUR_WORDS = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_STRUCTURE_SIZE_ONE_WORD = 0x00000001 +NVC6C0_SET_REPORT_SEMAPHORE_D_FLUSH_DISABLE = (2, 2) +NVC6C0_SET_REPORT_SEMAPHORE_D_FLUSH_DISABLE_FALSE = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_FLUSH_DISABLE_TRUE = 0x00000001 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_ENABLE = (3, 3) +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_ENABLE_FALSE = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_ENABLE_TRUE = 0x00000001 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP = (11, 9) +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_ADD = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_MIN = 0x00000001 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_MAX = 0x00000002 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_INC = 0x00000003 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_DEC = 0x00000004 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_AND = 0x00000005 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_OR = 0x00000006 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_XOR = 0x00000007 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_FORMAT = (18, 17) +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVC6C0_SET_REPORT_SEMAPHORE_D_CONDITIONAL_TRAP = (19, 19) +NVC6C0_SET_REPORT_SEMAPHORE_D_CONDITIONAL_TRAP_FALSE = 0x00000000 +NVC6C0_SET_REPORT_SEMAPHORE_D_CONDITIONAL_TRAP_TRUE = 0x00000001 +NVC6C0_SET_TRAP_HANDLER_A = 0x25f8 +NVC6C0_SET_TRAP_HANDLER_A_ADDRESS_UPPER = (16, 0) +NVC6C0_SET_TRAP_HANDLER_B = 0x25fc +NVC6C0_SET_TRAP_HANDLER_B_ADDRESS_LOWER = (31, 0) +NVC6C0_SET_BINDLESS_TEXTURE = 0x2608 +NVC6C0_SET_BINDLESS_TEXTURE_CONSTANT_BUFFER_SLOT_SELECT = (2, 0) +NVC6C0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE = lambda i: (0x32f4+(i)*4) # type: ignore +NVC6C0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE_V = (31, 0) +NVC6C0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE_UPPER = lambda i: (0x3314+(i)*4) # type: ignore +NVC6C0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE_UPPER_V = (31, 0) +NVC6C0_ENABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER = 0x3334 +NVC6C0_ENABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_V = (0, 0) +NVC6C0_DISABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER = 0x3338 +NVC6C0_DISABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_V = (0, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_VALUE_UPPER = lambda i: (0x333c+(i)*4) # type: ignore +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_VALUE_UPPER_V = (31, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_VALUE = lambda i: (0x335c+(i)*4) # type: ignore +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_VALUE_V = (31, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_EVENT = lambda i: (0x337c+(i)*4) # type: ignore +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_EVENT_EVENT = (7, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A = lambda i: (0x339c+(i)*4) # type: ignore +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT0 = (1, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT0 = (4, 2) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT1 = (6, 5) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT1 = (9, 7) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT2 = (11, 10) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT2 = (14, 12) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT3 = (16, 15) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT3 = (19, 17) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT4 = (21, 20) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT4 = (24, 22) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT5 = (26, 25) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT5 = (29, 27) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_SPARE = (31, 30) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B = lambda i: (0x33bc+(i)*4) # type: ignore +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_EDGE = (0, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_MODE = (2, 1) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_WINDOWED = (3, 3) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_FUNC = (19, 4) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_TRAP_CONTROL = 0x33dc +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_TRAP_CONTROL_MASK = (7, 0) +NVC6C0_START_SHADER_PERFORMANCE_COUNTER = 0x33e0 +NVC6C0_START_SHADER_PERFORMANCE_COUNTER_COUNTER_MASK = (7, 0) +NVC6C0_STOP_SHADER_PERFORMANCE_COUNTER = 0x33e4 +NVC6C0_STOP_SHADER_PERFORMANCE_COUNTER_COUNTER_MASK = (7, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_SCTL_FILTER = 0x33e8 +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_SCTL_FILTER_V = (31, 0) +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CORE_MIO_FILTER = 0x33ec +NVC6C0_SET_SHADER_PERFORMANCE_COUNTER_CORE_MIO_FILTER_V = (31, 0) +NVC6C0_SET_MME_SHADOW_SCRATCH = lambda i: (0x3400+(i)*4) # type: ignore +NVC6C0_SET_MME_SHADOW_SCRATCH_V = (31, 0) +BLACKWELL_COMPUTE_A = 0xCDC0 +NVCDC0_SET_OBJECT = 0x0000 +NVCDC0_SET_OBJECT_CLASS_ID = (15, 0) +NVCDC0_SET_OBJECT_ENGINE_ID = (20, 16) +NVCDC0_NO_OPERATION = 0x0100 +NVCDC0_NO_OPERATION_V = (31, 0) +NVCDC0_SET_NOTIFY_A = 0x0104 +NVCDC0_SET_NOTIFY_A_ADDRESS_UPPER = (24, 0) +NVCDC0_SET_NOTIFY_B = 0x0108 +NVCDC0_SET_NOTIFY_B_ADDRESS_LOWER = (31, 0) +NVCDC0_NOTIFY = 0x010c +NVCDC0_NOTIFY_TYPE = (31, 0) +NVCDC0_NOTIFY_TYPE_WRITE_ONLY = 0x00000000 +NVCDC0_NOTIFY_TYPE_WRITE_THEN_AWAKEN = 0x00000001 +NVCDC0_WAIT_FOR_IDLE = 0x0110 +NVCDC0_WAIT_FOR_IDLE_V = (31, 0) +NVCDC0_LOAD_MME_INSTRUCTION_RAM_POINTER = 0x0114 +NVCDC0_LOAD_MME_INSTRUCTION_RAM_POINTER_V = (31, 0) +NVCDC0_LOAD_MME_INSTRUCTION_RAM = 0x0118 +NVCDC0_LOAD_MME_INSTRUCTION_RAM_V = (31, 0) +NVCDC0_LOAD_MME_START_ADDRESS_RAM_POINTER = 0x011c +NVCDC0_LOAD_MME_START_ADDRESS_RAM_POINTER_V = (31, 0) +NVCDC0_LOAD_MME_START_ADDRESS_RAM = 0x0120 +NVCDC0_LOAD_MME_START_ADDRESS_RAM_V = (31, 0) +NVCDC0_SET_MME_SHADOW_RAM_CONTROL = 0x0124 +NVCDC0_SET_MME_SHADOW_RAM_CONTROL_MODE = (1, 0) +NVCDC0_SET_MME_SHADOW_RAM_CONTROL_MODE_METHOD_TRACK = 0x00000000 +NVCDC0_SET_MME_SHADOW_RAM_CONTROL_MODE_METHOD_TRACK_WITH_FILTER = 0x00000001 +NVCDC0_SET_MME_SHADOW_RAM_CONTROL_MODE_METHOD_PASSTHROUGH = 0x00000002 +NVCDC0_SET_MME_SHADOW_RAM_CONTROL_MODE_METHOD_REPLAY = 0x00000003 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_A = 0x0130 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_A_OFFSET_UPPER = (7, 0) +NVCDC0_SET_GLOBAL_RENDER_ENABLE_B = 0x0134 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_B_OFFSET_LOWER = (31, 0) +NVCDC0_SET_GLOBAL_RENDER_ENABLE_C = 0x0138 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_C_MODE = (2, 0) +NVCDC0_SET_GLOBAL_RENDER_ENABLE_C_MODE_FALSE = 0x00000000 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_C_MODE_TRUE = 0x00000001 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_C_MODE_CONDITIONAL = 0x00000002 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL = 0x00000003 +NVCDC0_SET_GLOBAL_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL = 0x00000004 +NVCDC0_SEND_GO_IDLE = 0x013c +NVCDC0_SEND_GO_IDLE_V = (31, 0) +NVCDC0_PM_TRIGGER = 0x0140 +NVCDC0_PM_TRIGGER_V = (31, 0) +NVCDC0_PM_TRIGGER_WFI = 0x0144 +NVCDC0_PM_TRIGGER_WFI_V = (31, 0) +NVCDC0_FE_ATOMIC_SEQUENCE_BEGIN = 0x0148 +NVCDC0_FE_ATOMIC_SEQUENCE_BEGIN_V = (31, 0) +NVCDC0_FE_ATOMIC_SEQUENCE_END = 0x014c +NVCDC0_FE_ATOMIC_SEQUENCE_END_V = (31, 0) +NVCDC0_SET_INSTRUMENTATION_METHOD_HEADER = 0x0150 +NVCDC0_SET_INSTRUMENTATION_METHOD_HEADER_V = (31, 0) +NVCDC0_SET_INSTRUMENTATION_METHOD_DATA = 0x0154 +NVCDC0_SET_INSTRUMENTATION_METHOD_DATA_V = (31, 0) +NVCDC0_SET_REPORT_SEMAPHORE_PAYLOAD_LOWER = 0x0158 +NVCDC0_SET_REPORT_SEMAPHORE_PAYLOAD_LOWER_PAYLOAD_LOWER = (31, 0) +NVCDC0_SET_REPORT_SEMAPHORE_PAYLOAD_UPPER = 0x015c +NVCDC0_SET_REPORT_SEMAPHORE_PAYLOAD_UPPER_PAYLOAD_UPPER = (31, 0) +NVCDC0_SET_REPORT_SEMAPHORE_ADDRESS_LOWER = 0x0160 +NVCDC0_SET_REPORT_SEMAPHORE_ADDRESS_LOWER_LOWER = (31, 0) +NVCDC0_SET_REPORT_SEMAPHORE_ADDRESS_UPPER = 0x0164 +NVCDC0_SET_REPORT_SEMAPHORE_ADDRESS_UPPER_UPPER = (24, 0) +NVCDC0_REPORT_SEMAPHORE_EXECUTE = 0x0168 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_OPERATION = (1, 0) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_OPERATION_RELEASE = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_OPERATION_ACQUIRE = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_OPERATION_REPORT_ONLY = 0x00000002 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_OPERATION_TRAP = 0x00000003 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_AWAKEN_ENABLE = (2, 2) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_AWAKEN_ENABLE_FALSE = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_AWAKEN_ENABLE_TRUE = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_STRUCTURE_SIZE = (4, 3) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_STRUCTURE_SIZE_SEMAPHORE_FOUR_WORDS = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_STRUCTURE_SIZE_SEMAPHORE_ONE_WORD = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_STRUCTURE_SIZE_SEMAPHORE_TWO_WORDS = 0x00000002 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_FLUSH_DISABLE = (5, 5) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_FLUSH_DISABLE_FALSE = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_FLUSH_DISABLE_TRUE = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_ENABLE = (6, 6) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_ENABLE_FALSE = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_ENABLE_TRUE = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP = (9, 7) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_ADD = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_MIN = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_MAX = 0x00000002 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_INC = 0x00000003 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_DEC = 0x00000004 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_AND = 0x00000005 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_OR = 0x00000006 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_OP_RED_XOR = 0x00000007 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_FORMAT = (11, 10) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_FORMAT_UNSIGNED = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_REDUCTION_FORMAT_SIGNED = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_PAYLOAD_SIZE64 = (12, 12) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_PAYLOAD_SIZE64_FALSE = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_PAYLOAD_SIZE64_TRUE = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_TRAP_TYPE = (14, 13) +NVCDC0_REPORT_SEMAPHORE_EXECUTE_TRAP_TYPE_TRAP_NONE = 0x00000000 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_TRAP_TYPE_TRAP_UNCONDITIONAL = 0x00000001 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_TRAP_TYPE_TRAP_CONDITIONAL = 0x00000002 +NVCDC0_REPORT_SEMAPHORE_EXECUTE_TRAP_TYPE_TRAP_CONDITIONAL_EXT = 0x00000003 +NVCDC0_LINE_LENGTH_IN = 0x0180 +NVCDC0_LINE_LENGTH_IN_VALUE = (31, 0) +NVCDC0_LINE_COUNT = 0x0184 +NVCDC0_LINE_COUNT_VALUE = (31, 0) +NVCDC0_OFFSET_OUT_UPPER = 0x0188 +NVCDC0_OFFSET_OUT_UPPER_VALUE = (24, 0) +NVCDC0_OFFSET_OUT = 0x018c +NVCDC0_OFFSET_OUT_VALUE = (31, 0) +NVCDC0_PITCH_OUT = 0x0190 +NVCDC0_PITCH_OUT_VALUE = (31, 0) +NVCDC0_LAUNCH_DMA = 0x01b0 +NVCDC0_LAUNCH_DMA_DST_MEMORY_LAYOUT = (0, 0) +NVCDC0_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR = 0x00000000 +NVCDC0_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH = 0x00000001 +NVCDC0_LAUNCH_DMA_COMPLETION_TYPE = (5, 4) +NVCDC0_LAUNCH_DMA_COMPLETION_TYPE_FLUSH_DISABLE = 0x00000000 +NVCDC0_LAUNCH_DMA_COMPLETION_TYPE_FLUSH_ONLY = 0x00000001 +NVCDC0_LAUNCH_DMA_COMPLETION_TYPE_RELEASE_SEMAPHORE = 0x00000002 +NVCDC0_LAUNCH_DMA_INTERRUPT_TYPE = (9, 8) +NVCDC0_LAUNCH_DMA_INTERRUPT_TYPE_NONE = 0x00000000 +NVCDC0_LAUNCH_DMA_INTERRUPT_TYPE_INTERRUPT = 0x00000001 +NVCDC0_LAUNCH_DMA_SEMAPHORE_STRUCT_SIZE = (12, 12) +NVCDC0_LAUNCH_DMA_SEMAPHORE_STRUCT_SIZE_FOUR_WORDS = 0x00000000 +NVCDC0_LAUNCH_DMA_SEMAPHORE_STRUCT_SIZE_ONE_WORD = 0x00000001 +NVCDC0_LAUNCH_DMA_REDUCTION_ENABLE = (1, 1) +NVCDC0_LAUNCH_DMA_REDUCTION_ENABLE_FALSE = 0x00000000 +NVCDC0_LAUNCH_DMA_REDUCTION_ENABLE_TRUE = 0x00000001 +NVCDC0_LAUNCH_DMA_REDUCTION_OP = (15, 13) +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_ADD = 0x00000000 +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_MIN = 0x00000001 +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_MAX = 0x00000002 +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_INC = 0x00000003 +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_DEC = 0x00000004 +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_AND = 0x00000005 +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_OR = 0x00000006 +NVCDC0_LAUNCH_DMA_REDUCTION_OP_RED_XOR = 0x00000007 +NVCDC0_LAUNCH_DMA_REDUCTION_FORMAT = (3, 2) +NVCDC0_LAUNCH_DMA_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVCDC0_LAUNCH_DMA_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVCDC0_LAUNCH_DMA_SYSMEMBAR_DISABLE = (6, 6) +NVCDC0_LAUNCH_DMA_SYSMEMBAR_DISABLE_FALSE = 0x00000000 +NVCDC0_LAUNCH_DMA_SYSMEMBAR_DISABLE_TRUE = 0x00000001 +NVCDC0_LOAD_INLINE_DATA = 0x01b4 +NVCDC0_LOAD_INLINE_DATA_V = (31, 0) +NVCDC0_SET_I2M_SEMAPHORE_A = 0x01dc +NVCDC0_SET_I2M_SEMAPHORE_A_OFFSET_UPPER = (24, 0) +NVCDC0_SET_I2M_SEMAPHORE_B = 0x01e0 +NVCDC0_SET_I2M_SEMAPHORE_B_OFFSET_LOWER = (31, 0) +NVCDC0_SET_I2M_SEMAPHORE_C = 0x01e4 +NVCDC0_SET_I2M_SEMAPHORE_C_PAYLOAD = (31, 0) +NVCDC0_SET_SM_SCG_CONTROL = 0x01e8 +NVCDC0_SET_SM_SCG_CONTROL_COMPUTE_IN_GRAPHICS = (0, 0) +NVCDC0_SET_SM_SCG_CONTROL_COMPUTE_IN_GRAPHICS_FALSE = 0x00000000 +NVCDC0_SET_SM_SCG_CONTROL_COMPUTE_IN_GRAPHICS_TRUE = 0x00000001 +NVCDC0_SET_MME_SWITCH_STATE = 0x01ec +NVCDC0_SET_MME_SWITCH_STATE_VALID = (0, 0) +NVCDC0_SET_MME_SWITCH_STATE_VALID_FALSE = 0x00000000 +NVCDC0_SET_MME_SWITCH_STATE_VALID_TRUE = 0x00000001 +NVCDC0_SET_MME_SWITCH_STATE_SAVE_MACRO = (11, 4) +NVCDC0_SET_MME_SWITCH_STATE_RESTORE_MACRO = (19, 12) +NVCDC0_SET_I2M_SPARE_NOOP00 = 0x01f0 +NVCDC0_SET_I2M_SPARE_NOOP00_V = (31, 0) +NVCDC0_SET_I2M_SPARE_NOOP01 = 0x01f4 +NVCDC0_SET_I2M_SPARE_NOOP01_V = (31, 0) +NVCDC0_SET_I2M_SPARE_NOOP02 = 0x01f8 +NVCDC0_SET_I2M_SPARE_NOOP02_V = (31, 0) +NVCDC0_SET_I2M_SPARE_NOOP03 = 0x01fc +NVCDC0_SET_I2M_SPARE_NOOP03_V = (31, 0) +NVCDC0_SET_VALID_SPAN_OVERFLOW_AREA_A = 0x0200 +NVCDC0_SET_VALID_SPAN_OVERFLOW_AREA_A_ADDRESS_UPPER = (7, 0) +NVCDC0_SET_VALID_SPAN_OVERFLOW_AREA_B = 0x0204 +NVCDC0_SET_VALID_SPAN_OVERFLOW_AREA_B_ADDRESS_LOWER = (31, 0) +NVCDC0_SET_VALID_SPAN_OVERFLOW_AREA_C = 0x0208 +NVCDC0_SET_VALID_SPAN_OVERFLOW_AREA_C_SIZE = (31, 0) +NVCDC0_PERFMON_TRANSFER = 0x0210 +NVCDC0_PERFMON_TRANSFER_V = (31, 0) +NVCDC0_INVALIDATE_SHADER_CACHES = 0x021c +NVCDC0_INVALIDATE_SHADER_CACHES_INSTRUCTION = (0, 0) +NVCDC0_INVALIDATE_SHADER_CACHES_INSTRUCTION_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_INSTRUCTION_TRUE = 0x00000001 +NVCDC0_INVALIDATE_SHADER_CACHES_DATA = (4, 4) +NVCDC0_INVALIDATE_SHADER_CACHES_DATA_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_DATA_TRUE = 0x00000001 +NVCDC0_INVALIDATE_SHADER_CACHES_CONSTANT = (12, 12) +NVCDC0_INVALIDATE_SHADER_CACHES_CONSTANT_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_CONSTANT_TRUE = 0x00000001 +NVCDC0_INVALIDATE_SHADER_CACHES_LOCKS = (1, 1) +NVCDC0_INVALIDATE_SHADER_CACHES_LOCKS_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_LOCKS_TRUE = 0x00000001 +NVCDC0_INVALIDATE_SHADER_CACHES_FLUSH_DATA = (2, 2) +NVCDC0_INVALIDATE_SHADER_CACHES_FLUSH_DATA_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_FLUSH_DATA_TRUE = 0x00000001 +NVCDC0_SET_RESERVED_SW_METHOD00 = 0x0220 +NVCDC0_SET_RESERVED_SW_METHOD00_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD01 = 0x0224 +NVCDC0_SET_RESERVED_SW_METHOD01_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD02 = 0x0228 +NVCDC0_SET_RESERVED_SW_METHOD02_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD03 = 0x022c +NVCDC0_SET_RESERVED_SW_METHOD03_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD04 = 0x0230 +NVCDC0_SET_RESERVED_SW_METHOD04_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD05 = 0x0234 +NVCDC0_SET_RESERVED_SW_METHOD05_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD06 = 0x0238 +NVCDC0_SET_RESERVED_SW_METHOD06_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD07 = 0x023c +NVCDC0_SET_RESERVED_SW_METHOD07_V = (31, 0) +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI = 0x0244 +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_LINES = (0, 0) +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_LINES_ALL = 0x00000000 +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_LINES_ONE = 0x00000001 +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI_TAG = (25, 4) +NVCDC0_SET_CWD_REF_COUNTER = 0x0248 +NVCDC0_SET_CWD_REF_COUNTER_SELECT = (5, 0) +NVCDC0_SET_CWD_REF_COUNTER_VALUE = (23, 8) +NVCDC0_SET_RESERVED_SW_METHOD08 = 0x024c +NVCDC0_SET_RESERVED_SW_METHOD08_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD09 = 0x0250 +NVCDC0_SET_RESERVED_SW_METHOD09_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD10 = 0x0254 +NVCDC0_SET_RESERVED_SW_METHOD10_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD11 = 0x0258 +NVCDC0_SET_RESERVED_SW_METHOD11_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD12 = 0x025c +NVCDC0_SET_RESERVED_SW_METHOD12_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD13 = 0x0260 +NVCDC0_SET_RESERVED_SW_METHOD13_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD14 = 0x0264 +NVCDC0_SET_RESERVED_SW_METHOD14_V = (31, 0) +NVCDC0_SET_RESERVED_SW_METHOD15 = 0x0268 +NVCDC0_SET_RESERVED_SW_METHOD15_V = (31, 0) +NVCDC0_SET_COMPUTE_CLASS_VERSION = 0x0280 +NVCDC0_SET_COMPUTE_CLASS_VERSION_CURRENT = (15, 0) +NVCDC0_SET_COMPUTE_CLASS_VERSION_OLDEST_SUPPORTED = (31, 16) +NVCDC0_CHECK_COMPUTE_CLASS_VERSION = 0x0284 +NVCDC0_CHECK_COMPUTE_CLASS_VERSION_CURRENT = (15, 0) +NVCDC0_CHECK_COMPUTE_CLASS_VERSION_OLDEST_SUPPORTED = (31, 16) +NVCDC0_SET_QMD_VERSION = 0x0288 +NVCDC0_SET_QMD_VERSION_CURRENT = (15, 0) +NVCDC0_SET_QMD_VERSION_OLDEST_SUPPORTED = (31, 16) +NVCDC0_INVALIDATE_SKED_CACHES = 0x0298 +NVCDC0_INVALIDATE_SKED_CACHES_FORCE_IGNORE_VEID = (0, 0) +NVCDC0_INVALIDATE_SKED_CACHES_FORCE_IGNORE_VEID_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SKED_CACHES_FORCE_IGNORE_VEID_TRUE = 0x00000001 +NVCDC0_SET_SHADER_SHARED_MEMORY_WINDOW_A = 0x02a0 +NVCDC0_SET_SHADER_SHARED_MEMORY_WINDOW_A_BASE_ADDRESS_UPPER = (16, 0) +NVCDC0_SET_SHADER_SHARED_MEMORY_WINDOW_B = 0x02a4 +NVCDC0_SET_SHADER_SHARED_MEMORY_WINDOW_B_BASE_ADDRESS = (31, 0) +NVCDC0_SET_CWD_SLOT_COUNT = 0x02b0 +NVCDC0_SET_CWD_SLOT_COUNT_V = (7, 0) +NVCDC0_SEND_PCAS_A = 0x02b4 +NVCDC0_SEND_PCAS_A_QMD_ADDRESS_SHIFTED8 = (31, 0) +NVCDC0_SEND_PCAS_B = 0x02b8 +NVCDC0_SEND_PCAS_B_FROM = (23, 0) +NVCDC0_SEND_PCAS_B_DELTA = (31, 24) +NVCDC0_SEND_SIGNALING_PCAS_B = 0x02bc +NVCDC0_SEND_SIGNALING_PCAS_B_INVALIDATE = (0, 0) +NVCDC0_SEND_SIGNALING_PCAS_B_INVALIDATE_FALSE = 0x00000000 +NVCDC0_SEND_SIGNALING_PCAS_B_INVALIDATE_TRUE = 0x00000001 +NVCDC0_SEND_SIGNALING_PCAS_B_SCHEDULE = (1, 1) +NVCDC0_SEND_SIGNALING_PCAS_B_SCHEDULE_FALSE = 0x00000000 +NVCDC0_SEND_SIGNALING_PCAS_B_SCHEDULE_TRUE = 0x00000001 +NVCDC0_SEND_SIGNALING_PCAS2_B = 0x02c0 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION = (3, 0) +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_NOP = 0x00000000 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE = 0x00000001 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_SCHEDULE = 0x00000002 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE_COPY_SCHEDULE = 0x00000003 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INCREMENT_PUT = 0x00000006 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_DECREMENT_DEPENDENCE = 0x00000007 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_PREFETCH = 0x00000008 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_PREFETCH_SCHEDULE = 0x00000009 +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE_PREFETCH_COPY_SCHEDULE = 0x0000000A +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INVALIDATE_PREFETCH_COPY_FORCE_REQUIRE_SCHEDULING = 0x0000000B +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INCREMENT_DEPENDENCE = 0x0000000C +NVCDC0_SEND_SIGNALING_PCAS2_B_PCAS_ACTION_INCREMENT_CWD_REF_COUNTER = 0x0000000D +NVCDC0_SEND_SIGNALING_PCAS2_B_SELECT = (13, 8) +NVCDC0_SEND_SIGNALING_PCAS2_B_OFFSET_MINUS_ONE = (23, 14) +NVCDC0_SET_SKED_CACHE_CONTROL = 0x02cc +NVCDC0_SET_SKED_CACHE_CONTROL_IGNORE_VEID = (0, 0) +NVCDC0_SET_SKED_CACHE_CONTROL_IGNORE_VEID_FALSE = 0x00000000 +NVCDC0_SET_SKED_CACHE_CONTROL_IGNORE_VEID_TRUE = 0x00000001 +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A = 0x02e4 +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A_SIZE_UPPER = (7, 0) +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_B = 0x02e8 +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_B_SIZE_LOWER = (31, 0) +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C = 0x02ec +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C_MAX_SM_COUNT = (8, 0) +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C_MAX_PLURAL_TPC_GPC_COUNT = (23, 16) +NVCDC0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C_MAX_SINGLETON_TPC_GPC_COUNT = (31, 24) +NVCDC0_SET_SPA_VERSION = 0x0310 +NVCDC0_SET_SPA_VERSION_MINOR = (7, 0) +NVCDC0_SET_SPA_VERSION_MAJOR = (15, 8) +NVCDC0_SET_INLINE_QMD_ADDRESS_A = 0x0318 +NVCDC0_SET_INLINE_QMD_ADDRESS_A_QMD_ADDRESS_SHIFTED8_UPPER = (8, 0) +NVCDC0_SET_INLINE_QMD_ADDRESS_A_INLINE_SIZE = (30, 30) +NVCDC0_SET_INLINE_QMD_ADDRESS_A_INLINE_SIZE_INLINE_256 = 0x00000000 +NVCDC0_SET_INLINE_QMD_ADDRESS_A_INLINE_SIZE_INLINE_384 = 0x00000001 +NVCDC0_SET_INLINE_QMD_ADDRESS_B = 0x031c +NVCDC0_SET_INLINE_QMD_ADDRESS_B_QMD_ADDRESS_SHIFTED8_LOWER = (31, 0) +NVCDC0_LOAD_INLINE_QMD_DATA = lambda i: (0x0320+(i)*4) # type: ignore +NVCDC0_LOAD_INLINE_QMD_DATA_V = (31, 0) +NVCDC0_SET_FALCON00 = 0x0500 +NVCDC0_SET_FALCON00_V = (31, 0) +NVCDC0_SET_FALCON01 = 0x0504 +NVCDC0_SET_FALCON01_V = (31, 0) +NVCDC0_SET_FALCON02 = 0x0508 +NVCDC0_SET_FALCON02_V = (31, 0) +NVCDC0_SET_FALCON03 = 0x050c +NVCDC0_SET_FALCON03_V = (31, 0) +NVCDC0_SET_FALCON04 = 0x0510 +NVCDC0_SET_FALCON04_V = (31, 0) +NVCDC0_SET_FALCON05 = 0x0514 +NVCDC0_SET_FALCON05_V = (31, 0) +NVCDC0_SET_FALCON06 = 0x0518 +NVCDC0_SET_FALCON06_V = (31, 0) +NVCDC0_SET_FALCON07 = 0x051c +NVCDC0_SET_FALCON07_V = (31, 0) +NVCDC0_SET_FALCON08 = 0x0520 +NVCDC0_SET_FALCON08_V = (31, 0) +NVCDC0_SET_FALCON09 = 0x0524 +NVCDC0_SET_FALCON09_V = (31, 0) +NVCDC0_SET_FALCON10 = 0x0528 +NVCDC0_SET_FALCON10_V = (31, 0) +NVCDC0_SET_FALCON11 = 0x052c +NVCDC0_SET_FALCON11_V = (31, 0) +NVCDC0_SET_FALCON12 = 0x0530 +NVCDC0_SET_FALCON12_V = (31, 0) +NVCDC0_SET_FALCON13 = 0x0534 +NVCDC0_SET_FALCON13_V = (31, 0) +NVCDC0_SET_FALCON14 = 0x0538 +NVCDC0_SET_FALCON14_V = (31, 0) +NVCDC0_SET_FALCON15 = 0x053c +NVCDC0_SET_FALCON15_V = (31, 0) +NVCDC0_SET_MME_MEM_ADDRESS_A = 0x0550 +NVCDC0_SET_MME_MEM_ADDRESS_A_UPPER = (24, 0) +NVCDC0_SET_MME_MEM_ADDRESS_B = 0x0554 +NVCDC0_SET_MME_MEM_ADDRESS_B_LOWER = (31, 0) +NVCDC0_SET_MME_DATA_RAM_ADDRESS = 0x0558 +NVCDC0_SET_MME_DATA_RAM_ADDRESS_WORD = (31, 0) +NVCDC0_MME_DMA_READ = 0x055c +NVCDC0_MME_DMA_READ_LENGTH = (31, 0) +NVCDC0_MME_DMA_READ_FIFOED = 0x0560 +NVCDC0_MME_DMA_READ_FIFOED_LENGTH = (31, 0) +NVCDC0_MME_DMA_WRITE = 0x0564 +NVCDC0_MME_DMA_WRITE_LENGTH = (31, 0) +NVCDC0_MME_DMA_REDUCTION = 0x0568 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP = (2, 0) +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_ADD = 0x00000000 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_MIN = 0x00000001 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_MAX = 0x00000002 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_INC = 0x00000003 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_DEC = 0x00000004 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_AND = 0x00000005 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_OR = 0x00000006 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_OP_RED_XOR = 0x00000007 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_FORMAT = (5, 4) +NVCDC0_MME_DMA_REDUCTION_REDUCTION_FORMAT_UNSIGNED = 0x00000000 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_FORMAT_SIGNED = 0x00000001 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_SIZE = (8, 8) +NVCDC0_MME_DMA_REDUCTION_REDUCTION_SIZE_FOUR_BYTES = 0x00000000 +NVCDC0_MME_DMA_REDUCTION_REDUCTION_SIZE_EIGHT_BYTES = 0x00000001 +NVCDC0_MME_DMA_SYSMEMBAR = 0x056c +NVCDC0_MME_DMA_SYSMEMBAR_V = (0, 0) +NVCDC0_MME_DMA_SYNC = 0x0570 +NVCDC0_MME_DMA_SYNC_VALUE = (31, 0) +NVCDC0_EVENT_START_TRIGGER = 0x074c +NVCDC0_EVENT_START_TRIGGER_V = (31, 0) +NVCDC0_EVENT_STOP_TRIGGER = 0x075c +NVCDC0_EVENT_STOP_TRIGGER_V = (31, 0) +NVCDC0_SET_SHADER_LOCAL_MEMORY_A = 0x0790 +NVCDC0_SET_SHADER_LOCAL_MEMORY_A_ADDRESS_UPPER = (24, 0) +NVCDC0_SET_SHADER_LOCAL_MEMORY_B = 0x0794 +NVCDC0_SET_SHADER_LOCAL_MEMORY_B_ADDRESS_LOWER = (31, 0) +NVCDC0_SET_TEXTURE_HEADER_VERSION = 0x07ac +NVCDC0_SET_TEXTURE_HEADER_VERSION_MAJOR = (7, 0) +NVCDC0_SET_SHADER_LOCAL_MEMORY_WINDOW_A = 0x07b0 +NVCDC0_SET_SHADER_LOCAL_MEMORY_WINDOW_A_BASE_ADDRESS_UPPER = (16, 0) +NVCDC0_SET_SHADER_LOCAL_MEMORY_WINDOW_B = 0x07b4 +NVCDC0_SET_SHADER_LOCAL_MEMORY_WINDOW_B_BASE_ADDRESS = (31, 0) +NVCDC0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_A = 0x07b8 +NVCDC0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_A_LOWER = (31, 0) +NVCDC0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_B = 0x07bc +NVCDC0_SET_MONITORED_FENCE_SIGNAL_ADDRESS_BASE_B_UPPER = (24, 0) +NVCDC0_THROTTLE_SM = 0x07fc +NVCDC0_THROTTLE_SM_MULTIPLY_ADD = (0, 0) +NVCDC0_THROTTLE_SM_MULTIPLY_ADD_FALSE = 0x00000000 +NVCDC0_THROTTLE_SM_MULTIPLY_ADD_TRUE = 0x00000001 +NVCDC0_SET_SHADER_CACHE_CONTROL = 0x0d94 +NVCDC0_SET_SHADER_CACHE_CONTROL_ICACHE_PREFETCH_ENABLE = (0, 0) +NVCDC0_SET_SHADER_CACHE_CONTROL_ICACHE_PREFETCH_ENABLE_FALSE = 0x00000000 +NVCDC0_SET_SHADER_CACHE_CONTROL_ICACHE_PREFETCH_ENABLE_TRUE = 0x00000001 +NVCDC0_SET_SCG_COMPUTE_SCHEDULING_PARAMETERS = lambda i: (0x0da0+(i)*4) # type: ignore +NVCDC0_SET_SCG_COMPUTE_SCHEDULING_PARAMETERS_V = (31, 0) +NVCDC0_SET_SM_TIMEOUT_INTERVAL = 0x0de4 +NVCDC0_SET_SM_TIMEOUT_INTERVAL_COUNTER_BIT = (5, 0) +NVCDC0_MME_DMA_WRITE_METHOD_BARRIER = 0x0dec +NVCDC0_MME_DMA_WRITE_METHOD_BARRIER_V = (0, 0) +NVCDC0_EVENT_STOP_TRIGGER_WFI = 0x0ffc +NVCDC0_EVENT_STOP_TRIGGER_WFI_V = (31, 0) +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI = 0x1288 +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_LINES = (0, 0) +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_LINES_ALL = 0x00000000 +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_LINES_ONE = 0x00000001 +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_NO_WFI_TAG = (25, 4) +NVCDC0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT = 0x12a8 +NVCDC0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT_ALL = (0, 0) +NVCDC0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT_ALL_FALSE = 0x00000000 +NVCDC0_ACTIVATE_PERF_SETTINGS_FOR_COMPUTE_CONTEXT_ALL_TRUE = 0x00000001 +NVCDC0_INVALIDATE_SAMPLER_CACHE = 0x1330 +NVCDC0_INVALIDATE_SAMPLER_CACHE_LINES = (0, 0) +NVCDC0_INVALIDATE_SAMPLER_CACHE_LINES_ALL = 0x00000000 +NVCDC0_INVALIDATE_SAMPLER_CACHE_LINES_ONE = 0x00000001 +NVCDC0_INVALIDATE_SAMPLER_CACHE_TAG = (25, 4) +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE = 0x1334 +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_LINES = (0, 0) +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_LINES_ALL = 0x00000000 +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_LINES_ONE = 0x00000001 +NVCDC0_INVALIDATE_TEXTURE_HEADER_CACHE_TAG = (25, 4) +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE = 0x1338 +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_LINES = (0, 0) +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_LINES_ALL = 0x00000000 +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_LINES_ONE = 0x00000001 +NVCDC0_INVALIDATE_TEXTURE_DATA_CACHE_TAG = (25, 4) +NVCDC0_INVALIDATE_SAMPLER_CACHE_NO_WFI = 0x1424 +NVCDC0_INVALIDATE_SAMPLER_CACHE_NO_WFI_LINES = (0, 0) +NVCDC0_INVALIDATE_SAMPLER_CACHE_NO_WFI_LINES_ALL = 0x00000000 +NVCDC0_INVALIDATE_SAMPLER_CACHE_NO_WFI_LINES_ONE = 0x00000001 +NVCDC0_INVALIDATE_SAMPLER_CACHE_NO_WFI_TAG = (25, 4) +NVCDC0_SET_SHADER_EXCEPTIONS = 0x1528 +NVCDC0_SET_SHADER_EXCEPTIONS_ENABLE = (0, 0) +NVCDC0_SET_SHADER_EXCEPTIONS_ENABLE_FALSE = 0x00000000 +NVCDC0_SET_SHADER_EXCEPTIONS_ENABLE_TRUE = 0x00000001 +NVCDC0_SET_RENDER_ENABLE_A = 0x1550 +NVCDC0_SET_RENDER_ENABLE_A_OFFSET_UPPER = (7, 0) +NVCDC0_SET_RENDER_ENABLE_B = 0x1554 +NVCDC0_SET_RENDER_ENABLE_B_OFFSET_LOWER = (31, 0) +NVCDC0_SET_RENDER_ENABLE_C = 0x1558 +NVCDC0_SET_RENDER_ENABLE_C_MODE = (2, 0) +NVCDC0_SET_RENDER_ENABLE_C_MODE_FALSE = 0x00000000 +NVCDC0_SET_RENDER_ENABLE_C_MODE_TRUE = 0x00000001 +NVCDC0_SET_RENDER_ENABLE_C_MODE_CONDITIONAL = 0x00000002 +NVCDC0_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL = 0x00000003 +NVCDC0_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL = 0x00000004 +NVCDC0_SET_TEX_SAMPLER_POOL_A = 0x155c +NVCDC0_SET_TEX_SAMPLER_POOL_A_OFFSET_UPPER = (24, 0) +NVCDC0_SET_TEX_SAMPLER_POOL_B = 0x1560 +NVCDC0_SET_TEX_SAMPLER_POOL_B_OFFSET_LOWER = (31, 0) +NVCDC0_SET_TEX_SAMPLER_POOL_C = 0x1564 +NVCDC0_SET_TEX_SAMPLER_POOL_C_MAXIMUM_INDEX = (31, 0) +NVCDC0_SET_TEX_HEADER_POOL_A = 0x1574 +NVCDC0_SET_TEX_HEADER_POOL_A_OFFSET_UPPER = (24, 0) +NVCDC0_SET_TEX_HEADER_POOL_B = 0x1578 +NVCDC0_SET_TEX_HEADER_POOL_B_OFFSET_LOWER = (31, 0) +NVCDC0_SET_TEX_HEADER_POOL_C = 0x157c +NVCDC0_SET_TEX_HEADER_POOL_C_MAXIMUM_INDEX = (31, 0) +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI = 0x1698 +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_INSTRUCTION = (0, 0) +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_INSTRUCTION_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_INSTRUCTION_TRUE = 0x00000001 +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_GLOBAL_DATA = (4, 4) +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_GLOBAL_DATA_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_GLOBAL_DATA_TRUE = 0x00000001 +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_CONSTANT = (12, 12) +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_CONSTANT_FALSE = 0x00000000 +NVCDC0_INVALIDATE_SHADER_CACHES_NO_WFI_CONSTANT_TRUE = 0x00000001 +NVCDC0_SET_RENDER_ENABLE_OVERRIDE = 0x1944 +NVCDC0_SET_RENDER_ENABLE_OVERRIDE_MODE = (1, 0) +NVCDC0_SET_RENDER_ENABLE_OVERRIDE_MODE_USE_RENDER_ENABLE = 0x00000000 +NVCDC0_SET_RENDER_ENABLE_OVERRIDE_MODE_ALWAYS_RENDER = 0x00000001 +NVCDC0_SET_RENDER_ENABLE_OVERRIDE_MODE_NEVER_RENDER = 0x00000002 +NVCDC0_PIPE_NOP = 0x1a2c +NVCDC0_PIPE_NOP_V = (31, 0) +NVCDC0_SET_SPARE00 = 0x1a30 +NVCDC0_SET_SPARE00_V = (31, 0) +NVCDC0_SET_SPARE01 = 0x1a34 +NVCDC0_SET_SPARE01_V = (31, 0) +NVCDC0_SET_SPARE02 = 0x1a38 +NVCDC0_SET_SPARE02_V = (31, 0) +NVCDC0_SET_SPARE03 = 0x1a3c +NVCDC0_SET_SPARE03_V = (31, 0) +NVCDC0_SET_REPORT_SEMAPHORE_A = 0x1b00 +NVCDC0_SET_REPORT_SEMAPHORE_A_OFFSET_UPPER = (24, 0) +NVCDC0_SET_REPORT_SEMAPHORE_B = 0x1b04 +NVCDC0_SET_REPORT_SEMAPHORE_B_OFFSET_LOWER = (31, 0) +NVCDC0_SET_REPORT_SEMAPHORE_C = 0x1b08 +NVCDC0_SET_REPORT_SEMAPHORE_C_PAYLOAD = (31, 0) +NVCDC0_SET_REPORT_SEMAPHORE_D = 0x1b0c +NVCDC0_SET_REPORT_SEMAPHORE_D_OPERATION = (1, 0) +NVCDC0_SET_REPORT_SEMAPHORE_D_OPERATION_RELEASE = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_OPERATION_TRAP = 0x00000003 +NVCDC0_SET_REPORT_SEMAPHORE_D_AWAKEN_ENABLE = (20, 20) +NVCDC0_SET_REPORT_SEMAPHORE_D_AWAKEN_ENABLE_FALSE = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_AWAKEN_ENABLE_TRUE = 0x00000001 +NVCDC0_SET_REPORT_SEMAPHORE_D_STRUCTURE_SIZE = (28, 28) +NVCDC0_SET_REPORT_SEMAPHORE_D_STRUCTURE_SIZE_FOUR_WORDS = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_STRUCTURE_SIZE_ONE_WORD = 0x00000001 +NVCDC0_SET_REPORT_SEMAPHORE_D_FLUSH_DISABLE = (2, 2) +NVCDC0_SET_REPORT_SEMAPHORE_D_FLUSH_DISABLE_FALSE = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_FLUSH_DISABLE_TRUE = 0x00000001 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_ENABLE = (3, 3) +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_ENABLE_FALSE = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_ENABLE_TRUE = 0x00000001 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP = (11, 9) +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_ADD = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_MIN = 0x00000001 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_MAX = 0x00000002 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_INC = 0x00000003 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_DEC = 0x00000004 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_AND = 0x00000005 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_OR = 0x00000006 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_OP_RED_XOR = 0x00000007 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_FORMAT = (18, 17) +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_FORMAT_UNSIGNED_32 = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_REDUCTION_FORMAT_SIGNED_32 = 0x00000001 +NVCDC0_SET_REPORT_SEMAPHORE_D_CONDITIONAL_TRAP = (19, 19) +NVCDC0_SET_REPORT_SEMAPHORE_D_CONDITIONAL_TRAP_FALSE = 0x00000000 +NVCDC0_SET_REPORT_SEMAPHORE_D_CONDITIONAL_TRAP_TRUE = 0x00000001 +NVCDC0_SET_TRAP_HANDLER_A = 0x25f8 +NVCDC0_SET_TRAP_HANDLER_A_ADDRESS_UPPER = (24, 0) +NVCDC0_SET_TRAP_HANDLER_B = 0x25fc +NVCDC0_SET_TRAP_HANDLER_B_ADDRESS_LOWER = (31, 0) +NVCDC0_SET_BINDLESS_TEXTURE = 0x2608 +NVCDC0_SET_BINDLESS_TEXTURE_CONSTANT_BUFFER_SLOT_SELECT = (2, 0) +NVCDC0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE = lambda i: (0x32f4+(i)*4) # type: ignore +NVCDC0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE_V = (31, 0) +NVCDC0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE_UPPER = lambda i: (0x3314+(i)*4) # type: ignore +NVCDC0_SET_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_VALUE_UPPER_V = (31, 0) +NVCDC0_ENABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER = 0x3334 +NVCDC0_ENABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_V = (0, 0) +NVCDC0_DISABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER = 0x3338 +NVCDC0_DISABLE_SHADER_PERFORMANCE_SNAPSHOT_COUNTER_V = (0, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_VALUE_UPPER = lambda i: (0x333c+(i)*4) # type: ignore +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_VALUE_UPPER_V = (31, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_VALUE = lambda i: (0x335c+(i)*4) # type: ignore +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_VALUE_V = (31, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_EVENT = lambda i: (0x337c+(i)*4) # type: ignore +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_EVENT_EVENT = (7, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A = lambda i: (0x339c+(i)*4) # type: ignore +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT0 = (1, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT0 = (4, 2) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT1 = (6, 5) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT1 = (9, 7) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT2 = (11, 10) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT2 = (14, 12) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT3 = (16, 15) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT3 = (19, 17) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT4 = (21, 20) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT4 = (24, 22) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_EVENT5 = (26, 25) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_BIT_SELECT5 = (29, 27) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_A_SPARE = (31, 30) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B = lambda i: (0x33bc+(i)*4) # type: ignore +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_EDGE = (0, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_MODE = (2, 1) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_WINDOWED = (3, 3) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CONTROL_B_FUNC = (19, 4) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_TRAP_CONTROL = 0x33dc +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_TRAP_CONTROL_MASK = (7, 0) +NVCDC0_START_SHADER_PERFORMANCE_COUNTER = 0x33e0 +NVCDC0_START_SHADER_PERFORMANCE_COUNTER_COUNTER_MASK = (7, 0) +NVCDC0_STOP_SHADER_PERFORMANCE_COUNTER = 0x33e4 +NVCDC0_STOP_SHADER_PERFORMANCE_COUNTER_COUNTER_MASK = (7, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_SCTL_FILTER = 0x33e8 +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_SCTL_FILTER_V = (31, 0) +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CORE_MIO_FILTER = 0x33ec +NVCDC0_SET_SHADER_PERFORMANCE_COUNTER_CORE_MIO_FILTER_V = (31, 0) +NVCDC0_SET_MME_SHADOW_SCRATCH = lambda i: (0x3400+(i)*4) # type: ignore +NVCDC0_SET_MME_SHADOW_SCRATCH_V = (31, 0) +NVCDC0_CALL_MME_MACRO = lambda j: (0x3800+(j)*8) # type: ignore +NVCDC0_CALL_MME_MACRO_V = (31, 0) +NVCDC0_CALL_MME_DATA = lambda j: (0x3804+(j)*8) # type: ignore +NVCDC0_CALL_MME_DATA_V = (31, 0) +AMPERE_DMA_COPY_A = (0x0000C6B5) +NVC6B5_NOP = (0x00000100) +NVC6B5_NOP_PARAMETER = (31, 0) +NVC6B5_PM_TRIGGER = (0x00000140) +NVC6B5_PM_TRIGGER_V = (31, 0) +NVC6B5_SET_SEMAPHORE_A = (0x00000240) +NVC6B5_SET_SEMAPHORE_A_UPPER = (16, 0) +NVC6B5_SET_SEMAPHORE_B = (0x00000244) +NVC6B5_SET_SEMAPHORE_B_LOWER = (31, 0) +NVC6B5_SET_SEMAPHORE_PAYLOAD = (0x00000248) +NVC6B5_SET_SEMAPHORE_PAYLOAD_PAYLOAD = (31, 0) +NVC6B5_SET_RENDER_ENABLE_A = (0x00000254) +NVC6B5_SET_RENDER_ENABLE_A_UPPER = (7, 0) +NVC6B5_SET_RENDER_ENABLE_B = (0x00000258) +NVC6B5_SET_RENDER_ENABLE_B_LOWER = (31, 0) +NVC6B5_SET_RENDER_ENABLE_C = (0x0000025C) +NVC6B5_SET_RENDER_ENABLE_C_MODE = (2, 0) +NVC6B5_SET_RENDER_ENABLE_C_MODE_FALSE = (0x00000000) +NVC6B5_SET_RENDER_ENABLE_C_MODE_TRUE = (0x00000001) +NVC6B5_SET_RENDER_ENABLE_C_MODE_CONDITIONAL = (0x00000002) +NVC6B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL = (0x00000003) +NVC6B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL = (0x00000004) +NVC6B5_SET_SRC_PHYS_MODE = (0x00000260) +NVC6B5_SET_SRC_PHYS_MODE_TARGET = (1, 0) +NVC6B5_SET_SRC_PHYS_MODE_TARGET_LOCAL_FB = (0x00000000) +NVC6B5_SET_SRC_PHYS_MODE_TARGET_COHERENT_SYSMEM = (0x00000001) +NVC6B5_SET_SRC_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM = (0x00000002) +NVC6B5_SET_SRC_PHYS_MODE_TARGET_PEERMEM = (0x00000003) +NVC6B5_SET_SRC_PHYS_MODE_BASIC_KIND = (5, 2) +NVC6B5_SET_SRC_PHYS_MODE_PEER_ID = (8, 6) +NVC6B5_SET_SRC_PHYS_MODE_FLA = (9, 9) +NVC6B5_SET_DST_PHYS_MODE = (0x00000264) +NVC6B5_SET_DST_PHYS_MODE_TARGET = (1, 0) +NVC6B5_SET_DST_PHYS_MODE_TARGET_LOCAL_FB = (0x00000000) +NVC6B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM = (0x00000001) +NVC6B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM = (0x00000002) +NVC6B5_SET_DST_PHYS_MODE_TARGET_PEERMEM = (0x00000003) +NVC6B5_SET_DST_PHYS_MODE_BASIC_KIND = (5, 2) +NVC6B5_SET_DST_PHYS_MODE_PEER_ID = (8, 6) +NVC6B5_SET_DST_PHYS_MODE_FLA = (9, 9) +NVC6B5_LAUNCH_DMA = (0x00000300) +NVC6B5_LAUNCH_DMA_DATA_TRANSFER_TYPE = (1, 0) +NVC6B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE = (0x00000000) +NVC6B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED = (0x00000001) +NVC6B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED = (0x00000002) +NVC6B5_LAUNCH_DMA_FLUSH_ENABLE = (2, 2) +NVC6B5_LAUNCH_DMA_FLUSH_ENABLE_FALSE = (0x00000000) +NVC6B5_LAUNCH_DMA_FLUSH_ENABLE_TRUE = (0x00000001) +NVC6B5_LAUNCH_DMA_FLUSH_TYPE = (25, 25) +NVC6B5_LAUNCH_DMA_FLUSH_TYPE_SYS = (0x00000000) +NVC6B5_LAUNCH_DMA_FLUSH_TYPE_GL = (0x00000001) +NVC6B5_LAUNCH_DMA_SEMAPHORE_TYPE = (4, 3) +NVC6B5_LAUNCH_DMA_SEMAPHORE_TYPE_NONE = (0x00000000) +NVC6B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_ONE_WORD_SEMAPHORE = (0x00000001) +NVC6B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_FOUR_WORD_SEMAPHORE = (0x00000002) +NVC6B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_CONDITIONAL_INTR_SEMAPHORE = (0x00000003) +NVC6B5_LAUNCH_DMA_INTERRUPT_TYPE = (6, 5) +NVC6B5_LAUNCH_DMA_INTERRUPT_TYPE_NONE = (0x00000000) +NVC6B5_LAUNCH_DMA_INTERRUPT_TYPE_BLOCKING = (0x00000001) +NVC6B5_LAUNCH_DMA_INTERRUPT_TYPE_NON_BLOCKING = (0x00000002) +NVC6B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT = (7, 7) +NVC6B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_BLOCKLINEAR = (0x00000000) +NVC6B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_PITCH = (0x00000001) +NVC6B5_LAUNCH_DMA_DST_MEMORY_LAYOUT = (8, 8) +NVC6B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR = (0x00000000) +NVC6B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH = (0x00000001) +NVC6B5_LAUNCH_DMA_MULTI_LINE_ENABLE = (9, 9) +NVC6B5_LAUNCH_DMA_MULTI_LINE_ENABLE_FALSE = (0x00000000) +NVC6B5_LAUNCH_DMA_MULTI_LINE_ENABLE_TRUE = (0x00000001) +NVC6B5_LAUNCH_DMA_REMAP_ENABLE = (10, 10) +NVC6B5_LAUNCH_DMA_REMAP_ENABLE_FALSE = (0x00000000) +NVC6B5_LAUNCH_DMA_REMAP_ENABLE_TRUE = (0x00000001) +NVC6B5_LAUNCH_DMA_FORCE_RMWDISABLE = (11, 11) +NVC6B5_LAUNCH_DMA_FORCE_RMWDISABLE_FALSE = (0x00000000) +NVC6B5_LAUNCH_DMA_FORCE_RMWDISABLE_TRUE = (0x00000001) +NVC6B5_LAUNCH_DMA_SRC_TYPE = (12, 12) +NVC6B5_LAUNCH_DMA_SRC_TYPE_VIRTUAL = (0x00000000) +NVC6B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL = (0x00000001) +NVC6B5_LAUNCH_DMA_DST_TYPE = (13, 13) +NVC6B5_LAUNCH_DMA_DST_TYPE_VIRTUAL = (0x00000000) +NVC6B5_LAUNCH_DMA_DST_TYPE_PHYSICAL = (0x00000001) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION = (17, 14) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMIN = (0x00000000) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMAX = (0x00000001) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IXOR = (0x00000002) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IAND = (0x00000003) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IOR = (0x00000004) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IADD = (0x00000005) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INC = (0x00000006) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_DEC = (0x00000007) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FADD = (0x0000000A) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN = (18, 18) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_SIGNED = (0x00000000) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_UNSIGNED = (0x00000001) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE = (19, 19) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_FALSE = (0x00000000) +NVC6B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_TRUE = (0x00000001) +NVC6B5_LAUNCH_DMA_VPRMODE = (23, 22) +NVC6B5_LAUNCH_DMA_VPRMODE_VPR_NONE = (0x00000000) +NVC6B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID = (0x00000001) +NVC6B5_LAUNCH_DMA_RESERVED_START_OF_COPY = (24, 24) +NVC6B5_LAUNCH_DMA_DISABLE_PLC = (26, 26) +NVC6B5_LAUNCH_DMA_DISABLE_PLC_FALSE = (0x00000000) +NVC6B5_LAUNCH_DMA_DISABLE_PLC_TRUE = (0x00000001) +NVC6B5_LAUNCH_DMA_RESERVED_ERR_CODE = (31, 28) +NVC6B5_OFFSET_IN_UPPER = (0x00000400) +NVC6B5_OFFSET_IN_UPPER_UPPER = (16, 0) +NVC6B5_OFFSET_IN_LOWER = (0x00000404) +NVC6B5_OFFSET_IN_LOWER_VALUE = (31, 0) +NVC6B5_OFFSET_OUT_UPPER = (0x00000408) +NVC6B5_OFFSET_OUT_UPPER_UPPER = (16, 0) +NVC6B5_OFFSET_OUT_LOWER = (0x0000040C) +NVC6B5_OFFSET_OUT_LOWER_VALUE = (31, 0) +NVC6B5_PITCH_IN = (0x00000410) +NVC6B5_PITCH_IN_VALUE = (31, 0) +NVC6B5_PITCH_OUT = (0x00000414) +NVC6B5_PITCH_OUT_VALUE = (31, 0) +NVC6B5_LINE_LENGTH_IN = (0x00000418) +NVC6B5_LINE_LENGTH_IN_VALUE = (31, 0) +NVC6B5_LINE_COUNT = (0x0000041C) +NVC6B5_LINE_COUNT_VALUE = (31, 0) +NVC6B5_SET_REMAP_CONST_A = (0x00000700) +NVC6B5_SET_REMAP_CONST_A_V = (31, 0) +NVC6B5_SET_REMAP_CONST_B = (0x00000704) +NVC6B5_SET_REMAP_CONST_B_V = (31, 0) +NVC6B5_SET_REMAP_COMPONENTS = (0x00000708) +NVC6B5_SET_REMAP_COMPONENTS_DST_X = (2, 0) +NVC6B5_SET_REMAP_COMPONENTS_DST_X_SRC_X = (0x00000000) +NVC6B5_SET_REMAP_COMPONENTS_DST_X_SRC_Y = (0x00000001) +NVC6B5_SET_REMAP_COMPONENTS_DST_X_SRC_Z = (0x00000002) +NVC6B5_SET_REMAP_COMPONENTS_DST_X_SRC_W = (0x00000003) +NVC6B5_SET_REMAP_COMPONENTS_DST_X_CONST_A = (0x00000004) +NVC6B5_SET_REMAP_COMPONENTS_DST_X_CONST_B = (0x00000005) +NVC6B5_SET_REMAP_COMPONENTS_DST_X_NO_WRITE = (0x00000006) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y = (6, 4) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y_SRC_X = (0x00000000) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Y = (0x00000001) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Z = (0x00000002) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y_SRC_W = (0x00000003) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y_CONST_A = (0x00000004) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y_CONST_B = (0x00000005) +NVC6B5_SET_REMAP_COMPONENTS_DST_Y_NO_WRITE = (0x00000006) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z = (10, 8) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z_SRC_X = (0x00000000) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Y = (0x00000001) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Z = (0x00000002) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z_SRC_W = (0x00000003) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z_CONST_A = (0x00000004) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z_CONST_B = (0x00000005) +NVC6B5_SET_REMAP_COMPONENTS_DST_Z_NO_WRITE = (0x00000006) +NVC6B5_SET_REMAP_COMPONENTS_DST_W = (14, 12) +NVC6B5_SET_REMAP_COMPONENTS_DST_W_SRC_X = (0x00000000) +NVC6B5_SET_REMAP_COMPONENTS_DST_W_SRC_Y = (0x00000001) +NVC6B5_SET_REMAP_COMPONENTS_DST_W_SRC_Z = (0x00000002) +NVC6B5_SET_REMAP_COMPONENTS_DST_W_SRC_W = (0x00000003) +NVC6B5_SET_REMAP_COMPONENTS_DST_W_CONST_A = (0x00000004) +NVC6B5_SET_REMAP_COMPONENTS_DST_W_CONST_B = (0x00000005) +NVC6B5_SET_REMAP_COMPONENTS_DST_W_NO_WRITE = (0x00000006) +NVC6B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE = (17, 16) +NVC6B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_ONE = (0x00000000) +NVC6B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_TWO = (0x00000001) +NVC6B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_THREE = (0x00000002) +NVC6B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_FOUR = (0x00000003) +NVC6B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS = (21, 20) +NVC6B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_ONE = (0x00000000) +NVC6B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_TWO = (0x00000001) +NVC6B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_THREE = (0x00000002) +NVC6B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_FOUR = (0x00000003) +NVC6B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS = (25, 24) +NVC6B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_ONE = (0x00000000) +NVC6B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_TWO = (0x00000001) +NVC6B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_THREE = (0x00000002) +NVC6B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_FOUR = (0x00000003) +NVC6B5_SET_DST_BLOCK_SIZE = (0x0000070C) +NVC6B5_SET_DST_BLOCK_SIZE_WIDTH = (3, 0) +NVC6B5_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB = (0x00000000) +NVC6B5_SET_DST_BLOCK_SIZE_HEIGHT = (7, 4) +NVC6B5_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB = (0x00000000) +NVC6B5_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS = (0x00000001) +NVC6B5_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS = (0x00000002) +NVC6B5_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS = (0x00000003) +NVC6B5_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS = (0x00000004) +NVC6B5_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS = (0x00000005) +NVC6B5_SET_DST_BLOCK_SIZE_DEPTH = (11, 8) +NVC6B5_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB = (0x00000000) +NVC6B5_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS = (0x00000001) +NVC6B5_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS = (0x00000002) +NVC6B5_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS = (0x00000003) +NVC6B5_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS = (0x00000004) +NVC6B5_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS = (0x00000005) +NVC6B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT = (15, 12) +NVC6B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 = (0x00000001) +NVC6B5_SET_DST_WIDTH = (0x00000710) +NVC6B5_SET_DST_WIDTH_V = (31, 0) +NVC6B5_SET_DST_HEIGHT = (0x00000714) +NVC6B5_SET_DST_HEIGHT_V = (31, 0) +NVC6B5_SET_DST_DEPTH = (0x00000718) +NVC6B5_SET_DST_DEPTH_V = (31, 0) +NVC6B5_SET_DST_LAYER = (0x0000071C) +NVC6B5_SET_DST_LAYER_V = (31, 0) +NVC6B5_SET_DST_ORIGIN = (0x00000720) +NVC6B5_SET_DST_ORIGIN_X = (15, 0) +NVC6B5_SET_DST_ORIGIN_Y = (31, 16) +NVC6B5_SET_SRC_BLOCK_SIZE = (0x00000728) +NVC6B5_SET_SRC_BLOCK_SIZE_WIDTH = (3, 0) +NVC6B5_SET_SRC_BLOCK_SIZE_WIDTH_ONE_GOB = (0x00000000) +NVC6B5_SET_SRC_BLOCK_SIZE_HEIGHT = (7, 4) +NVC6B5_SET_SRC_BLOCK_SIZE_HEIGHT_ONE_GOB = (0x00000000) +NVC6B5_SET_SRC_BLOCK_SIZE_HEIGHT_TWO_GOBS = (0x00000001) +NVC6B5_SET_SRC_BLOCK_SIZE_HEIGHT_FOUR_GOBS = (0x00000002) +NVC6B5_SET_SRC_BLOCK_SIZE_HEIGHT_EIGHT_GOBS = (0x00000003) +NVC6B5_SET_SRC_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS = (0x00000004) +NVC6B5_SET_SRC_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS = (0x00000005) +NVC6B5_SET_SRC_BLOCK_SIZE_DEPTH = (11, 8) +NVC6B5_SET_SRC_BLOCK_SIZE_DEPTH_ONE_GOB = (0x00000000) +NVC6B5_SET_SRC_BLOCK_SIZE_DEPTH_TWO_GOBS = (0x00000001) +NVC6B5_SET_SRC_BLOCK_SIZE_DEPTH_FOUR_GOBS = (0x00000002) +NVC6B5_SET_SRC_BLOCK_SIZE_DEPTH_EIGHT_GOBS = (0x00000003) +NVC6B5_SET_SRC_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS = (0x00000004) +NVC6B5_SET_SRC_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS = (0x00000005) +NVC6B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT = (15, 12) +NVC6B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 = (0x00000001) +NVC6B5_SET_SRC_WIDTH = (0x0000072C) +NVC6B5_SET_SRC_WIDTH_V = (31, 0) +NVC6B5_SET_SRC_HEIGHT = (0x00000730) +NVC6B5_SET_SRC_HEIGHT_V = (31, 0) +NVC6B5_SET_SRC_DEPTH = (0x00000734) +NVC6B5_SET_SRC_DEPTH_V = (31, 0) +NVC6B5_SET_SRC_LAYER = (0x00000738) +NVC6B5_SET_SRC_LAYER_V = (31, 0) +NVC6B5_SET_SRC_ORIGIN = (0x0000073C) +NVC6B5_SET_SRC_ORIGIN_X = (15, 0) +NVC6B5_SET_SRC_ORIGIN_Y = (31, 16) +NVC6B5_SRC_ORIGIN_X = (0x00000744) +NVC6B5_SRC_ORIGIN_X_VALUE = (31, 0) +NVC6B5_SRC_ORIGIN_Y = (0x00000748) +NVC6B5_SRC_ORIGIN_Y_VALUE = (31, 0) +NVC6B5_DST_ORIGIN_X = (0x0000074C) +NVC6B5_DST_ORIGIN_X_VALUE = (31, 0) +NVC6B5_DST_ORIGIN_Y = (0x00000750) +NVC6B5_DST_ORIGIN_Y_VALUE = (31, 0) +NVC6B5_PM_TRIGGER_END = (0x00001114) +NVC6B5_PM_TRIGGER_END_V = (31, 0) +BLACKWELL_DMA_COPY_A = (0x0000C9B5) +UVM_IOCTL_BASE = lambda i: i # type: ignore +UVM_RESERVE_VA = UVM_IOCTL_BASE(1) +UVM_RELEASE_VA = UVM_IOCTL_BASE(2) +UVM_REGION_COMMIT = UVM_IOCTL_BASE(3) +UVM_REGION_DECOMMIT = UVM_IOCTL_BASE(4) +UVM_REGION_SET_STREAM = UVM_IOCTL_BASE(5) +UVM_SET_STREAM_RUNNING = UVM_IOCTL_BASE(6) +UVM_MAX_STREAMS_PER_IOCTL_CALL = 32 +UVM_SET_STREAM_STOPPED = UVM_IOCTL_BASE(7) +UVM_ADD_SESSION = UVM_IOCTL_BASE(10) +UVM_MAX_COUNTERS_PER_IOCTL_CALL = 32 +UVM_ENABLE_COUNTERS = UVM_IOCTL_BASE(12) +UVM_MAP_COUNTER = UVM_IOCTL_BASE(13) +UVM_CREATE_EVENT_QUEUE = UVM_IOCTL_BASE(14) +UVM_REMOVE_EVENT_QUEUE = UVM_IOCTL_BASE(15) +UVM_MAP_EVENT_QUEUE = UVM_IOCTL_BASE(16) +UVM_EVENT_CTRL = UVM_IOCTL_BASE(17) +UVM_GET_GPU_UUID_TABLE = UVM_IOCTL_BASE(20) +UVM_REGISTER_GPU_VASPACE = UVM_IOCTL_BASE(25) +UVM_UNREGISTER_GPU_VASPACE = UVM_IOCTL_BASE(26) +UVM_REGISTER_CHANNEL = UVM_IOCTL_BASE(27) +UVM_UNREGISTER_CHANNEL = UVM_IOCTL_BASE(28) +UVM_ENABLE_PEER_ACCESS = UVM_IOCTL_BASE(29) +UVM_DISABLE_PEER_ACCESS = UVM_IOCTL_BASE(30) +UVM_MAP_EXTERNAL_ALLOCATION = UVM_IOCTL_BASE(33) +UVM_FREE = UVM_IOCTL_BASE(34) +UVM_MEM_MAP = UVM_IOCTL_BASE(35) +UVM_REGISTER_GPU = UVM_IOCTL_BASE(37) +UVM_UNREGISTER_GPU = UVM_IOCTL_BASE(38) +UVM_PAGEABLE_MEM_ACCESS = UVM_IOCTL_BASE(39) +UVM_SET_PREFERRED_LOCATION = UVM_IOCTL_BASE(42) +UVM_UNSET_PREFERRED_LOCATION = UVM_IOCTL_BASE(43) +UVM_ENABLE_READ_DUPLICATION = UVM_IOCTL_BASE(44) +UVM_DISABLE_READ_DUPLICATION = UVM_IOCTL_BASE(45) +UVM_SET_ACCESSED_BY = UVM_IOCTL_BASE(46) +UVM_UNSET_ACCESSED_BY = UVM_IOCTL_BASE(47) +UVM_MIGRATE_FLAG_ASYNC = 0x00000001 +UVM_MIGRATE_FLAG_SKIP_CPU_MAP = 0x00000002 +UVM_MIGRATE_FLAG_NO_GPU_VA_SPACE = 0x00000004 +UVM_MIGRATE_FLAGS_TEST_ALL = (UVM_MIGRATE_FLAG_SKIP_CPU_MAP | UVM_MIGRATE_FLAG_NO_GPU_VA_SPACE) +UVM_MIGRATE_FLAGS_ALL = (UVM_MIGRATE_FLAG_ASYNC | UVM_MIGRATE_FLAGS_TEST_ALL) +UVM_MIGRATE = UVM_IOCTL_BASE(51) +UVM_TOOLS_INIT_EVENT_TRACKER = UVM_IOCTL_BASE(56) +UVM_TOOLS_SET_NOTIFICATION_THRESHOLD = UVM_IOCTL_BASE(57) +UVM_TOOLS_EVENT_QUEUE_ENABLE_EVENTS = UVM_IOCTL_BASE(58) +UVM_TOOLS_EVENT_QUEUE_DISABLE_EVENTS = UVM_IOCTL_BASE(59) +UVM_TOOLS_ENABLE_COUNTERS = UVM_IOCTL_BASE(60) +UVM_TOOLS_DISABLE_COUNTERS = UVM_IOCTL_BASE(61) +UVM_TOOLS_READ_PROCESS_MEMORY = UVM_IOCTL_BASE(62) +UVM_TOOLS_WRITE_PROCESS_MEMORY = UVM_IOCTL_BASE(63) +UVM_TOOLS_GET_PROCESSOR_UUID_TABLE = UVM_IOCTL_BASE(64) +UVM_MAP_DYNAMIC_PARALLELISM_REGION = UVM_IOCTL_BASE(65) +UVM_UNMAP_EXTERNAL = UVM_IOCTL_BASE(66) +UVM_TOOLS_FLUSH_EVENTS = UVM_IOCTL_BASE(67) +UVM_ALLOC_SEMAPHORE_POOL = UVM_IOCTL_BASE(68) +UVM_CLEAN_UP_ZOMBIE_RESOURCES = UVM_IOCTL_BASE(69) +UVM_PAGEABLE_MEM_ACCESS_ON_GPU = UVM_IOCTL_BASE(70) +UVM_POPULATE_PAGEABLE = UVM_IOCTL_BASE(71) +UVM_POPULATE_PAGEABLE_FLAG_ALLOW_MANAGED = 0x00000001 +UVM_POPULATE_PAGEABLE_FLAG_SKIP_PROT_CHECK = 0x00000002 +UVM_POPULATE_PAGEABLE_FLAG_ALLOW_SPECIAL = 0x00000004 +UVM_POPULATE_PAGEABLE_FLAGS_INTERNAL = UVM_POPULATE_PAGEABLE_FLAG_ALLOW_SPECIAL +UVM_POPULATE_PAGEABLE_FLAGS_TEST = (UVM_POPULATE_PAGEABLE_FLAG_ALLOW_MANAGED | UVM_POPULATE_PAGEABLE_FLAG_SKIP_PROT_CHECK) +UVM_POPULATE_PAGEABLE_FLAGS_ALL = (UVM_POPULATE_PAGEABLE_FLAGS_INTERNAL | UVM_POPULATE_PAGEABLE_FLAGS_TEST) +UVM_VALIDATE_VA_RANGE = UVM_IOCTL_BASE(72) +UVM_CREATE_EXTERNAL_RANGE = UVM_IOCTL_BASE(73) +UVM_MAP_EXTERNAL_SPARSE = UVM_IOCTL_BASE(74) +UVM_MM_INITIALIZE = UVM_IOCTL_BASE(75) +UVM_TOOLS_INIT_EVENT_TRACKER_V2 = UVM_IOCTL_BASE(76) +UVM_TOOLS_GET_PROCESSOR_UUID_TABLE_V2 = UVM_IOCTL_BASE(77) +UVM_ALLOC_DEVICE_P2P = UVM_IOCTL_BASE(78) +UVM_CLEAR_ALL_ACCESS_COUNTERS = UVM_IOCTL_BASE(79) +UVM_DISCARD = UVM_IOCTL_BASE(80) +UVM_QUERY_RESIDENCY = UVM_IOCTL_BASE(81) +UVM_IS_DMA_BUF_IMPORT_SUPPORTED = UVM_IOCTL_BASE(82) +UVM_IMPORT_DMA_BUF = UVM_IOCTL_BASE(83) +UVM_MAP_DMA_BUF = UVM_IOCTL_BASE(84) +UVM_UNMAP_DMA_BUF = UVM_IOCTL_BASE(85) +UVM_IS_8_SUPPORTED = UVM_IOCTL_BASE(2047) +UVM_INITIALIZE = 0x30000001 +UVM_DEINITIALIZE = 0x30000002 +NV_PFAULT_MMU_ENG_ID_GRAPHICS = 64 +NV_PFAULT_MMU_ENG_ID_DISPLAY = 1 +NV_PFAULT_MMU_ENG_ID_GSP = 2 +NV_PFAULT_MMU_ENG_ID_IFB = 9 +NV_PFAULT_MMU_ENG_ID_FLA = 4 +NV_PFAULT_MMU_ENG_ID_BAR1 = 128 +NV_PFAULT_MMU_ENG_ID_BAR2 = 192 +NV_PFAULT_MMU_ENG_ID_SEC = 14 +NV_PFAULT_MMU_ENG_ID_PERF = 8 +NV_PFAULT_MMU_ENG_ID_NVDEC = 25 +NV_PFAULT_MMU_ENG_ID_NVDEC0 = 25 +NV_PFAULT_MMU_ENG_ID_NVDEC1 = 26 +NV_PFAULT_MMU_ENG_ID_NVDEC2 = 27 +NV_PFAULT_MMU_ENG_ID_NVDEC3 = 28 +NV_PFAULT_MMU_ENG_ID_NVDEC4 = 29 +NV_PFAULT_MMU_ENG_ID_NVJPG0 = 30 +NV_PFAULT_MMU_ENG_ID_GRCOPY = 15 +NV_PFAULT_MMU_ENG_ID_CE0 = 15 +NV_PFAULT_MMU_ENG_ID_CE1 = 16 +NV_PFAULT_MMU_ENG_ID_CE2 = 17 +NV_PFAULT_MMU_ENG_ID_CE3 = 18 +NV_PFAULT_MMU_ENG_ID_CE4 = 19 +NV_PFAULT_MMU_ENG_ID_CE5 = 20 +NV_PFAULT_MMU_ENG_ID_CE6 = 21 +NV_PFAULT_MMU_ENG_ID_CE7 = 22 +NV_PFAULT_MMU_ENG_ID_CE8 = 23 +NV_PFAULT_MMU_ENG_ID_CE9 = 24 +NV_PFAULT_MMU_ENG_ID_PWR_PMU = 6 +NV_PFAULT_MMU_ENG_ID_PTP = 3 +NV_PFAULT_MMU_ENG_ID_NVENC0 = 11 +NV_PFAULT_MMU_ENG_ID_NVENC1 = 12 +NV_PFAULT_MMU_ENG_ID_NVENC2 = 13 +NV_PFAULT_MMU_ENG_ID_OFA0 = 10 +NV_PFAULT_MMU_ENG_ID_PHYSICAL = 31 +NV_PFAULT_MMU_ENG_ID_HOST0 = 32 +NV_PFAULT_MMU_ENG_ID_HOST1 = 33 +NV_PFAULT_MMU_ENG_ID_HOST2 = 34 +NV_PFAULT_MMU_ENG_ID_HOST3 = 35 +NV_PFAULT_MMU_ENG_ID_HOST4 = 36 +NV_PFAULT_MMU_ENG_ID_HOST5 = 37 +NV_PFAULT_MMU_ENG_ID_HOST6 = 38 +NV_PFAULT_MMU_ENG_ID_HOST7 = 39 +NV_PFAULT_MMU_ENG_ID_HOST8 = 40 +NV_PFAULT_MMU_ENG_ID_HOST9 = 41 +NV_PFAULT_MMU_ENG_ID_HOST10 = 42 +NV_PFAULT_MMU_ENG_ID_HOST11 = 43 +NV_PFAULT_MMU_ENG_ID_HOST12 = 44 +NV_PFAULT_MMU_ENG_ID_HOST13 = 45 +NV_PFAULT_MMU_ENG_ID_HOST14 = 46 +NV_PFAULT_MMU_ENG_ID_HOST15 = 47 +NV_PFAULT_MMU_ENG_ID_HOST16 = 48 +NV_PFAULT_MMU_ENG_ID_HOST17 = 49 +NV_PFAULT_MMU_ENG_ID_HOST18 = 50 +NV_PFAULT_MMU_ENG_ID_HOST19 = 51 +NV_PFAULT_MMU_ENG_ID_HOST20 = 52 +NV_PFAULT_MMU_ENG_ID_HOST21 = 53 +NV_PFAULT_MMU_ENG_ID_HOST22 = 54 +NV_PFAULT_MMU_ENG_ID_HOST23 = 55 +NV_PFAULT_MMU_ENG_ID_HOST24 = 56 +NV_PFAULT_MMU_ENG_ID_HOST25 = 57 +NV_PFAULT_MMU_ENG_ID_HOST26 = 58 +NV_PFAULT_MMU_ENG_ID_HOST27 = 59 +NV_PFAULT_MMU_ENG_ID_HOST28 = 60 +NV_PFAULT_MMU_ENG_ID_HOST29 = 61 +NV_PFAULT_MMU_ENG_ID_HOST30 = 62 +NV_PFAULT_MMU_ENG_ID_HOST31 = 63 +NV_PFAULT_MMU_ENG_ID_BAR1_FN0 = 128 +NV_PFAULT_MMU_ENG_ID_BAR1_FN1 = 129 +NV_PFAULT_MMU_ENG_ID_BAR1_FN2 = 130 +NV_PFAULT_MMU_ENG_ID_BAR1_FN3 = 131 +NV_PFAULT_MMU_ENG_ID_BAR1_FN4 = 132 +NV_PFAULT_MMU_ENG_ID_BAR1_FN5 = 133 +NV_PFAULT_MMU_ENG_ID_BAR1_FN6 = 134 +NV_PFAULT_MMU_ENG_ID_BAR1_FN7 = 135 +NV_PFAULT_MMU_ENG_ID_BAR1_FN8 = 136 +NV_PFAULT_MMU_ENG_ID_BAR1_FN9 = 137 +NV_PFAULT_MMU_ENG_ID_BAR1_FN10 = 138 +NV_PFAULT_MMU_ENG_ID_BAR1_FN11 = 139 +NV_PFAULT_MMU_ENG_ID_BAR1_FN12 = 140 +NV_PFAULT_MMU_ENG_ID_BAR1_FN13 = 141 +NV_PFAULT_MMU_ENG_ID_BAR1_FN14 = 142 +NV_PFAULT_MMU_ENG_ID_BAR1_FN15 = 143 +NV_PFAULT_MMU_ENG_ID_BAR1_FN16 = 144 +NV_PFAULT_MMU_ENG_ID_BAR1_FN17 = 145 +NV_PFAULT_MMU_ENG_ID_BAR1_FN18 = 146 +NV_PFAULT_MMU_ENG_ID_BAR1_FN19 = 147 +NV_PFAULT_MMU_ENG_ID_BAR1_FN20 = 148 +NV_PFAULT_MMU_ENG_ID_BAR1_FN21 = 149 +NV_PFAULT_MMU_ENG_ID_BAR1_FN22 = 150 +NV_PFAULT_MMU_ENG_ID_BAR1_FN23 = 151 +NV_PFAULT_MMU_ENG_ID_BAR1_FN24 = 152 +NV_PFAULT_MMU_ENG_ID_BAR1_FN25 = 153 +NV_PFAULT_MMU_ENG_ID_BAR1_FN26 = 154 +NV_PFAULT_MMU_ENG_ID_BAR1_FN27 = 155 +NV_PFAULT_MMU_ENG_ID_BAR1_FN28 = 156 +NV_PFAULT_MMU_ENG_ID_BAR1_FN29 = 157 +NV_PFAULT_MMU_ENG_ID_BAR1_FN30 = 158 +NV_PFAULT_MMU_ENG_ID_BAR1_FN31 = 159 +NV_PFAULT_MMU_ENG_ID_BAR1_FN32 = 160 +NV_PFAULT_MMU_ENG_ID_BAR1_FN33 = 161 +NV_PFAULT_MMU_ENG_ID_BAR1_FN34 = 162 +NV_PFAULT_MMU_ENG_ID_BAR1_FN35 = 163 +NV_PFAULT_MMU_ENG_ID_BAR1_FN36 = 164 +NV_PFAULT_MMU_ENG_ID_BAR1_FN37 = 165 +NV_PFAULT_MMU_ENG_ID_BAR1_FN38 = 166 +NV_PFAULT_MMU_ENG_ID_BAR1_FN39 = 167 +NV_PFAULT_MMU_ENG_ID_BAR1_FN40 = 168 +NV_PFAULT_MMU_ENG_ID_BAR1_FN41 = 169 +NV_PFAULT_MMU_ENG_ID_BAR1_FN42 = 170 +NV_PFAULT_MMU_ENG_ID_BAR1_FN43 = 171 +NV_PFAULT_MMU_ENG_ID_BAR1_FN44 = 172 +NV_PFAULT_MMU_ENG_ID_BAR1_FN45 = 173 +NV_PFAULT_MMU_ENG_ID_BAR1_FN46 = 174 +NV_PFAULT_MMU_ENG_ID_BAR1_FN47 = 175 +NV_PFAULT_MMU_ENG_ID_BAR1_FN48 = 176 +NV_PFAULT_MMU_ENG_ID_BAR1_FN49 = 177 +NV_PFAULT_MMU_ENG_ID_BAR1_FN50 = 178 +NV_PFAULT_MMU_ENG_ID_BAR1_FN51 = 179 +NV_PFAULT_MMU_ENG_ID_BAR1_FN52 = 180 +NV_PFAULT_MMU_ENG_ID_BAR1_FN53 = 181 +NV_PFAULT_MMU_ENG_ID_BAR1_FN54 = 182 +NV_PFAULT_MMU_ENG_ID_BAR1_FN55 = 183 +NV_PFAULT_MMU_ENG_ID_BAR1_FN56 = 184 +NV_PFAULT_MMU_ENG_ID_BAR1_FN57 = 185 +NV_PFAULT_MMU_ENG_ID_BAR1_FN58 = 186 +NV_PFAULT_MMU_ENG_ID_BAR1_FN59 = 187 +NV_PFAULT_MMU_ENG_ID_BAR1_FN60 = 188 +NV_PFAULT_MMU_ENG_ID_BAR1_FN61 = 189 +NV_PFAULT_MMU_ENG_ID_BAR1_FN62 = 190 +NV_PFAULT_MMU_ENG_ID_BAR1_FN63 = 191 +NV_PFAULT_MMU_ENG_ID_BAR2_FN0 = 192 +NV_PFAULT_MMU_ENG_ID_BAR2_FN1 = 193 +NV_PFAULT_MMU_ENG_ID_BAR2_FN2 = 194 +NV_PFAULT_MMU_ENG_ID_BAR2_FN3 = 195 +NV_PFAULT_MMU_ENG_ID_BAR2_FN4 = 196 +NV_PFAULT_MMU_ENG_ID_BAR2_FN5 = 197 +NV_PFAULT_MMU_ENG_ID_BAR2_FN6 = 198 +NV_PFAULT_MMU_ENG_ID_BAR2_FN7 = 199 +NV_PFAULT_MMU_ENG_ID_BAR2_FN8 = 200 +NV_PFAULT_MMU_ENG_ID_BAR2_FN9 = 201 +NV_PFAULT_MMU_ENG_ID_BAR2_FN10 = 202 +NV_PFAULT_MMU_ENG_ID_BAR2_FN11 = 203 +NV_PFAULT_MMU_ENG_ID_BAR2_FN12 = 204 +NV_PFAULT_MMU_ENG_ID_BAR2_FN13 = 205 +NV_PFAULT_MMU_ENG_ID_BAR2_FN14 = 206 +NV_PFAULT_MMU_ENG_ID_BAR2_FN15 = 207 +NV_PFAULT_MMU_ENG_ID_BAR2_FN16 = 208 +NV_PFAULT_MMU_ENG_ID_BAR2_FN17 = 209 +NV_PFAULT_MMU_ENG_ID_BAR2_FN18 = 210 +NV_PFAULT_MMU_ENG_ID_BAR2_FN19 = 211 +NV_PFAULT_MMU_ENG_ID_BAR2_FN20 = 212 +NV_PFAULT_MMU_ENG_ID_BAR2_FN21 = 213 +NV_PFAULT_MMU_ENG_ID_BAR2_FN22 = 214 +NV_PFAULT_MMU_ENG_ID_BAR2_FN23 = 215 +NV_PFAULT_MMU_ENG_ID_BAR2_FN24 = 216 +NV_PFAULT_MMU_ENG_ID_BAR2_FN25 = 217 +NV_PFAULT_MMU_ENG_ID_BAR2_FN26 = 218 +NV_PFAULT_MMU_ENG_ID_BAR2_FN27 = 219 +NV_PFAULT_MMU_ENG_ID_BAR2_FN28 = 220 +NV_PFAULT_MMU_ENG_ID_BAR2_FN29 = 221 +NV_PFAULT_MMU_ENG_ID_BAR2_FN30 = 222 +NV_PFAULT_MMU_ENG_ID_BAR2_FN31 = 223 +NV_PFAULT_MMU_ENG_ID_BAR2_FN32 = 224 +NV_PFAULT_MMU_ENG_ID_BAR2_FN33 = 225 +NV_PFAULT_MMU_ENG_ID_BAR2_FN34 = 226 +NV_PFAULT_MMU_ENG_ID_BAR2_FN35 = 227 +NV_PFAULT_MMU_ENG_ID_BAR2_FN36 = 228 +NV_PFAULT_MMU_ENG_ID_BAR2_FN37 = 229 +NV_PFAULT_MMU_ENG_ID_BAR2_FN38 = 230 +NV_PFAULT_MMU_ENG_ID_BAR2_FN39 = 231 +NV_PFAULT_MMU_ENG_ID_BAR2_FN40 = 232 +NV_PFAULT_MMU_ENG_ID_BAR2_FN41 = 233 +NV_PFAULT_MMU_ENG_ID_BAR2_FN42 = 234 +NV_PFAULT_MMU_ENG_ID_BAR2_FN43 = 235 +NV_PFAULT_MMU_ENG_ID_BAR2_FN44 = 236 +NV_PFAULT_MMU_ENG_ID_BAR2_FN45 = 237 +NV_PFAULT_MMU_ENG_ID_BAR2_FN46 = 238 +NV_PFAULT_MMU_ENG_ID_BAR2_FN47 = 239 +NV_PFAULT_MMU_ENG_ID_BAR2_FN48 = 240 +NV_PFAULT_MMU_ENG_ID_BAR2_FN49 = 241 +NV_PFAULT_MMU_ENG_ID_BAR2_FN50 = 242 +NV_PFAULT_MMU_ENG_ID_BAR2_FN51 = 243 +NV_PFAULT_MMU_ENG_ID_BAR2_FN52 = 244 +NV_PFAULT_MMU_ENG_ID_BAR2_FN53 = 245 +NV_PFAULT_MMU_ENG_ID_BAR2_FN54 = 246 +NV_PFAULT_MMU_ENG_ID_BAR2_FN55 = 247 +NV_PFAULT_MMU_ENG_ID_BAR2_FN56 = 248 +NV_PFAULT_MMU_ENG_ID_BAR2_FN57 = 249 +NV_PFAULT_MMU_ENG_ID_BAR2_FN58 = 250 +NV_PFAULT_MMU_ENG_ID_BAR2_FN59 = 251 +NV_PFAULT_MMU_ENG_ID_BAR2_FN60 = 252 +NV_PFAULT_MMU_ENG_ID_BAR2_FN61 = 253 +NV_PFAULT_MMU_ENG_ID_BAR2_FN62 = 254 +NV_PFAULT_MMU_ENG_ID_BAR2_FN63 = 255 +NV_PFAULT_FAULT_TYPE = (4, 0) +NV_PFAULT_FAULT_TYPE_PDE = 0x00000000 +NV_PFAULT_FAULT_TYPE_PDE_SIZE = 0x00000001 +NV_PFAULT_FAULT_TYPE_PTE = 0x00000002 +NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION = 0x00000003 +NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK = 0x00000004 +NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION = 0x00000005 +NV_PFAULT_FAULT_TYPE_RO_VIOLATION = 0x00000006 +NV_PFAULT_FAULT_TYPE_WO_VIOLATION = 0x00000007 +NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION = 0x00000008 +NV_PFAULT_FAULT_TYPE_WORK_CREATION = 0x00000009 +NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE = 0x0000000a +NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE = 0x0000000b +NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND = 0x0000000c +NV_PFAULT_FAULT_TYPE_REGION_VIOLATION = 0x0000000d +NV_PFAULT_FAULT_TYPE_POISONED = 0x0000000e +NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION = 0x0000000f +NV_PFAULT_CLIENT = (14, 8) +NV_PFAULT_CLIENT_GPC_T1_0 = 0x00000000 +NV_PFAULT_CLIENT_GPC_T1_1 = 0x00000001 +NV_PFAULT_CLIENT_GPC_T1_2 = 0x00000002 +NV_PFAULT_CLIENT_GPC_T1_3 = 0x00000003 +NV_PFAULT_CLIENT_GPC_T1_4 = 0x00000004 +NV_PFAULT_CLIENT_GPC_T1_5 = 0x00000005 +NV_PFAULT_CLIENT_GPC_T1_6 = 0x00000006 +NV_PFAULT_CLIENT_GPC_T1_7 = 0x00000007 +NV_PFAULT_CLIENT_GPC_PE_0 = 0x00000008 +NV_PFAULT_CLIENT_GPC_PE_1 = 0x00000009 +NV_PFAULT_CLIENT_GPC_PE_2 = 0x0000000A +NV_PFAULT_CLIENT_GPC_PE_3 = 0x0000000B +NV_PFAULT_CLIENT_GPC_PE_4 = 0x0000000C +NV_PFAULT_CLIENT_GPC_PE_5 = 0x0000000D +NV_PFAULT_CLIENT_GPC_PE_6 = 0x0000000E +NV_PFAULT_CLIENT_GPC_PE_7 = 0x0000000F +NV_PFAULT_CLIENT_GPC_RAST = 0x00000010 +NV_PFAULT_CLIENT_GPC_GCC = 0x00000011 +NV_PFAULT_CLIENT_GPC_GPCCS = 0x00000012 +NV_PFAULT_CLIENT_GPC_PROP_0 = 0x00000013 +NV_PFAULT_CLIENT_GPC_PROP_1 = 0x00000014 +NV_PFAULT_CLIENT_GPC_PROP_2 = 0x00000015 +NV_PFAULT_CLIENT_GPC_PROP_3 = 0x00000016 +NV_PFAULT_CLIENT_GPC_T1_8 = 0x00000021 +NV_PFAULT_CLIENT_GPC_T1_9 = 0x00000022 +NV_PFAULT_CLIENT_GPC_T1_10 = 0x00000023 +NV_PFAULT_CLIENT_GPC_T1_11 = 0x00000024 +NV_PFAULT_CLIENT_GPC_T1_12 = 0x00000025 +NV_PFAULT_CLIENT_GPC_T1_13 = 0x00000026 +NV_PFAULT_CLIENT_GPC_T1_14 = 0x00000027 +NV_PFAULT_CLIENT_GPC_T1_15 = 0x00000028 +NV_PFAULT_CLIENT_GPC_TPCCS_0 = 0x00000029 +NV_PFAULT_CLIENT_GPC_TPCCS_1 = 0x0000002A +NV_PFAULT_CLIENT_GPC_TPCCS_2 = 0x0000002B +NV_PFAULT_CLIENT_GPC_TPCCS_3 = 0x0000002C +NV_PFAULT_CLIENT_GPC_TPCCS_4 = 0x0000002D +NV_PFAULT_CLIENT_GPC_TPCCS_5 = 0x0000002E +NV_PFAULT_CLIENT_GPC_TPCCS_6 = 0x0000002F +NV_PFAULT_CLIENT_GPC_TPCCS_7 = 0x00000030 +NV_PFAULT_CLIENT_GPC_PE_8 = 0x00000031 +NV_PFAULT_CLIENT_GPC_PE_9 = 0x00000032 +NV_PFAULT_CLIENT_GPC_TPCCS_8 = 0x00000033 +NV_PFAULT_CLIENT_GPC_TPCCS_9 = 0x00000034 +NV_PFAULT_CLIENT_GPC_T1_16 = 0x00000035 +NV_PFAULT_CLIENT_GPC_T1_17 = 0x00000036 +NV_PFAULT_CLIENT_GPC_T1_18 = 0x00000037 +NV_PFAULT_CLIENT_GPC_T1_19 = 0x00000038 +NV_PFAULT_CLIENT_GPC_PE_10 = 0x00000039 +NV_PFAULT_CLIENT_GPC_PE_11 = 0x0000003A +NV_PFAULT_CLIENT_GPC_TPCCS_10 = 0x0000003B +NV_PFAULT_CLIENT_GPC_TPCCS_11 = 0x0000003C +NV_PFAULT_CLIENT_GPC_T1_20 = 0x0000003D +NV_PFAULT_CLIENT_GPC_T1_21 = 0x0000003E +NV_PFAULT_CLIENT_GPC_T1_22 = 0x0000003F +NV_PFAULT_CLIENT_GPC_T1_23 = 0x00000040 +NV_PFAULT_CLIENT_GPC_PE_12 = 0x00000041 +NV_PFAULT_CLIENT_GPC_PE_13 = 0x00000042 +NV_PFAULT_CLIENT_GPC_TPCCS_12 = 0x00000043 +NV_PFAULT_CLIENT_GPC_TPCCS_13 = 0x00000044 +NV_PFAULT_CLIENT_GPC_T1_24 = 0x00000045 +NV_PFAULT_CLIENT_GPC_T1_25 = 0x00000046 +NV_PFAULT_CLIENT_GPC_T1_26 = 0x00000047 +NV_PFAULT_CLIENT_GPC_T1_27 = 0x00000048 +NV_PFAULT_CLIENT_GPC_PE_14 = 0x00000049 +NV_PFAULT_CLIENT_GPC_PE_15 = 0x0000004A +NV_PFAULT_CLIENT_GPC_TPCCS_14 = 0x0000004B +NV_PFAULT_CLIENT_GPC_TPCCS_15 = 0x0000004C +NV_PFAULT_CLIENT_GPC_T1_28 = 0x0000004D +NV_PFAULT_CLIENT_GPC_T1_29 = 0x0000004E +NV_PFAULT_CLIENT_GPC_T1_30 = 0x0000004F +NV_PFAULT_CLIENT_GPC_T1_31 = 0x00000050 +NV_PFAULT_CLIENT_GPC_PE_16 = 0x00000051 +NV_PFAULT_CLIENT_GPC_PE_17 = 0x00000052 +NV_PFAULT_CLIENT_GPC_TPCCS_16 = 0x00000053 +NV_PFAULT_CLIENT_GPC_TPCCS_17 = 0x00000054 +NV_PFAULT_CLIENT_GPC_T1_32 = 0x00000055 +NV_PFAULT_CLIENT_GPC_T1_33 = 0x00000056 +NV_PFAULT_CLIENT_GPC_T1_34 = 0x00000057 +NV_PFAULT_CLIENT_GPC_T1_35 = 0x00000058 +NV_PFAULT_CLIENT_GPC_PE_18 = 0x00000059 +NV_PFAULT_CLIENT_GPC_PE_19 = 0x0000005A +NV_PFAULT_CLIENT_GPC_TPCCS_18 = 0x0000005B +NV_PFAULT_CLIENT_GPC_TPCCS_19 = 0x0000005C +NV_PFAULT_CLIENT_GPC_T1_36 = 0x0000005D +NV_PFAULT_CLIENT_GPC_T1_37 = 0x0000005E +NV_PFAULT_CLIENT_GPC_T1_38 = 0x0000005F +NV_PFAULT_CLIENT_GPC_T1_39 = 0x00000060 +NV_PFAULT_CLIENT_GPC_ROP_0 = 0x00000070 +NV_PFAULT_CLIENT_GPC_ROP_1 = 0x00000071 +NV_PFAULT_CLIENT_GPC_ROP_2 = 0x00000072 +NV_PFAULT_CLIENT_GPC_ROP_3 = 0x00000073 +NV_PFAULT_CLIENT_GPC_GPM = 0x00000017 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_0 = 0x00000018 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_1 = 0x00000019 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_2 = 0x0000001A +NV_PFAULT_CLIENT_GPC_LTP_UTLB_3 = 0x0000001B +NV_PFAULT_CLIENT_GPC_LTP_UTLB_4 = 0x0000001C +NV_PFAULT_CLIENT_GPC_LTP_UTLB_5 = 0x0000001D +NV_PFAULT_CLIENT_GPC_LTP_UTLB_6 = 0x0000001E +NV_PFAULT_CLIENT_GPC_LTP_UTLB_7 = 0x0000001F +NV_PFAULT_CLIENT_GPC_RGG_UTLB = 0x00000020 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_8 = 0x00000031 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_9 = 0x00000032 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_10 = 0x00000033 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_11 = 0x00000034 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_12 = 0x00000035 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_13 = 0x00000036 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_14 = 0x00000037 +NV_PFAULT_CLIENT_GPC_LTP_UTLB_15 = 0x00000038 +NV_PFAULT_CLIENT_HUB_VIP = 0x00000000 +NV_PFAULT_CLIENT_HUB_CE0 = 0x00000001 +NV_PFAULT_CLIENT_HUB_CE1 = 0x00000002 +NV_PFAULT_CLIENT_HUB_DNISO = 0x00000003 +NV_PFAULT_CLIENT_HUB_DISPNISO = 0x00000003 +NV_PFAULT_CLIENT_HUB_FE0 = 0x00000004 +NV_PFAULT_CLIENT_HUB_FE = 0x00000004 +NV_PFAULT_CLIENT_HUB_FECS0 = 0x00000005 +NV_PFAULT_CLIENT_HUB_FECS = 0x00000005 +NV_PFAULT_CLIENT_HUB_HOST = 0x00000006 +NV_PFAULT_CLIENT_HUB_HOST_CPU = 0x00000007 +NV_PFAULT_CLIENT_HUB_HOST_CPU_NB = 0x00000008 +NV_PFAULT_CLIENT_HUB_ISO = 0x00000009 +NV_PFAULT_CLIENT_HUB_MMU = 0x0000000A +NV_PFAULT_CLIENT_HUB_NVDEC0 = 0x0000000B +NV_PFAULT_CLIENT_HUB_NVDEC = 0x0000000B +NV_PFAULT_CLIENT_HUB_NVENC1 = 0x0000000D +NV_PFAULT_CLIENT_HUB_NISO = 0x0000000E +NV_PFAULT_CLIENT_HUB_ACTRS = 0x0000000E +NV_PFAULT_CLIENT_HUB_P2P = 0x0000000F +NV_PFAULT_CLIENT_HUB_PD = 0x00000010 +NV_PFAULT_CLIENT_HUB_PERF0 = 0x00000011 +NV_PFAULT_CLIENT_HUB_PERF = 0x00000011 +NV_PFAULT_CLIENT_HUB_PMU = 0x00000012 +NV_PFAULT_CLIENT_HUB_RASTERTWOD = 0x00000013 +NV_PFAULT_CLIENT_HUB_SCC = 0x00000014 +NV_PFAULT_CLIENT_HUB_SCC_NB = 0x00000015 +NV_PFAULT_CLIENT_HUB_SEC = 0x00000016 +NV_PFAULT_CLIENT_HUB_SSYNC = 0x00000017 +NV_PFAULT_CLIENT_HUB_GRCOPY = 0x00000018 +NV_PFAULT_CLIENT_HUB_CE2 = 0x00000018 +NV_PFAULT_CLIENT_HUB_XV = 0x00000019 +NV_PFAULT_CLIENT_HUB_MMU_NB = 0x0000001A +NV_PFAULT_CLIENT_HUB_NVENC0 = 0x0000001B +NV_PFAULT_CLIENT_HUB_NVENC = 0x0000001B +NV_PFAULT_CLIENT_HUB_DFALCON = 0x0000001C +NV_PFAULT_CLIENT_HUB_SKED0 = 0x0000001D +NV_PFAULT_CLIENT_HUB_SKED = 0x0000001D +NV_PFAULT_CLIENT_HUB_AFALCON = 0x0000001E +NV_PFAULT_CLIENT_HUB_DONT_CARE = 0x0000001F +NV_PFAULT_CLIENT_HUB_HSCE0 = 0x00000020 +NV_PFAULT_CLIENT_HUB_HSCE1 = 0x00000021 +NV_PFAULT_CLIENT_HUB_HSCE2 = 0x00000022 +NV_PFAULT_CLIENT_HUB_HSCE3 = 0x00000023 +NV_PFAULT_CLIENT_HUB_HSCE4 = 0x00000024 +NV_PFAULT_CLIENT_HUB_HSCE5 = 0x00000025 +NV_PFAULT_CLIENT_HUB_HSCE6 = 0x00000026 +NV_PFAULT_CLIENT_HUB_HSCE7 = 0x00000027 +NV_PFAULT_CLIENT_HUB_HSCE8 = 0x00000028 +NV_PFAULT_CLIENT_HUB_HSCE9 = 0x00000029 +NV_PFAULT_CLIENT_HUB_HSHUB = 0x0000002A +NV_PFAULT_CLIENT_HUB_PTP_X0 = 0x0000002B +NV_PFAULT_CLIENT_HUB_PTP_X1 = 0x0000002C +NV_PFAULT_CLIENT_HUB_PTP_X2 = 0x0000002D +NV_PFAULT_CLIENT_HUB_PTP_X3 = 0x0000002E +NV_PFAULT_CLIENT_HUB_PTP_X4 = 0x0000002F +NV_PFAULT_CLIENT_HUB_PTP_X5 = 0x00000030 +NV_PFAULT_CLIENT_HUB_PTP_X6 = 0x00000031 +NV_PFAULT_CLIENT_HUB_PTP_X7 = 0x00000032 +NV_PFAULT_CLIENT_HUB_NVENC2 = 0x00000033 +NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER0 = 0x00000034 +NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER1 = 0x00000035 +NV_PFAULT_CLIENT_HUB_DWBIF = 0x00000036 +NV_PFAULT_CLIENT_HUB_FBFALCON = 0x00000037 +NV_PFAULT_CLIENT_HUB_CE_SHIM = 0x00000038 +NV_PFAULT_CLIENT_HUB_GSP = 0x00000039 +NV_PFAULT_CLIENT_HUB_NVDEC1 = 0x0000003A +NV_PFAULT_CLIENT_HUB_NVDEC2 = 0x0000003B +NV_PFAULT_CLIENT_HUB_NVJPG0 = 0x0000003C +NV_PFAULT_CLIENT_HUB_NVDEC3 = 0x0000003D +NV_PFAULT_CLIENT_HUB_NVDEC4 = 0x0000003E +NV_PFAULT_CLIENT_HUB_OFA0 = 0x0000003F +NV_PFAULT_CLIENT_HUB_HSCE10 = 0x00000040 +NV_PFAULT_CLIENT_HUB_HSCE11 = 0x00000041 +NV_PFAULT_CLIENT_HUB_HSCE12 = 0x00000042 +NV_PFAULT_CLIENT_HUB_HSCE13 = 0x00000043 +NV_PFAULT_CLIENT_HUB_HSCE14 = 0x00000044 +NV_PFAULT_CLIENT_HUB_HSCE15 = 0x00000045 +NV_PFAULT_CLIENT_HUB_PTP_X8 = 0x00000046 +NV_PFAULT_CLIENT_HUB_PTP_X9 = 0x00000047 +NV_PFAULT_CLIENT_HUB_PTP_X10 = 0x00000048 +NV_PFAULT_CLIENT_HUB_PTP_X11 = 0x00000049 +NV_PFAULT_CLIENT_HUB_PTP_X12 = 0x0000004A +NV_PFAULT_CLIENT_HUB_PTP_X13 = 0x0000004B +NV_PFAULT_CLIENT_HUB_PTP_X14 = 0x0000004C +NV_PFAULT_CLIENT_HUB_PTP_X15 = 0x0000004D +NV_PFAULT_CLIENT_HUB_FE1 = 0x0000004E +NV_PFAULT_CLIENT_HUB_FE2 = 0x0000004F +NV_PFAULT_CLIENT_HUB_FE3 = 0x00000050 +NV_PFAULT_CLIENT_HUB_FE4 = 0x00000051 +NV_PFAULT_CLIENT_HUB_FE5 = 0x00000052 +NV_PFAULT_CLIENT_HUB_FE6 = 0x00000053 +NV_PFAULT_CLIENT_HUB_FE7 = 0x00000054 +NV_PFAULT_CLIENT_HUB_FECS1 = 0x00000055 +NV_PFAULT_CLIENT_HUB_FECS2 = 0x00000056 +NV_PFAULT_CLIENT_HUB_FECS3 = 0x00000057 +NV_PFAULT_CLIENT_HUB_FECS4 = 0x00000058 +NV_PFAULT_CLIENT_HUB_FECS5 = 0x00000059 +NV_PFAULT_CLIENT_HUB_FECS6 = 0x0000005A +NV_PFAULT_CLIENT_HUB_FECS7 = 0x0000005B +NV_PFAULT_CLIENT_HUB_SKED1 = 0x0000005C +NV_PFAULT_CLIENT_HUB_SKED2 = 0x0000005D +NV_PFAULT_CLIENT_HUB_SKED3 = 0x0000005E +NV_PFAULT_CLIENT_HUB_SKED4 = 0x0000005F +NV_PFAULT_CLIENT_HUB_SKED5 = 0x00000060 +NV_PFAULT_CLIENT_HUB_SKED6 = 0x00000061 +NV_PFAULT_CLIENT_HUB_SKED7 = 0x00000062 +NV_PFAULT_CLIENT_HUB_ESC = 0x00000063 +NV_PFAULT_ACCESS_TYPE = (19, 16) +NV_PFAULT_ACCESS_TYPE_READ = 0x00000000 +NV_PFAULT_ACCESS_TYPE_WRITE = 0x00000001 +NV_PFAULT_ACCESS_TYPE_ATOMIC = 0x00000002 +NV_PFAULT_ACCESS_TYPE_PREFETCH = 0x00000003 +NV_PFAULT_ACCESS_TYPE_VIRT_READ = 0x00000000 +NV_PFAULT_ACCESS_TYPE_VIRT_WRITE = 0x00000001 +NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC = 0x00000002 +NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_STRONG = 0x00000002 +NV_PFAULT_ACCESS_TYPE_VIRT_PREFETCH = 0x00000003 +NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_WEAK = 0x00000004 +NV_PFAULT_ACCESS_TYPE_PHYS_READ = 0x00000008 +NV_PFAULT_ACCESS_TYPE_PHYS_WRITE = 0x00000009 +NV_PFAULT_ACCESS_TYPE_PHYS_ATOMIC = 0x0000000a +NV_PFAULT_ACCESS_TYPE_PHYS_PREFETCH = 0x0000000b +NV_PFAULT_MMU_CLIENT_TYPE = (20, 20) +NV_PFAULT_MMU_CLIENT_TYPE_GPC = 0x00000000 +NV_PFAULT_MMU_CLIENT_TYPE_HUB = 0x00000001 +NV_PFAULT_GPC_ID = (28, 24) +NV_PFAULT_PROTECTED_MODE = (29, 29) +NV_PFAULT_REPLAYABLE_FAULT_EN = (30, 30) +NV_PFAULT_VALID = (31, 31) +NV_ESC_RM_ALLOC_MEMORY = 0x27 +NV_ESC_RM_ALLOC_OBJECT = 0x28 +NV_ESC_RM_FREE = 0x29 +NV_ESC_RM_CONTROL = 0x2A +NV_ESC_RM_ALLOC = 0x2B +NV_ESC_RM_DUP_OBJECT = 0x34 +NV_ESC_RM_SHARE = 0x35 +NV_ESC_RM_I2C_ACCESS = 0x39 +NV_ESC_RM_IDLE_CHANNELS = 0x41 +NV_ESC_RM_VID_HEAP_CONTROL = 0x4A +NV_ESC_RM_ACCESS_REGISTRY = 0x4D +NV_ESC_RM_MAP_MEMORY = 0x4E +NV_ESC_RM_UNMAP_MEMORY = 0x4F +NV_ESC_RM_GET_EVENT_DATA = 0x52 +NV_ESC_RM_ALLOC_CONTEXT_DMA2 = 0x54 +NV_ESC_RM_ADD_VBLANK_CALLBACK = 0x56 +NV_ESC_RM_MAP_MEMORY_DMA = 0x57 +NV_ESC_RM_UNMAP_MEMORY_DMA = 0x58 +NV_ESC_RM_BIND_CONTEXT_DMA = 0x59 +NV_ESC_RM_EXPORT_OBJECT_TO_FD = 0x5C +NV_ESC_RM_IMPORT_OBJECT_FROM_FD = 0x5D +NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO = 0x5E +NV_ESC_RM_LOCKLESS_DIAGNOSTIC = 0x5F +NV_RM_API_VERSION_STRING_LENGTH = 64 +NV_RM_API_VERSION_CMD_STRICT = 0 +NV_RM_API_VERSION_CMD_RELAXED = '1' +NV_RM_API_VERSION_CMD_QUERY = '2' +NV_RM_API_VERSION_REPLY_UNRECOGNIZED = 0 +NV_RM_API_VERSION_REPLY_RECOGNIZED = 1 +NV_DMABUF_EXPORT_MAX_HANDLES = 128 +NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT = 0 +NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE = 1 +NV_IOCTL_MAGIC = 'F' +NV_IOCTL_BASE = 200 +NV_ESC_CARD_INFO = (NV_IOCTL_BASE + 0) +NV_ESC_REGISTER_FD = (NV_IOCTL_BASE + 1) +NV_ESC_ALLOC_OS_EVENT = (NV_IOCTL_BASE + 6) +NV_ESC_FREE_OS_EVENT = (NV_IOCTL_BASE + 7) +NV_ESC_STATUS_CODE = (NV_IOCTL_BASE + 9) +NV_ESC_CHECK_VERSION_STR = (NV_IOCTL_BASE + 10) +NV_ESC_IOCTL_XFER_CMD = (NV_IOCTL_BASE + 11) +NV_ESC_ATTACH_GPUS_TO_FD = (NV_IOCTL_BASE + 12) +NV_ESC_QUERY_DEVICE_INTR = (NV_IOCTL_BASE + 13) +NV_ESC_SYS_PARAMS = (NV_IOCTL_BASE + 14) +NV_ESC_EXPORT_TO_DMABUF_FD = (NV_IOCTL_BASE + 17) +NV_ESC_WAIT_OPEN_COMPLETE = (NV_IOCTL_BASE + 18) +__aligned = lambda n: __attribute__((aligned(n))) # type: ignore +NV_ESC_NUMA_INFO = (NV_IOCTL_BASE + 15) +NV_ESC_SET_NUMA_STATUS = (NV_IOCTL_BASE + 16) +NV_IOCTL_NUMA_INFO_MAX_OFFLINE_ADDRESSES = 64 +NV_IOCTL_NUMA_STATUS_DISABLED = 0 +NV_IOCTL_NUMA_STATUS_OFFLINE = 1 +NV_IOCTL_NUMA_STATUS_ONLINE_IN_PROGRESS = 2 +NV_IOCTL_NUMA_STATUS_ONLINE = 3 +NV_IOCTL_NUMA_STATUS_ONLINE_FAILED = 4 +NV_IOCTL_NUMA_STATUS_OFFLINE_IN_PROGRESS = 5 +NV_IOCTL_NUMA_STATUS_OFFLINE_FAILED = 6 +NVOS04_FLAGS_CHANNEL_TYPE = (1, 0) +NVOS04_FLAGS_CHANNEL_TYPE_PHYSICAL = 0x00000000 +NVOS04_FLAGS_CHANNEL_TYPE_VIRTUAL = 0x00000001 +NVOS04_FLAGS_CHANNEL_TYPE_PHYSICAL_FOR_VIRTUAL = 0x00000002 +NVOS04_FLAGS_VPR = (2, 2) +NVOS04_FLAGS_VPR_FALSE = 0x00000000 +NVOS04_FLAGS_VPR_TRUE = 0x00000001 +NVOS04_FLAGS_CC_SECURE = (2, 2) +NVOS04_FLAGS_CC_SECURE_FALSE = 0x00000000 +NVOS04_FLAGS_CC_SECURE_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_SKIP_MAP_REFCOUNTING = (3, 3) +NVOS04_FLAGS_CHANNEL_SKIP_MAP_REFCOUNTING_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_SKIP_MAP_REFCOUNTING_TRUE = 0x00000001 +NVOS04_FLAGS_GROUP_CHANNEL_RUNQUEUE = (4, 4) +NVOS04_FLAGS_GROUP_CHANNEL_RUNQUEUE_DEFAULT = 0x00000000 +NVOS04_FLAGS_GROUP_CHANNEL_RUNQUEUE_ONE = 0x00000001 +NVOS04_FLAGS_PRIVILEGED_CHANNEL = (5, 5) +NVOS04_FLAGS_PRIVILEGED_CHANNEL_FALSE = 0x00000000 +NVOS04_FLAGS_PRIVILEGED_CHANNEL_TRUE = 0x00000001 +NVOS04_FLAGS_DELAY_CHANNEL_SCHEDULING = (6, 6) +NVOS04_FLAGS_DELAY_CHANNEL_SCHEDULING_FALSE = 0x00000000 +NVOS04_FLAGS_DELAY_CHANNEL_SCHEDULING_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_DENY_PHYSICAL_MODE_CE = (7, 7) +NVOS04_FLAGS_CHANNEL_DENY_PHYSICAL_MODE_CE_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_DENY_PHYSICAL_MODE_CE_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_USERD_INDEX_VALUE = (10, 8) +NVOS04_FLAGS_CHANNEL_USERD_INDEX_FIXED = (11, 11) +NVOS04_FLAGS_CHANNEL_USERD_INDEX_FIXED_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_USERD_INDEX_FIXED_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_USERD_INDEX_PAGE_VALUE = (20, 12) +NVOS04_FLAGS_CHANNEL_USERD_INDEX_PAGE_FIXED = (21, 21) +NVOS04_FLAGS_CHANNEL_USERD_INDEX_PAGE_FIXED_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_USERD_INDEX_PAGE_FIXED_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_DENY_AUTH_LEVEL_PRIV = (22, 22) +NVOS04_FLAGS_CHANNEL_DENY_AUTH_LEVEL_PRIV_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_DENY_AUTH_LEVEL_PRIV_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER = (23, 23) +NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_CLIENT_MAP_FIFO = (24, 24) +NVOS04_FLAGS_CHANNEL_CLIENT_MAP_FIFO_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_CLIENT_MAP_FIFO_TRUE = 0x00000001 +NVOS04_FLAGS_SET_EVICT_LAST_CE_PREFETCH_CHANNEL = (25, 25) +NVOS04_FLAGS_SET_EVICT_LAST_CE_PREFETCH_CHANNEL_FALSE = 0x00000000 +NVOS04_FLAGS_SET_EVICT_LAST_CE_PREFETCH_CHANNEL_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_VGPU_PLUGIN_CONTEXT = (26, 26) +NVOS04_FLAGS_CHANNEL_VGPU_PLUGIN_CONTEXT_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_VGPU_PLUGIN_CONTEXT_TRUE = 0x00000001 +NVOS04_FLAGS_CHANNEL_PBDMA_ACQUIRE_TIMEOUT = (27, 27) +NVOS04_FLAGS_CHANNEL_PBDMA_ACQUIRE_TIMEOUT_FALSE = 0x00000000 +NVOS04_FLAGS_CHANNEL_PBDMA_ACQUIRE_TIMEOUT_TRUE = 0x00000001 +NVOS04_FLAGS_GROUP_CHANNEL_THREAD = (29, 28) +NVOS04_FLAGS_GROUP_CHANNEL_THREAD_DEFAULT = 0x00000000 +NVOS04_FLAGS_GROUP_CHANNEL_THREAD_ONE = 0x00000001 +NVOS04_FLAGS_GROUP_CHANNEL_THREAD_TWO = 0x00000002 +NVOS04_FLAGS_MAP_CHANNEL = (30, 30) +NVOS04_FLAGS_MAP_CHANNEL_FALSE = 0x00000000 +NVOS04_FLAGS_MAP_CHANNEL_TRUE = 0x00000001 +NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC = (31, 31) +NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC_FALSE = 0x00000000 +NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC_TRUE = 0x00000001 +CC_CHAN_ALLOC_IV_SIZE_DWORD = 3 +CC_CHAN_ALLOC_NONCE_SIZE_DWORD = 8 +NV_CHANNEL_ALLOC_PARAMS_MESSAGE_ID = (0x906f) +FILE_DEVICE_NV = 0x00008000 +NV_IOCTL_FCT_BASE = 0x00000800 +NVOS_MAX_SUBDEVICES = 8 +UNIFIED_NV_STATUS = 1 +NVOS_STATUS = NV_STATUS +NVOS_STATUS_SUCCESS = NV_OK +NVOS_STATUS_ERROR_CARD_NOT_PRESENT = NV_ERR_CARD_NOT_PRESENT +NVOS_STATUS_ERROR_DUAL_LINK_INUSE = NV_ERR_DUAL_LINK_INUSE +NVOS_STATUS_ERROR_GENERIC = NV_ERR_GENERIC +NVOS_STATUS_ERROR_GPU_NOT_FULL_POWER = NV_ERR_GPU_NOT_FULL_POWER +NVOS_STATUS_ERROR_ILLEGAL_ACTION = NV_ERR_ILLEGAL_ACTION +NVOS_STATUS_ERROR_IN_USE = NV_ERR_STATE_IN_USE +NVOS_STATUS_ERROR_INSUFFICIENT_RESOURCES = NV_ERR_INSUFFICIENT_RESOURCES +NVOS_STATUS_ERROR_INSUFFICIENT_ZBC_ENTRY = NV_ERR_INSUFFICIENT_ZBC_ENTRY +NVOS_STATUS_ERROR_INVALID_ACCESS_TYPE = NV_ERR_INVALID_ACCESS_TYPE +NVOS_STATUS_ERROR_INVALID_ARGUMENT = NV_ERR_INVALID_ARGUMENT +NVOS_STATUS_ERROR_INVALID_BASE = NV_ERR_INVALID_BASE +NVOS_STATUS_ERROR_INVALID_CHANNEL = NV_ERR_INVALID_CHANNEL +NVOS_STATUS_ERROR_INVALID_CLASS = NV_ERR_INVALID_CLASS +NVOS_STATUS_ERROR_INVALID_CLIENT = NV_ERR_INVALID_CLIENT +NVOS_STATUS_ERROR_INVALID_COMMAND = NV_ERR_INVALID_COMMAND +NVOS_STATUS_ERROR_INVALID_DATA = NV_ERR_INVALID_DATA +NVOS_STATUS_ERROR_INVALID_DEVICE = NV_ERR_INVALID_DEVICE +NVOS_STATUS_ERROR_INVALID_DMA_SPECIFIER = NV_ERR_INVALID_DMA_SPECIFIER +NVOS_STATUS_ERROR_INVALID_EVENT = NV_ERR_INVALID_EVENT +NVOS_STATUS_ERROR_INVALID_FLAGS = NV_ERR_INVALID_FLAGS +NVOS_STATUS_ERROR_INVALID_FUNCTION = NV_ERR_INVALID_FUNCTION +NVOS_STATUS_ERROR_INVALID_HEAP = NV_ERR_INVALID_HEAP +NVOS_STATUS_ERROR_INVALID_INDEX = NV_ERR_INVALID_INDEX +NVOS_STATUS_ERROR_INVALID_LIMIT = NV_ERR_INVALID_LIMIT +NVOS_STATUS_ERROR_INVALID_METHOD = NV_ERR_INVALID_METHOD +NVOS_STATUS_ERROR_INVALID_OBJECT_BUFFER = NV_ERR_BUFFER_TOO_SMALL +NVOS_STATUS_ERROR_INVALID_OBJECT_ERROR = NV_ERR_INVALID_OBJECT +NVOS_STATUS_ERROR_INVALID_OBJECT_HANDLE = NV_ERR_INVALID_OBJECT_HANDLE +NVOS_STATUS_ERROR_INVALID_OBJECT_NEW = NV_ERR_INVALID_OBJECT_NEW +NVOS_STATUS_ERROR_INVALID_OBJECT_OLD = NV_ERR_INVALID_OBJECT_OLD +NVOS_STATUS_ERROR_INVALID_OBJECT_PARENT = NV_ERR_INVALID_OBJECT_PARENT +NVOS_STATUS_ERROR_INVALID_OFFSET = NV_ERR_INVALID_OFFSET +NVOS_STATUS_ERROR_INVALID_OWNER = NV_ERR_INVALID_OWNER +NVOS_STATUS_ERROR_INVALID_PARAM_STRUCT = NV_ERR_INVALID_PARAM_STRUCT +NVOS_STATUS_ERROR_INVALID_PARAMETER = NV_ERR_INVALID_PARAMETER +NVOS_STATUS_ERROR_INVALID_POINTER = NV_ERR_INVALID_POINTER +NVOS_STATUS_ERROR_INVALID_REGISTRY_KEY = NV_ERR_INVALID_REGISTRY_KEY +NVOS_STATUS_ERROR_INVALID_STATE = NV_ERR_INVALID_STATE +NVOS_STATUS_ERROR_INVALID_STRING_LENGTH = NV_ERR_INVALID_STRING_LENGTH +NVOS_STATUS_ERROR_INVALID_XLATE = NV_ERR_INVALID_XLATE +NVOS_STATUS_ERROR_IRQ_NOT_FIRING = NV_ERR_IRQ_NOT_FIRING +NVOS_STATUS_ERROR_MULTIPLE_MEMORY_TYPES = NV_ERR_MULTIPLE_MEMORY_TYPES +NVOS_STATUS_ERROR_NOT_SUPPORTED = NV_ERR_NOT_SUPPORTED +NVOS_STATUS_ERROR_OPERATING_SYSTEM = NV_ERR_OPERATING_SYSTEM +NVOS_STATUS_ERROR_LIB_RM_VERSION_MISMATCH = NV_ERR_LIB_RM_VERSION_MISMATCH +NVOS_STATUS_ERROR_PROTECTION_FAULT = NV_ERR_PROTECTION_FAULT +NVOS_STATUS_ERROR_TIMEOUT = NV_ERR_TIMEOUT +NVOS_STATUS_ERROR_TOO_MANY_PRIMARIES = NV_ERR_TOO_MANY_PRIMARIES +NVOS_STATUS_ERROR_IRQ_EDGE_TRIGGERED = NV_ERR_IRQ_EDGE_TRIGGERED +NVOS_STATUS_ERROR_INVALID_OPERATION = NV_ERR_INVALID_OPERATION +NVOS_STATUS_ERROR_NOT_COMPATIBLE = NV_ERR_NOT_COMPATIBLE +NVOS_STATUS_ERROR_MORE_PROCESSING_REQUIRED = NV_WARN_MORE_PROCESSING_REQUIRED +NVOS_STATUS_ERROR_INSUFFICIENT_PERMISSIONS = NV_ERR_INSUFFICIENT_PERMISSIONS +NVOS_STATUS_ERROR_TIMEOUT_RETRY = NV_ERR_TIMEOUT_RETRY +NVOS_STATUS_ERROR_NOT_READY = NV_ERR_NOT_READY +NVOS_STATUS_ERROR_GPU_IS_LOST = NV_ERR_GPU_IS_LOST +NVOS_STATUS_ERROR_IN_FULLCHIP_RESET = NV_ERR_GPU_IN_FULLCHIP_RESET +NVOS_STATUS_ERROR_INVALID_LOCK_STATE = NV_ERR_INVALID_LOCK_STATE +NVOS_STATUS_ERROR_INVALID_ADDRESS = NV_ERR_INVALID_ADDRESS +NVOS_STATUS_ERROR_INVALID_IRQ_LEVEL = NV_ERR_INVALID_IRQ_LEVEL +NVOS_STATUS_ERROR_MEMORY_TRAINING_FAILED = NV_ERR_MEMORY_TRAINING_FAILED +NVOS_STATUS_ERROR_BUSY_RETRY = NV_ERR_BUSY_RETRY +NVOS_STATUS_ERROR_INSUFFICIENT_POWER = NV_ERR_INSUFFICIENT_POWER +NVOS_STATUS_ERROR_OBJECT_NOT_FOUND = NV_ERR_OBJECT_NOT_FOUND +NVOS_STATUS_ERROR_RESOURCE_LOST = NV_ERR_RESOURCE_LOST +NVOS_STATUS_ERROR_BUFFER_TOO_SMALL = NV_ERR_BUFFER_TOO_SMALL +NVOS_STATUS_ERROR_RESET_REQUIRED = NV_ERR_RESET_REQUIRED +NVOS_STATUS_ERROR_INVALID_REQUEST = NV_ERR_INVALID_REQUEST +NVOS_STATUS_ERROR_PRIV_SEC_VIOLATION = NV_ERR_PRIV_SEC_VIOLATION +NVOS_STATUS_ERROR_GPU_IN_DEBUG_MODE = NV_ERR_GPU_IN_DEBUG_MODE +NVOS_STATUS_ERROR_ALREADY_SIGNALLED = NV_ERR_ALREADY_SIGNALLED +NV01_FREE = (0x00000000) +NV01_ROOT = (0x0) +NV01_ROOT_NON_PRIV = (0x00000001) +NV01_ROOT_CLIENT = (0x00000041) +NV01_ALLOC_MEMORY = (0x00000002) +NVOS02_FLAGS_PHYSICALITY = (7, 4) +NVOS02_FLAGS_PHYSICALITY_CONTIGUOUS = (0x00000000) +NVOS02_FLAGS_PHYSICALITY_NONCONTIGUOUS = (0x00000001) +NVOS02_FLAGS_LOCATION = (11, 8) +NVOS02_FLAGS_LOCATION_PCI = (0x00000000) +NVOS02_FLAGS_LOCATION_VIDMEM = (0x00000002) +NVOS02_FLAGS_COHERENCY = (15, 12) +NVOS02_FLAGS_COHERENCY_UNCACHED = (0x00000000) +NVOS02_FLAGS_COHERENCY_CACHED = (0x00000001) +NVOS02_FLAGS_COHERENCY_WRITE_COMBINE = (0x00000002) +NVOS02_FLAGS_COHERENCY_WRITE_THROUGH = (0x00000003) +NVOS02_FLAGS_COHERENCY_WRITE_PROTECT = (0x00000004) +NVOS02_FLAGS_COHERENCY_WRITE_BACK = (0x00000005) +NVOS02_FLAGS_ALLOC = (17, 16) +NVOS02_FLAGS_ALLOC_NONE = (0x00000001) +NVOS02_FLAGS_GPU_CACHEABLE = (18, 18) +NVOS02_FLAGS_GPU_CACHEABLE_NO = (0x00000000) +NVOS02_FLAGS_GPU_CACHEABLE_YES = (0x00000001) +NVOS02_FLAGS_KERNEL_MAPPING = (19, 19) +NVOS02_FLAGS_KERNEL_MAPPING_NO_MAP = (0x00000000) +NVOS02_FLAGS_KERNEL_MAPPING_MAP = (0x00000001) +NVOS02_FLAGS_ALLOC_NISO_DISPLAY = (20, 20) +NVOS02_FLAGS_ALLOC_NISO_DISPLAY_NO = (0x00000000) +NVOS02_FLAGS_ALLOC_NISO_DISPLAY_YES = (0x00000001) +NVOS02_FLAGS_ALLOC_USER_READ_ONLY = (21, 21) +NVOS02_FLAGS_ALLOC_USER_READ_ONLY_NO = (0x00000000) +NVOS02_FLAGS_ALLOC_USER_READ_ONLY_YES = (0x00000001) +NVOS02_FLAGS_ALLOC_DEVICE_READ_ONLY = (22, 22) +NVOS02_FLAGS_ALLOC_DEVICE_READ_ONLY_NO = (0x00000000) +NVOS02_FLAGS_ALLOC_DEVICE_READ_ONLY_YES = (0x00000001) +NVOS02_FLAGS_PEER_MAP_OVERRIDE = (23, 23) +NVOS02_FLAGS_PEER_MAP_OVERRIDE_DEFAULT = (0x00000000) +NVOS02_FLAGS_PEER_MAP_OVERRIDE_REQUIRED = (0x00000001) +NVOS02_FLAGS_ALLOC_TYPE_SYNCPOINT = (24, 24) +NVOS02_FLAGS_ALLOC_TYPE_SYNCPOINT_APERTURE = (0x00000001) +NVOS02_FLAGS_MEMORY_PROTECTION = (26, 25) +NVOS02_FLAGS_MEMORY_PROTECTION_DEFAULT = (0x00000000) +NVOS02_FLAGS_MEMORY_PROTECTION_PROTECTED = (0x00000001) +NVOS02_FLAGS_MEMORY_PROTECTION_UNPROTECTED = (0x00000002) +NVOS02_FLAGS_REGISTER_MEMDESC_TO_PHYS_RM = (27, 27) +NVOS02_FLAGS_REGISTER_MEMDESC_TO_PHYS_RM_FALSE = (0x00000000) +NVOS02_FLAGS_REGISTER_MEMDESC_TO_PHYS_RM_TRUE = (0x00000001) +NVOS02_FLAGS_MAPPING = (31, 30) +NVOS02_FLAGS_MAPPING_DEFAULT = (0x00000000) +NVOS02_FLAGS_MAPPING_NO_MAP = (0x00000001) +NVOS02_FLAGS_MAPPING_NEVER_MAP = (0x00000002) +NVOS03_FLAGS_ACCESS = (1, 0) +NVOS03_FLAGS_ACCESS_READ_WRITE = (0x00000000) +NVOS03_FLAGS_ACCESS_READ_ONLY = (0x00000001) +NVOS03_FLAGS_ACCESS_WRITE_ONLY = (0x00000002) +NVOS03_FLAGS_PREALLOCATE = (2, 2) +NVOS03_FLAGS_PREALLOCATE_DISABLE = (0x00000000) +NVOS03_FLAGS_PREALLOCATE_ENABLE = (0x00000001) +NVOS03_FLAGS_GPU_MAPPABLE = (15, 15) +NVOS03_FLAGS_GPU_MAPPABLE_DISABLE = (0x00000000) +NVOS03_FLAGS_GPU_MAPPABLE_ENABLE = (0x00000001) +NVOS03_FLAGS_PTE_KIND_BL_OVERRIDE = (16, 16) +NVOS03_FLAGS_PTE_KIND_BL_OVERRIDE_FALSE = (0x00000000) +NVOS03_FLAGS_PTE_KIND_BL_OVERRIDE_TRUE = (0x00000001) +NVOS03_FLAGS_PTE_KIND = (17, 16) +NVOS03_FLAGS_PTE_KIND_NONE = (0x00000000) +NVOS03_FLAGS_PTE_KIND_BL = (0x00000001) +NVOS03_FLAGS_PTE_KIND_PITCH = (0x00000002) +NVOS03_FLAGS_TYPE = (23, 20) +NVOS03_FLAGS_TYPE_NOTIFIER = (0x00000001) +NVOS03_FLAGS_MAPPING = (20, 20) +NVOS03_FLAGS_MAPPING_NONE = (0x00000000) +NVOS03_FLAGS_MAPPING_KERNEL = (0x00000001) +NVOS03_FLAGS_CACHE_SNOOP = (28, 28) +NVOS03_FLAGS_CACHE_SNOOP_ENABLE = (0x00000000) +NVOS03_FLAGS_CACHE_SNOOP_DISABLE = (0x00000001) +NVOS03_FLAGS_HASH_TABLE = (29, 29) +NVOS03_FLAGS_HASH_TABLE_ENABLE = (0x00000000) +NVOS03_FLAGS_HASH_TABLE_DISABLE = (0x00000001) +NV01_ALLOC_OBJECT = (0x00000005) +NV01_EVENT_KERNEL_CALLBACK = (0x00000078) +NV01_EVENT_OS_EVENT = (0x00000079) +NV01_EVENT_WIN32_EVENT = NV01_EVENT_OS_EVENT +NV01_EVENT_KERNEL_CALLBACK_EX = (0x0000007E) +NV01_EVENT_BROADCAST = (0x80000000) +NV01_EVENT_PERMIT_NON_ROOT_EVENT_KERNEL_CALLBACK_CREATION = (0x40000000) +NV01_EVENT_SUBDEVICE_SPECIFIC = (0x20000000) +NV01_EVENT_WITHOUT_EVENT_DATA = (0x10000000) +NV01_EVENT_NONSTALL_INTR = (0x08000000) +NV01_EVENT_CLIENT_RM = (0x04000000) +NV04_I2C_ACCESS = (0x00000013) +NVOS_I2C_ACCESS_MAX_BUFFER_SIZE = 2048 +NVOS20_COMMAND_unused0001 = 0x0001 +NVOS20_COMMAND_unused0002 = 0x0002 +NVOS20_COMMAND_STRING_PRINT = 0x0003 +NV04_ALLOC = (0x00000015) +NVOS64_FLAGS_NONE = (0x00000000) +NVOS64_FLAGS_FINN_SERIALIZED = (0x00000001) +NVOS65_PARAMETERS_VERSION_MAGIC = 0x77FEF81E +NV04_IDLE_CHANNELS = (0x0000001E) +NVOS30_FLAGS_BEHAVIOR = (3, 0) +NVOS30_FLAGS_BEHAVIOR_SPIN = (0x00000000) +NVOS30_FLAGS_BEHAVIOR_SLEEP = (0x00000001) +NVOS30_FLAGS_BEHAVIOR_QUERY = (0x00000002) +NVOS30_FLAGS_BEHAVIOR_FORCE_BUSY_CHECK = (0x00000003) +NVOS30_FLAGS_CHANNEL = (7, 4) +NVOS30_FLAGS_CHANNEL_LIST = (0x00000000) +NVOS30_FLAGS_CHANNEL_SINGLE = (0x00000001) +NVOS30_FLAGS_IDLE = (30, 8) +NVOS30_FLAGS_IDLE_PUSH_BUFFER = (0x00000001) +NVOS30_FLAGS_IDLE_CACHE1 = (0x00000002) +NVOS30_FLAGS_IDLE_GRAPHICS = (0x00000004) +NVOS30_FLAGS_IDLE_MPEG = (0x00000008) +NVOS30_FLAGS_IDLE_MOTION_ESTIMATION = (0x00000010) +NVOS30_FLAGS_IDLE_VIDEO_PROCESSOR = (0x00000020) +NVOS30_FLAGS_IDLE_MSPDEC = (0x00000020) +NVOS30_FLAGS_IDLE_BITSTREAM_PROCESSOR = (0x00000040) +NVOS30_FLAGS_IDLE_MSVLD = (0x00000040) +NVOS30_FLAGS_IDLE_NVDEC0 = NVOS30_FLAGS_IDLE_MSVLD +NVOS30_FLAGS_IDLE_CIPHER_DMA = (0x00000080) +NVOS30_FLAGS_IDLE_SEC = (0x00000080) +NVOS30_FLAGS_IDLE_CALLBACKS = (0x00000100) +NVOS30_FLAGS_IDLE_MSPPP = (0x00000200) +NVOS30_FLAGS_IDLE_CE0 = (0x00000400) +NVOS30_FLAGS_IDLE_CE1 = (0x00000800) +NVOS30_FLAGS_IDLE_CE2 = (0x00001000) +NVOS30_FLAGS_IDLE_CE3 = (0x00002000) +NVOS30_FLAGS_IDLE_CE4 = (0x00004000) +NVOS30_FLAGS_IDLE_CE5 = (0x00008000) +NVOS30_FLAGS_IDLE_VIC = (0x00010000) +NVOS30_FLAGS_IDLE_MSENC = (0x00020000) +NVOS30_FLAGS_IDLE_NVENC0 = NVOS30_FLAGS_IDLE_MSENC +NVOS30_FLAGS_IDLE_NVENC1 = (0x00040000) +NVOS30_FLAGS_IDLE_NVENC2 = (0x00080000) +NVOS30_FLAGS_IDLE_NVJPG = (0x00100000) +NVOS30_FLAGS_IDLE_NVDEC1 = (0x00200000) +NVOS30_FLAGS_IDLE_NVDEC2 = (0x00400000) +NVOS30_FLAGS_IDLE_ACTIVECHANNELS = (0x00800000) +NVOS30_FLAGS_IDLE_ALL_ENGINES = (NVOS30_FLAGS_IDLE_GRAPHICS | NVOS30_FLAGS_IDLE_MPEG | NVOS30_FLAGS_IDLE_MOTION_ESTIMATION | NVOS30_FLAGS_IDLE_VIDEO_PROCESSOR | NVOS30_FLAGS_IDLE_BITSTREAM_PROCESSOR | NVOS30_FLAGS_IDLE_CIPHER_DMA | NVOS30_FLAGS_IDLE_MSPDEC | NVOS30_FLAGS_IDLE_NVDEC0 | NVOS30_FLAGS_IDLE_SEC | NVOS30_FLAGS_IDLE_MSPPP | NVOS30_FLAGS_IDLE_CE0 | NVOS30_FLAGS_IDLE_CE1 | NVOS30_FLAGS_IDLE_CE2 | NVOS30_FLAGS_IDLE_CE3 | NVOS30_FLAGS_IDLE_CE4 | NVOS30_FLAGS_IDLE_CE5 | NVOS30_FLAGS_IDLE_NVENC0 | NVOS30_FLAGS_IDLE_NVENC1 | NVOS30_FLAGS_IDLE_NVENC2 | NVOS30_FLAGS_IDLE_VIC | NVOS30_FLAGS_IDLE_NVJPG | NVOS30_FLAGS_IDLE_NVDEC1 | NVOS30_FLAGS_IDLE_NVDEC2) +NVOS30_FLAGS_WAIT_FOR_ELPG_ON = (31, 31) +NVOS30_FLAGS_WAIT_FOR_ELPG_ON_NO = (0x00000000) +NVOS30_FLAGS_WAIT_FOR_ELPG_ON_YES = (0x00000001) +NV04_VID_HEAP_CONTROL = (0x00000020) +NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS = 0 +NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY = 1 +NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY = 2 +NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR = 3 +NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE = 4 +NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR = 5 +NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR = 6 +NVOS32_DESCRIPTOR_TYPE_KERNEL_VIRTUAL_ADDRESS = 7 +NVOS32_FUNCTION_ALLOC_SIZE = 2 +NVOS32_FUNCTION_FREE = 3 +NVOS32_FUNCTION_INFO = 5 +NVOS32_FUNCTION_ALLOC_TILED_PITCH_HEIGHT = 6 +NVOS32_FUNCTION_ALLOC_SIZE_RANGE = 14 +NVOS32_FUNCTION_REACQUIRE_COMPR = 15 +NVOS32_FUNCTION_RELEASE_COMPR = 16 +NVOS32_FUNCTION_GET_MEM_ALIGNMENT = 18 +NVOS32_FUNCTION_HW_ALLOC = 19 +NVOS32_FUNCTION_HW_FREE = 20 +NVOS32_FUNCTION_ALLOC_OS_DESCRIPTOR = 27 +NVOS32_IVC_HEAP_NUMBER_DONT_ALLOCATE_ON_IVC_HEAP = 0 +NV_RM_OS32_ALLOC_OS_DESCRIPTOR_WITH_OS32_ATTR = 1 +NVOS32_DELETE_RESOURCES_ALL = 0 +NVOS32_TYPE_IMAGE = 0 +NVOS32_TYPE_DEPTH = 1 +NVOS32_TYPE_TEXTURE = 2 +NVOS32_TYPE_VIDEO = 3 +NVOS32_TYPE_FONT = 4 +NVOS32_TYPE_CURSOR = 5 +NVOS32_TYPE_DMA = 6 +NVOS32_TYPE_INSTANCE = 7 +NVOS32_TYPE_PRIMARY = 8 +NVOS32_TYPE_ZCULL = 9 +NVOS32_TYPE_UNUSED = 10 +NVOS32_TYPE_SHADER_PROGRAM = 11 +NVOS32_TYPE_OWNER_RM = 12 +NVOS32_TYPE_NOTIFIER = 13 +NVOS32_TYPE_RESERVED = 14 +NVOS32_TYPE_PMA = 15 +NVOS32_TYPE_STENCIL = 16 +NVOS32_TYPE_SYNCPOINT = 17 +NVOS32_NUM_MEM_TYPES = 18 +NVOS32_ATTR_NONE = 0x00000000 +NVOS32_ATTR_DEPTH = (2, 0) +NVOS32_ATTR_DEPTH_UNKNOWN = 0x00000000 +NVOS32_ATTR_DEPTH_8 = 0x00000001 +NVOS32_ATTR_DEPTH_16 = 0x00000002 +NVOS32_ATTR_DEPTH_24 = 0x00000003 +NVOS32_ATTR_DEPTH_32 = 0x00000004 +NVOS32_ATTR_DEPTH_64 = 0x00000005 +NVOS32_ATTR_DEPTH_128 = 0x00000006 +NVOS32_ATTR_COMPR_COVG = (3, 3) +NVOS32_ATTR_COMPR_COVG_DEFAULT = 0x00000000 +NVOS32_ATTR_COMPR_COVG_PROVIDED = 0x00000001 +NVOS32_ATTR_AA_SAMPLES = (7, 4) +NVOS32_ATTR_AA_SAMPLES_1 = 0x00000000 +NVOS32_ATTR_AA_SAMPLES_2 = 0x00000001 +NVOS32_ATTR_AA_SAMPLES_4 = 0x00000002 +NVOS32_ATTR_AA_SAMPLES_4_ROTATED = 0x00000003 +NVOS32_ATTR_AA_SAMPLES_6 = 0x00000004 +NVOS32_ATTR_AA_SAMPLES_8 = 0x00000005 +NVOS32_ATTR_AA_SAMPLES_16 = 0x00000006 +NVOS32_ATTR_AA_SAMPLES_4_VIRTUAL_8 = 0x00000007 +NVOS32_ATTR_AA_SAMPLES_4_VIRTUAL_16 = 0x00000008 +NVOS32_ATTR_AA_SAMPLES_8_VIRTUAL_16 = 0x00000009 +NVOS32_ATTR_AA_SAMPLES_8_VIRTUAL_32 = 0x0000000A +NVOS32_ATTR_GPU_CACHE_SNOOPABLE = (9, 8) +NVOS32_ATTR_GPU_CACHE_SNOOPABLE_MAPPING = 0x00000000 +NVOS32_ATTR_GPU_CACHE_SNOOPABLE_OFF = 0x00000001 +NVOS32_ATTR_GPU_CACHE_SNOOPABLE_ON = 0x00000002 +NVOS32_ATTR_GPU_CACHE_SNOOPABLE_INVALID = 0x00000003 +NVOS32_ATTR_ZCULL = (11, 10) +NVOS32_ATTR_ZCULL_NONE = 0x00000000 +NVOS32_ATTR_ZCULL_REQUIRED = 0x00000001 +NVOS32_ATTR_ZCULL_ANY = 0x00000002 +NVOS32_ATTR_ZCULL_SHARED = 0x00000003 +NVOS32_ATTR_COMPR = (13, 12) +NVOS32_ATTR_COMPR_NONE = 0x00000000 +NVOS32_ATTR_COMPR_REQUIRED = 0x00000001 +NVOS32_ATTR_COMPR_ANY = 0x00000002 +NVOS32_ATTR_COMPR_PLC_REQUIRED = NVOS32_ATTR_COMPR_REQUIRED +NVOS32_ATTR_COMPR_PLC_ANY = NVOS32_ATTR_COMPR_ANY +NVOS32_ATTR_COMPR_DISABLE_PLC_ANY = 0x00000003 +NVOS32_ATTR_ALLOCATE_FROM_RESERVED_HEAP = (14, 14) +NVOS32_ATTR_ALLOCATE_FROM_RESERVED_HEAP_NO = 0x00000000 +NVOS32_ATTR_ALLOCATE_FROM_RESERVED_HEAP_YES = 0x00000001 +NVOS32_ATTR_FORMAT = (17, 16) +NVOS32_ATTR_FORMAT_LOW_FIELD = 16 +NVOS32_ATTR_FORMAT_HIGH_FIELD = 17 +NVOS32_ATTR_FORMAT_PITCH = 0x00000000 +NVOS32_ATTR_FORMAT_SWIZZLED = 0x00000001 +NVOS32_ATTR_FORMAT_BLOCK_LINEAR = 0x00000002 +NVOS32_ATTR_Z_TYPE = (18, 18) +NVOS32_ATTR_Z_TYPE_FIXED = 0x00000000 +NVOS32_ATTR_Z_TYPE_FLOAT = 0x00000001 +NVOS32_ATTR_ZS_PACKING = (21, 19) +NVOS32_ATTR_ZS_PACKING_S8 = 0x00000000 +NVOS32_ATTR_ZS_PACKING_Z24S8 = 0x00000000 +NVOS32_ATTR_ZS_PACKING_S8Z24 = 0x00000001 +NVOS32_ATTR_ZS_PACKING_Z32 = 0x00000002 +NVOS32_ATTR_ZS_PACKING_Z24X8 = 0x00000003 +NVOS32_ATTR_ZS_PACKING_X8Z24 = 0x00000004 +NVOS32_ATTR_ZS_PACKING_Z32_X24S8 = 0x00000005 +NVOS32_ATTR_ZS_PACKING_X8Z24_X24S8 = 0x00000006 +NVOS32_ATTR_ZS_PACKING_Z16 = 0x00000007 +NVOS32_ATTR_COLOR_PACKING = NVOS32_ATTR_ZS_PACKING +NVOS32_ATTR_COLOR_PACKING_A8R8G8B8 = 0x00000000 +NVOS32_ATTR_COLOR_PACKING_X8R8G8B8 = 0x00000001 +NVOS32_ATTR_PAGE_SIZE = (24, 23) +NVOS32_ATTR_PAGE_SIZE_DEFAULT = 0x00000000 +NVOS32_ATTR_PAGE_SIZE_4KB = 0x00000001 +NVOS32_ATTR_PAGE_SIZE_BIG = 0x00000002 +NVOS32_ATTR_PAGE_SIZE_HUGE = 0x00000003 +NVOS32_ATTR_LOCATION = (26, 25) +NVOS32_ATTR_LOCATION_VIDMEM = 0x00000000 +NVOS32_ATTR_LOCATION_PCI = 0x00000001 +NVOS32_ATTR_LOCATION_ANY = 0x00000003 +NVOS32_ATTR_PHYSICALITY = (28, 27) +NVOS32_ATTR_PHYSICALITY_DEFAULT = 0x00000000 +NVOS32_ATTR_PHYSICALITY_NONCONTIGUOUS = 0x00000001 +NVOS32_ATTR_PHYSICALITY_CONTIGUOUS = 0x00000002 +NVOS32_ATTR_PHYSICALITY_ALLOW_NONCONTIGUOUS = 0x00000003 +NVOS32_ATTR_COHERENCY = (31, 29) +NVOS32_ATTR_COHERENCY_UNCACHED = 0x00000000 +NVOS32_ATTR_COHERENCY_CACHED = 0x00000001 +NVOS32_ATTR_COHERENCY_WRITE_COMBINE = 0x00000002 +NVOS32_ATTR_COHERENCY_WRITE_THROUGH = 0x00000003 +NVOS32_ATTR_COHERENCY_WRITE_PROTECT = 0x00000004 +NVOS32_ATTR_COHERENCY_WRITE_BACK = 0x00000005 +NVOS32_ATTR2_NONE = 0x00000000 +NVOS32_ATTR2_ZBC = (1, 0) +NVOS32_ATTR2_ZBC_DEFAULT = 0x00000000 +NVOS32_ATTR2_ZBC_PREFER_NO_ZBC = 0x00000001 +NVOS32_ATTR2_ZBC_PREFER_ZBC = 0x00000002 +NVOS32_ATTR2_ZBC_REQUIRE_ONLY_ZBC = 0x00000003 +NVOS32_ATTR2_ZBC_INVALID = 0x00000003 +NVOS32_ATTR2_GPU_CACHEABLE = (3, 2) +NVOS32_ATTR2_GPU_CACHEABLE_DEFAULT = 0x00000000 +NVOS32_ATTR2_GPU_CACHEABLE_YES = 0x00000001 +NVOS32_ATTR2_GPU_CACHEABLE_NO = 0x00000002 +NVOS32_ATTR2_GPU_CACHEABLE_INVALID = 0x00000003 +NVOS32_ATTR2_P2P_GPU_CACHEABLE = (5, 4) +NVOS32_ATTR2_P2P_GPU_CACHEABLE_DEFAULT = 0x00000000 +NVOS32_ATTR2_P2P_GPU_CACHEABLE_YES = 0x00000001 +NVOS32_ATTR2_P2P_GPU_CACHEABLE_NO = 0x00000002 +NVOS32_ATTR2_32BIT_POINTER = (6, 6) +NVOS32_ATTR2_32BIT_POINTER_DISABLE = 0x00000000 +NVOS32_ATTR2_32BIT_POINTER_ENABLE = 0x00000001 +NVOS32_ATTR2_FIXED_NUMA_NODE_ID = (7, 7) +NVOS32_ATTR2_FIXED_NUMA_NODE_ID_NO = 0x00000000 +NVOS32_ATTR2_FIXED_NUMA_NODE_ID_YES = 0x00000001 +NVOS32_ATTR2_SMMU_ON_GPU = (9, 8) +NVOS32_ATTR2_SMMU_ON_GPU_DEFAULT = 0x00000000 +NVOS32_ATTR2_SMMU_ON_GPU_DISABLE = 0x00000001 +NVOS32_ATTR2_SMMU_ON_GPU_ENABLE = 0x00000002 +NVOS32_ATTR2_USE_SCANOUT_CARVEOUT = (10, 10) +NVOS32_ATTR2_USE_SCANOUT_CARVEOUT_FALSE = 0x00000000 +NVOS32_ATTR2_USE_SCANOUT_CARVEOUT_TRUE = 0x00000001 +NVOS32_ATTR2_ALLOC_COMPCACHELINE_ALIGN = (11, 11) +NVOS32_ATTR2_ALLOC_COMPCACHELINE_ALIGN_OFF = 0x0 +NVOS32_ATTR2_ALLOC_COMPCACHELINE_ALIGN_ON = 0x1 +NVOS32_ATTR2_ALLOC_COMPCACHELINE_ALIGN_DEFAULT = NVOS32_ATTR2_ALLOC_COMPCACHELINE_ALIGN_OFF +NVOS32_ATTR2_PRIORITY = (13, 12) +NVOS32_ATTR2_PRIORITY_DEFAULT = 0x0 +NVOS32_ATTR2_PRIORITY_HIGH = 0x1 +NVOS32_ATTR2_PRIORITY_LOW = 0x2 +NVOS32_ATTR2_INTERNAL = (14, 14) +NVOS32_ATTR2_INTERNAL_NO = 0x0 +NVOS32_ATTR2_INTERNAL_YES = 0x1 +NVOS32_ATTR2_PREFER_2C = (15, 15) +NVOS32_ATTR2_PREFER_2C_NO = 0x00000000 +NVOS32_ATTR2_PREFER_2C_YES = 0x00000001 +NVOS32_ATTR2_NISO_DISPLAY = (16, 16) +NVOS32_ATTR2_NISO_DISPLAY_NO = 0x00000000 +NVOS32_ATTR2_NISO_DISPLAY_YES = 0x00000001 +NVOS32_ATTR2_ZBC_SKIP_ZBCREFCOUNT = (17, 17) +NVOS32_ATTR2_ZBC_SKIP_ZBCREFCOUNT_NO = 0x00000000 +NVOS32_ATTR2_ZBC_SKIP_ZBCREFCOUNT_YES = 0x00000001 +NVOS32_ATTR2_ISO = (18, 18) +NVOS32_ATTR2_ISO_NO = 0x00000000 +NVOS32_ATTR2_ISO_YES = 0x00000001 +NVOS32_ATTR2_BLACKLIST = (19, 19) +NVOS32_ATTR2_BLACKLIST_ON = 0x00000000 +NVOS32_ATTR2_BLACKLIST_OFF = 0x00000001 +NVOS32_ATTR2_PAGE_OFFLINING = (19, 19) +NVOS32_ATTR2_PAGE_OFFLINING_ON = 0x00000000 +NVOS32_ATTR2_PAGE_OFFLINING_OFF = 0x00000001 +NVOS32_ATTR2_PAGE_SIZE_HUGE = (21, 20) +NVOS32_ATTR2_PAGE_SIZE_HUGE_DEFAULT = 0x00000000 +NVOS32_ATTR2_PAGE_SIZE_HUGE_2MB = 0x00000001 +NVOS32_ATTR2_PAGE_SIZE_HUGE_512MB = 0x00000002 +NVOS32_ATTR2_PAGE_SIZE_HUGE_256GB = 0x00000003 +NVOS32_ATTR2_PROTECTION_USER = (22, 22) +NVOS32_ATTR2_PROTECTION_USER_READ_WRITE = 0x00000000 +NVOS32_ATTR2_PROTECTION_USER_READ_ONLY = 0x00000001 +NVOS32_ATTR2_PROTECTION_DEVICE = (23, 23) +NVOS32_ATTR2_PROTECTION_DEVICE_READ_WRITE = 0x00000000 +NVOS32_ATTR2_PROTECTION_DEVICE_READ_ONLY = 0x00000001 +NVOS32_ATTR2_MEMORY_PROTECTION = (26, 25) +NVOS32_ATTR2_MEMORY_PROTECTION_DEFAULT = 0x00000000 +NVOS32_ATTR2_MEMORY_PROTECTION_PROTECTED = 0x00000001 +NVOS32_ATTR2_MEMORY_PROTECTION_UNPROTECTED = 0x00000002 +NVOS32_ATTR2_ALLOCATE_FROM_SUBHEAP = (27, 27) +NVOS32_ATTR2_ALLOCATE_FROM_SUBHEAP_NO = 0x00000000 +NVOS32_ATTR2_ALLOCATE_FROM_SUBHEAP_YES = 0x00000001 +NVOS32_ATTR2_ENABLE_LOCALIZED_MEMORY = (30, 29) +NVOS32_ATTR2_ENABLE_LOCALIZED_MEMORY_DEFAULT = 0x00000000 +NVOS32_ATTR2_ENABLE_LOCALIZED_MEMORY_UGPU0 = 0x00000001 +NVOS32_ATTR2_ENABLE_LOCALIZED_MEMORY_UGPU1 = 0x00000002 +NVOS32_ATTR2_ENABLE_LOCALIZED_MEMORY_UGPU_COUNT = 2 +NVOS32_ATTR2_REGISTER_MEMDESC_TO_PHYS_RM = (31, 31) +NVOS32_ATTR2_REGISTER_MEMDESC_TO_PHYS_RM_FALSE = 0x00000000 +NVOS32_ATTR2_REGISTER_MEMDESC_TO_PHYS_RM_TRUE = 0x00000001 +NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT = 0x00000001 +NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP = 0x00000002 +NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN = 0x00000004 +NVOS32_ALLOC_FLAGS_FORCE_ALIGN_HOST_PAGE = 0x00000008 +NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE = 0x00000010 +NVOS32_ALLOC_FLAGS_BANK_HINT = 0x00000020 +NVOS32_ALLOC_FLAGS_BANK_FORCE = 0x00000040 +NVOS32_ALLOC_FLAGS_ALIGNMENT_HINT = 0x00000080 +NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE = 0x00000100 +NVOS32_ALLOC_FLAGS_BANK_GROW_UP = 0x00000000 +NVOS32_ALLOC_FLAGS_BANK_GROW_DOWN = 0x00000200 +NVOS32_ALLOC_FLAGS_LAZY = 0x00000400 +NVOS32_ALLOC_FLAGS_FORCE_REVERSE_ALLOC = 0x00000800 +NVOS32_ALLOC_FLAGS_NO_SCANOUT = 0x00001000 +NVOS32_ALLOC_FLAGS_PITCH_FORCE = 0x00002000 +NVOS32_ALLOC_FLAGS_MEMORY_HANDLE_PROVIDED = 0x00004000 +NVOS32_ALLOC_FLAGS_MAP_NOT_REQUIRED = 0x00008000 +NVOS32_ALLOC_FLAGS_PERSISTENT_VIDMEM = 0x00010000 +NVOS32_ALLOC_FLAGS_USE_BEGIN_END = 0x00020000 +NVOS32_ALLOC_FLAGS_TURBO_CIPHER_ENCRYPTED = 0x00040000 +NVOS32_ALLOC_FLAGS_VIRTUAL = 0x00080000 +NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX = 0x00100000 +NVOS32_ALLOC_FLAGS_ZCULL_COVG_SPECIFIED = 0x00200000 +NVOS32_ALLOC_FLAGS_EXTERNALLY_MANAGED = 0x00400000 +NVOS32_ALLOC_FLAGS_FORCE_DEDICATED_PDE = 0x00800000 +NVOS32_ALLOC_FLAGS_PROTECTED = 0x01000000 +NVOS32_ALLOC_FLAGS_KERNEL_MAPPING_MAP = 0x02000000 +NVOS32_ALLOC_FLAGS_MAXIMIZE_ADDRESS_SPACE = 0x02000000 +NVOS32_ALLOC_FLAGS_SPARSE = 0x04000000 +NVOS32_ALLOC_FLAGS_USER_READ_ONLY = 0x04000000 +NVOS32_ALLOC_FLAGS_DEVICE_READ_ONLY = 0x08000000 +NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC = 0x10000000 +NVOS32_ALLOC_FLAGS_PREFER_PTES_IN_SYSMEMORY = 0x20000000 +NVOS32_ALLOC_FLAGS_SKIP_ALIGN_PAD = 0x40000000 +NVOS32_ALLOC_FLAGS_WPR1 = 0x40000000 +NVOS32_ALLOC_FLAGS_ZCULL_DONT_ALLOCATE_SHARED_1X = 0x80000000 +NVOS32_ALLOC_FLAGS_WPR2 = 0x80000000 +NVOS32_ALLOC_INTERNAL_FLAGS_CLIENTALLOC = 0x00000001 +NVOS32_ALLOC_INTERNAL_FLAGS_SKIP_SCRUB = 0x00000004 +NVOS32_ALLOC_FLAGS_MAXIMIZE_4GB_ADDRESS_SPACE = NVOS32_ALLOC_FLAGS_MAXIMIZE_ADDRESS_SPACE +NVOS32_ALLOC_FLAGS_VIRTUAL_ONLY = ( NVOS32_ALLOC_FLAGS_VIRTUAL | NVOS32_ALLOC_FLAGS_LAZY | NVOS32_ALLOC_FLAGS_EXTERNALLY_MANAGED | NVOS32_ALLOC_FLAGS_SPARSE | NVOS32_ALLOC_FLAGS_MAXIMIZE_ADDRESS_SPACE | NVOS32_ALLOC_FLAGS_PREFER_PTES_IN_SYSMEMORY ) +NVOS32_ALLOC_COMPR_COVG_SCALE = 10 +NVOS32_ALLOC_COMPR_COVG_BITS = (1, 0) +NVOS32_ALLOC_COMPR_COVG_BITS_DEFAULT = 0x00000000 +NVOS32_ALLOC_COMPR_COVG_BITS_1 = 0x00000001 +NVOS32_ALLOC_COMPR_COVG_BITS_2 = 0x00000002 +NVOS32_ALLOC_COMPR_COVG_BITS_4 = 0x00000003 +NVOS32_ALLOC_COMPR_COVG_MAX = (11, 2) +NVOS32_ALLOC_COMPR_COVG_MIN = (21, 12) +NVOS32_ALLOC_COMPR_COVG_START = (31, 22) +NVOS32_ALLOC_ZCULL_COVG_FORMAT = (3, 0) +NVOS32_ALLOC_ZCULL_COVG_FORMAT_LOW_RES_Z = 0x00000000 +NVOS32_ALLOC_ZCULL_COVG_FORMAT_HIGH_RES_Z = 0x00000002 +NVOS32_ALLOC_ZCULL_COVG_FORMAT_LOW_RES_ZS = 0x00000003 +NVOS32_ALLOC_ZCULL_COVG_FALLBACK = (4, 4) +NVOS32_ALLOC_ZCULL_COVG_FALLBACK_DISALLOW = 0x00000000 +NVOS32_ALLOC_ZCULL_COVG_FALLBACK_ALLOW = 0x00000001 +NVOS32_ALLOC_COMPTAG_OFFSET_START = (19, 0) +NVOS32_ALLOC_COMPTAG_OFFSET_START_DEFAULT = 0x00000000 +NVOS32_ALLOC_COMPTAG_OFFSET_USAGE = (31, 30) +NVOS32_ALLOC_COMPTAG_OFFSET_USAGE_DEFAULT = 0x00000000 +NVOS32_ALLOC_COMPTAG_OFFSET_USAGE_OFF = 0x00000000 +NVOS32_ALLOC_COMPTAG_OFFSET_USAGE_FIXED = 0x00000001 +NVOS32_ALLOC_COMPTAG_OFFSET_USAGE_MIN = 0x00000002 +NVOS32_REALLOC_FLAGS_GROW_ALLOCATION = 0x00000000 +NVOS32_REALLOC_FLAGS_SHRINK_ALLOCATION = 0x00000001 +NVOS32_REALLOC_FLAGS_REALLOC_UP = 0x00000000 +NVOS32_REALLOC_FLAGS_REALLOC_DOWN = 0x00000002 +NVOS32_RELEASE_COMPR_FLAGS_MEMORY_HANDLE_PROVIDED = 0x000000001 +NVOS32_REACQUIRE_COMPR_FLAGS_MEMORY_HANDLE_PROVIDED = 0x000000001 +NVOS32_FREE_FLAGS_MEMORY_HANDLE_PROVIDED = 0x00000001 +NVOS32_DUMP_FLAGS_TYPE = (1, 0) +NVOS32_DUMP_FLAGS_TYPE_FB = 0x00000000 +NVOS32_DUMP_FLAGS_TYPE_CLIENT_PD = 0x00000001 +NVOS32_DUMP_FLAGS_TYPE_CLIENT_VA = 0x00000002 +NVOS32_DUMP_FLAGS_TYPE_CLIENT_VAPTE = 0x00000003 +NVOS32_BLOCK_TYPE_FREE = 0xFFFFFFFF +NVOS32_INVALID_BLOCK_FREE_OFFSET = 0xFFFFFFFF +NVOS32_MEM_TAG_NONE = 0x00000000 +NV04_MAP_MEMORY = (0x00000021) +NV04_MAP_MEMORY_FLAGS_NONE = (0x00000000) +NV04_MAP_MEMORY_FLAGS_USER = (0x00004000) +NVOS33_FLAGS_ACCESS = (1, 0) +NVOS33_FLAGS_ACCESS_READ_WRITE = (0x00000000) +NVOS33_FLAGS_ACCESS_READ_ONLY = (0x00000001) +NVOS33_FLAGS_ACCESS_WRITE_ONLY = (0x00000002) +NVOS33_FLAGS_PERSISTENT = (4, 4) +NVOS33_FLAGS_PERSISTENT_DISABLE = (0x00000000) +NVOS33_FLAGS_PERSISTENT_ENABLE = (0x00000001) +NVOS33_FLAGS_SKIP_SIZE_CHECK = (8, 8) +NVOS33_FLAGS_SKIP_SIZE_CHECK_DISABLE = (0x00000000) +NVOS33_FLAGS_SKIP_SIZE_CHECK_ENABLE = (0x00000001) +NVOS33_FLAGS_MEM_SPACE = (14, 14) +NVOS33_FLAGS_MEM_SPACE_CLIENT = (0x00000000) +NVOS33_FLAGS_MEM_SPACE_USER = (0x00000001) +NVOS33_FLAGS_MAPPING = (16, 15) +NVOS33_FLAGS_MAPPING_DEFAULT = (0x00000000) +NVOS33_FLAGS_MAPPING_DIRECT = (0x00000001) +NVOS33_FLAGS_MAPPING_REFLECTED = (0x00000002) +NVOS33_FLAGS_FIFO_MAPPING = (17, 17) +NVOS33_FLAGS_FIFO_MAPPING_DEFAULT = (0x00000000) +NVOS33_FLAGS_FIFO_MAPPING_ENABLE = (0x00000001) +NVOS33_FLAGS_MAP_FIXED = (18, 18) +NVOS33_FLAGS_MAP_FIXED_DISABLE = (0x00000000) +NVOS33_FLAGS_MAP_FIXED_ENABLE = (0x00000001) +NVOS33_FLAGS_RESERVE_ON_UNMAP = (19, 19) +NVOS33_FLAGS_RESERVE_ON_UNMAP_DISABLE = (0x00000000) +NVOS33_FLAGS_RESERVE_ON_UNMAP_ENABLE = (0x00000001) +NVOS33_FLAGS_BUS = (21, 20) +NVOS33_FLAGS_BUS_DEFAULT = 0 +NVOS33_FLAGS_BUS_COHERENT_LINK = 1 +NVOS33_FLAGS_BUS_PCIE = 2 +NVOS33_FLAGS_OS_DESCRIPTOR = (22, 22) +NVOS33_FLAGS_OS_DESCRIPTOR_DISABLE = (0x00000000) +NVOS33_FLAGS_OS_DESCRIPTOR_ENABLE = (0x00000001) +NVOS33_FLAGS_CACHING_TYPE = (25, 23) +NVOS33_FLAGS_CACHING_TYPE_CACHED = 0 +NVOS33_FLAGS_CACHING_TYPE_UNCACHED = 1 +NVOS33_FLAGS_CACHING_TYPE_WRITECOMBINED = 2 +NVOS33_FLAGS_CACHING_TYPE_WRITEBACK = 5 +NVOS33_FLAGS_CACHING_TYPE_DEFAULT = 6 +NVOS33_FLAGS_CACHING_TYPE_UNCACHED_WEAK = 7 +NVOS33_FLAGS_ALLOW_MAPPING_ON_HCC = (26, 26) +NVOS33_FLAGS_ALLOW_MAPPING_ON_HCC_NO = (0x00000000) +NVOS33_FLAGS_ALLOW_MAPPING_ON_HCC_YES = (0x00000001) +NV04_UNMAP_MEMORY = (0x00000022) +NV04_ACCESS_REGISTRY = (0x00000026) +NVOS38_ACCESS_TYPE_READ_DWORD = 1 +NVOS38_ACCESS_TYPE_WRITE_DWORD = 2 +NVOS38_ACCESS_TYPE_READ_BINARY = 6 +NVOS38_ACCESS_TYPE_WRITE_BINARY = 7 +NVOS38_MAX_REGISTRY_STRING_LENGTH = 256 +NVOS38_MAX_REGISTRY_BINARY_LENGTH = 256 +NV04_ALLOC_CONTEXT_DMA = (0x00000027) +NV04_GET_EVENT_DATA = (0x00000028) +NVSIM01_BUS_XACT = (0x0000002C) +NV04_MAP_MEMORY_DMA = (0x0000002E) +NVOS46_FLAGS_ACCESS = (1, 0) +NVOS46_FLAGS_ACCESS_READ_WRITE = (0x00000000) +NVOS46_FLAGS_ACCESS_READ_ONLY = (0x00000001) +NVOS46_FLAGS_ACCESS_WRITE_ONLY = (0x00000002) +NVOS46_FLAGS_32BIT_POINTER = (2, 2) +NVOS46_FLAGS_32BIT_POINTER_DISABLE = (0x00000000) +NVOS46_FLAGS_32BIT_POINTER_ENABLE = (0x00000001) +NVOS46_FLAGS_PAGE_KIND = (3, 3) +NVOS46_FLAGS_PAGE_KIND_PHYSICAL = (0x00000000) +NVOS46_FLAGS_PAGE_KIND_VIRTUAL = (0x00000001) +NVOS46_FLAGS_CACHE_SNOOP = (4, 4) +NVOS46_FLAGS_CACHE_SNOOP_DISABLE = (0x00000000) +NVOS46_FLAGS_CACHE_SNOOP_ENABLE = (0x00000001) +NVOS46_FLAGS_KERNEL_MAPPING = (5, 5) +NVOS46_FLAGS_KERNEL_MAPPING_NONE = (0x00000000) +NVOS46_FLAGS_KERNEL_MAPPING_ENABLE = (0x00000001) +NVOS46_FLAGS_SHADER_ACCESS = (7, 6) +NVOS46_FLAGS_SHADER_ACCESS_DEFAULT = (0x00000000) +NVOS46_FLAGS_SHADER_ACCESS_READ_ONLY = (0x00000001) +NVOS46_FLAGS_SHADER_ACCESS_WRITE_ONLY = (0x00000002) +NVOS46_FLAGS_SHADER_ACCESS_READ_WRITE = (0x00000003) +NVOS46_FLAGS_PAGE_SIZE = (11, 8) +NVOS46_FLAGS_PAGE_SIZE_DEFAULT = (0x00000000) +NVOS46_FLAGS_PAGE_SIZE_4KB = (0x00000001) +NVOS46_FLAGS_PAGE_SIZE_BIG = (0x00000002) +NVOS46_FLAGS_PAGE_SIZE_BOTH = (0x00000003) +NVOS46_FLAGS_PAGE_SIZE_HUGE = (0x00000004) +NVOS46_FLAGS_PAGE_SIZE_512M = (0x00000005) +NVOS46_FLAGS_PAGE_SIZE_256G = (0x00000006) +NVOS46_FLAGS_SYSTEM_L3_ALLOC = (13, 13) +NVOS46_FLAGS_SYSTEM_L3_ALLOC_DEFAULT = (0x00000000) +NVOS46_FLAGS_SYSTEM_L3_ALLOC_ENABLE_HINT = (0x00000001) +NVOS46_FLAGS_DMA_OFFSET_GROWS = (14, 14) +NVOS46_FLAGS_DMA_OFFSET_GROWS_UP = (0x00000000) +NVOS46_FLAGS_DMA_OFFSET_GROWS_DOWN = (0x00000001) +NVOS46_FLAGS_DMA_OFFSET_FIXED = (15, 15) +NVOS46_FLAGS_DMA_OFFSET_FIXED_FALSE = (0x00000000) +NVOS46_FLAGS_DMA_OFFSET_FIXED_TRUE = (0x00000001) +NVOS46_FLAGS_DISABLE_ENCRYPTION = (16, 16) +NVOS46_FLAGS_DISABLE_ENCRYPTION_FALSE = (0x00000000) +NVOS46_FLAGS_DISABLE_ENCRYPTION_TRUE = (0x00000001) +NVOS46_FLAGS_GPU_CACHEABLE = (18, 17) +NVOS46_FLAGS_GPU_CACHEABLE_DEFAULT = (0x00000000) +NVOS46_FLAGS_GPU_CACHEABLE_YES = (0x00000001) +NVOS46_FLAGS_GPU_CACHEABLE_NO = (0x00000002) +NVOS46_FLAGS_GPU_CACHEABLE_INVALID = (0x00000003) +NVOS46_FLAGS_PAGE_KIND_OVERRIDE = (19, 19) +NVOS46_FLAGS_PAGE_KIND_OVERRIDE_NO = (0x00000000) +NVOS46_FLAGS_PAGE_KIND_OVERRIDE_YES = (0x00000001) +NVOS46_FLAGS_P2P = (27, 20) +NVOS46_FLAGS_P2P_ENABLE = (21, 20) +NVOS46_FLAGS_P2P_ENABLE_NO = (0x00000000) +NVOS46_FLAGS_P2P_ENABLE_YES = (0x00000001) +NVOS46_FLAGS_P2P_ENABLE_NONE = NVOS46_FLAGS_P2P_ENABLE_NO +NVOS46_FLAGS_P2P_ENABLE_SLI = NVOS46_FLAGS_P2P_ENABLE_YES +NVOS46_FLAGS_P2P_ENABLE_NOSLI = (0x00000002) +NVOS46_FLAGS_P2P_SUBDEVICE_ID = (24, 22) +NVOS46_FLAGS_P2P_SUBDEV_ID_SRC = NVOS46_FLAGS_P2P_SUBDEVICE_ID +NVOS46_FLAGS_P2P_SUBDEV_ID_TGT = (27, 25) +NVOS46_FLAGS_TLB_LOCK = (28, 28) +NVOS46_FLAGS_TLB_LOCK_DISABLE = (0x00000000) +NVOS46_FLAGS_TLB_LOCK_ENABLE = (0x00000001) +NVOS46_FLAGS_DMA_UNICAST_REUSE_ALLOC = (29, 29) +NVOS46_FLAGS_DMA_UNICAST_REUSE_ALLOC_FALSE = (0x00000000) +NVOS46_FLAGS_DMA_UNICAST_REUSE_ALLOC_TRUE = (0x00000001) +NVOS46_FLAGS_ENABLE_FORCE_COMPRESSED_MAP = (30, 30) +NVOS46_FLAGS_ENABLE_FORCE_COMPRESSED_MAP_FALSE = (0x00000000) +NVOS46_FLAGS_ENABLE_FORCE_COMPRESSED_MAP_TRUE = (0x00000001) +NVOS46_FLAGS_DEFER_TLB_INVALIDATION = (31, 31) +NVOS46_FLAGS_DEFER_TLB_INVALIDATION_FALSE = (0x00000000) +NVOS46_FLAGS_DEFER_TLB_INVALIDATION_TRUE = (0x00000001) +NVOS46_FLAGS2_GPU_CACHE_SNOOP = (1, 0) +NVOS46_FLAGS2_GPU_CACHE_SNOOP_DEFAULT = (0x00000000) +NVOS46_FLAGS2_GPU_CACHE_SNOOP_ENABLE = (0x00000001) +NVOS46_FLAGS2_GPU_CACHE_SNOOP_DISABLE = (0x00000002) +NV04_UNMAP_MEMORY_DMA = (0x0000002F) +NVOS47_FLAGS_DEFER_TLB_INVALIDATION = (0, 0) +NVOS47_FLAGS_DEFER_TLB_INVALIDATION_FALSE = (0x00000000) +NVOS47_FLAGS_DEFER_TLB_INVALIDATION_TRUE = (0x00000001) +NV04_BIND_CONTEXT_DMA = (0x00000031) +NV04_CONTROL = (0x00000036) +NVOS54_FLAGS_NONE = (0x00000000) +NVOS54_FLAGS_IRQL_RAISED = (0x00000001) +NVOS54_FLAGS_LOCK_BYPASS = (0x00000002) +NVOS54_FLAGS_FINN_SERIALIZED = (0x00000004) +NV04_DUP_OBJECT = (0x00000037) +NV04_DUP_HANDLE_FLAGS_NONE = (0x00000000) +NV04_DUP_HANDLE_FLAGS_REJECT_KERNEL_DUP_PRIVILEGE = (0x00000001) +NV04_UPDATE_DEVICE_MAPPING_INFO = (0x00000038) +NV04_SHARE = (0x0000003E) +NVPOWERSTATE_STAGE_NONE = 0 +NVPOWERSTATE_STAGE_WAIT_FOR_GFW_BOOT_OK = 1 +NVPOWERSTATE_STAGE_INIT_LIBOS_LOGGING_STRUCTURES = 2 +NVPOWERSTATE_STAGE_GSP_PREPARE_FOR_BOOTSTRAP = 3 +NVPOWERSTATE_STAGE_GSP_BOOTSTRAP = 4 +NVPOWERSTATE_STAGE_BOOT_GSP_RM_PROXY = 5 +NVPOWERSTATE_STAGE_VBIOS_HANDLE_SECURE_BOOT = 6 +NVPOWERSTATE_STAGE_RESTORE_PCIE_CONFIG_REGISTERS = 7 +NVPOWERSTATE_STAGE_GCX_BOOT_TIMER_CB_SCHEDULE = 8 +NVPOWERSTATE_STAGE_POWER_MANAGEMENT_RESUME_PRE_LOAD_PHYSICAL_UNATTACHED = 9 +NVPOWERSTATE_STAGE_PMS_EXPECTED_CHECKPOINT_DONE = 10 +NVPOWERSTATE_STAGE_POLL_FOR_NVLINK_READY = 11 +NVPOWERSTATE_STAGE_CE_STATE_PRE_LOAD = 12 +NVPOWERSTATE_STAGE_LOAD_PROXY_UCODE_EARLY_INIT = 13 +NVPOWERSTATE_STAGE_RESTORE_NON_WPR_REGION = 14 +NVPOWERSTATE_STAGE_PMU_20_OS_BOOTSTRAP = 15 +NVPOWERSTATE_STAGE_STATE_PRE_LOAD_ENGINE = 16 +NVPOWERSTATE_STAGE_STATE_PRE_LOAD_UNKNOWN = 17 +NVPOWERSTATE_STAGE_STATE_LOAD_ENGINE = 18 +NVPOWERSTATE_STAGE_STATE_LOAD_UNKNOWN = 19 +NVPOWERSTATE_STAGE_STATE_LOAD_PHYSICAL = 20 +NVPOWERSTATE_STAGE_STATE_POST_LOAD_ENGINE = 21 +NVPOWERSTATE_STAGE_STATE_POST_LOAD_UNKNOWN = 22 +NVPOWERSTATE_STAGE_GSP_PREPARE_SUSPEND_RESUME_DATA = 23 +NVPOWERSTATE_STAGE_MC_POINTER_NULL = 24 +NVPOWERSTATE_STAGE_SAVE_PCIE_CONFIG_REGISTERS = 25 +NVPOWERSTATE_STAGE_STATE_PRE_UNLOAD_ENGINE = 26 +NVPOWERSTATE_STAGE_STATE_PRE_UNLOAD_UNKNOWN = 27 +NVPOWERSTATE_STAGE_STATE_UNLOAD_ENGINE = 28 +NVPOWERSTATE_STAGE_STATE_UNLOAD_UNKNOWN = 29 +NVPOWERSTATE_STAGE_STATE_POST_UNLOAD_ENGINE = 30 +NVPOWERSTATE_STAGE_STATE_POST_UNLOAD_UNKNOWN = 31 +NVPOWERSTATE_STAGE_GSP_UNLOAD_RM = 32 +NVPOWERSTATE_STAGE_MONITOR_STATE_0 = 33 +NVPOWERSTATE_STAGE_MONITOR_STATE_1 = 34 +NVPOWERSTATE_STAGE_MONITOR_STATE_HIBERNATE = 35 +NVPOWERSTATE_STAGE_SET_POWER_STATE_UNKNOWN = 36 +NVPOWERSTATE_STAGE__COUNT = 37 +NV_DEVICE_ALLOCATION_SZNAME_MAXLEN = 128 +NV_DEVICE_ALLOCATION_FLAGS_NONE = (0x00000000) +NV_DEVICE_ALLOCATION_FLAGS_MAP_PTE_GLOBALLY = (0x00000001) +NV_DEVICE_ALLOCATION_FLAGS_MINIMIZE_PTETABLE_SIZE = (0x00000002) +NV_DEVICE_ALLOCATION_FLAGS_RETRY_PTE_ALLOC_IN_SYS = (0x00000004) +NV_DEVICE_ALLOCATION_FLAGS_VASPACE_SIZE = (0x00000008) +NV_DEVICE_ALLOCATION_FLAGS_MAP_PTE = (0x00000010) +NV_DEVICE_ALLOCATION_FLAGS_VASPACE_IS_TARGET = (0x00000020) +NV_DEVICE_ALLOCATION_FLAGS_VASPACE_SHARED_MANAGEMENT = (0x00000100) +NV_DEVICE_ALLOCATION_FLAGS_VASPACE_BIG_PAGE_SIZE_64k = (0x00000200) +NV_DEVICE_ALLOCATION_FLAGS_VASPACE_BIG_PAGE_SIZE_128k = (0x00000400) +NV_DEVICE_ALLOCATION_FLAGS_RESTRICT_RESERVED_VALIMITS = (0x00000800) +NV_DEVICE_ALLOCATION_FLAGS_VASPACE_PTABLE_PMA_MANAGED = (0x00001000) +NV_DEVICE_ALLOCATION_FLAGS_HOST_VGPU_DEVICE = (0x00002000) +NV_DEVICE_ALLOCATION_FLAGS_PLUGIN_CONTEXT = (0x00004000) +NV_DEVICE_ALLOCATION_FLAGS_VASPACE_REQUIRE_FIXED_OFFSET = (0x00008000) +NV_DEVICE_ALLOCATION_VAMODE_OPTIONAL_MULTIPLE_VASPACES = (0x00000000) +NV_DEVICE_ALLOCATION_VAMODE_SINGLE_VASPACE = (0x00000001) +NV_DEVICE_ALLOCATION_VAMODE_MULTIPLE_VASPACES = (0x00000002) +NV_CHANNELGPFIFO_NOTIFICATION_TYPE_ERROR = 0x00000000 +NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 0x00000001 +NV_CHANNELGPFIFO_NOTIFICATION_TYPE_KEY_ROTATION_STATUS = 0x00000002 +NV_CHANNELGPFIFO_NOTIFICATION_TYPE__SIZE_1 = 3 +NV_CHANNELGPFIFO_NOTIFICATION_STATUS_VALUE = (14, 0) +NV_CHANNELGPFIFO_NOTIFICATION_STATUS_IN_PROGRESS = (15, 15) +NV_CHANNELGPFIFO_NOTIFICATION_STATUS_IN_PROGRESS_TRUE = 0x1 +NV_CHANNELGPFIFO_NOTIFICATION_STATUS_IN_PROGRESS_FALSE = 0x0 +NV50VAIO_CHANNELDMA_ALLOCATION_FLAGS_CONNECT_PB_AT_GRAB = (1, 1) +NV50VAIO_CHANNELDMA_ALLOCATION_FLAGS_CONNECT_PB_AT_GRAB_YES = 0x00000000 +NV50VAIO_CHANNELDMA_ALLOCATION_FLAGS_CONNECT_PB_AT_GRAB_NO = 0x00000001 +NV_SWRUNLIST_QOS_INTR_NONE = 0x00000000 +NV_BSP_ALLOCATION_PARAMETERS = NV_NVDEC_ALLOCATION_PARAMETERS +NV_VP_ALLOCATION_FLAGS_STANDARD_UCODE = (0x00000000) +NV_VP_ALLOCATION_FLAGS_STATIC_UCODE = (0x00000001) +NV_VP_ALLOCATION_FLAGS_DYNAMIC_UCODE = (0x00000002) +NV_VP_ALLOCATION_FLAGS_AVP_CLIENT_VIDEO = (0x00000000) +NV_VP_ALLOCATION_FLAGS_AVP_CLIENT_AUDIO = (0x00000001) +NV_MSENC_ALLOCATION_PARAMETERS = NV_NVENC_ALLOCATION_PARAMETERS +NV04_ADD_VBLANK_CALLBACK = (0x0000003D) +NV_VASPACE_ALLOCATION_FLAGS_NONE = (0x00000000) +NV_VASPACE_ALLOCATION_FLAGS_MINIMIZE_PTETABLE_SIZE = BIT(0) +NV_VASPACE_ALLOCATION_FLAGS_RETRY_PTE_ALLOC_IN_SYS = BIT(1) +NV_VASPACE_ALLOCATION_FLAGS_SHARED_MANAGEMENT = BIT(2) +NV_VASPACE_ALLOCATION_FLAGS_IS_EXTERNALLY_OWNED = BIT(3) +NV_VASPACE_ALLOCATION_FLAGS_ENABLE_NVLINK_ATS = BIT(4) +NV_VASPACE_ALLOCATION_FLAGS_ENABLE_PAGE_FAULTING = BIT(6) +NV_VASPACE_ALLOCATION_FLAGS_VA_INTERNAL_LIMIT = BIT(7) +NV_VASPACE_ALLOCATION_FLAGS_ALLOW_ZERO_ADDRESS = BIT(8) +NV_VASPACE_ALLOCATION_FLAGS_IS_FLA = BIT(9) +NV_VASPACE_ALLOCATION_FLAGS_SKIP_SCRUB_MEMPOOL = BIT(10) +NV_VASPACE_ALLOCATION_FLAGS_OPTIMIZE_PTETABLE_MEMPOOL_USAGE = BIT(11) +NV_VASPACE_ALLOCATION_FLAGS_REQUIRE_FIXED_OFFSET = BIT(12) +NV_VASPACE_ALLOCATION_FLAGS_PTETABLE_HEAP_MANAGED = BIT(13) +NV_VASPACE_ALLOCATION_FLAGS_ENABLE_NVLINK_ATS_TEST = BIT(14) +NV_VASPACE_ALLOCATION_INDEX_GPU_NEW = 0x00 +NV_VASPACE_ALLOCATION_INDEX_GPU_HOST = 0x01 +NV_VASPACE_ALLOCATION_INDEX_GPU_GLOBAL = 0x02 +NV_VASPACE_ALLOCATION_INDEX_GPU_DEVICE = 0x03 +NV_VASPACE_ALLOCATION_INDEX_GPU_FLA = 0x04 +NV_VASPACE_ALLOCATION_INDEX_GPU_MAX = 0x05 +NV_VASPACE_BIG_PAGE_SIZE_64K = (64 * 1024) +NV_VASPACE_BIG_PAGE_SIZE_128K = (128 * 1024) +NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT = (1, 0) +NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_SYNC = (0x00000000) +NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_ASYNC = (0x00000001) +NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_SPECIFIED = (0x00000002) +NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_ASYNC_PREFER_LOWER = (0x00000003) +NV_CTXSHARE_ALLOCATION_SUBCTXID_SUBCTXID = (30, 0) +NV_CTXSHARE_ALLOCATION_SUBCTXID_ASYNC_PREFER_LOWER_ALLOCATION = (31, 31) +NV_CTXSHARE_ALLOCATION_SUBCTXID_ASYNC_PREFER_LOWER_ALLOCATION_SUCCESS = (0x00000001) +NV_CTXSHARE_ALLOCATION_SUBCTXID_ASYNC_PREFER_LOWER_ALLOCATION_FAIL = (0x00000000) +NV_TIMEOUT_CONTROL_CMD_SET_DEVICE_TIMEOUT = (0x00000002) +NV_TIMEOUT_CONTROL_CMD_RESET_DEVICE_TIMEOUT = (0x00000003) +NVC36F_CTRL_RESERVED = (0x00) +NVC36F_CTRL_GPFIFO = (0x01) +NVC36F_CTRL_INTERNAL = (0x03) +NVC36F_CTRL_CMD_NULL = (0xc36f0000) +NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN = (0xc36f0108) +NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS_MESSAGE_ID = (0x8) +NVC36F_CTRL_CMD_GPFIFO_FAULT_METHOD_BUFFER_MAX_RUNQUEUES = 0x2 +NVC36F_CTRL_CMD_GPFIFO_UPDATE_FAULT_METHOD_BUFFER = (0xc36f0109) +NVC36F_CTRL_GPFIFO_UPDATE_FAULT_METHOD_BUFFER_PARAMS_MESSAGE_ID = (0x9) +NVC36F_CTRL_CMD_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX = (0xc36f010a) +NVC36F_CTRL_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX_PARAMS_MESSAGE_ID = (0xA) +NVC36F_CTRL_CMD_INTERNAL_GPFIFO_GET_WORK_SUBMIT_TOKEN = (0xc36f0301) +NVC36F_CTRL_INTERNAL_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS_MESSAGE_ID = (0x1) +NVCB33_CTRL_RESERVED = (0x00) +NVCB33_CTRL_CONF_COMPUTE = (0x01) +NV_CONF_COMPUTE_CTRL_CMD_NULL = (0xcb330000) +NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_CAPABILITIES = (0xcb330101) +NV_CONF_COMPUTE_SYSTEM_CPU_CAPABILITY_NONE = 0 +NV_CONF_COMPUTE_SYSTEM_CPU_CAPABILITY_AMD_SEV = 1 +NV_CONF_COMPUTE_SYSTEM_CPU_CAPABILITY_INTEL_TDX = 2 +NV_CONF_COMPUTE_SYSTEM_CPU_CAPABILITY_AMD_SEV_SNP = 3 +NV_CONF_COMPUTE_SYSTEM_CPU_CAPABILITY_AMD_SNP_VTOM = 4 +NV_CONF_COMPUTE_SYSTEM_GPUS_CAPABILITY_NONE = 0 +NV_CONF_COMPUTE_SYSTEM_GPUS_CAPABILITY_APM = 1 +NV_CONF_COMPUTE_SYSTEM_GPUS_CAPABILITY_HCC = 2 +NV_CONF_COMPUTE_SYSTEM_GPUS_CAPABILITY_TDISP = 3 +NV_CONF_COMPUTE_SYSTEM_ENVIRONMENT_UNAVAILABLE = 0 +NV_CONF_COMPUTE_SYSTEM_ENVIRONMENT_SIM = 1 +NV_CONF_COMPUTE_SYSTEM_ENVIRONMENT_PROD = 2 +NV_CONF_COMPUTE_SYSTEM_FEATURE_DISABLED = 0 +NV_CONF_COMPUTE_SYSTEM_FEATURE_APM_ENABLED = 1 +NV_CONF_COMPUTE_SYSTEM_FEATURE_HCC_ENABLED = 2 +NV_CONF_COMPUTE_SYSTEM_FEATURE_TDISP_ENABLED = 3 +NV_CONF_COMPUTE_SYSTEM_DEVTOOLS_MODE_DISABLED = 0 +NV_CONF_COMPUTE_SYSTEM_DEVTOOLS_MODE_ENABLED = 1 +NV_CONF_COMPUTE_SYSTEM_MULTI_GPU_MODE_NONE = 0 +NV_CONF_COMPUTE_SYSTEM_MULTI_GPU_MODE_PROTECTED_PCIE = 1 +NV_CONF_COMPUTE_SYSTEM_MULTI_GPU_MODE_NVLE = 2 +NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_GET_GPUS_STATE = (0xcb330104) +NV_CONF_COMPUTE_CTRL_CMD_SYSTEM_SET_GPUS_STATE = (0xcb330105) +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_VIDMEM_SIZE = (0xcb330106) +NV_CONF_COMPUTE_CTRL_CMD_GPU_SET_VIDMEM_SIZE = (0xcb330107) +NV_CONF_COMPUTE_CTRL_CMD_GET_NUM_SUPPORTED_CC_SECURE_CHANNELS = (0xcb330108) +NV_CONF_COMPUTE_CTRL_CMD_GET_NUM_SUPPORTED_CC_SECURE_CHANNELS_PARAMS_MESSAGE_ID = (0x8) +NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_CERTIFICATE = (0xcb330109) +NV_CONF_COMPUTE_CERT_CHAIN_MAX_SIZE = 0x1000 +NV_CONF_COMPUTE_ATTESTATION_CERT_CHAIN_MAX_SIZE = 0x1400 +NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_CERTIFICATE_PARAMS_MESSAGE_ID = (0x9) +NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_ATTESTATION_REPORT = (0xcb33010a) +NV_CONF_COMPUTE_GPU_ATTESTATION_REPORT_MAX_SIZE = 0x2000 +NV_CONF_COMPUTE_GPU_CEC_ATTESTATION_REPORT_MAX_SIZE = 0x1000 +NV_CONF_COMPUTE_NONCE_SIZE = 0x20 +NV_CONF_COMPUTE_CTRL_CMD_GET_GPU_ATTESTATION_REPORT_PARAMS_MESSAGE_ID = (0xA) +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_NUM_SECURE_CHANNELS = (0xcb33010b) +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_NUM_SECURE_CHANNELS_PARAMS_MESSAGE_ID = (0xB) +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_KEY_ROTATION_STATE = (0xcb33010c) +NV_CONF_COMPUTE_CTRL_CMD_GPU_KEY_ROTATION_DISABLED = 0 +NV_CONF_COMPUTE_CTRL_CMD_GPU_KEY_ROTATION_KERN_ENABLED = 1 +NV_CONF_COMPUTE_CTRL_CMD_GPU_KEY_ROTATION_USER_ENABLED = 2 +NV_CONF_COMPUTE_CTRL_CMD_GPU_KEY_ROTATION_BOTH_ENABLED = 3 +NV_CONF_COMPUTE_CTRL_CMD_GPU_GET_KEY_ROTATION_STATE_PARAMS_MESSAGE_ID = (0xC) +NVA06C_CTRL_RESERVED = (0x00) +NVA06C_CTRL_GPFIFO = (0x01) +NVA06C_CTRL_INTERNAL = (0x02) +NVA06C_CTRL_CMD_NULL = (0xa06c0000) +NVA06C_CTRL_CMD_GPFIFO_SCHEDULE = (0xa06c0101) +NVA06C_CTRL_GPFIFO_SCHEDULE_PARAMS_MESSAGE_ID = (0x1) +NVA06C_CTRL_CMD_BIND = (0xa06c0102) +NVA06C_CTRL_BIND_PARAMS_MESSAGE_ID = (0x2) +NVA06C_CTRL_CMD_SET_TIMESLICE = (0xa06c0103) +NVA06C_CTRL_SET_TIMESLICE_PARAMS_MESSAGE_ID = (0x3) +NVA06C_CTRL_CMD_GET_TIMESLICE = (0xa06c0104) +NVA06C_CTRL_GET_TIMESLICE_PARAMS_MESSAGE_ID = (0x4) +NVA06C_CTRL_CMD_PREEMPT = (0xa06c0105) +NVA06C_CTRL_PREEMPT_PARAMS_MESSAGE_ID = (0x5) +NVA06C_CTRL_CMD_PREEMPT_MAX_MANUAL_TIMEOUT_US = (1000000) +NVA06C_CTRL_CMD_GET_INFO = (0xa06c0106) +NVA06C_CTRL_GET_INFO_PARAMS_MESSAGE_ID = (0x6) +NVA06C_CTRL_CMD_SET_INTERLEAVE_LEVEL = (0xa06c0107) +NVA06C_CTRL_INTERLEAVE_LEVEL_LOW = (0x00000000) +NVA06C_CTRL_INTERLEAVE_LEVEL_MEDIUM = (0x00000001) +NVA06C_CTRL_INTERLEAVE_LEVEL_HIGH = (0x00000002) +NVA06C_CTRL_SET_INTERLEAVE_LEVEL_PARAMS_MESSAGE_ID = (0x7) +NVA06C_CTRL_CMD_GET_INTERLEAVE_LEVEL = (0xa06c0108) +NVA06C_CTRL_GET_INTERLEAVE_LEVEL_PARAMS_MESSAGE_ID = (0x8) +NVA06C_CTRL_CMD_PROGRAM_VIDMEM_PROMOTE = (0xa06c0109) +NVA06C_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS_MESSAGE_ID = (0x9) +NVA06C_CTRL_CMD_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS = (0xa06c010a) +NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_MAX_RUNQUEUES = (NVC36F_CTRL_CMD_GPFIFO_FAULT_METHOD_BUFFER_MAX_RUNQUEUES) +NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS_MESSAGE_ID = (0xa) +NVA06C_CTRL_CMD_MAKE_REALTIME = (0xa06c0110) +NVA06C_CTRL_MAKE_REALTIME_PARAMS_MESSAGE_ID = (0x10) +NVA06C_CTRL_CMD_INTERNAL_GPFIFO_SCHEDULE = (0xa06c0201) +NVA06C_CTRL_INTERNAL_GPFIFO_SCHEDULE_PARAMS_MESSAGE_ID = (0x1) +NVA06C_CTRL_CMD_INTERNAL_SET_TIMESLICE = (0xa06c0202) +NVA06C_CTRL_INTERNAL_SET_TIMESLICE_PARAMS_MESSAGE_ID = (0x2) +GMMU_FMT_MAX_LEVELS = 6 +NV90F1_CTRL_RESERVED = (0x00) +NV90F1_CTRL_VASPACE = (0x01) +NV90F1_CTRL_CMD_NULL = (0x90f10000) +NV90F1_CTRL_CMD_VASPACE_GET_GMMU_FORMAT = (0x90f10101) +NV90F1_CTRL_VASPACE_GET_GMMU_FORMAT_PARAMS_MESSAGE_ID = (0x1) +NV90F1_CTRL_CMD_VASPACE_GET_PAGE_LEVEL_INFO = (0x90f10102) +NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS_MESSAGE_ID = (0x2) +NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_FLAG_NONE = 0x0 +NV90F1_CTRL_CMD_VASPACE_RESERVE_ENTRIES = (0x90f10103) +NV90F1_CTRL_VASPACE_RESERVE_ENTRIES_PARAMS_MESSAGE_ID = (0x3) +NV90F1_CTRL_CMD_VASPACE_RELEASE_ENTRIES = (0x90f10104) +NV90F1_CTRL_VASPACE_RELEASE_ENTRIES_PARAMS_MESSAGE_ID = (0x4) +NV90F1_CTRL_CMD_VASPACE_GET_PAGE_LEVEL_INFO_VERIF = (0x90f10105) +NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_VERIF_PARAMS_MESSAGE_ID = (0x5) +NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES = (0x90f10106) +NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS_MESSAGE_ID = (0x6) +NV90F1_CTRL_CMD_VASPACE_GET_HOST_RM_MANAGED_SIZE = (0x90f10107) +NV90F1_CTRL_VASPACE_GET_HOST_RM_MANAGED_SIZE_PARAMS_MESSAGE_ID = (0x7) +NV90F1_CTRL_CMD_VASPACE_GET_VAS_HEAP_INFO = (0x90f10108) +NV90F1_CTRL_VASPACE_GET_VAS_HEAP_INFO_PARAMS_MESSAGE_ID = (0x8) +NVA06F_CTRL_CMD_GPFIFO_SCHEDULE = (0xa06f0103) +NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS_MESSAGE_ID = (0x3) +NVA06F_CTRL_CMD_BIND = (0xa06f0104) +NVA06F_CTRL_BIND_PARAMS_MESSAGE_ID = (0x4) +NVA06F_CTRL_CMD_SET_ERROR_NOTIFIER = (0xa06f0108) +NVA06F_CTRL_SET_ERROR_NOTIFIER_PARAMS_MESSAGE_ID = (0x8) +NVA06F_CTRL_CMD_SET_INTERLEAVE_LEVEL = (0xa06f0109) +NVA06F_CTRL_SET_INTERLEAVE_LEVEL_PARAMS_MESSAGE_ID = (0x9) +NVA06F_CTRL_CMD_GET_INTERLEAVE_LEVEL = (0xa06f0110) +NVA06F_CTRL_GET_INTERLEAVE_LEVEL_PARAMS_MESSAGE_ID = (0x10) +NVA06F_CTRL_CMD_RESTART_RUNLIST = (0xa06f0111) +NVA06F_CTRL_RESTART_RUNLIST_PARAMS_MESSAGE_ID = (0x11) +NVA06F_CTRL_CMD_STOP_CHANNEL = (0xa06f0112) +NVA06F_CTRL_STOP_CHANNEL_PARAMS_MESSAGE_ID = (0x12) +NVA06F_CTRL_CMD_GET_CONTEXT_ID = (0xa06f0113) +NVA06F_CTRL_GET_CONTEXT_ID_PARAMS_MESSAGE_ID = (0x13) +NV0000_CTRL_RESERVED = (0x00) +NV0000_CTRL_SYSTEM = (0x01) +NV0000_CTRL_GPU = (0x02) +NV0000_CTRL_GSYNC = (0x03) +NV0000_CTRL_DIAG = (0x04) +NV0000_CTRL_EVENT = (0x05) +NV0000_CTRL_NVD = (0x06) +NV0000_CTRL_SWINSTR = (0x07) +NV0000_CTRL_PROC = (0x09) +NV0000_CTRL_SYNC_GPU_BOOST = (0x0A) +NV0000_CTRL_GPUACCT = (0x0B) +NV0000_CTRL_VGPU = (0x0C) +NV0000_CTRL_CLIENT = (0x0D) +NV0000_CTRL_OS_WINDOWS = (0x3F) +NV0000_CTRL_OS_MACOS = (0x3E) +NV0000_CTRL_OS_UNIX = (0x3D) +NV0000_CTRL_CMD_NULL = (0x0) +NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE = (0xd01) +NV0000_CTRL_CLIENT_GET_ADDR_SPACE_TYPE_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_INVALID = 0x00000000 +NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_SYSMEM = 0x00000001 +NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_VIDMEM = 0x00000002 +NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_REGMEM = 0x00000003 +NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC = 0x00000004 +NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE_FABRIC_MC = 0x00000005 +NV0000_CTRL_CMD_CLIENT_GET_HANDLE_INFO = (0xd02) +NV0000_CTRL_CLIENT_GET_HANDLE_INFO_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_CMD_CLIENT_GET_HANDLE_INFO_INDEX_INVALID = 0x00000000 +NV0000_CTRL_CMD_CLIENT_GET_HANDLE_INFO_INDEX_PARENT = 0x00000001 +NV0000_CTRL_CMD_CLIENT_GET_HANDLE_INFO_INDEX_CLASSID = 0x00000002 +NV0000_CTRL_CMD_CLIENT_GET_ACCESS_RIGHTS = (0xd03) +NV0000_CTRL_CLIENT_GET_ACCESS_RIGHTS_PARAMS_MESSAGE_ID = (0x3) +NV0000_CTRL_CMD_CLIENT_SET_INHERITED_SHARE_POLICY = (0xd04) +NV0000_CTRL_CLIENT_SET_INHERITED_SHARE_POLICY_PARAMS_MESSAGE_ID = (0x4) +NV0000_CTRL_CMD_CLIENT_GET_CHILD_HANDLE = (0xd05) +NV0000_CTRL_CMD_CLIENT_GET_CHILD_HANDLE_PARAMS_MESSAGE_ID = (0x5) +NV0000_CTRL_CMD_CLIENT_SHARE_OBJECT = (0xd06) +NV0000_CTRL_CLIENT_SHARE_OBJECT_PARAMS_MESSAGE_ID = (0x6) +NV0000_CTRL_CMD_CLIENT_OBJECTS_ARE_DUPLICATES = (0xd07) +NV0000_CTRL_CLIENT_OBJECTS_ARE_DUPLICATES_PARAMS_MESSAGE_ID = (0x7) +NV0000_CTRL_CMD_CLIENT_SUBSCRIBE_TO_IMEX_CHANNEL = (0xd08) +NV0000_CTRL_CLIENT_SUBSCRIBE_TO_IMEX_CHANNEL_PARAMS_MESSAGE_ID = (0x8) +NV0000_CTRL_CMD_DIAG_GET_LOCK_METER_STATE = (0x480) +NV0000_CTRL_DIAG_GET_LOCK_METER_STATE_PARAMS_MESSAGE_ID = (0x80) +NV0000_CTRL_DIAG_GET_LOCK_METER_STATE_DISABLED = (0x00000000) +NV0000_CTRL_DIAG_GET_LOCK_METER_STATE_ENABLED = (0x00000001) +NV0000_CTRL_DIAG_LOCK_METER_MAX_TABLE_ENTRIES = (0x20000) +NV0000_CTRL_CMD_DIAG_SET_LOCK_METER_STATE = (0x481) +NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_PARAMS_MESSAGE_ID = (0x81) +NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_DISABLE = (0x00000000) +NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_ENABLE = (0x00000001) +NV0000_CTRL_DIAG_SET_LOCK_METER_STATE_RESET = (0x00000002) +NV0000_CTRL_DIAG_LOCK_METER_ENTRY_FILENAME_LENGTH = (0xc) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ACQUIRE_SEMA = (0x00000001) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ACQUIRE_SEMA_FORCED = (0x00000002) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ACQUIRE_SEMA_COND = (0x00000003) +NV0000_CTRL_DIAG_LOCK_METER_TAG_RELEASE_SEMA = (0x00000004) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ACQUIRE_API = (0x00000010) +NV0000_CTRL_DIAG_LOCK_METER_TAG_RELEASE_API = (0x00000011) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ACQUIRE_GPUS = (0x00000020) +NV0000_CTRL_DIAG_LOCK_METER_TAG_RELEASE_GPUS = (0x00000021) +NV0000_CTRL_DIAG_LOCK_METER_TAG_DATA = (0x00000100) +NV0000_CTRL_DIAG_LOCK_METER_TAG_RMCTRL = (0x00001000) +NV0000_CTRL_DIAG_LOCK_METER_TAG_CFG_GETEX = (0x00002002) +NV0000_CTRL_DIAG_LOCK_METER_TAG_CFG_SETEX = (0x00002003) +NV0000_CTRL_DIAG_LOCK_METER_TAG_VIDHEAP = (0x00003000) +NV0000_CTRL_DIAG_LOCK_METER_TAG_MAPMEM = (0x00003001) +NV0000_CTRL_DIAG_LOCK_METER_TAG_UNMAPMEM = (0x00003002) +NV0000_CTRL_DIAG_LOCK_METER_TAG_MAPMEM_DMA = (0x00003003) +NV0000_CTRL_DIAG_LOCK_METER_TAG_UNMAPMEM_DMA = (0x00003004) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ALLOC = (0x00004000) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ALLOC_MEM = (0x00004001) +NV0000_CTRL_DIAG_LOCK_METER_TAG_DUP_OBJECT = (0x00004010) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_CLIENT = (0x00005000) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_DEVICE = (0x00005001) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_SUBDEVICE = (0x00005002) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_SUBDEVICE_DIAG = (0x00005003) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_DISP = (0x00005004) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_DISP_CMN = (0x00005005) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_CHANNEL = (0x00005006) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_CHANNEL_MPEG = (0x00005007) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_CHANNEL_DISP = (0x00005008) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_MEMORY = (0x00005009) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_FBMEM = (0x0000500A) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_OBJECT = (0x0000500B) +NV0000_CTRL_DIAG_LOCK_METER_TAG_FREE_EVENT = (0x0000500C) +NV0000_CTRL_DIAG_LOCK_METER_TAG_IDLE_CHANNELS = (0x00006000) +NV0000_CTRL_DIAG_LOCK_METER_TAG_BIND_CTXDMA = (0x00007000) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ALLOC_CTXDMA = (0x00007001) +NV0000_CTRL_DIAG_LOCK_METER_TAG_ISR = (0x0000F000) +NV0000_CTRL_DIAG_LOCK_METER_TAG_DPC = (0x0000F00F) +NV0000_CTRL_CMD_DIAG_GET_LOCK_METER_ENTRIES = (0x485) +NV0000_CTRL_DIAG_GET_LOCK_METER_ENTRIES_MAX = (0x40) +NV0000_CTRL_DIAG_GET_LOCK_METER_ENTRIES_PARAMS_MESSAGE_ID = (0x85) +NV0000_CTRL_CMD_DIAG_PROFILE_RPC = (0x488) +NV0000_CTRL_DIAG_PROFILE_RPC_PARAMS_MESSAGE_ID = (0x88) +NV0000_CTRL_PROFILE_RPC_CMD_DISABLE = (0x00000000) +NV0000_CTRL_PROFILE_RPC_CMD_ENABLE = (0x00000001) +NV0000_CTRL_PROFILE_RPC_CMD_RESET = (0x00000002) +NV0000_CTRL_CMD_DIAG_DUMP_RPC = (0x489) +NV0000_CTRL_DIAG_RPC_MAX_ENTRIES = (100) +NV0000_CTRL_DIAG_DUMP_RPC_PARAMS_MESSAGE_ID = (0x89) +NV0000_NOTIFIERS_DISPLAY_CHANGE = (0) +NV0000_NOTIFIERS_VGPU_UNBIND_EVENT = (1) +NV0000_NOTIFIERS_VGPU_BIND_EVENT = (2) +NV0000_NOTIFIERS_GPU_BIND_UNBIND_EVENT = (3) +NV0000_NOTIFIERS_MAXCOUNT = (4) +NV0000_CTRL_CMD_EVENT_SET_NOTIFICATION = (0x501) +NV0000_CTRL_EVENT_SET_NOTIFICATION_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE = (0x00000000) +NV0000_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE = (0x00000001) +NV0000_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT = (0x00000002) +NV0000_CTRL_CMD_GET_SYSTEM_EVENT_DATA = (0x502) +NV0000_CTRL_GET_SYSTEM_EVENT_DATA_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_CMD_GPU_GET_ATTACHED_IDS = (0x201) +NV0000_CTRL_GPU_MAX_ATTACHED_GPUS = 32 +NV0000_CTRL_GPU_INVALID_ID = (0xffffffff) +NV0000_CTRL_GPU_GET_ATTACHED_IDS_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_CMD_GPU_GET_ID_INFO = (0x202) +NV0000_CTRL_GPU_MAX_SZNAME = 128 +NV0000_CTRL_NO_NUMA_NODE = (-1) +NV0000_CTRL_GPU_GET_ID_INFO_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_SLI_STATUS_OK = (0x00000000) +NV0000_CTRL_SLI_STATUS_OS_NOT_SUPPORTED = (0x00000002) +NV0000_CTRL_SLI_STATUS_GPU_NOT_SUPPORTED = (0x00000040) +NV0000_CTRL_SLI_STATUS_INVALID_GPU_COUNT = (0x00000001) +NV0000_CTRL_CMD_GPU_GET_ID_INFO_V2 = (0x205) +NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS_MESSAGE_ID = (0x5) +NV0000_CTRL_GPU_ID_INFO_IN_USE = (0, 0) +NV0000_CTRL_GPU_ID_INFO_IN_USE_FALSE = (0x00000000) +NV0000_CTRL_GPU_ID_INFO_IN_USE_TRUE = (0x00000001) +NV0000_CTRL_GPU_ID_INFO_LINKED_INTO_SLI_DEVICE = (1, 1) +NV0000_CTRL_GPU_ID_INFO_LINKED_INTO_SLI_DEVICE_FALSE = (0x00000000) +NV0000_CTRL_GPU_ID_INFO_LINKED_INTO_SLI_DEVICE_TRUE = (0x00000001) +NV0000_CTRL_GPU_ID_INFO_MOBILE = (2, 2) +NV0000_CTRL_GPU_ID_INFO_MOBILE_FALSE = (0x00000000) +NV0000_CTRL_GPU_ID_INFO_MOBILE_TRUE = (0x00000001) +NV0000_CTRL_GPU_ID_INFO_BOOT_MASTER = (3, 3) +NV0000_CTRL_GPU_ID_INFO_BOOT_MASTER_FALSE = (0x00000000) +NV0000_CTRL_GPU_ID_INFO_BOOT_MASTER_TRUE = (0x00000001) +NV0000_CTRL_GPU_ID_INFO_SOC = (5, 5) +NV0000_CTRL_GPU_ID_INFO_SOC_FALSE = (0x00000000) +NV0000_CTRL_GPU_ID_INFO_SOC_TRUE = (0x00000001) +NV0000_CTRL_GPU_ID_INFO_ATS_ENABLED = (6, 6) +NV0000_CTRL_GPU_ID_INFO_ATS_ENABLED_FALSE = (0x00000000) +NV0000_CTRL_GPU_ID_INFO_ATS_ENABLED_TRUE = (0x00000001) +NV0000_CTRL_GPU_ID_INFO_SOC_TYPE = (8, 7) +NV0000_CTRL_GPU_ID_INFO_SOC_TYPE_NONE = (0x00000000) +NV0000_CTRL_GPU_ID_INFO_SOC_TYPE_DISPLAY = (0x00000001) +NV0000_CTRL_GPU_ID_INFO_SOC_TYPE_IGPU = (0x00000002) +NV0000_CTRL_GPU_ID_INFO_SOC_TYPE_DISPLAY_AND_IGPU = (0x00000003) +NV0000_CTRL_CMD_GPU_GET_INIT_STATUS = (0x203) +NV0000_CTRL_GPU_GET_INIT_STATUS_PARAMS_MESSAGE_ID = (0x3) +NV0000_CTRL_CMD_GPU_GET_DEVICE_IDS = (0x204) +NV0000_CTRL_GPU_GET_DEVICE_IDS_PARAMS_MESSAGE_ID = (0x4) +NV0000_CTRL_CMD_GPU_GET_PROBED_IDS = (0x214) +NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS_MESSAGE_ID = (0x14) +NV0000_CTRL_GPU_PROBED_ID_FLAGS_SOC_DISPLAY = (0, 0) +NV0000_CTRL_GPU_PROBED_ID_FLAGS_SOC_DISPLAY_FALSE = (0x00000000) +NV0000_CTRL_GPU_PROBED_ID_FLAGS_SOC_DISPLAY_TRUE = (0x00000001) +NV0000_CTRL_CMD_GPU_GET_PCI_INFO = (0x21b) +NV0000_CTRL_GPU_GET_PCI_INFO_PARAMS_MESSAGE_ID = (0x1B) +NV0000_CTRL_CMD_GPU_ATTACH_IDS = (0x215) +NV0000_CTRL_GPU_ATTACH_ALL_PROBED_IDS = (0x0000ffff) +NV0000_CTRL_GPU_ATTACH_IDS_PARAMS_MESSAGE_ID = (0x15) +NV0000_CTRL_CMD_GPU_DETACH_IDS = (0x216) +NV0000_CTRL_GPU_DETACH_ALL_ATTACHED_IDS = (0x0000ffff) +NV0000_CTRL_GPU_DETACH_IDS_PARAMS_MESSAGE_ID = (0x16) +NV0000_CTRL_CMD_GPU_GET_VIDEO_LINKS = (0x219) +NV0000_CTRL_GPU_MAX_VIDEO_LINKS = 8 +NV0000_CTRL_GPU_GET_VIDEO_LINKS_PARAMS_MESSAGE_ID = (0x19) +NV0000_CTRL_CMD_GPU_GET_UUID_INFO = (0x274) +NV0000_GPU_MAX_GID_LENGTH = (0x00000100) +NV0000_CTRL_GPU_GET_UUID_INFO_PARAMS_MESSAGE_ID = (0x74) +NV0000_CTRL_CMD_GPU_GET_UUID_INFO_FLAGS_FORMAT = (1, 0) +NV0000_CTRL_CMD_GPU_GET_UUID_INFO_FLAGS_FORMAT_ASCII = (0x00000000) +NV0000_CTRL_CMD_GPU_GET_UUID_INFO_FLAGS_FORMAT_BINARY = (0x00000002) +NV0000_CTRL_CMD_GPU_GET_UUID_INFO_FLAGS_TYPE = (2, 2) +NV0000_CTRL_CMD_GPU_GET_UUID_INFO_FLAGS_TYPE_SHA1 = (0x00000000) +NV0000_CTRL_CMD_GPU_GET_UUID_INFO_FLAGS_TYPE_SHA256 = (0x00000001) +NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID = (0x275) +NV0000_CTRL_GPU_GET_UUID_FROM_GPU_ID_PARAMS_MESSAGE_ID = (0x75) +NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID_FLAGS_FORMAT = (1, 0) +NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID_FLAGS_FORMAT_ASCII = (0x00000000) +NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID_FLAGS_FORMAT_BINARY = (0x00000002) +NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID_FLAGS_TYPE = (2, 2) +NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID_FLAGS_TYPE_SHA1 = (0x00000000) +NV0000_CTRL_CMD_GPU_GET_UUID_FROM_GPU_ID_FLAGS_TYPE_SHA256 = (0x00000001) +NV0000_CTRL_CMD_GPU_MODIFY_DRAIN_STATE = (0x278) +NV0000_CTRL_GPU_DRAIN_STATE_DISABLED = (0x00000000) +NV0000_CTRL_GPU_DRAIN_STATE_ENABLED = (0x00000001) +NV0000_CTRL_GPU_DRAIN_STATE_FLAG_REMOVE_DEVICE = (0x00000001) +NV0000_CTRL_GPU_DRAIN_STATE_FLAG_LINK_DISABLE = (0x00000002) +NV0000_CTRL_GPU_MODIFY_DRAIN_STATE_PARAMS_MESSAGE_ID = (0x78) +NV0000_CTRL_CMD_GPU_QUERY_DRAIN_STATE = (0x279) +NV0000_CTRL_GPU_QUERY_DRAIN_STATE_PARAMS_MESSAGE_ID = (0x79) +NV0000_CTRL_CMD_GPU_DISCOVER = (0x27a) +NV0000_CTRL_CMD_GPU_GET_MEMOP_ENABLE = (0x27b) +NV0000_CTRL_GPU_GET_MEMOP_ENABLE_PARAMS_MESSAGE_ID = (0x7B) +NV0000_CTRL_GPU_FLAGS_MEMOP_ENABLE = (0x00000001) +NV0000_CTRL_GPU_LEGACY_CONFIG_MAX_PARAM_DATA = 0x00000175 +NV0000_CTRL_GPU_LEGACY_CONFIG_MAX_PROPERTIES_IN = 6 +NV0000_CTRL_GPU_LEGACY_CONFIG_MAX_PROPERTIES_OUT = 5 +NV0000_CTRL_CMD_IDLE_CHANNELS = (0x283) +NV0000_CTRL_GPU_IDLE_CHANNELS_PARAMS_MESSAGE_ID = (0x83) +NV0000_CTRL_GPU_IMAGE_TYPE_GSP = (0x00000001) +NV0000_CTRL_GPU_IMAGE_TYPE_GSP_LOG = (0x00000002) +NV0000_CTRL_GPU_IMAGE_TYPE_BINDATA_IMAGE = (0x00000003) +NV0000_CTRL_GPU_IMAGE_TYPE_UCODES = (0x00000004) +NV0000_CTRL_CMD_PUSH_UCODE_IMAGE = (0x285) +NV0000_CTRL_GPU_PUSH_UCODE_IMAGE_PARAMS_MESSAGE_ID = (0x85) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_SETTING_LEGACY = (2, 0) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_SETTING_LINK_COUNT = (15, 3) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_FULL = (0x00) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_OFF = (0x01) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_MIN = (0x02) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_HALF = (0x03) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_3QUARTER = (0x04) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_LINK_COUNT = (0x05) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_SCOPE_UNSET = (0x00) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_SCOPE_PER_NODE = (0x01) +NV0000_CTRL_CMD_GPU_NVLINK_BW_MODE_SCOPE_PER_GPU = (0x02) +NV0000_CTRL_CMD_GPU_SET_NVLINK_BW_MODE = (0x286) +NV0000_CTRL_GPU_SET_NVLINK_BW_MODE_PARAMS_MESSAGE_ID = (0x86) +NV0000_CTRL_CMD_GPU_GET_NVLINK_BW_MODE = (0x287) +NV0000_CTRL_GPU_GET_NVLINK_BW_MODE_PARAMS_MESSAGE_ID = (0x87) +NV0000_CTRL_CMD_GPU_GET_ACTIVE_DEVICE_IDS = (0x288) +NV0000_CTRL_GPU_MAX_ACTIVE_DEVICES = 256 +NV0000_CTRL_GPU_GET_ACTIVE_DEVICE_IDS_PARAMS_MESSAGE_ID = (0x88) +NV0000_CTRL_CMD_GPU_ASYNC_ATTACH_ID = (0x289) +NV0000_CTRL_GPU_ASYNC_ATTACH_ID_PARAMS_MESSAGE_ID = (0x89) +NV0000_CTRL_CMD_GPU_WAIT_ATTACH_ID = (0x290) +NV0000_CTRL_GPU_WAIT_ATTACH_ID_PARAMS_MESSAGE_ID = (0x90) +NV0000_CTRL_CMD_GPU_PCI_READ_WRITE_DATA_SIZE_08 = 0 +NV0000_CTRL_CMD_GPU_PCI_READ_WRITE_DATA_SIZE_16 = 1 +NV0000_CTRL_CMD_GPU_PCI_READ_WRITE_DATA_SIZE_32 = 2 +NV0000_CTRL_CMD_GPU_PCI_READ_WRITE = (0x291) +NV0000_CTRL_GPU_PCI_READ_WRITE_PARAMS_MESSAGE_ID = (0x91) +NV0000_CTRL_CMD_GPUACCT_SET_ACCOUNTING_STATE = (0xb01) +NV0000_CTRL_GPU_ACCOUNTING_STATE_ENABLED = (0x00000000) +NV0000_CTRL_GPU_ACCOUNTING_STATE_DISABLED = (0x00000001) +NV0000_CTRL_GPUACCT_SET_ACCOUNTING_STATE_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_CMD_GPUACCT_GET_ACCOUNTING_STATE = (0xb02) +NV0000_CTRL_GPUACCT_GET_ACCOUNTING_STATE_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_CMD_GPUACCT_GET_PROC_ACCOUNTING_INFO = (0xb03) +NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_PARAMS_MESSAGE_ID = (0x3) +NV0000_CTRL_CMD_GPUACCT_GET_ACCOUNTING_PIDS = (0xb04) +NV0000_GPUACCT_PID_MAX_COUNT = 4000 +NV0000_CTRL_GPUACCT_GET_ACCOUNTING_PIDS_PARAMS_MESSAGE_ID = (0x4) +NV0000_CTRL_CMD_GPUACCT_CLEAR_ACCOUNTING_DATA = (0xb05) +NV0000_CTRL_GPUACCT_CLEAR_ACCOUNTING_DATA_PARAMS_MESSAGE_ID = (0x5) +NV0000_CTRL_CMD_GPUACCT_GET_PROC_ACCOUNTING_INFO_V2 = (0xb06) +NV0000_CTRL_GPUACCT_GET_PROC_ACCOUNTING_INFO_V2_PARAMS_MESSAGE_ID = (0x6) +NV0000_CTRL_CMD_GSYNC_GET_ATTACHED_IDS = (0x301) +NV0000_CTRL_GSYNC_GET_ATTACHED_IDS_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_GSYNC_INVALID_ID = (0xffffffff) +NV0000_CTRL_CMD_GSYNC_GET_ID_INFO = (0x302) +NV0000_CTRL_GSYNC_GET_ID_INFO_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_NVD_DUMP_COMPONENT_SYS = (0x400) +NV0000_CTRL_NVD_DUMP_COMPONENT_NVLOG = (0x800) +NV0000_CTRL_NVD_DUMP_COMPONENT_RESERVED = (0xB00) +NV0000_CTRL_CMD_NVD_GET_DUMP_SIZE = (0x601) +NV0000_CTRL_NVD_GET_DUMP_SIZE_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_NVD_MAX_DUMP_SIZE = (1000000) +NV0000_CTRL_CMD_NVD_GET_DUMP = (0x602) +NV0000_CTRL_NVD_GET_DUMP_PARAMS_MESSAGE_ID = (0x2) +NV0000_NVD_CPU_TIME_CLK_ID_DEFAULT = (0x00000000) +NV0000_NVD_CPU_TIME_CLK_ID_OSTIME = (0x00000001) +NV0000_NVD_CPU_TIME_CLK_ID_TSC = (0x00000002) +NV0000_NVD_CPU_TIME_CLK_ID_PLATFORM_API = (0x00000003) +NV0000_CTRL_CMD_NVD_GET_TIMESTAMP = (0x603) +NV0000_CTRL_NVD_GET_TIMESTAMP_PARAMS_MESSAGE_ID = (0x3) +NV0000_CTRL_CMD_NVD_GET_NVLOG_INFO = (0x604) +NV0000_CTRL_NVD_MAX_RUNTIME_SIZES = (16) +NV0000_CTRL_NVD_SIGNATURE_SIZE = (4) +NV0000_CTRL_NVD_MAX_BUFFERS = (3840) +NV0000_CTRL_NVD_GET_NVLOG_INFO_PARAMS_MESSAGE_ID = (0x4) +NV0000_CTRL_NVD_RUNTIME_SIZE_UNUSED = (0) +NV0000_CTRL_NVD_RUNTIME_SIZE_INT = (1) +NV0000_CTRL_NVD_RUNTIME_SIZE_LONG_LONG = (2) +NV0000_CTRL_NVD_RUNTIME_SIZE_STRING = (3) +NV0000_CTRL_NVD_RUNTIME_SIZE_PTR = (4) +NV0000_CTRL_NVD_RUNTIME_SIZE_CHAR = (5) +NV0000_CTRL_NVD_RUNTIME_SIZE_FLOAT = (6) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_BUFFER_INFO = (7, 0) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_BUFFER_SIZE = (23, 8) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_BUFFER_SIZE_DISABLE = (0x00000000) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_BUFFER_SIZE_DEFAULT = (0x00000004) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_RUNTIME_LEVEL = (28, 25) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_TIMESTAMP = (30, 29) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_TIMESTAMP_NONE = (0x00000000) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_TIMESTAMP_32 = (0x00000001) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_TIMESTAMP_64 = (0x00000002) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_TIMESTAMP_32_DIFF = (0x00000003) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_INITED = (31, 31) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_INITED_NO = (0x00000000) +NV0000_CTRL_NVD_NVLOG_INFO_PRINTFLAGS_INITED_YES = (0x00000001) +NV0000_CTRL_CMD_NVD_GET_NVLOG_BUFFER_INFO = (0x605) +NV0000_CTRL_NVD_GET_NVLOG_BUFFER_INFO_PARAMS_MESSAGE_ID = (0x5) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_PAUSE = (0, 0) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_PAUSE_NO = (0x00000000) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_PAUSE_YES = (0x00000001) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_DISABLED = (0, 0) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_DISABLED_NO = (0x00000000) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_DISABLED_YES = (0x00000001) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_TYPE = (1, 1) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_TYPE_RING = (0x00000000) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_TYPE_NOWRAP = (0x00000001) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_EXPANDABLE = (2, 2) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_EXPANDABLE_NO = (0x00000000) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_EXPANDABLE_YES = (0x00000001) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_NONPAGED = (3, 3) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_NONPAGED_NO = (0x00000000) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_NONPAGED_YES = (0x00000001) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_LOCKING = (5, 4) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_LOCKING_NONE = (0x00000000) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_LOCKING_STATE = (0x00000001) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_LOCKING_FULL = (0x00000002) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_OCA = (6, 6) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_OCA_NO = (0x00000000) +NV0000_CTRL_NVD_NVLOG_BUFFER_INFO_FLAGS_OCA_YES = (0x00000001) +NV0000_CTRL_CMD_NVD_GET_NVLOG = (0x606) +NV0000_CTRL_NVLOG_MAX_BLOCK_SIZE = (4000) +NV0000_CTRL_NVD_GET_NVLOG_PARAMS_MESSAGE_ID = (0x6) +NV0000_CTRL_CMD_NVD_GET_RCERR_RPT = (0x607) +NV0000_CTRL_CMD_NVD_RCERR_RPT_MAX_ENTRIES = 200 +NV0000_CTRL_CMD_NVD_RCERR_RPT_TYPE_TEST = 0 +NV0000_CTRL_CMD_NVD_RCERR_RPT_TYPE_GRSTATUS = 1 +NV0000_CTRL_CMD_NVD_RCERR_RPT_TYPE_GPCSTATUS = 2 +NV0000_CTRL_CMD_NVD_RCERR_RPT_TYPE_MMU_FAULT_STATUS = 3 +NV0000_CTRL_CMD_NVD_RCERR_RPT_TYPE_RC_ERROR = 4 +NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_EMPTY = 0x00000000 +NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_OVERFLOWED = 0x00000001 +NV0000_CTRL_CMD_NVD_RCERR_RPT_REG_MAX_PSEDO_REG = 0x0000000f +NV0000_CTRL_CMD_NVD_RCERR_RPT_FLAGS_POS_FIRST = 0x00000001 +NV0000_CTRL_CMD_NVD_RCERR_RPT_FLAGS_POS_LAST = 0x00000002 +NV0000_CTRL_CMD_NVD_RCERR_RPT_FLAGS_RANGE_VALID = 0x00000004 +NV0000_CTRL_CMD_NVD_RCERR_RPT_FLAGS_DATA_VALID = 0x00000008 +TPC_REG_ATTR = lambda gpcId,tpcId: ((gpcId << 8) | (tpcId)) # type: ignore +ROP_REG_ATTR = lambda gpcId,ropId: ((gpcId << 8) | (ropId)) # type: ignore +SM_REG_ATTR = lambda gpcId,tpcId,smId: ((((gpcId) << 16) | ((tpcId) << 8)) | (smId)) # type: ignore +NV0000_CTRL_CMD_NVD_RCERR_RPT_ANY_PROCESS_ID = 0x00000000 +NV0000_CTRL_CMD_NVD_RCERR_RPT_ANY_OWNER_ID = 0xFFFFFFFF +NV0000_CTRL_CMD_NVD_GET_RCERR_RPT_PARAMS_MESSAGE_ID = (0x7) +NV0000_CTRL_CMD_NVD_GET_DPC_ISR_TS = (0x608) +NV0000_CTRL_NVD_GET_DPC_ISR_TS_PARAMS_MESSAGE_ID = (0x8) +NV0000_CTRL_SET_SUB_PROCESS_ID_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_DISABLE_SUB_PROCESS_USERD_ISOLATION_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_CMD_SET_SUB_PROCESS_ID = (0x901) +NV0000_CTRL_CMD_DISABLE_SUB_PROCESS_USERD_ISOLATION = (0x902) +NV0000_SYNC_GPU_BOOST_MAX_GROUPS = (0x10) +NV0000_SYNC_GPU_BOOST_INVALID_GROUP_ID = 0xFFFFFFFF +NV0000_CTRL_CMD_SYNC_GPU_BOOST_INFO = (0xa01) +NV0000_SYNC_GPU_BOOST_INFO_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_CMD_SYNC_GPU_BOOST_GROUP_CREATE = (0xa02) +NV0000_SYNC_GPU_BOOST_GROUP_CREATE_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_CMD_SYNC_GPU_BOOST_GROUP_DESTROY = (0xa03) +NV0000_SYNC_GPU_BOOST_GROUP_DESTROY_PARAMS_MESSAGE_ID = (0x3) +NV0000_CTRL_CMD_SYNC_GPU_BOOST_GROUP_INFO = (0xa04) +NV0000_SYNC_GPU_BOOST_GROUP_INFO_PARAMS_MESSAGE_ID = (0x4) +NV0000_CTRL_CMD_SYSTEM_GET_FEATURES = (0x1f0) +NV0000_CTRL_SYSTEM_GET_FEATURES_PARAMS_MESSAGE_ID = (0xF0) +NV0000_CTRL_SYSTEM_GET_FEATURES_SLI = (0, 0) +NV0000_CTRL_SYSTEM_GET_FEATURES_SLI_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_FEATURES_SLI_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_FEATURES_UUID_BASED_MEM_SHARING = (3, 3) +NV0000_CTRL_SYSTEM_GET_FEATURES_UUID_BASED_MEM_SHARING_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_FEATURES_UUID_BASED_MEM_SHARING_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_FEATURES_RM_TEST_ONLY_CODE_ENABLED = (4, 4) +NV0000_CTRL_SYSTEM_GET_FEATURES_RM_TEST_ONLY_CODE_ENABLED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_FEATURES_RM_TEST_ONLY_CODE_ENABLED_TRUE = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION = (0x101) +NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS_MESSAGE_ID = (0x1) +NV0000_CTRL_CMD_SYSTEM_GET_CPU_INFO = (0x102) +NV0000_CTRL_SYSTEM_GET_CPU_INFO_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_SYSTEM_CPU_FAMILY = (3, 0) +NV0000_CTRL_SYSTEM_CPU_EXTENDED_FAMILY = (11, 4) +NV0000_CTRL_SYSTEM_CPU_MODEL = (3, 0) +NV0000_CTRL_SYSTEM_CPU_EXTENDED_MODEL = (7, 4) +NV0000_CTRL_SYSTEM_CPU_ID_AMD_FAMILY = 0xF +NV0000_CTRL_SYSTEM_CPU_ID_AMD_EXTENDED_FAMILY = 0xA +NV0000_CTRL_SYSTEM_CPU_ID_AMD_MODEL = 0x0 +NV0000_CTRL_SYSTEM_CPU_ID_AMD_EXTENDED_MODEL = 0x4 +NV0000_CTRL_SYSTEM_CPU_ID_INTEL_FAMILY = 0x6 +NV0000_CTRL_SYSTEM_CPU_ID_INTEL_EXTENDED_FAMILY = 0x0 +NV0000_CTRL_SYSTEM_CPU_ID_INTEL_CORE_S_MODEL = 0x7 +NV0000_CTRL_SYSTEM_CPU_ID_INTEL_CORE_P_MODEL = 0xA +NV0000_CTRL_SYSTEM_CPU_ID_INTEL_EXTENDED_MODEL = 0x9 +NV0000_CTRL_SYSTEM_CPU_TYPE_UNKNOWN = (0x00000000) +NV0000_CTRL_SYSTEM_CPU_TYPE_P5 = (0x00000001) +NV0000_CTRL_SYSTEM_CPU_TYPE_P55 = (0x00000002) +NV0000_CTRL_SYSTEM_CPU_TYPE_P6 = (0x00000003) +NV0000_CTRL_SYSTEM_CPU_TYPE_P2 = (0x00000004) +NV0000_CTRL_SYSTEM_CPU_TYPE_P2XC = (0x00000005) +NV0000_CTRL_SYSTEM_CPU_TYPE_CELA = (0x00000006) +NV0000_CTRL_SYSTEM_CPU_TYPE_P3 = (0x00000007) +NV0000_CTRL_SYSTEM_CPU_TYPE_P3_INTL2 = (0x00000008) +NV0000_CTRL_SYSTEM_CPU_TYPE_P4 = (0x00000009) +NV0000_CTRL_SYSTEM_CPU_TYPE_CORE2 = (0x00000010) +NV0000_CTRL_SYSTEM_CPU_TYPE_CELN_M16H = (0x00000011) +NV0000_CTRL_SYSTEM_CPU_TYPE_CORE2_EXTRM = (0x00000012) +NV0000_CTRL_SYSTEM_CPU_TYPE_ATOM = (0x00000013) +NV0000_CTRL_SYSTEM_CPU_TYPE_XEON_SPR = (0x00000014) +NV0000_CTRL_SYSTEM_CPU_TYPE_K5 = (0x00000030) +NV0000_CTRL_SYSTEM_CPU_TYPE_K6 = (0x00000031) +NV0000_CTRL_SYSTEM_CPU_TYPE_K62 = (0x00000032) +NV0000_CTRL_SYSTEM_CPU_TYPE_K63 = (0x00000033) +NV0000_CTRL_SYSTEM_CPU_TYPE_K7 = (0x00000034) +NV0000_CTRL_SYSTEM_CPU_TYPE_K8 = (0x00000035) +NV0000_CTRL_SYSTEM_CPU_TYPE_K10 = (0x00000036) +NV0000_CTRL_SYSTEM_CPU_TYPE_K11 = (0x00000037) +NV0000_CTRL_SYSTEM_CPU_TYPE_RYZEN = (0x00000038) +NV0000_CTRL_SYSTEM_CPU_TYPE_C6 = (0x00000060) +NV0000_CTRL_SYSTEM_CPU_TYPE_C62 = (0x00000061) +NV0000_CTRL_SYSTEM_CPU_TYPE_GX = (0x00000070) +NV0000_CTRL_SYSTEM_CPU_TYPE_M1 = (0x00000071) +NV0000_CTRL_SYSTEM_CPU_TYPE_M2 = (0x00000072) +NV0000_CTRL_SYSTEM_CPU_TYPE_MGX = (0x00000073) +NV0000_CTRL_SYSTEM_CPU_TYPE_TM_CRUSOE = (0x00000080) +NV0000_CTRL_SYSTEM_CPU_TYPE_PPC603 = (0x00000090) +NV0000_CTRL_SYSTEM_CPU_TYPE_PPC604 = (0x00000091) +NV0000_CTRL_SYSTEM_CPU_TYPE_PPC750 = (0x00000092) +NV0000_CTRL_SYSTEM_CPU_TYPE_POWERN = (0x00000093) +NV0000_CTRL_SYSTEM_CPU_TYPE_ARM_UNKNOWN = (0xA0000000) +NV0000_CTRL_SYSTEM_CPU_TYPE_ARM_A9 = (0xA0000009) +NV0000_CTRL_SYSTEM_CPU_TYPE_ARM_A15 = (0xA000000F) +NV0000_CTRL_SYSTEM_CPU_TYPE_NV_DENVER_1_0 = (0xA0001000) +NV0000_CTRL_SYSTEM_CPU_TYPE_NV_DENVER_2_0 = (0xA0002000) +NV0000_CTRL_SYSTEM_CPU_TYPE_ARMV8A_GENERIC = (0xA00FF000) +NV0000_CTRL_SYSTEM_CPU_TYPE_ARMV9A_GENERIC = (0xA00FF001) +NV0000_CTRL_SYSTEM_CPU_CAP_MMX = (0x00000001) +NV0000_CTRL_SYSTEM_CPU_CAP_SSE = (0x00000002) +NV0000_CTRL_SYSTEM_CPU_CAP_3DNOW = (0x00000004) +NV0000_CTRL_SYSTEM_CPU_CAP_SSE2 = (0x00000008) +NV0000_CTRL_SYSTEM_CPU_CAP_SFENCE = (0x00000010) +NV0000_CTRL_SYSTEM_CPU_CAP_WRITE_COMBINING = (0x00000020) +NV0000_CTRL_SYSTEM_CPU_CAP_ALTIVEC = (0x00000040) +NV0000_CTRL_SYSTEM_CPU_CAP_PUT_NEEDS_IO = (0x00000080) +NV0000_CTRL_SYSTEM_CPU_CAP_NEEDS_WC_WORKAROUND = (0x00000100) +NV0000_CTRL_SYSTEM_CPU_CAP_3DNOW_EXT = (0x00000200) +NV0000_CTRL_SYSTEM_CPU_CAP_MMX_EXT = (0x00000400) +NV0000_CTRL_SYSTEM_CPU_CAP_CMOV = (0x00000800) +NV0000_CTRL_SYSTEM_CPU_CAP_CLFLUSH = (0x00001000) +NV0000_CTRL_SYSTEM_CPU_CAP_NEEDS_WAR_190854 = (0x00002000) +NV0000_CTRL_SYSTEM_CPU_CAP_SSE3 = (0x00004000) +NV0000_CTRL_SYSTEM_CPU_CAP_NEEDS_WAR_124888 = (0x00008000) +NV0000_CTRL_SYSTEM_CPU_CAP_HT_CAPABLE = (0x00010000) +NV0000_CTRL_SYSTEM_CPU_CAP_SSE41 = (0x00020000) +NV0000_CTRL_SYSTEM_CPU_CAP_SSE42 = (0x00040000) +NV0000_CTRL_SYSTEM_CPU_CAP_AVX = (0x00080000) +NV0000_CTRL_SYSTEM_CPU_CAP_ERMS = (0x00100000) +NV0000_CTRL_CMD_SYSTEM_GET_CHIPSET_INFO = (0x104) +NV0000_SYSTEM_MAX_CHIPSET_STRING_LENGTH = (0x0000020) +NV0000_SYSTEM_CHIPSET_INVALID_ID = (0xffff) +NV0000_CTRL_SYSTEM_GET_CHIPSET_INFO_PARAMS_MESSAGE_ID = (0x4) +NV0000_CTRL_SYSTEM_CHIPSET_FLAG_HAS_RESIZABLE_BAR_ISSUE = (0, 0) +NV0000_CTRL_SYSTEM_CHIPSET_FLAG_HAS_RESIZABLE_BAR_ISSUE_NO = (0x00000000) +NV0000_CTRL_SYSTEM_CHIPSET_FLAG_HAS_RESIZABLE_BAR_ISSUE_YES = (0x00000001) +NV0000_CTRL_SYSTEM_CHIPSET_FLAG_BAR1_UNALIGNED_ACCESS = (1, 1) +NV0000_CTRL_SYSTEM_CHIPSET_FLAG_BAR1_UNALIGNED_ACCESS_NO = (0x00000000) +NV0000_CTRL_SYSTEM_CHIPSET_FLAG_BAR1_UNALIGNED_ACCESS_YES = (0x00000001) +NV0000_CTRL_SYSTEM_GET_VRR_COOKIE_PRESENT = (0x107) +NV0000_CTRL_SYSTEM_GET_VRR_COOKIE_PRESENT_PARAMS_MESSAGE_ID = (0x7) +NV0000_CTRL_CMD_SYSTEM_GET_LOCK_TIMES = (0x109) +NV0000_CTRL_SYSTEM_GET_LOCK_TIMES_PARAMS_MESSAGE_ID = (0x9) +NV0000_CTRL_CMD_SYSTEM_GET_CLASSLIST = (0x108) +NV0000_CTRL_SYSTEM_MAX_CLASSLIST_SIZE = (32) +NV0000_CTRL_SYSTEM_GET_CLASSLIST_PARAMS_MESSAGE_ID = (0x8) +NV0000_CTRL_CMD_SYSTEM_NOTIFY_EVENT = (0x110) +NV0000_CTRL_SYSTEM_NOTIFY_EVENT_PARAMS_MESSAGE_ID = (0x10) +NV0000_CTRL_SYSTEM_EVENT_TYPE_LID_STATE = (0x00000000) +NV0000_CTRL_SYSTEM_EVENT_TYPE_POWER_SOURCE = (0x00000001) +NV0000_CTRL_SYSTEM_EVENT_TYPE_DOCK_STATE = (0x00000002) +NV0000_CTRL_SYSTEM_EVENT_TYPE_TRUST_LID = (0x00000003) +NV0000_CTRL_SYSTEM_EVENT_TYPE_TRUST_DOCK = (0x00000004) +NV0000_CTRL_SYSTEM_EVENT_DATA_LID_OPEN = (0x00000000) +NV0000_CTRL_SYSTEM_EVENT_DATA_LID_CLOSED = (0x00000001) +NV0000_CTRL_SYSTEM_EVENT_DATA_POWER_BATTERY = (0x00000000) +NV0000_CTRL_SYSTEM_EVENT_DATA_POWER_AC = (0x00000001) +NV0000_CTRL_SYSTEM_EVENT_DATA_UNDOCKED = (0x00000000) +NV0000_CTRL_SYSTEM_EVENT_DATA_DOCKED = (0x00000001) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_LID_DSM = (0x00000000) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_LID_DCS = (0x00000001) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_LID_NVIF = (0x00000002) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_LID_ACPI = (0x00000003) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_LID_POLL = (0x00000004) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_LID_COUNT = (0x5) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_DOCK_DSM = (0x00000000) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_DOCK_DCS = (0x00000001) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_DOCK_NVIF = (0x00000002) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_DOCK_ACPI = (0x00000003) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_DOCK_POLL = (0x00000004) +NV0000_CTRL_SYSTEM_EVENT_DATA_TRUST_DOCK_COUNT = (0x5) +NV0000_CTRL_SYSTEM_EVENT_DATA_FORCED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_EVENT_DATA_FORCED_TRUE = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GET_PLATFORM_TYPE = (0x111) +NV0000_CTRL_CMD_SYSTEM_GET_PLATFORM_TYPE_PARAMS_MESSAGE_ID = (0x11) +NV0000_CTRL_SYSTEM_GET_PLATFORM_TYPE_DESKTOP = (0x000000) +NV0000_CTRL_SYSTEM_GET_PLATFORM_TYPE_MOBILE_GENERIC = (0x000001) +NV0000_CTRL_SYSTEM_GET_PLATFORM_TYPE_MOBILE_TOSHIBA = (0x000002) +NV0000_CTRL_SYSTEM_GET_PLATFORM_TYPE_SOC = (0x000003) +NV0000_CTRL_CMD_SYSTEM_DEBUG_RMMSG_CTRL = (0x121) +NV0000_CTRL_SYSTEM_DEBUG_RMMSG_SIZE = 512 +NV0000_CTRL_SYSTEM_DEBUG_RMMSG_CTRL_CMD_GET = (0x00000000) +NV0000_CTRL_SYSTEM_DEBUG_RMMSG_CTRL_CMD_SET = (0x00000001) +NV0000_CTRL_SYSTEM_DEBUG_RMMSG_CTRL_PARAMS_MESSAGE_ID = (0x21) +NV0000_CTRL_CMD_SYSTEM_GPS_CONTROL = (0x122) +NV0000_CTRL_SYSTEM_GPS_CONTROL_PARAMS_MESSAGE_ID = (0x22) +NV0000_CTRL_CMD_SYSTEM_GPS_INVALID = (0xFFFF) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_GET_INIT = (0x0000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_SET_EXEC = (0x0001) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_GET_EXEC = (0x0002) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_SET_ACTIONS = (0x0003) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_GET_ACTIONS = (0x0004) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_SET_LOGIC = (0x0005) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_GET_LOGIC = (0x0006) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_SET_PREFERENCE = (0x0007) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_GET_PREFERENCE = (0x0008) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_SET_GPU2CPU_LIMIT = (0x0009) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_GET_GPU2CPU_LIMIT = (0x000A) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_SET_PMU_GPS_STATE = (0x000B) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_GET_PMU_GPS_STATE = (0x000C) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_SET_MAX_POWER = (0x0100) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_MAX_POWER = (0x0101) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_SET_COOLING_BUDGET = (0x0102) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_COOLING_BUDGET = (0x0103) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_SET_INTEGRAL_PERIOD = (0x0104) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_INTEGRAL_PERIOD = (0x0105) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_SET_RULESET = (0x0106) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_RULESET = (0x0107) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_RULE_COUNT = (0x0108) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_SET_APP_BOOST = (0x0109) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_APP_BOOST = (0x010A) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_SET_PWR_SUPPLY_MODE = (0x010B) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_PWR_SUPPLY_MODE = (0x010C) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_VCT_SUPPORT_INFO = (0x010D) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_GET_SUPPORTED_FUNCTIONS = (0x010E) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER = (0x0200) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_DELTA = (0x0201) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_FUTURE = (0x0202) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_LTMAVG = (0x0203) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_INTEGRAL = (0x0204) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_BURDEN = (0x0205) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_INTERMEDIATE = (0x0206) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_SENSOR_PARAMETERS = (0x0210) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_TEMP = (0x0220) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_TEMP_DELTA = (0x0221) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_TEMP_FUTURE = (0x0222) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_PSTATE = (0x0240) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_PSTATE_CAP = (0x0241) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_PSTATE_MIN = (0x0242) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_PSTATE_MAX = (0x0243) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_PSTATE_ACTION = (0x0244) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_PSTATE_SLFM_PRESENT = (0x0245) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_SET_POWER_SIM_STATE = (0x0250) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_SIM_STATE = (0x0251) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_SET_POWER_SIM_DATA = (0x0252) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_GET_POWER_SIM_DATA = (0x0253) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_INIT_USING_SBIOS_AND_ACK = (0x0320) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_PING_SBIOS_FOR_EVENT = (0x0321) +NV0000_CTRL_CMD_SYSTEM_GPS_LOCALE_INVALID = (0xFFFF) +NV0000_CTRL_CMD_SYSTEM_GPS_LOCALE_SYSTEM = (0x0000) +NV0000_CTRL_CMD_SYSTEM_GPS_LOCALE_CPU = lambda i: (0x0100+((i)%0x100)) # type: ignore +NV0000_CTRL_CMD_SYSTEM_GPS_LOCALE_GPU = lambda i: (0x0200+((i)%0x100)) # type: ignore +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_INVALID = (0x80000000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_INIT_NO = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_INIT_YES = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_EXEC_STOP = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_EXEC_START = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_ACTIONS_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_ACTIONS_ON = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_LOGIC_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_LOGIC_FUZZY = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_LOGIC_DETERMINISTIC = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_PREFERENCE_CPU = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_PREFERENCE_GPU = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_PREFERENCE_BOTH = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_NO_MAP_OVERRIDE = (0xFFFFFFFF) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_PMU_GPS_STATE_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_CMD_DEF_PMU_GPS_STATE_ON = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_PWR_SUPPLY_REAL = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_PWR_SUPPLY_FAKE_AC = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_PWR_SUPPLY_FAKE_BATT = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_FUNC_SUPPORT = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_FUNC_VENTURASTATUS = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_FUNC_GETPSS = (0x00000004) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_FUNC_SETPPC = (0x00000008) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_FUNC_GETPPC = (0x00000010) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_FUNC_VENTURACB = (0x00000020) +NV0000_CTRL_CMD_SYSTEM_GPS_SYS_DEF_FUNC_SYSPARAMS = (0x00000040) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_DEC_TO_P0 = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_DEC_BY_1 = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_DO_NOTHING = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_SET_CURRENT = (0x00000003) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_INC_BY_1 = (0x00000004) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_INC_BY_2 = (0x00000005) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_INC_TO_LFM = (0x00000006) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_ACTION_INC_TO_SLFM = (0x00000007) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_SLFM_PRESENT_NO = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_SLFM_PRESENT_YES = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_POWER_SIM_STATE_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_GPS_DATA_DEF_POWER_SIM_STATE_ON = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_GPS_BATCH_CONTROL = (0x123) +NV0000_CTRL_CMD_SYSTEM_GPS_BATCH_COMMAND_MAX = (16) +NV0000_CTRL_SYSTEM_GPS_BATCH_CONTROL_PARAMS_MESSAGE_ID = (0x23) +NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS = (0x127) +NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS = 32 +NV0000_CTRL_SYSTEM_MAX_ATTACHED_GPUS_SQUARED = 1024 +NV0000_CTRL_SYSTEM_MAX_P2P_GROUP_GPUS = 8 +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INVALID_PEER = 0xffffffff +NV0000_CTRL_P2P_CAPS_INDEX_READ = 0 +NV0000_CTRL_P2P_CAPS_INDEX_WRITE = 1 +NV0000_CTRL_P2P_CAPS_INDEX_NVLINK = 2 +NV0000_CTRL_P2P_CAPS_INDEX_ATOMICS = 3 +NV0000_CTRL_P2P_CAPS_INDEX_PROP = 4 +NV0000_CTRL_P2P_CAPS_INDEX_LOOPBACK = 5 +NV0000_CTRL_P2P_CAPS_INDEX_PCI = 6 +NV0000_CTRL_P2P_CAPS_INDEX_C2C = 7 +NV0000_CTRL_P2P_CAPS_INDEX_PCI_BAR1 = 8 +NV0000_CTRL_P2P_CAPS_INDEX_TABLE_SIZE = 9 +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PARAMS_MESSAGE_ID = (0x27) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_WRITES_SUPPORTED = (0, 0) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_WRITES_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_WRITES_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_READS_SUPPORTED = (1, 1) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_READS_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_READS_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PROP_SUPPORTED = (2, 2) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PROP_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PROP_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_NVLINK_SUPPORTED = (3, 3) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_NVLINK_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_NVLINK_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_ATOMICS_SUPPORTED = (4, 4) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_ATOMICS_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_ATOMICS_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_LOOPBACK_SUPPORTED = (5, 5) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_LOOPBACK_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_LOOPBACK_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PCI_SUPPORTED = (6, 6) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PCI_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PCI_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_WRITES_SUPPORTED = (7, 7) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_WRITES_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_WRITES_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_READS_SUPPORTED = (8, 8) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_READS_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_READS_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_ATOMICS_SUPPORTED = (9, 9) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_ATOMICS_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_ATOMICS_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_NVLINK_SUPPORTED = (10, 10) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_NVLINK_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INDIRECT_NVLINK_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_C2C_SUPPORTED = (12, 12) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_C2C_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_C2C_SUPPORTED_TRUE = (0x00000001) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PCI_BAR1_SUPPORTED = (13, 13) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PCI_BAR1_SUPPORTED_FALSE = (0x00000000) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PCI_BAR1_SUPPORTED_TRUE = (0x00000001) +NV0000_P2P_CAPS_STATUS_OK = (0x00) +NV0000_P2P_CAPS_STATUS_CHIPSET_NOT_SUPPORTED = (0x01) +NV0000_P2P_CAPS_STATUS_GPU_NOT_SUPPORTED = (0x02) +NV0000_P2P_CAPS_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED = (0x03) +NV0000_P2P_CAPS_STATUS_DISABLED_BY_REGKEY = (0x04) +NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED = (0x05) +NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_V2 = (0x12b) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_V2_PARAMS_MESSAGE_ID = (0x2B) +NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_MATRIX = (0x13a) +NV0000_CTRL_SYSTEM_GET_P2P_CAPS_MATRIX_PARAMS_MESSAGE_ID = (0x3A) +NV0000_CTRL_CMD_SYSTEM_GPS_CTRL = (0x12a) +NV0000_CTRL_SYSTEM_GPS_CTRL_PARAMS_MESSAGE_ID = (0x2A) +NV0000_CTRL_GPS_PSHARE_PARAMS_PSP_CURRENT_VERSION = (0x00010000) +NV0000_CTRL_GPS_CMD_TYPE_GET_THERM_LIMIT = (0x00000002) +NV0000_CTRL_GPS_INPUT_SENSOR_INDEX = (0x00000000) +NV0000_CTRL_GPS_RESULT_THERMAL_LIMIT = (0x00000000) +NV0000_CTRL_GPS_RESULT_MIN_LIMIT = (0x00000001) +NV0000_CTRL_GPS_RESULT_MAX_LIMIT = (0x00000002) +NV0000_CTRL_GPS_RESULT_LIMIT_SOURCE = (0x00000003) +NV0000_CTRL_GPS_CMD_TYPE_SET_THERM_LIMIT = (0x00000003) +NV0000_CTRL_GPS_INPUT_THERMAL_LIMIT = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_CTRL_DOWN_N_DELTA = (0x00000004) +NV0000_CTRL_GPS_RESULT_TEMP_CTRL_DOWN_N_DELTA = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_CTRL_DOWN_N_DELTA = (0x00000005) +NV0000_CTRL_GPS_INPUT_TEMP_CTRL_DOWN_N_DELTA = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_CTRL_HOLD_DELTA = (0x00000006) +NV0000_CTRL_GPS_RESULT_TEMP_CTRL_HOLD_DELTA = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_CTRL_HOLD_DELTA = (0x00000007) +NV0000_CTRL_GPS_INPUT_TEMP_CTRL_HOLD_DELTA = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_CTRL_UP_DELTA = (0x00000008) +NV0000_CTRL_GPS_RESULT_TEMP_CTRL_UP_DELTA = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_CTRL_UP_DELTA = (0x00000009) +NV0000_CTRL_GPS_INPUT_TEMP_CTRL_UP_DELTA = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_CTRL_ENGAGE_DELTA = (0x0000000A) +NV0000_CTRL_GPS_RESULT_TEMP_CTRL_ENGAGE_DELTA = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_CTRL_ENGAGE_DELTA = (0x0000000B) +NV0000_CTRL_GPS_INPUT_TEMP_CTRL_ENGAGE_DELTA = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_CTRL_DISENGAGE_DELTA = (0x0000000C) +NV0000_CTRL_GPS_RESULT_TEMP_CTRL_DISENGAGE_DELTA = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_CTRL_DISENGAGE_DELTA = (0x0000000D) +NV0000_CTRL_GPS_INPUT_TEMP_CTRL_DISENGAGE_DELTA = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_CTRL_STATUS = (0x00000016) +NV0000_CTRL_GPS_RESULT_TEMP_CTRL_STATUS = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_CTRL_STATUS = (0x00000017) +NV0000_CTRL_GPS_INPUT_TEMP_CTRL_STATUS = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_CPU_GET_UTIL_AVG_NUM = (0x00000018) +NV0000_CTRL_GPS_RESULT_CPU_SET_UTIL_AVG_NUM = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_CPU_SET_UTIL_AVG_NUM = (0x00000019) +NV0000_CTRL_GPS_INPUT_CPU_GET_UTIL_AVG_NUM = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_PERF_SENSOR = (0x0000001A) +NV0000_CTRL_GPS_INPUT_NEXT_EXPECTED_POLL = (0x00000001) +NV0000_CTRL_GPS_RESULT_PERF_SENSOR_VALUE = (0x00000000) +NV0000_CTRL_GPS_RESULT_PERF_SENSOR_AVAILABLE = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_CALL_ACPI = (0x0000001B) +NV0000_CTRL_GPS_INPUT_ACPI_CMD = (0x00000000) +NV0000_CTRL_GPS_INPUT_ACPI_PARAM_IN = (0x00000001) +NV0000_CTRL_GPS_OUTPUT_ACPI_RESULT_1 = (0x00000000) +NV0000_CTRL_GPS_OUTPUT_ACPI_RESULT_2 = (0x00000001) +NV0000_CTRL_GPS_OUTPUT_ACPI_PSHAREPARAM_STATUS = (0x00000000) +NV0000_CTRL_GPS_OUTPUT_ACPI_PSHAREPARAM_VERSION = (0x00000001) +NV0000_CTRL_GPS_OUTPUT_ACPI_PSHAREPARAM_SZ = (0x00000002) +NV0000_CTRL_GPS_OUTPUT_ACPI_PSS_SZ = (0x00000000) +NV0000_CTRL_GPS_OUTPUT_ACPI_PSS_COUNT = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_SET_IGPU_TURBO = (0x0000001C) +NV0000_CTRL_GPS_INPUT_SET_IGPU_TURBO = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_PERIOD = (0x00000026) +NV0000_CTRL_GPS_INPUT_TEMP_PERIOD = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_PERIOD = (0x00000027) +NV0000_CTRL_GPS_RESULT_TEMP_PERIOD = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_NUDGE_FACTOR = (0x00000028) +NV0000_CTRL_GPS_INPUT_TEMP_NUDGE_UP = (0x00000000) +NV0000_CTRL_GPS_INPUT_TEMP_NUDGE_DOWN = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_NUDGE_FACTOR = (0x00000029) +NV0000_CTRL_GPS_RESULT_TEMP_NUDGE_UP = (0x00000000) +NV0000_CTRL_GPS_RESULT_TEMP_NUDGE_DOWN = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_THRESHOLD_SAMPLES = (0x0000002A) +NV0000_CTRL_GPS_INPUT_TEMP_THRESHOLD_SAMPLE_HOLD = (0x00000000) +NV0000_CTRL_GPS_INPUT_TEMP_THRESHOLD_SAMPLE_STEP = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_THRESHOLD_SAMPLES = (0x0000002B) +NV0000_CTRL_GPS_RESULT_TEMP_THRESHOLD_SAMPLE_HOLD = (0x00000000) +NV0000_CTRL_GPS_RESULT_TEMP_THRESHOLD_SAMPLE_STEP = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_SET_TEMP_PERF_LIMITS = (0x0000002C) +NV0000_CTRL_GPS_INPUT_TEMP_PERF_LIMIT_UPPER = (0x00000000) +NV0000_CTRL_GPS_INPUT_TEMP_PERF_LIMIT_LOWER = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_GET_TEMP_PERF_LIMITS = (0x0000002D) +NV0000_CTRL_GPS_RESULT_TEMP_PERF_LIMIT_UPPER = (0x00000000) +NV0000_CTRL_GPS_RESULT_TEMP_PERF_LIMIT_LOWER = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_SET_PM1_AVAILABLE = (0x0000002E) +NV0000_CTRL_GPS_INPUT_PM1_AVAILABLE = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_PM1_AVAILABLE = (0x0000002F) +NV0000_CTRL_GPS_OUTPUT_PM1_AVAILABLE = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_CPU_PACKAGE_LIMITS = (0x00000044) +NV0000_CTRL_GPS_CMD_TYPE_GET_CPU_PACKAGE_LIMITS_PL1 = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_CPU_PACKAGE_LIMITS_PL2 = (0x00000001) +NV0000_CTRL_GPS_CMD_TYPE_SET_CPU_PACKAGE_LIMITS = (0x00000045) +NV0000_CTRL_GPS_CMD_TYPE_SET_CPU_PACKAGE_LIMITS_PL1 = (0x00000000) +NV0000_CTRL_GPS_CMD_TYPE_GET_CPU_FREQ_LIMIT = (0x00000046) +NV0000_CTRL_GPS_CMD_TYPE_GET_CPU_FREQ_LIMIT_MHZ = (0) +NV0000_CTRL_GPS_CMD_TYPE_SET_CPU_FREQ_LIMIT = (0x00000047) +NV0000_CTRL_GPS_CMD_TYPE_SET_CPU_FREQ_LIMIT_MHZ = (0) +NV0000_CTRL_GPS_CMD_TYPE_GET_PPM = (0x00000048) +NV0000_CTRL_GPS_CMD_TYPE_GET_PPM_INDEX = (0) +NV0000_CTRL_GPS_CMD_TYPE_GET_PPM_AVAILABLE_MASK = (1) +NV0000_CTRL_GPS_CMD_TYPE_SET_PPM = (0x00000049) +NV0000_CTRL_GPS_CMD_TYPE_SET_PPM_INDEX = (0) +NV0000_CTRL_GPS_CMD_TYPE_SET_PPM_INDEX_MAX = (2) +NV0000_CTRL_GPS_PPM_INDEX = (7, 0) +NV0000_CTRL_GPS_PPM_INDEX_MAXPERF = (0) +NV0000_CTRL_GPS_PPM_INDEX_BALANCED = (1) +NV0000_CTRL_GPS_PPM_INDEX_QUIET = (2) +NV0000_CTRL_GPS_PPM_INDEX_INVALID = (0xFF) +NV0000_CTRL_GPS_PPM_MASK = (15, 8) +NV0000_CTRL_GPS_PPM_MASK_INVALID = (0) +NV0000_CTRL_GPS_CMD_PS_STATUS_OFF = (0) +NV0000_CTRL_GPS_CMD_PS_STATUS_ON = (1) +GPS_MAX_COUNTERS_PER_BLOCK = 32 +NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSOR_COUNTERS_PARAMS_MESSAGE_ID = (0x29) +NV0000_CTRL_CMD_SYSTEM_GPS_GET_PERF_SENSORS = (0x12c) +NV0000_CTRL_SYSTEM_GPS_GET_PERF_SENSORS_PARAMS_MESSAGE_ID = (0x2C) +NV0000_CTRL_CMD_SYSTEM_GPS_GET_EXTENDED_PERF_SENSORS = (0x12e) +NV0000_CTRL_SYSTEM_GPS_GET_EXTENDED_PERF_SENSORS_PARAMS_MESSAGE_ID = (0x2E) +GPS_MAX_ACPI_OUTPUT_BUFFER_SIZE = 288 +NV0000_CTRL_SYSTEM_GPS_CALL_ACPI_PARAMS_MESSAGE_ID = (0x2D) +NV0000_CTRL_CMD_SYSTEM_GPS_CALL_ACPI = (0x12d) +NV0000_CTRL_SYSTEM_PARAM_TGPU = (0x00000000) +NV0000_CTRL_SYSTEM_PARAM_PDTS = (0x00000001) +NV0000_CTRL_SYSTEM_PARAM_SFAN = (0x00000002) +NV0000_CTRL_SYSTEM_PARAM_SKNT = (0x00000003) +NV0000_CTRL_SYSTEM_PARAM_CPUE = (0x00000004) +NV0000_CTRL_SYSTEM_PARAM_TMP1 = (0x00000005) +NV0000_CTRL_SYSTEM_PARAM_TMP2 = (0x00000006) +NV0000_CTRL_SYSTEM_PARAM_CTGP = (0x00000007) +NV0000_CTRL_SYSTEM_PARAM_PPMD = (0x00000008) +NV0000_CTRL_SYSTEM_PARAM_COUNT = (0x00000009) +NV0000_CTRL_CMD_SYSTEM_EXECUTE_ACPI_METHOD = (0x130) +NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_PARAMS_MESSAGE_ID = (0x30) +NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_DSM_NVOP_OPTIMUSCAPS = (0x00000000) +NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_DSM_NVOP_OPTIMUSFLAG = (0x00000001) +NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_DSM_JT_CAPS = (0x00000002) +NV0000_CTRL_SYSTEM_EXECUTE_ACPI_METHOD_DSM_JT_PLATPOLICY = (0x00000003) +NV0000_CTRL_CMD_SYSTEM_ENABLE_ETW_EVENTS = (0x131) +NV0000_CTRL_SYSTEM_ENABLE_ETW_EVENTS_PARAMS_MESSAGE_ID = (0x31) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_ALL = (0x00000001) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_NOFREQ = (0x00000002) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_FLUSH = (0x00000004) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_PERF = (0x00000010) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_ELPG = (0x00000020) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_NVDPS = (0x00000040) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_POWER = (0x00000080) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_DISP = (0x00000100) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_RMAPI = (0x00000200) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_INTR = (0x00000400) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_LOCK = (0x00000800) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_RCJOURNAL = (0x00001000) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_GENERIC = (0x00002000) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_THERM = (0x00004000) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_GPS = (0x00008000) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_PCIE = (0x00010000) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_NVTELEMETRY = (0x00020000) +NV0000_CTRL_CMD_SYSTEM_GPS_GET_FRM_DATA = (0x12f) +NV0000_CTRL_SYSTEM_GPS_FRM_DATA_SAMPLE_SIZE = 64 +NV0000_CTRL_SYSTEM_GPS_GET_FRM_DATA_PARAMS_MESSAGE_ID = (0x2F) +NV0000_CTRL_CMD_SYSTEM_GPS_SET_FRM_DATA = (0x132) +NV0000_CTRL_SYSTEM_GPS_SET_FRM_DATA_PARAMS_MESSAGE_ID = (0x32) +NV0000_CTRL_CMD_SYSTEM_GET_VGX_SYSTEM_INFO_BUFFER_SIZE = 256 +NV0000_CTRL_CMD_SYSTEM_GET_VGX_SYSTEM_INFO = (0x133) +NV0000_CTRL_SYSTEM_GET_VGX_SYSTEM_INFO_PARAMS_MESSAGE_ID = (0x33) +NV0000_CTRL_CMD_SYSTEM_GET_GPUS_POWER_STATUS = (0x134) +NV0000_CTRL_SYSTEM_GET_GPUS_POWER_STATUS_PARAMS_MESSAGE_ID = (0x34) +NV0000_CTRL_SYSTEM_GPU_EXTERNAL_POWER_STATUS_CONNECTED = 0 +NV0000_CTRL_SYSTEM_GPU_EXTERNAL_POWER_STATUS_NOT_CONNECTED = 1 +NV0000_CTRL_CMD_SYSTEM_GET_PRIVILEGED_STATUS = (0x135) +NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_PARAMS_MESSAGE_ID = (0x35) +NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_PRIV_USER_FLAG = (0x00000001) +NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_KERNEL_HANDLE_FLAG = (0x00000002) +NV0000_CTRL_SYSTEM_GET_PRIVILEGED_STATUS_PRIV_HANDLE_FLAG = (0x00000004) +NV0000_CTRL_CMD_SYSTEM_GET_FABRIC_STATUS = (0x136) +NV0000_CTRL_SYSTEM_GET_FABRIC_STATUS_PARAMS_MESSAGE_ID = (0x36) +NV0000_CTRL_VGPU_GET_VGPU_VERSION = (0x137) +NV0000_CTRL_VGPU_GET_VGPU_VERSION_PARAMS_MESSAGE_ID = (0x37) +NV0000_CTRL_VGPU_SET_VGPU_VERSION = (0x138) +NV0000_CTRL_VGPU_SET_VGPU_VERSION_PARAMS_MESSAGE_ID = (0x38) +NV0000_CTRL_CMD_SYSTEM_GET_RM_INSTANCE_ID = (0x139) +NV0000_CTRL_SYSTEM_GET_RM_INSTANCE_ID_PARAMS_MESSAGE_ID = (0x39) +NV0000_CTRL_CMD_SYSTEM_NVPCF_GET_POWER_MODE_INFO = (0x13b) +NVPCF_CTRL_SYSPWRLIMIT_TYPE_BASE = 1 +NV0000_CTRL_SYSTEM_POWER_INFO_INDEX_MAX_SIZE = 32 +NV0000_CTRL_CMD_SYSTEM_GET_SYSTEM_POWER_LIMIT_MESSAGE_ID = (0x48) +NV0000_CTRL_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS_MESSAGE_ID = (0x3B) +CONTROLLER_FILTER_TYPE_EMWA = 0 +CONTROLLER_FILTER_TYPE_MOVING_MAX = 1 +NVPCF0100_CTRL_CONFIG_DSM_1X_FUNC_GET_SUPPORTED_CASE = 0 +NVPCF0100_CTRL_CONFIG_DSM_1X_FUNC_GET_DYNAMIC_CASE = 1 +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_SUPPORTED_CASE = 2 +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_DYNAMIC_CASE = 3 +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_STATIC_CASE = 4 +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_DC_SYSTEM_POWER_LIMITS_CASE = 5 +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_CPU_TDP_LIMIT_CONTROL_CASE = 6 +NVPCF0100_CTRL_CONFIG_DSM_1X_FUNC_GET_SUPPORTED = (0x00000000) +NVPCF0100_CTRL_CONFIG_DSM_1X_FUNC_GET_DYNAMIC_PARAMS = (0x00000002) +NVPCF0100_CTRL_CONFIG_DSM_FUNC_GET_SUPPORTED_IS_SUPPORTED = (0, 0) +NVPCF0100_CTRL_CONFIG_DSM_FUNC_GET_SUPPORTED_IS_SUPPORTED_YES = 1 +NVPCF0100_CTRL_CONFIG_DSM_FUNC_GET_SUPPORTED_IS_SUPPORTED_NO = 0 +NVPCF0100_CTRL_CONFIG_DSM_FUNC_GET_DC_SYSTEM_POWER_LIMITS_IS_SUPPORTED = (8, 8) +NVPCF0100_CTRL_CONFIG_DSM_FUNC_GET_DC_SYSTEM_POWER_LIMITS_IS_SUPPORTED_YES = 1 +NVPCF0100_CTRL_CONFIG_DSM_FUNC_GET_DC_SYSTEM_POWER_LIMITS_IS_SUPPORTED_NO = 0 +NVPCF0100_CTRL_CONFIG_DSM_FUNC_CPU_TDP_LIMIT_CONTROL_IS_SUPPORTED = (9, 9) +NVPCF0100_CTRL_CONFIG_DSM_FUNC_CPU_TDP_LIMIT_CONTROL_IS_SUPPORTED_YES = 1 +NVPCF0100_CTRL_CONFIG_DSM_FUNC_CPU_TDP_LIMIT_CONTROL_IS_SUPPORTED_NO = 0 +NVPCF0100_CTRL_CONFIG_DSM_2X_VERSION = (0x00000200) +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_SUPPORTED = (0x00000000) +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_STATIC_CONFIG_TABLES = (0x00000001) +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_DYNAMIC_PARAMS = (0x00000002) +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_DC_SYSTEM_POWER_LIMITS_TABLE = (0x00000008) +NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_CPU_TDP_LIMIT_CONTROL = (0x00000009) +NVPCF0100_CTRL_CONFIG_2X_BUFF_SIZE_MAX = (255) +NV0000_CTRL_CMD_SYSTEM_SYNC_EXTERNAL_FABRIC_MGMT = (0x13c) +NV0000_CTRL_CMD_SYSTEM_SYNC_EXTERNAL_FABRIC_MGMT_PARAMS_MESSAGE_ID = (0x3C) +NV0000_CTRL_CMD_SYSTEM_GET_CLIENT_DATABASE_INFO = (0x13d) +NV0000_CTRL_SYSTEM_GET_CLIENT_DATABASE_INFO_PARAMS_MESSAGE_ID = (0x3D) +NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_MAX_STRING_SIZE = 256 +NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION_V2 = (0x13e) +NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_PARAMS_MESSAGE_ID = (0x3E) +NV0000_CTRL_CMD_SYSTEM_RMCTRL_CACHE_MODE_CTRL = (0x13f) +NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_PARAMS_MESSAGE_ID = (0x3F) +NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_CMD_GET = (0x00000000) +NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_CMD_SET = (0x00000001) +NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE = (0x00000000) +NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_ENABLE = (0x00000001) +NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_VERIFY_ONLY = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CONTROL = (0x140) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CONTROL_PARAMS_MESSAGE_ID = (0x40) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_INVALID = (0xFFFF) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_GET_INIT = (0x0000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_SET_EXEC = (0x0001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_GET_EXEC = (0x0002) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_SET_ACTIONS = (0x0003) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_GET_ACTIONS = (0x0004) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_SET_LOGIC = (0x0005) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_GET_LOGIC = (0x0006) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_SET_PREFERENCE = (0x0007) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_GET_PREFERENCE = (0x0008) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_SET_GPU2CPU_LIMIT = (0x0009) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_GET_GPU2CPU_LIMIT = (0x000A) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_SET_PMU_PFM_REQ_HNDLR_STATE = (0x000B) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_GET_PMU_PFM_REQ_HNDLR_STATE = (0x000C) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_SET_MAX_POWER = (0x0100) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_MAX_POWER = (0x0101) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_SET_COOLING_BUDGET = (0x0102) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_COOLING_BUDGET = (0x0103) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_SET_INTEGRAL_PERIOD = (0x0104) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_INTEGRAL_PERIOD = (0x0105) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_SET_RULESET = (0x0106) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_RULESET = (0x0107) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_RULE_COUNT = (0x0108) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_SET_APP_BOOST = (0x0109) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_APP_BOOST = (0x010A) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_SET_PWR_SUPPLY_MODE = (0x010B) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_PWR_SUPPLY_MODE = (0x010C) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_VCT_SUPPORT_INFO = (0x010D) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_GET_SUPPORTED_FUNCTIONS = (0x010E) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER = (0x0200) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_DELTA = (0x0201) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_FUTURE = (0x0202) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_LTMAVG = (0x0203) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_INTEGRAL = (0x0204) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_BURDEN = (0x0205) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_INTERMEDIATE = (0x0206) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_SENSOR_PARAMETERS = (0x0210) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_TEMP = (0x0220) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_TEMP_DELTA = (0x0221) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_TEMP_FUTURE = (0x0222) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_PSTATE = (0x0240) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_PSTATE_CAP = (0x0241) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_PSTATE_MIN = (0x0242) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_PSTATE_MAX = (0x0243) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_PSTATE_ACTION = (0x0244) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_PSTATE_SLFM_PRESENT = (0x0245) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_SET_POWER_SIM_STATE = (0x0250) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_SIM_STATE = (0x0251) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_SET_POWER_SIM_DATA = (0x0252) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_GET_POWER_SIM_DATA = (0x0253) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_INIT_USING_SBIOS_AND_ACK = (0x0320) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_PING_SBIOS_FOR_EVENT = (0x0321) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_LOCALE_INVALID = (0xFFFF) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_LOCALE_SYSTEM = (0x0000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_LOCALE_CPU = lambda i: (0x0100+((i)%0x100)) # type: ignore +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_LOCALE_GPU = lambda i: (0x0200+((i)%0x100)) # type: ignore +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_INVALID = (0x80000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_INIT_NO = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_INIT_YES = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_EXEC_STOP = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_EXEC_START = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_ACTIONS_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_ACTIONS_ON = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_LOGIC_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_LOGIC_FUZZY = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_LOGIC_DETERMINISTIC = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_PREFERENCE_CPU = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_PREFERENCE_GPU = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_PREFERENCE_BOTH = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_NO_MAP_OVERRIDE = (0xFFFFFFFF) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_PMU_PFM_REQ_HNDLR_STATE_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CMD_DEF_PMU_PFM_REQ_HNDLR_STATE_ON = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_PWR_SUPPLY_REAL = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_PWR_SUPPLY_FAKE_AC = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_PWR_SUPPLY_FAKE_BATT = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_FUNC_SUPPORT = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_FUNC_VENTURASTATUS = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_FUNC_GETPSS = (0x00000004) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_FUNC_SETPPC = (0x00000008) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_FUNC_GETPPC = (0x00000010) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_FUNC_VENTURACB = (0x00000020) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SYS_DEF_FUNC_SYSPARAMS = (0x00000040) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_DEC_TO_P0 = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_DEC_BY_1 = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_DO_NOTHING = (0x00000002) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_SET_CURRENT = (0x00000003) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_INC_BY_1 = (0x00000004) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_INC_BY_2 = (0x00000005) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_INC_TO_LFM = (0x00000006) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_ACTION_INC_TO_SLFM = (0x00000007) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_SLFM_PRESENT_NO = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_SLFM_PRESENT_YES = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_POWER_SIM_STATE_OFF = (0x00000000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_DATA_DEF_POWER_SIM_STATE_ON = (0x00000001) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL = (0x141) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_BATCH_COMMAND_MAX = (16) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_BATCH_CONTROL_PARAMS_MESSAGE_ID = (0x41) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CTRL = (0x142) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CTRL_PARAMS_MESSAGE_ID = (0x42) +NV0000_CTRL_PFM_REQ_HNDLR_PSHARE_PARAMS_PSP_CURRENT_VERSION = (0x00010000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_THERM_LIMIT = (0x00000002) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_SENSOR_INDEX = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_THERMAL_LIMIT = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_MIN_LIMIT = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_MAX_LIMIT = (0x00000002) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_LIMIT_SOURCE = (0x00000003) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_THERM_LIMIT = (0x00000003) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_THERMAL_LIMIT = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_CTRL_DOWN_N_DELTA = (0x00000004) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_CTRL_DOWN_N_DELTA = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_CTRL_DOWN_N_DELTA = (0x00000005) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_CTRL_DOWN_N_DELTA = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_CTRL_HOLD_DELTA = (0x00000006) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_CTRL_HOLD_DELTA = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_CTRL_HOLD_DELTA = (0x00000007) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_CTRL_HOLD_DELTA = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_CTRL_UP_DELTA = (0x00000008) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_CTRL_UP_DELTA = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_CTRL_UP_DELTA = (0x00000009) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_CTRL_UP_DELTA = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_CTRL_ENGAGE_DELTA = (0x0000000A) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_CTRL_ENGAGE_DELTA = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_CTRL_ENGAGE_DELTA = (0x0000000B) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_CTRL_ENGAGE_DELTA = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_CTRL_DISENGAGE_DELTA = (0x0000000C) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_CTRL_DISENGAGE_DELTA = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_CTRL_DISENGAGE_DELTA = (0x0000000D) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_CTRL_DISENGAGE_DELTA = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_CTRL_STATUS = (0x00000016) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_CTRL_STATUS = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_CTRL_STATUS = (0x00000017) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_CTRL_STATUS = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_CPU_GET_UTIL_AVG_NUM = (0x00000018) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_CPU_SET_UTIL_AVG_NUM = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_CPU_SET_UTIL_AVG_NUM = (0x00000019) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_CPU_GET_UTIL_AVG_NUM = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_PERF_SENSOR = (0x0000001A) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_NEXT_EXPECTED_POLL = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_PERF_SENSOR_VALUE = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_PERF_SENSOR_AVAILABLE = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_CALL_ACPI = (0x0000001B) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_ACPI_CMD = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_ACPI_PARAM_IN = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_ACPI_RESULT_1 = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_ACPI_RESULT_2 = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_ACPI_PSHAREPARAM_STATUS = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_ACPI_PSHAREPARAM_VERSION = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_ACPI_PSHAREPARAM_SZ = (0x00000002) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_ACPI_PSS_SZ = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_ACPI_PSS_COUNT = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_IGPU_TURBO = (0x0000001C) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_SET_IGPU_TURBO = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_PERIOD = (0x00000026) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_PERIOD = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_PERIOD = (0x00000027) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_PERIOD = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_NUDGE_FACTOR = (0x00000028) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_NUDGE_UP = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_NUDGE_DOWN = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_NUDGE_FACTOR = (0x00000029) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_NUDGE_UP = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_NUDGE_DOWN = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_THRESHOLD_SAMPLES = (0x0000002A) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_THRESHOLD_SAMPLE_HOLD = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_THRESHOLD_SAMPLE_STEP = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_THRESHOLD_SAMPLES = (0x0000002B) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_THRESHOLD_SAMPLE_HOLD = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_THRESHOLD_SAMPLE_STEP = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_TEMP_PERF_LIMITS = (0x0000002C) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_PERF_LIMIT_UPPER = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_TEMP_PERF_LIMIT_LOWER = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_TEMP_PERF_LIMITS = (0x0000002D) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_PERF_LIMIT_UPPER = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_RESULT_TEMP_PERF_LIMIT_LOWER = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_PM1_AVAILABLE = (0x0000002E) +NV0000_CTRL_PFM_REQ_HNDLR_INPUT_PM1_AVAILABLE = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_PM1_AVAILABLE = (0x0000002F) +NV0000_CTRL_PFM_REQ_HNDLR_OUTPUT_PM1_AVAILABLE = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_CPU_PACKAGE_LIMITS = (0x00000044) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_CPU_PACKAGE_LIMITS_PL1 = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_CPU_PACKAGE_LIMITS_PL2 = (0x00000001) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_CPU_PACKAGE_LIMITS = (0x00000045) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_CPU_PACKAGE_LIMITS_PL1 = (0x00000000) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_CPU_FREQ_LIMIT = (0x00000046) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_CPU_FREQ_LIMIT_MHZ = (0) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_CPU_FREQ_LIMIT = (0x00000047) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_CPU_FREQ_LIMIT_MHZ = (0) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_PPM = (0x00000048) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_PPM_INDEX = (0) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_GET_PPM_AVAILABLE_MASK = (1) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_PPM = (0x00000049) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_PPM_INDEX = (0) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_TYPE_SET_PPM_INDEX_MAX = (2) +NV0000_CTRL_PFM_REQ_HNDLR_PPM_INDEX = (7, 0) +NV0000_CTRL_PFM_REQ_HNDLR_PPM_INDEX_MAXPERF = (0) +NV0000_CTRL_PFM_REQ_HNDLR_PPM_INDEX_BALANCED = (1) +NV0000_CTRL_PFM_REQ_HNDLR_PPM_INDEX_QUIET = (2) +NV0000_CTRL_PFM_REQ_HNDLR_PPM_INDEX_INVALID = (0xFF) +NV0000_CTRL_PFM_REQ_HNDLR_PPM_MASK = (15, 8) +NV0000_CTRL_PFM_REQ_HNDLR_PPM_MASK_INVALID = (0) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_PS_STATUS_OFF = (0) +NV0000_CTRL_PFM_REQ_HNDLR_CMD_PS_STATUS_ON = (1) +PFM_REQ_HNDLR_MAX_COUNTERS_PER_BLOCK = 32 +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSORS = (0x146) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_PERF_SENSORS_PARAMS_MESSAGE_ID = (0x46) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_GET_EXTENDED_PERF_SENSORS = (0x147) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_EXTENDED_PERF_SENSORS_PARAMS_MESSAGE_ID = (0x47) +PFM_REQ_HNDLR_MAX_ACPI_OUTPUT_BUFFER_SIZE = 288 +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_CALL_ACPI_PARAMS_MESSAGE_ID = (0x43) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_CALL_ACPI = (0x143) +NV0000_CTRL_SYSTEM_RMTRACE_MODULE_PFM_REQ_HNDLR = (0x00008000) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_GET_FRM_DATA = (0x144) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_FRM_DATA_SAMPLE_SIZE = 64 +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_GET_FRM_DATA_PARAMS_MESSAGE_ID = (0x44) +NV0000_CTRL_CMD_SYSTEM_PFM_REQ_HNDLR_SET_FRM_DATA = (0x145) +NV0000_CTRL_SYSTEM_PFM_REQ_HNDLR_SET_FRM_DATA_PARAMS_MESSAGE_ID = (0x45) +NV0000_CTRL_CMD_SYSTEM_READ_CPER = (0x149) +NV0000_CTRL_SYSTEM_READ_CPER_ACCESS_TYPE_GPU = (1) +READ_CPER_BUFFER_SIZE = 1024 +NV0000_CTRL_SYSTEM_READ_CPER_PARAMS_MESSAGE_ID = (0x49) +NV0000_CTRL_CMD_SYSTEM_SOC_SET_PERF_LIMITS = (0x150) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMIT_ID_TEST_PERF_LEVELS = (0) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMIT_ID_ISMODEPOSSIBLE = (1) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMIT_ID_MCLK_SWITCH = (2) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMIT_ID_MODS = (3) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMIT_BW_RESOURCE_PERF_LEVEL = (0) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMIT_BW_RESOURCE_DISPCLK = (1) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMIT_BW_RESOURCE_HUBCLK = (2) +NV0000_CTRL_SYSTEM_SOC_PERF_MAX_LIMITS = (16) +NV0000_CTRL_SYSTEM_SOC_SET_PERF_LIMITS_PARAMS_MESSAGE_ID = (0x150) +NV0000_CTRL_CMD_OS_UNIX_FLUSH_USER_CACHE = (0x3d02) +NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH = (0x00000001) +NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_INVALIDATE = (0x00000002) +NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH_INVALIDATE = (0x00000003) +NV0000_CTRL_CMD_OS_UNIX_GET_CONTROL_FILE_DESCRIPTOR = (0x3d04) +NV0000_CTRL_CMD_OS_UNIX_EXPORT_OBJECT_TO_FD = (0x3d05) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS_MESSAGE_ID = (0x5) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS_EMPTY_FD = (0, 0) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS_EMPTY_FD_FALSE = (0x00000000) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS_EMPTY_FD_TRUE = (0x00000001) +NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_FROM_FD = (0x3d06) +NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS_MESSAGE_ID = (0x6) +NV0000_CTRL_CMD_OS_GET_GPU_INFO = (0x3d07) +NV0000_CTRL_CMD_OS_UNIX_GET_EXPORT_OBJECT_INFO = (0x3d08) +NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE = 64 +NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS_MESSAGE_ID = (0x8) +NV0000_CTRL_CMD_OS_UNIX_REFRESH_RMAPI_DEVICE_LIST = (0x3d09) +NV0000_CTRL_CMD_OS_UNIX_CREATE_EXPORT_OBJECT_FD = (0x3d0a) +NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_BUFFER_SIZE = NV0000_OS_UNIX_EXPORT_OBJECT_FD_BUFFER_SIZE +NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS_MESSAGE_ID = (0xA) +NV0000_CTRL_CMD_OS_UNIX_EXPORT_OBJECTS_TO_FD = (0x3d0b) +NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS = 512 +NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS_MESSAGE_ID = (0xB) +NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECTS_FROM_FD = (0x3d0c) +NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS = 128 +NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_NONE = 0 +NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_VIDMEM = 1 +NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_SYSMEM = 2 +NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC = 3 +NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_FABRIC_MC = 4 +NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS_MESSAGE_ID = (0xC) +NV0000_CTRL_CMD_OS_UNIX_MEMACCT_LIMIT_MAX = (0xFFFFFFFFFFFFFFFF) +NV0000_CTRL_CMD_OS_UNIX_MEMACCT_CURRENT_PROCESS = (-1) +NV0000_CTRL_OS_UNIX_MEMACCT_SET_LIMITS_PARAMS_MESSAGE_ID = (0xD) +NV0000_CTRL_OS_UNIX_CMD_MEMACCT_SET_LIMITS = (0x3d0d) +NV0000_CTRL_OS_UNIX_MEMACCT_GET_LIMITS_PARAMS_MESSAGE_ID = (0xE) +NV0000_CTRL_OS_UNIX_CMD_MEMACCT_GET_LIMITS = (0x3d0e) +NV0000_CTRL_CMD_OS_UNIX_MEMACCT_IMPL_NONE = 0 +NV0000_CTRL_CMD_OS_UNIX_MEMACCT_IMPL_FALLBACK = 1 +NV0000_CTRL_CMD_OS_UNIX_MEMACCT_IMPL_OS = 2 +NV0000_CTRL_OS_UNIX_MEMACCT_GET_IMPL_PARAMS_MESSAGE_ID = (0xF) +NV0000_CTRL_OS_UNIX_CMD_MEMACCT_GET_IMPL = (0x3d0f) +NV0000_BUSDEVICE_DOMAIN = (31, 16) +NV0000_BUSDEVICE_BUS = (15, 8) +NV0000_BUSDEVICE_DEVICE = (7, 0) +NV0000_CTRL_CMD_VGPU_CREATE_DEVICE = (0xc02) +NV0000_CTRL_VGPU_CREATE_DEVICE_PARAMS_MESSAGE_ID = (0x2) +NV0000_CTRL_CMD_VGPU_GET_INSTANCES = (0xc03) +NV0000_CTRL_VGPU_GET_INSTANCES_PARAMS_MESSAGE_ID = (0x3) +NV0000_CTRL_CMD_VGPU_DELETE_DEVICE = (0xc04) +NV0000_CTRL_VGPU_DELETE_DEVICE_PARAMS_MESSAGE_ID = (0x4) +NV0000_CTRL_CMD_VGPU_VFIO_NOTIFY_RM_STATUS = (0xc05) +NV0000_CTRL_VGPU_VFIO_NOTIFY_RM_STATUS_PARAMS_MESSAGE_ID = (0x5) +NV0000_CTRL_CMD_GPU_UPDATE_SYSFS_NODE = (0x206) +NV0000_CTRL_GPU_UPDATE_SYSFS_NODE_PARAMS_MESSAGE_ID = (0x6) +NV0080_CTRL_RESERVED = (0x00) +NV0080_CTRL_BIF = (0x01) +NV0080_CTRL_GPU = (0x02) +NV0080_CTRL_CLK = (0x10) +NV0080_CTRL_GR = (0x11) +NV0080_CTRL_CIPHER = (0x12) +NV0080_CTRL_FB = (0x13) +NV0080_CTRL_HOST = (0x14) +NV0080_CTRL_VIDEO = (0x15) +NV0080_CTRL_FIFO = (0x17) +NV0080_CTRL_DMA = (0x18) +NV0080_CTRL_PERF = (0x19) +NV0080_CTRL_PERF_LEGACY_NON_PRIVILEGED = (0x99) +NV0080_CTRL_NVENC = (0x1B) +NV0080_CTRL_NVDEC = (0x1C) +NV0080_CTRL_RC = (0x1D) +NV0080_CTRL_OS_UNIX = (0x1E) +NV0080_CTRL_NVJPG = (0x1F) +NV0080_CTRL_INTERNAL = (0x20) +NV0080_CTRL_NVLINK = (0x21) +NV0080_CTRL_CMD_NULL = (0x800000) +NV0080_CTRL_CMD_BIF_RESET = (0x800102) +NV0080_CTRL_BIF_RESET_PARAMS_MESSAGE_ID = (0x2) +NV0080_CTRL_BIF_RESET_FLAGS_TYPE = (4, 0) +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_SW_RESET = 0x1 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_SBR = 0x2 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_FUNDAMENTAL = 0x3 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_BOOT_DEVICE_FUSE = 0x4 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_BOOT_DEVICE = 0x5 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_PEX = 0x6 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_OOBHUB_TRIGGER = 0x7 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_BASE = 0x8 +NV0080_CTRL_BIF_RESET_FLAGS_TYPE_CXL_RESET = 0x9 +NV0080_CTRL_CMD_BIF_ASPM_CYA_UPDATE = (0x800105) +NV0080_CTRL_BIF_ASPM_CYA_UPDATE_PARAMS_MESSAGE_ID = (0x5) +NV0080_CTRL_CMD_BIF_GET_PCIE_POWER_CONTROL_MASK = (0x800106) +NV0080_CTRL_CMD_BIF_GET_PCIE_POWER_CONTROL_MASK_PARAMS_MESSAGE_ID = (0x6) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_VALID = (0, 0) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_VALID_FALSE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_VALID_TRUE = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ENCRYPTED = (2, 1) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ENCRYPTED_FALSE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ENCRYPTED_TRUE = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ENCRYPTED_NOT_SUPPORTED = (0x00000002) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_APERTURE = (6, 3) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_APERTURE_VIDEO_MEMORY = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_APERTURE_PEER_MEMORY = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_APERTURE_SYSTEM_COHERENT_MEMORY = (0x00000002) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_APERTURE_SYSTEM_NON_COHERENT_MEMORY = (0x00000003) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_COMPTAGS = (10, 7) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_COMPTAGS_NONE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_COMPTAGS_1 = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_COMPTAGS_2 = (0x00000002) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_COMPTAGS_4 = (0x00000004) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_GPU_CACHED = (12, 11) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_GPU_CACHED_FALSE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_GPU_CACHED_TRUE = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_GPU_CACHED_NOT_SUPPORTED = (0x00000002) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_SHADER_ACCESS = (14, 13) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_SHADER_ACCESS_READ_WRITE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_SHADER_ACCESS_READ_ONLY = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_SHADER_ACCESS_WRITE_ONLY = (0x00000002) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_SHADER_ACCESS_NOT_SUPPORTED = (0x00000003) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_READ_ONLY = (15, 15) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_READ_ONLY_FALSE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_READ_ONLY_TRUE = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ATOMIC = (16, 16) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ATOMIC_DISABLE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ATOMIC_ENABLE = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ACCESS_COUNTING = (17, 17) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ACCESS_COUNTING_DISABLE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_ACCESS_COUNTING_ENABLE = (0x00000001) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_PRIVILEGED = (18, 18) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_PRIVILEGED_FALSE = (0x00000000) +NV0080_CTRL_DMA_PTE_INFO_PARAMS_FLAGS_PRIVILEGED_TRUE = (0x00000001) +NV0080_CTRL_CMD_DMA_GET_PTE_INFO = (0x801801) +NV0080_CTRL_DMA_GET_PTE_INFO_PTE_BLOCKS = 5 +NV0080_CTRL_DMA_GET_PTE_INFO_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_CMD_DMA_SET_PTE_INFO = (0x80180a) +NV0080_CTRL_DMA_SET_PTE_INFO_PTE_BLOCKS = 5 +NV0080_CTRL_DMA_SET_PTE_INFO_PARAMS_MESSAGE_ID = (0xA) +NV0080_CTRL_CMD_DMA_FILL_PTE_MEM = (0x801802) +NV0080_CTRL_DMA_FILL_PTE_MEM_PARAMS_MESSAGE_ID = (0x2) +NV0080_CTRL_CMD_DMA_FLUSH = (0x801805) +NV0080_CTRL_DMA_FLUSH_PARAMS_MESSAGE_ID = (0x5) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_L2 = (0, 0) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_L2_DISABLE = (0x00000000) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_L2_ENABLE = (0x00000001) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_COMPTAG = (1, 1) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_COMPTAG_DISABLE = (0x00000000) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_COMPTAG_ENABLE = (0x00000001) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_FB = (2, 2) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_FB_DISABLE = (0x00000000) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_FB_ENABLE = (0x00000001) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_L2_INVALIDATE = (4, 3) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_L2_INVALIDATE_SYSMEM = (0x00000001) +NV0080_CTRL_DMA_FLUSH_TARGET_UNIT_L2_INVALIDATE_PEERMEM = (0x00000002) +NV0080_CTRL_CMD_DMA_ADV_SCHED_GET_VA_CAPS = (0x801806) +NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_MAX_NUM_PAGE_TABLE_FORMATS = (16) +NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS_MESSAGE_ID = (0x6) +NV0080_CTRL_CMD_DMA_ADV_SCHED_GET_VA_CAPS_WITH_VA_RANGE_LO = 0x1 +NV0080_CTRL_CMD_DMA_GET_PDE_INFO = (0x801809) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PTE_ADDR_SPACE_VIDEO_MEMORY = (0x00000000) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PTE_ADDR_SPACE_SYSTEM_COHERENT_MEMORY = (0x00000001) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PTE_ADDR_SPACE_SYSTEM_NON_COHERENT_MEMORY = (0x00000002) +NV0080_CTRL_DMA_PDE_INFO_PTE_BLOCKS = 5 +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_MESSAGE_ID = (0x9) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PDE_ADDR_SPACE_VIDEO_MEMORY = (0x00000000) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PDE_ADDR_SPACE_SYSTEM_COHERENT_MEMORY = (0x00000001) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PDE_ADDR_SPACE_SYSTEM_NON_COHERENT_MEMORY = (0x00000002) +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PDE_SIZE_FULL = 1 +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PDE_SIZE_HALF = 2 +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PDE_SIZE_QUARTER = 3 +NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS_PDE_SIZE_EIGHTH = 4 +NV0080_CTRL_CMD_DMA_INVALIDATE_TLB = (0x80180c) +NV0080_CTRL_DMA_INVALIDATE_TLB_PARAMS_MESSAGE_ID = (0xC) +NV0080_CTRL_DMA_INVALIDATE_TLB_ALL = (0, 0) +NV0080_CTRL_DMA_INVALIDATE_TLB_ALL_FALSE = (0x00000000) +NV0080_CTRL_DMA_INVALIDATE_TLB_ALL_TRUE = (0x00000001) +NV0080_CTRL_CMD_DMA_GET_CAPS = (0x80180d) +NV0080_CTRL_DMA_CAPS_TBL_SIZE = 8 +NV0080_CTRL_DMA_GET_CAPS_PARAMS_MESSAGE_ID = (0xD) +NV0080_CTRL_CMD_DMA_SET_VA_SPACE_SIZE = (0x80180e) +NV0080_CTRL_DMA_SET_VA_SPACE_SIZE_PARAMS_MESSAGE_ID = (0xE) +NV0080_CTRL_DMA_SET_VA_SPACE_SIZE_MAX = (0xFFFFFFFFFFFFFFFF) +NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 = (0x80180f) +NV0080_CTRL_DMA_UPDATE_PDE_2_PT_APERTURE_INVALID = (0x00000000) +NV0080_CTRL_DMA_UPDATE_PDE_2_PT_APERTURE_VIDEO_MEMORY = (0x00000001) +NV0080_CTRL_DMA_UPDATE_PDE_2_PT_APERTURE_SYSTEM_COHERENT_MEMORY = (0x00000002) +NV0080_CTRL_DMA_UPDATE_PDE_2_PT_APERTURE_SYSTEM_NON_COHERENT_MEMORY = (0x00000003) +NV0080_CTRL_DMA_UPDATE_PDE_2_PT_IDX_SMALL = 0 +NV0080_CTRL_DMA_UPDATE_PDE_2_PT_IDX_BIG = 1 +NV0080_CTRL_DMA_UPDATE_PDE_2_PT_IDX__SIZE = 2 +NV0080_CTRL_DMA_UPDATE_PDE_2_PARAMS_MESSAGE_ID = (0xF) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_FLUSH_PDE_CACHE = (0, 0) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_FLUSH_PDE_CACHE_FALSE = (0x00000000) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_FLUSH_PDE_CACHE_TRUE = (0x00000001) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_FORCE_OVERRIDE = (1, 1) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_FORCE_OVERRIDE_FALSE = (0x00000000) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_FORCE_OVERRIDE_TRUE = (0x00000001) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_PDE_SIZE = (3, 2) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_PDE_SIZE_FULL = (0x00000000) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_PDE_SIZE_HALF = (0x00000001) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_PDE_SIZE_QUARTER = (0x00000002) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_PDE_SIZE_EIGHTH = (0x00000003) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_SPARSE = (4, 4) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_SPARSE_FALSE = (0x00000000) +NV0080_CTRL_DMA_UPDATE_PDE_2_FLAGS_SPARSE_TRUE = (0x00000001) +NV0080_CTRL_DMA_SET_DEFAULT_VASPACE = (0x801812) +NV0080_CTRL_DMA_SET_DEFAULT_VASPACE_PARAMS_MESSAGE_ID = (0x12) +NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY = (0x801813) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS_MESSAGE_ID = (0x13) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_APERTURE = (1, 0) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_APERTURE_VIDMEM = (0x00000000) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_APERTURE_SYSMEM_COH = (0x00000001) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_APERTURE_SYSMEM_NONCOH = (0x00000002) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_PRESERVE_PDES = (2, 2) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_PRESERVE_PDES_FALSE = (0x00000000) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_PRESERVE_PDES_TRUE = (0x00000001) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_ALL_CHANNELS = (3, 3) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_ALL_CHANNELS_FALSE = (0x00000000) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_ALL_CHANNELS_TRUE = (0x00000001) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_IGNORE_CHANNEL_BUSY = (4, 4) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_IGNORE_CHANNEL_BUSY_FALSE = (0x00000000) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_IGNORE_CHANNEL_BUSY_TRUE = (0x00000001) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_EXTEND_VASPACE = (5, 5) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_EXTEND_VASPACE_FALSE = (0x00000000) +NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_FLAGS_EXTEND_VASPACE_TRUE = (0x00000001) +NV0080_CTRL_CMD_DMA_UNSET_PAGE_DIRECTORY = (0x801814) +NV0080_CTRL_DMA_UNSET_PAGE_DIRECTORY_PARAMS_MESSAGE_ID = (0x14) +NV0080_CTRL_CMD_FB_GET_CAPS = (0x801301) +NV0080_CTRL_FB_GET_CAPS_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_FB_CAPS_TBL_SIZE = 3 +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO = (0x801306) +NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS_MESSAGE_ID = (0x6) +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_UNKNOWN = 0 +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_SYSMEM = 1 +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_FBMEM = 2 +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_POLICY_LEGACY = 0 +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_POLICY_1TO1 = 1 +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_POLICY_1TO4 = 2 +NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_POLICY_RAWMODE = 3 +NV0080_CTRL_CMD_FB_GET_CAPS_V2 = (0x801307) +NV0080_CTRL_FB_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x7) +NV0080_CTRL_CMD_FB_SET_DEFAULT_VIDMEM_PHYSICALITY = (0x801308) +NV0080_CTRL_FB_SET_DEFAULT_VIDMEM_PHYSICALITY_PARAMS_MESSAGE_ID = (0x8) +NV0080_CTRL_CMD_FIFO_GET_CAPS = (0x801701) +NV0080_CTRL_FIFO_GET_CAPS_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_FIFO_CAPS_TBL_SIZE = 2 +NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES = (0x801707) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID = (4, 0) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS = (0x00000000) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_VLD = (0x00000001) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_VIDEO = (0x00000002) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_MPEG = (0x00000003) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_CAPTURE = (0x00000004) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_DISPLAY = (0x00000005) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_ENCRYPTION = (0x00000006) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_POSTPROCESS = (0x00000007) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ZCULL = (0x00000008) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PM = (0x00000009) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COMPUTE_PREEMPT = (0x0000000a) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PREEMPT = (0x0000000b) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_SPILL = (0x0000000c) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL = (0x0000000d) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BETACB = (0x0000000e) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV = (0x0000000f) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PATCH = (0x00000010) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BUNDLE_CB = (0x00000011) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL_GLOBAL = (0x00000012) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB = (0x00000013) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV_CB_GLOBAL = (0x00000014) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_POOL = (0x00000015) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_CTRL_BLK = (0x00000016) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_FECS_EVENT = (0x00000017) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP = (0x00000018) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_SETUP = (0x00000019) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT = (0x0000001a) +NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS_MESSAGE_ID = (0x7) +NV0080_CTRL_CMD_FIFO_GET_CHANNELLIST = (0x80170d) +NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS_MESSAGE_ID = (0xD) +NV0080_CTRL_CMD_FIFO_GET_LATENCY_BUFFER_SIZE = (0x80170e) +NV0080_CTRL_FIFO_GET_LATENCY_BUFFER_SIZE_PARAMS_MESSAGE_ID = (0xE) +NV0080_CTRL_FIFO_GET_CHANNELLIST_INVALID_CHANNEL = (0xffffffff) +NV0080_CTRL_CMD_FIFO_SET_CHANNEL_PROPERTIES = (0x80170f) +NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_PARAMS_MESSAGE_ID = (0xF) +NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_INVALIDATE_PDB_TARGET = (0x00000004) +NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_RESETENGINECONTEXT = (0x00000005) +NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_NOOP = (0x00000007) +NV0080_CTRL_FIFO_SET_CHANNEL_PROPERTIES_RESETENGINECONTEXT_NOPREEMPT = (0x00000008) +NV0080_CTRL_CMD_FIFO_STOP_RUNLIST = (0x801711) +NV0080_CTRL_FIFO_STOP_RUNLIST_PARAMS_MESSAGE_ID = (0x11) +NV0080_CTRL_CMD_FIFO_START_RUNLIST = (0x801712) +NV0080_CTRL_FIFO_START_RUNLIST_PARAMS_MESSAGE_ID = (0x12) +NV0080_CTRL_CMD_FIFO_GET_CAPS_V2 = (0x801713) +NV0080_CTRL_FIFO_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x13) +NV0080_CTRL_CMD_FIFO_IDLE_CHANNELS = (0x801714) +NV0080_CTRL_CMD_FIFO_IDLE_CHANNELS_MAX_CHANNELS = 4096 +NV0080_CTRL_FIFO_IDLE_CHANNELS_PARAMS_MESSAGE_ID = (0x14) +NV0080_CTRL_CMD_GPU_GET_CLASSLIST = (0x800201) +NV0080_CTRL_GPU_GET_CLASSLIST_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_CMD_GPU_GET_NUM_SUBDEVICES = (0x800280) +NV0080_CTRL_GPU_GET_NUM_SUBDEVICES_PARAMS_MESSAGE_ID = (0x80) +NV0080_CTRL_CMD_GPU_GET_VIDLINK_ORDER = (0x800281) +NV0080_CTRL_GPU_GET_VIDLINK_ORDER_PARAMS_MESSAGE_ID = (0x81) +NV0080_CTRL_CMD_GPU_SET_DISPLAY_OWNER = (0x800282) +NV0080_CTRL_GPU_SET_DISPLAY_OWNER_PARAMS_MESSAGE_ID = (0x82) +NV0080_CTRL_CMD_GPU_GET_DISPLAY_OWNER = (0x800283) +NV0080_CTRL_GPU_GET_DISPLAY_OWNER_PARAMS_MESSAGE_ID = (0x83) +NV0080_CTRL_CMD_GPU_SET_VIDLINK = (0x800285) +NV0080_CTRL_GPU_SET_VIDLINK_PARAMS_MESSAGE_ID = (0x85) +NV0080_CTRL_GPU_SET_VIDLINK_ENABLE_FALSE = (0x00000000) +NV0080_CTRL_GPU_SET_VIDLINK_ENABLE_TRUE = (0x00000001) +NV0080_CTRL_CMD_GPU_VIDEO_POWERGATE_GET_STATUS = 0 +NV0080_CTRL_CMD_GPU_VIDEO_POWERGATE_POWERDOWN = 1 +NV0080_CTRL_CMD_GPU_VIDEO_POWERGATE_POWERUP = 2 +NV0080_CTRL_CMD_GPU_VIDEO_POWERGATE_STATUS_POWER_ON = 0 +NV0080_CTRL_CMD_GPU_VIDEO_POWERGATE_STATUS_POWERING_DOWN = 1 +NV0080_CTRL_CMD_GPU_VIDEO_POWERGATE_STATUS_GATED = 2 +NV0080_CTRL_CMD_GPU_VIDEO_POWERGATE_STATUS_POWERING_UP = 3 +NV0080_CTRL_CMD_GPU_MODIFY_SW_STATE_PERSISTENCE = (0x800287) +NV0080_CTRL_GPU_SW_STATE_PERSISTENCE_ENABLED = (0x00000000) +NV0080_CTRL_GPU_SW_STATE_PERSISTENCE_DISABLED = (0x00000001) +NV0080_CTRL_GPU_MODIFY_SW_STATE_PERSISTENCE_PARAMS_MESSAGE_ID = (0x87) +NV0080_CTRL_CMD_GPU_QUERY_SW_STATE_PERSISTENCE = (0x800288) +NV0080_CTRL_GPU_QUERY_SW_STATE_PERSISTENCE_PARAMS_MESSAGE_ID = (0x88) +NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE = (0x800289) +NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NONE = (0x00000000) +NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NMOS = (0x00000001) +NV0080_CTRL_GPU_VIRTUALIZATION_MODE_VGX = (0x00000002) +NV0080_CTRL_GPU_VIRTUALIZATION_MODE_HOST = (0x00000003) +NV0080_CTRL_GPU_VIRTUALIZATION_MODE_HOST_VGPU = NV0080_CTRL_GPU_VIRTUALIZATION_MODE_HOST +NV0080_CTRL_GPU_VIRTUALIZATION_MODE_HOST_VSGA = (0x00000004) +NV0080_CTRL_GPU_GET_VIRTUALIZATION_MODE_PARAMS_MESSAGE_ID = (0x89) +NV0080_CTRL_CMD_GPU_GET_SPARSE_TEXTURE_COMPUTE_MODE = (0x80028c) +NV0080_CTRL_GPU_GET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS_MESSAGE_ID = (0x8C) +NV0080_CTRL_CMD_GPU_SET_SPARSE_TEXTURE_COMPUTE_MODE = (0x80028d) +NV0080_CTRL_GPU_SET_SPARSE_TEXTURE_COMPUTE_MODE_PARAMS_MESSAGE_ID = (0x8D) +NV0080_CTRL_GPU_SPARSE_TEXTURE_COMPUTE_MODE_DEFAULT = 0 +NV0080_CTRL_GPU_SPARSE_TEXTURE_COMPUTE_MODE_OPTIMIZE_COMPUTE = 1 +NV0080_CTRL_GPU_SPARSE_TEXTURE_COMPUTE_MODE_OPTIMIZE_SPARSE_TEXTURE = 2 +NV0080_CTRL_CMD_GPU_GET_VGX_CAPS = (0x80028e) +NV0080_CTRL_GPU_GET_VGX_CAPS_PARAMS_MESSAGE_ID = (0x8E) +NV0080_CTRL_CMD_GPU_GET_SRIOV_CAPS = (0x800291) +NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS_MESSAGE_ID = (0x91) +NV0080_CTRL_GPU_CLASSLIST_MAX_SIZE = 200 +NV0080_CTRL_CMD_GPU_GET_CLASSLIST_V2 = (0x800292) +NV0080_CTRL_GPU_GET_CLASSLIST_V2_PARAMS_MESSAGE_ID = (0x92) +NV0080_CTRL_CMD_GPU_FIND_SUBDEVICE_HANDLE = (0x800293) +NV0080_CTRL_GPU_FIND_SUBDEVICE_HANDLE_PARAM_MESSAGE_ID = (0x93) +NV0080_CTRL_GPU_GET_BRAND_CAPS_QUADRO = NVBIT(0) +NV0080_CTRL_GPU_GET_BRAND_CAPS_NVS = NVBIT(1) +NV0080_CTRL_GPU_GET_BRAND_CAPS_TITAN = NVBIT(2) +NV0080_CTRL_CMD_GPU_GET_BRAND_CAPS = (0x800294) +NV0080_CTRL_GPU_GET_BRAND_CAPS_PARAMS_MESSAGE_ID = (0x94) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_64M = (1 << 6) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_128M = (1 << 7) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_256M = (1 << 8) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_512M = (1 << 9) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_1G = (1 << 10) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_2G = (1 << 11) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_4G = (1 << 12) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_8G = (1 << 13) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_16G = (1 << 14) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_32G = (1 << 15) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_64G = (1 << 16) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_128G = (1 << 17) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_256G = (1 << 18) +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_MIN = NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_64M +NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_MAX = NV0080_CTRL_GPU_VGPU_VF_BAR1_SIZE_256G +NV0080_CTRL_GPU_VGPU_NUM_VFS_INVALID = 0x0 +NV0080_CTRL_GPU_SET_VGPU_VF_BAR1_SIZE = (0x800296) +NV0080_CTRL_GPU_SET_VGPU_VF_BAR1_SIZE_PARAMS_MESSAGE_ID = (0x96) +NV0080_CTRL_CMD_GPU_SET_VGPU_HETEROGENEOUS_MODE = (0x800297) +NV0080_CTRL_GPU_SET_VGPU_HETEROGENEOUS_MODE_PARAMS_MESSAGE_ID = (0x97) +NV0080_CTRL_CMD_GPU_GET_VGPU_HETEROGENEOUS_MODE = (0x800298) +NV0080_CTRL_GPU_GET_VGPU_HETEROGENEOUS_MODE_PARAMS_MESSAGE_ID = (0x98) +NV0080_CTRL_CMD_GR_GET_CAPS = (0x801102) +NV0080_CTRL_GR_GET_CAPS_PARAMS_MESSAGE_ID = (0x2) +NV0080_CTRL_GR_CAPS_TBL_SIZE = 23 +NV0080_CTRL_GR_INFO_INDEX_MAXCLIPS = (0x00000000) +NV0080_CTRL_GR_INFO_INDEX_MIN_ATTRS_BUG_261894 = (0x00000001) +NV0080_CTRL_GR_INFO_XBUF_MAX_PSETS_PER_BANK = (0x00000002) +NV0080_CTRL_GR_INFO_INDEX_BUFFER_ALIGNMENT = (0x00000003) +NV0080_CTRL_GR_INFO_INDEX_SWIZZLE_ALIGNMENT = (0x00000004) +NV0080_CTRL_GR_INFO_INDEX_VERTEX_CACHE_SIZE = (0x00000005) +NV0080_CTRL_GR_INFO_INDEX_VPE_COUNT = (0x00000006) +NV0080_CTRL_GR_INFO_INDEX_SHADER_PIPE_COUNT = (0x00000007) +NV0080_CTRL_GR_INFO_INDEX_THREAD_STACK_SCALING_FACTOR = (0x00000008) +NV0080_CTRL_GR_INFO_INDEX_SHADER_PIPE_SUB_COUNT = (0x00000009) +NV0080_CTRL_GR_INFO_INDEX_SM_REG_BANK_COUNT = (0x0000000A) +NV0080_CTRL_GR_INFO_INDEX_SM_REG_BANK_REG_COUNT = (0x0000000B) +NV0080_CTRL_GR_INFO_INDEX_SM_VERSION = (0x0000000C) +NV0080_CTRL_GR_INFO_INDEX_MAX_WARPS_PER_SM = (0x0000000D) +NV0080_CTRL_GR_INFO_INDEX_MAX_THREADS_PER_WARP = (0x0000000E) +NV0080_CTRL_GR_INFO_INDEX_GEOM_GS_OBUF_ENTRIES = (0x0000000F) +NV0080_CTRL_GR_INFO_INDEX_GEOM_XBUF_ENTRIES = (0x00000010) +NV0080_CTRL_GR_INFO_INDEX_FB_MEMORY_REQUEST_GRANULARITY = (0x00000011) +NV0080_CTRL_GR_INFO_INDEX_HOST_MEMORY_REQUEST_GRANULARITY = (0x00000012) +NV0080_CTRL_GR_INFO_INDEX_MAX_SP_PER_SM = (0x00000013) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS = (0x00000014) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPS = (0x00000015) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_ZCULL_BANKS = (0x00000016) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_TPC_PER_GPC = (0x00000017) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_MIN_FBPS = (0x00000018) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_MXBAR_FBP_PORTS = (0x00000019) +NV0080_CTRL_GR_INFO_INDEX_TIMESLICE_ENABLED = (0x0000001A) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPAS = (0x0000001B) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_PES_PER_GPC = (0x0000001C) +NV0080_CTRL_GR_INFO_INDEX_GPU_CORE_COUNT = (0x0000001D) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_TPCS_PER_PES = (0x0000001E) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_MXBAR_HUB_PORTS = (0x0000001F) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_SM_PER_TPC = (0x00000020) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_HSHUB_FBP_PORTS = (0x00000021) +NV0080_CTRL_GR_INFO_INDEX_RT_CORE_COUNT = (0x00000022) +NV0080_CTRL_GR_INFO_INDEX_TENSOR_CORE_COUNT = (0x00000023) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GRS = (0x00000024) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTCS = (0x00000025) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTC_SLICES = (0x00000026) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCMMU_PER_GPC = (0x00000027) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTC_PER_FBP = (0x00000028) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_ROP_PER_GPC = (0x00000029) +NV0080_CTRL_GR_INFO_INDEX_FAMILY_MAX_TPC_PER_GPC = (0x0000002A) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPA_PER_FBP = (0x0000002B) +NV0080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT = (0x0000002C) +NV0080_CTRL_GR_INFO_INDEX_MAX_LEGACY_SUBCONTEXT_COUNT = (0x0000002D) +NV0080_CTRL_GR_INFO_INDEX_MAX_PER_ENGINE_SUBCONTEXT_COUNT = (0x0000002E) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_SLICES_PER_LTC = (0x00000032) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GFXC_SMC_ENGINES = (0x00000033) +NV0080_CTRL_GR_INFO_INDEX_RESERVED = (0x00000033) +NV0080_CTRL_GR_INFO_INDEX_GFX_CAPABILITIES = (0x00000034) +NV0080_CTRL_GR_INFO_INDEX_MAX_MIG_ENGINES = (0x00000035) +NV0080_CTRL_GR_INFO_INDEX_MAX_PARTITIONABLE_GPCS = (0x00000036) +NV0080_CTRL_GR_INFO_INDEX_LITTER_MIN_SUBCTX_PER_SMC_ENG = (0x00000037) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS_PER_DIELET = (0x00000038) +NV0080_CTRL_GR_INFO_INDEX_LITTER_MAX_NUM_SMC_ENGINES_PER_DIELET = (0x00000039) +NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_CPC_PER_GPC = (0x0000003A) +NV0080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_NVLINK_MASK = (0x0000003B) +NV0080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_C2C_MASK = (0x0000003C) +NV0080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_PCIE_MASK = (0x0000003D) +NV0080_CTRL_GR_INFO_INDEX_RESERVED1 = (0x0000003E) +NV0080_CTRL_GR_INFO_INDEX_MAX = (0x0000003E) +NV0080_CTRL_GR_INFO_MAX_SIZE = (0x3f) +NV0080_CTRL_CMD_GR_GET_INFO = (0x801104) +NV0080_CTRL_GR_GET_INFO_PARAMS_MESSAGE_ID = (0x4) +NV0080_CTRL_CMD_GR_GET_TPC_PARTITION_MODE = (0x801107) +NV0080_CTRL_CMD_GR_SET_TPC_PARTITION_MODE = (0x801108) +NV0080_CTRL_GR_GET_TPC_PARTITION_MODE_PARAMS_MESSAGE_ID = (0x7) +NV0080_CTRL_GR_SET_TPC_PARTITION_MODE_PARAMS_MESSAGE_ID = (0x8) +NV0080_CTRL_CMD_GR_GET_CAPS_V2 = (0x801109) +NV0080_CTRL_GR_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x9) +NV0080_CTRL_CMD_GR_GET_INFO_V2 = (0x801110) +NV0080_CTRL_GR_GET_INFO_V2_PARAMS_MESSAGE_ID = (0x10) +NV0080_CTRL_CMD_HOST_GET_CAPS = (0x801401) +NV0080_CTRL_HOST_GET_CAPS_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_HOST_CAPS_TBL_SIZE = 3 +NV0080_CTRL_CMD_HOST_GET_CAPS_V2 = (0x801402) +NV0080_CTRL_HOST_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x2) +NV0080_CTRL_CMD_INTERNAL_GR_GET_TPC_PARTITION_MODE = (0x802002) +NV0080_CTRL_CMD_INTERNAL_GR_GET_TPC_PARTITION_MODE_FINN_PARAMS_MESSAGE_ID = (0x2) +NV0080_CTRL_CMD_INTERNAL_GR_SET_TPC_PARTITION_MODE = (0x802003) +NV0080_CTRL_CMD_INTERNAL_GR_SET_TPC_PARTITION_MODE_FINN_PARAMS_MESSAGE_ID = (0x3) +NV0080_CTRL_CMD_INTERNAL_PERF_CUDA_LIMIT_SET_CONTROL = (0x802009) +NV0080_CTRL_CMD_INTERNAL_PERF_CUDA_LIMIT_DISABLE = (0x802004) +NV0080_CTRL_CMD_INTERNAL_PERF_SLI_GPU_BOOST_SYNC_SET_CONTROL = (0x802007) +NV0080_CTRL_CMD_INTERNAL_FIFO_RC_AND_PERMANENTLY_DISABLE_CHANNELS = (0x802008) +NV_FIFO_PERMANENTLY_DISABLE_CHANNELS_MAX_CLIENTS = 200 +NV0080_CTRL_INTERNAL_FIFO_RC_AND_PERMANENTLY_DISABLE_CHANNELS_PARAMS_MESSAGE_ID = (0x08) +NV0080_CTRL_CMD_INTERNAL_MEMSYS_SET_ZBC_REFERENCED = (0x80200a) +NV0080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS_MESSAGE_ID = (0x0A) +NV0080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS_MESSAGE_ID = (0x45) +NV0080_CTRL_CMD_INTERNAL_KGR_INIT_BUG4208224_WAR = (0x802046) +NV0080_CTRL_INTERNAL_KGR_INIT_BUG4208224_WAR_PARAMS_MESSAGE_ID = (0x46) +NV0080_CTRL_CMD_NVDEC_GET_CAPS = (0x801c01) +NV0080_CTRL_CMD_BSP_GET_CAPS = NV0080_CTRL_CMD_NVDEC_GET_CAPS +NV0080_CTRL_NVDEC_GET_CAPS_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_BSP_GET_CAPS_PARAMS = NV0080_CTRL_NVDEC_GET_CAPS_PARAMS +NV0080_CTRL_NVDEC_CAPS_TBL_SIZE = 8 +NV0080_CTRL_BSP_CAPS_TBL_SIZE = NV0080_CTRL_NVDEC_CAPS_TBL_SIZE +NV0080_CTRL_CMD_NVDEC_GET_CAPS_V2 = (0x801c02) +NV0080_CTRL_CMD_BSP_GET_CAPS_V2 = NV0080_CTRL_CMD_NVDEC_GET_CAPS_V2 +NV0080_CTRL_NVDEC_GET_CAPS_PARAMS_V2_MESSAGE_ID = (0x2) +NV0080_CTRL_BSP_GET_CAPS_PARAMS_V2 = NV0080_CTRL_NVDEC_GET_CAPS_PARAMS_V2 +NV0080_CTRL_CMD_NVENC_GET_CAPS = (0x801b01) +NV0080_CTRL_CMD_MSENC_GET_CAPS = NV0080_CTRL_CMD_NVENC_GET_CAPS +NV0080_CTRL_NVENC_GET_CAPS_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_MSENC_GET_CAPS_PARAMS = NV0080_CTRL_NVENC_GET_CAPS_PARAMS +NV0080_CTRL_NVENC_CAPS_TBL_SIZE = 6 +NV0080_CTRL_MSENC_CAPS_TBL_SIZE = NV0080_CTRL_NVENC_CAPS_TBL_SIZE +NV0080_CTRL_CMD_NVENC_GET_CAPS_V2 = (0x801b02) +NV0080_CTRL_CMD_MSENC_GET_CAPS_V2 = NV0080_CTRL_CMD_NVENC_GET_CAPS_V2 +NV0080_CTRL_NVENC_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x2) +NV0080_CTRL_MSENC_GET_CAPS_V2_PARAMS = NV0080_CTRL_NVENC_GET_CAPS_V2_PARAMS +NV0080_CTRL_NVJPG_CAPS_TBL_SIZE = 9 +NV0080_CTRL_CMD_NVJPG_GET_CAPS_V2 = (0x801f02) +NV0080_CTRL_NVJPG_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x2) +NV0080_CTRL_PERF_SLI_GPU_BOOST_SYNC_CONTROL_PARAMS_MESSAGE_ID = (0x7) +NV0080_CTRL_PERF_CUDA_LIMIT_CONTROL_PARAMS_MESSAGE_ID = (0x9) +NV0080_CTRL_CMD_PERF_CUDA_LIMIT_SET_CONTROL = (0x801909) +NV0080_CTRL_CMD_OS_UNIX_VT_SWITCH = (0x801e01) +NV0080_CTRL_OS_UNIX_VT_SWITCH_PARAMS_MESSAGE_ID = (0x1) +NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_SAVE_VT_STATE = (0x00000001) +NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_RESTORE_VT_STATE = (0x00000002) +NV0080_CTRL_OS_UNIX_VT_SWITCH_CMD_CONSOLE_RESTORED = (0x00000003) +NV0080_CTRL_CMD_OS_UNIX_VT_GET_FB_INFO = (0x801e02) +NV0080_CTRL_OS_UNIX_VT_GET_FB_INFO_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_RESERVED = (0x00) +NV2080_CTRL_GPU = (0x01) +NV2080_CTRL_GPU_LEGACY_NON_PRIVILEGED = (0x81) +NV2080_CTRL_FUSE = (0x02) +NV2080_CTRL_FUSE_LEGACY_NON_PRIVILEGED = (0x82) +NV2080_CTRL_EVENT = (0x03) +NV2080_CTRL_TIMER = (0x04) +NV2080_CTRL_THERMAL = (0x05) +NV2080_CTRL_THERMAL_LEGACY_PRIVILEGED = (0xc5) +NV2080_CTRL_THERMAL_LEGACY_NON_PRIVILEGED = (0x85) +NV2080_CTRL_I2C = (0x06) +NV2080_CTRL_EXTI2C = (0x07) +NV2080_CTRL_BIOS = (0x08) +NV2080_CTRL_CIPHER = (0x09) +NV2080_CTRL_INTERNAL = (0x0A) +NV2080_CTRL_CLK_LEGACY_PRIVILEGED = (0xd0) +NV2080_CTRL_CLK_LEGACY_NON_PRIVILEGED = (0x90) +NV2080_CTRL_CLK = (0x10) +NV2080_CTRL_FIFO = (0x11) +NV2080_CTRL_GR = (0x12) +NV2080_CTRL_FB = (0x13) +NV2080_CTRL_NOISE = (0x15) +NV2080_CTRL_MC = (0x17) +NV2080_CTRL_BUS = (0x18) +NV2080_CTRL_PERF_LEGACY_PRIVILEGED = (0xe0) +NV2080_CTRL_PERF_LEGACY_NON_PRIVILEGED = (0xa0) +NV2080_CTRL_PERF = (0x20) +NV2080_CTRL_NVIF = (0x21) +NV2080_CTRL_RC = (0x22) +NV2080_CTRL_GPIO = (0x23) +NV2080_CTRL_GPIO_LEGACY_NON_PRIVILEGED = (0xa3) +NV2080_CTRL_NVD = (0x24) +NV2080_CTRL_DMA = (0x25) +NV2080_CTRL_PMGR = (0x26) +NV2080_CTRL_PMGR_LEGACY_PRIVILEGED = (0xe6) +NV2080_CTRL_PMGR_LEGACY_NON_PRIVILEGED = (0xa6) +NV2080_CTRL_POWER = (0x27) +NV2080_CTRL_POWER_LEGACY_NON_PRIVILEGED = (0xa7) +NV2080_CTRL_LPWR = (0x28) +NV2080_CTRL_LPWR_LEGACY_NON_PRIVILEGED = (0xa8) +NV2080_CTRL_LPWR_LEGACY_PRIVILEGED = (0xe8) +NV2080_CTRL_ACR = (0x29) +NV2080_CTRL_CE = (0x2A) +NV2080_CTRL_SPI = (0x2B) +NV2080_CTRL_NVLINK = (0x30) +NV2080_CTRL_FLCN = (0x31) +NV2080_CTRL_VOLT = (0x32) +NV2080_CTRL_VOLT_LEGACY_PRIVILEGED = (0xf2) +NV2080_CTRL_VOLT_LEGACY_NON_PRIVILEGED = (0xb2) +NV2080_CTRL_RIST = (0x33) +NV2080_CTRL_RIST_LEGACY_PRIVILEGED = (0xf3) +NV2080_CTRL_RIST_LEGACY_NON_PRIVILEGED = (0xb3) +NV2080_CTRL_ECC = (0x34) +NV2080_CTRL_ECC_NON_PRIVILEGED = (0xb4) +NV2080_CTRL_FLA = (0x35) +NV2080_CTRL_GSP = (0x36) +NV2080_CTRL_NNE = (0x37) +NV2080_CTRL_NNE_LEGACY_NON_PRIVILEGED = (0xb7) +NV2080_CTRL_GRMGR = (0x38) +NV2080_CTRL_UCODE_FUZZER = (0x39) +NV2080_CTRL_DMABUF = (0x3A) +NV2080_CTRL_BIF = (0x3B) +NV2080_CTRL_OS_WINDOWS = (0x3F) +NV2080_CTRL_OS_MACOS = (0x3E) +NV2080_CTRL_OS_UNIX = (0x3D) +NV2080_CTRL_CMD_NULL = (0x20800000) +NV2080_CTRL_CMD_BIF_DATA_PARAMS_MAX = 32 +NV2080_CTRL_CMD_BIF_GET_DATA = (0x20803b01) +NV2080_CTRL_BIF_GET_DATA_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_BIOS_INFO_MAX_SIZE = (0x0000000F) +NV2080_CTRL_BIOS_INFO_INDEX_REVISION = (0x00000000) +NV2080_CTRL_BIOS_INFO_INDEX_OEM_REVISION = (0x00000001) +NV2080_CTRL_CMD_BIOS_GET_INFO = (0x20800802) +NV2080_CTRL_CMD_BIOS_GET_INFO_V2 = (0x20800810) +NV2080_CTRL_BIOS_GET_INFO_V2_PARAMS_MESSAGE_ID = (0x10) +NV2080_CTRL_BIOS_NBSI_MAX_REG_STRING_LENGTH = (0x00000100) +NV2080_CTRL_BIOS_NBSI_STRING_TYPE_ASCII = (0x00000000) +NV2080_CTRL_BIOS_NBSI_STRING_TYPE_UNICODE = (0x00000001) +NV2080_CTRL_BIOS_NBSI_STRING_TYPE_HASH = (0x00000002) +NV2080_CTRL_BIOS_NBSI_MODULE_ROOT = (0x00000000) +NV2080_CTRL_BIOS_NBSI_MODULE_RM = (0x00000001) +NV2080_CTRL_BIOS_NBSI_MODULE_DISPLAYDRIVER = (0x00000002) +NV2080_CTRL_BIOS_NBSI_MODULE_VIDEO = (0x00000003) +NV2080_CTRL_BIOS_NBSI_MODULE_CPL = (0x00000004) +NV2080_CTRL_BIOS_NBSI_MODULE_D3D = (0x00000005) +NV2080_CTRL_BIOS_NBSI_MODULE_OGL = (0x00000006) +NV2080_CTRL_BIOS_NBSI_MODULE_PMU = (0x00000007) +NV2080_CTRL_BIOS_NBSI_MODULE_MODE = (0x00000008) +NV2080_CTRL_BIOS_NBSI_NUM_MODULES = (0x00000009) +NV2080_CTRL_BIOS_NBSI_MODULE_UNKNOWN = (0x80000000) +NV2080_CTRL_BIOS_GET_NBSI_SUCCESS = (0x00000000) +NV2080_CTRL_BIOS_GET_NBSI_OVERRIDE = (0x00000001) +NV2080_CTRL_BIOS_GET_NBSI_BAD_HASH = (0xFFFFFFFA) +NV2080_CTRL_BIOS_GET_NBSI_APITEST_SUCCESS = (0xFFFFFFFB) +NV2080_CTRL_BIOS_GET_NBSI_BAD_TABLE = (0xFFFFFFFC) +NV2080_CTRL_BIOS_GET_NBSI_NO_TABLE = (0xFFFFFFFD) +NV2080_CTRL_BIOS_GET_NBSI_INCOMPLETE = (0xFFFFFFFE) +NV2080_CTRL_BIOS_GET_NBSI_NOT_FOUND = (0xFFFFFFFF) +NV2080_CTRL_CMD_BIOS_GET_NBSI = (0x20800803) +NV2080_CTRL_BIOS_GET_NBSI_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_BIOS_GET_NBSI_V2 = (0x2080080e) +NV2080_BIOS_GET_NBSI_MAX_RET_SIZE = (0x100) +NV2080_CTRL_BIOS_GET_NBSI_V2_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_CMD_BIOS_GET_NBSI_OBJ = (0x20800806) +NV2080_CTRL_BIOS_GET_NBSI_OBJ_PARAMS_MESSAGE_ID = (0x6) +GLOB_TYPE_GET_NBSI_DIR = 0xfffe +GLOB_TYPE_APITEST = 0xffff +GLOB_TYPE_GET_NBSI_ACPI_RAW = 0xfffd +NV2080_CTRL_CMD_BIOS_GET_SKU_INFO = (0x20800808) +NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_CMD_BIOS_GET_POST_TIME = (0x20800809) +NV2080_CTRL_CMD_BIOS_GET_POST_TIME_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_BIOS_GET_UEFI_SUPPORT = (0x2080080b) +NV2080_CTRL_BIOS_GET_UEFI_SUPPORT_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_PRESENCE = (1, 0) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_PRESENCE_NO = (0x00000000) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_PRESENCE_YES = (0x00000001) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_PRESENCE_PLACEHOLDER = (0x00000002) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_PRESENCE_HIDDEN = (0x00000003) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_RUNNING = (2, 2) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_RUNNING_FALSE = (0x00000000) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_RUNNING_TRUE = (0x00000001) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_IS_EFI_INIT = (3, 3) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_IS_EFI_INIT_FALSE = (0x00000000) +NV2080_CTRL_BIOS_UEFI_SUPPORT_FLAGS_IS_EFI_INIT_TRUE = (0x00000001) +NV2080_CTRL_BOARDOBJGRP_MASK_MASK_ELEMENT_BIT_SIZE = 32 +NV2080_CTRL_BOARDOBJGRP_MASK_MASK_ELEMENT_INDEX = lambda _bit: ((_bit) / NV2080_CTRL_BOARDOBJGRP_MASK_MASK_ELEMENT_BIT_SIZE) # type: ignore +NV2080_CTRL_BOARDOBJGRP_MASK_MASK_ELEMENT_OFFSET = lambda _bit: ((_bit) % NV2080_CTRL_BOARDOBJGRP_MASK_MASK_ELEMENT_BIT_SIZE) # type: ignore +NV2080_CTRL_BOARDOBJGRP_MASK_DATA_SIZE = lambda _bits: (NV2080_CTRL_BOARDOBJGRP_MASK_MASK_ELEMENT_INDEX((_bits) - 1) + 1) # type: ignore +NV2080_CTRL_BOARDOBJGRP_MASK_ARRAY_START_SIZE = 1 +NV2080_CTRL_BOARDOBJGRP_MASK_ARRAY_EXTENSION_SIZE = lambda _bits: (NV2080_CTRL_BOARDOBJGRP_MASK_DATA_SIZE(_bits) - (NV2080_CTRL_BOARDOBJGRP_MASK_ARRAY_START_SIZE)) # type: ignore +NV2080_CTRL_BOARDOBJGRP_MASK_E32_FOR_EACH_INDEX__DEPRECATED = lambda _index,_pMask: NV2080_CTRL_BOARDOBJGRP_MASK_FOR_EACH_INDEX__DEPRECATED_DO_NOT_USE( NV2080_CTRL_BOARDOBJGRP_E32_MAX_OBJECTS,_index,_pMask) # type: ignore +NV2080_CTRL_BOARDOBJGRP_MASK_E255_FOR_EACH_INDEX__DEPRECATED = lambda _index,_pMask: NV2080_CTRL_BOARDOBJGRP_MASK_FOR_EACH_INDEX__DEPRECATED_DO_NOT_USE( NV2080_CTRL_BOARDOBJGRP_E255_MAX_OBJECTS,_index,_pMask) # type: ignore +NV2080_CTRL_CMD_BUS_GET_PCI_INFO = (0x20801801) +NV2080_CTRL_BUS_GET_PCI_INFO_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_BUS_GET_PCI_INFO_V2 = (0x2080183c) +NV2080_CTRL_BUS_GET_PCI_INFO_V2_PARAMS_MESSAGE_ID = (0x3C) +NV2080_CTRL_BUS_INFO_INDEX_TYPE = (0x00000000) +NV2080_CTRL_BUS_INFO_INDEX_INTLINE = (0x00000001) +NV2080_CTRL_BUS_INFO_INDEX_CAPS = (0x00000002) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CAPS = (0x00000003) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_CAPS = (0x00000004) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_LINK_CAPS = (0x00000005) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_DOWNSTREAM_LINK_CAPS = (0x00000006) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CTRL_STATUS = (0x00000007) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_CTRL_STATUS = (0x00000008) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_LINK_CTRL_STATUS = (0x00000009) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_DOWNSTREAM_LINK_CTRL_STATUS = (0x0000000A) +NV2080_CTRL_BUS_INFO_INDEX_COHERENT_DMA_FLAGS = (0x0000000B) +NV2080_CTRL_BUS_INFO_INDEX_NONCOHERENT_DMA_FLAGS = (0x0000000C) +NV2080_CTRL_BUS_INFO_INDEX_GPU_GART_SIZE = (0x0000000D) +NV2080_CTRL_BUS_INFO_INDEX_GPU_GART_FLAGS = (0x0000000E) +NV2080_CTRL_BUS_INFO_INDEX_BUS_NUMBER = (0x0000000F) +NV2080_CTRL_BUS_INFO_INDEX_DEVICE_NUMBER = (0x00000010) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_ERRORS = (0x00000011) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_ERRORS = (0x00000012) +NV2080_CTRL_BUS_INFO_INDEX_INTERFACE_TYPE = (0x00000013) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GEN2_INFO = (0x00000014) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_AER = (0x00000015) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_LINK_CAPS = (0x00000016) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_LINK_CTRL_STATUS = (0x00000017) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_ASLM_STATUS = (0x00000018) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_LINK_WIDTH_SWITCH_ERROR_COUNT = (0x00000019) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_LINK_SPEED_SWITCH_ERROR_COUNT = (0x0000001A) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_CYA_ASPM = (0x0000001B) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_LINECODE_ERRORS = (0x0000001C) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CRC_ERRORS = (0x0000001D) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_NAKS_RECEIVED = (0x0000001E) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_FAILED_L0S_EXITS = (0x0000001F) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_LINECODE_ERRORS_CLEAR = (0x00000020) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CRC_ERRORS_CLEAR = (0x00000021) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_NAKS_RECEIVED_CLEAR = (0x00000022) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_FAILED_L0S_EXITS_CLEAR = (0x00000023) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CORRECTABLE_ERRORS = (0x00000024) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_NONFATAL_ERRORS = (0x00000025) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_FATAL_ERRORS = (0x00000026) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_UNSUPPORTED_REQUESTS = (0x00000027) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CORRECTABLE_ERRORS_CLEAR = (0x00000028) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_NONFATAL_ERRORS_CLEAR = (0x00000029) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_FATAL_ERRORS_CLEAR = (0x0000002A) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_UNSUPPORTED_REQUESTS_CLEAR = (0x0000002B) +NV2080_CTRL_BUS_INFO_INDEX_DOMAIN_NUMBER = (0x0000002C) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_GEN_INFO = (0x0000002D) +NV2080_CTRL_BUS_INFO_INDEX_GPU_INTERFACE_TYPE = (0x0000002E) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_GEN_INFO = (0x0000002F) +NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_GEN_INFO = (0x00000030) +NV2080_CTRL_BUS_INFO_INDEX_MSI_INFO = (0x00000031) +NV2080_CTRL_BUS_INFO_INDEX_GPU_GART_SIZE_HI = (0x00000032) +NV2080_CTRL_BUS_INFO_INDEX_SYSMEM_CONNECTION_TYPE = (0x00000033) +NV2080_CTRL_BUS_INFO_INDEX_MAX = NV2080_CTRL_BUS_INFO_INDEX_SYSMEM_CONNECTION_TYPE +NV2080_CTRL_BUS_INFO_MAX_LIST_SIZE = (0x00000034) +NV2080_CTRL_BUS_INFO_TYPE_PCI = (0x00000001) +NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS = (0x00000003) +NV2080_CTRL_BUS_INFO_TYPE_FPCI = (0x00000004) +NV2080_CTRL_BUS_INFO_TYPE_AXI = (0x00000008) +NV2080_CTRL_BUS_INFO_CAPS_NEED_IO_FLUSH = (0x00000001) +NV2080_CTRL_BUS_INFO_CAPS_CHIP_INTEGRATED = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED = (3, 0) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_2500MBPS = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_5000MBPS = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_8000MBPS = (0x00000003) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_16000MBPS = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_32000MBPS = (0x00000005) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_64000MBPS = (0x00000006) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_WIDTH = (9, 4) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_ASPM = (11, 10) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_ASPM_NONE = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_ASPM_L0S = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_ASPM_L0S_L1 = (0x00000003) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN = (15, 12) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN1 = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN2 = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN3 = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN4 = (0x00000003) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN5 = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN6 = (0x00000005) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_CURR_LEVEL = (19, 16) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_CURR_LEVEL_GEN1 = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_CURR_LEVEL_GEN2 = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_CURR_LEVEL_GEN3 = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_CURR_LEVEL_GEN4 = (0x00000003) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_CURR_LEVEL_GEN5 = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_CURR_LEVEL_GEN6 = (0x00000005) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GPU_GEN = (23, 20) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GPU_GEN_GEN1 = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GPU_GEN_GEN2 = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GPU_GEN_GEN3 = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GPU_GEN_GEN4 = (0x00000003) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GPU_GEN_GEN5 = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GPU_GEN_GEN6 = (0x00000005) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_SPEED_CHANGES = (24, 24) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_SPEED_CHANGES_ENABLED = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_SPEED_CHANGES_DISABLED = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_ASPM = (1, 0) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_ASPM_DISABLED = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_ASPM_L0S = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_ASPM_L1 = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_ASPM_L0S_L1 = (0x00000003) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED = (19, 16) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_2500MBPS = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_5000MBPS = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_8000MBPS = (0x00000003) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_16000MBPS = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_32000MBPS = (0x00000005) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_64000MBPS = (0x00000006) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH = (25, 20) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_UNDEFINED = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_X1 = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_X2 = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_X4 = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_X8 = (0x00000008) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_X12 = (0x0000000C) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_X16 = (0x00000010) +NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_WIDTH_X32 = (0x00000020) +NV2080_CTRL_BUS_INFO_COHERENT_DMA_FLAGS_CTXDMA = (0, 0) +NV2080_CTRL_BUS_INFO_COHERENT_DMA_FLAGS_CTXDMA_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_COHERENT_DMA_FLAGS_CTXDMA_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_COHERENT_DMA_FLAGS_GPUGART = (2, 2) +NV2080_CTRL_BUS_INFO_COHERENT_DMA_FLAGS_GPUGART_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_COHERENT_DMA_FLAGS_GPUGART_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_CTXDMA = (0, 0) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_CTXDMA_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_CTXDMA_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_GPUGART = (2, 2) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_GPUGART_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_GPUGART_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_COH_MODE = (3, 3) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_COH_MODE_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_NONCOHERENT_DMA_FLAGS_COH_MODE_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_GPU_GART_FLAGS_REQFLUSH = (0, 0) +NV2080_CTRL_BUS_INFO_GPU_GART_FLAGS_REQFLUSH_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_GPU_GART_FLAGS_REQFLUSH_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_GPU_GART_FLAGS_UNIFIED = (1, 1) +NV2080_CTRL_BUS_INFO_GPU_GART_FLAGS_UNIFIED_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_GPU_GART_FLAGS_UNIFIED_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_CORR_ERROR = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_NON_FATAL_ERROR = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_FATAL_ERROR = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_UNSUPP_REQUEST = (0x00000008) +NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_ENTERED_RECOVERY = (0x00000010) +NV2080_CTRL_BUS_INFO_PCIE_GEN2_INFO_CAP = (0, 0) +NV2080_CTRL_BUS_INFO_PCIE_GEN2_INFO_CAP_FALSE = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_GEN2_INFO_CAP_TRUE = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_GEN2_INFO_CURR_LEVEL = (1, 1) +NV2080_CTRL_BUS_INFO_PCIE_GEN2_INFO_CURR_LEVEL_GEN1 = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_GEN2_INFO_CURR_LEVEL_GEN2 = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_TRAINING_ERR = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_DLINK_PROTO_ERR = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_POISONED_TLP = (0x00000004) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_FC_PROTO_ERR = (0x00000008) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_TIMEOUT = (0x00000010) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_ABORT = (0x00000020) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNEXP_CPL = (0x00000040) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_RCVR_OVERFLOW = (0x00000080) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_MALFORMED_TLP = (0x00000100) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_ECRC_ERROR = (0x00000200) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNSUPPORTED_REQ = (0x00000400) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RCV_ERR = (0x00010000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_TLP = (0x00020000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_DLLP = (0x00040000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_ROLLOVER = (0x00080000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_TIMEOUT = (0x00100000) +NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_ADVISORY_NONFATAL = (0x00200000) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_PCIE = (0, 0) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_PCIE_ERROR = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_PCIE_PRESENT = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_SUPPORTED = (1, 1) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_CL_CAPABLE = (2, 2) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_CL_CAPABLE_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_CL_CAPABLE_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_OS_SUPPORTED = (3, 3) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_OS_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_OS_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_BR04 = (4, 4) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_BR04_MISSING = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_ASLM_STATUS_BR04_PRESENT = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM_VALID = (0, 0) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM_VALID_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM_VALID_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM = (2, 1) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM_DISABLED = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM_L0S = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM_L1 = (0x00000002) +NV2080_CTRL_BUS_INFO_PCIE_GPU_CYA_ASPM_L0S_L1 = (0x00000003) +NV2080_CTRL_BUS_INFO_MSI_STATUS = (0, 0) +NV2080_CTRL_BUS_INFO_MSI_STATUS_DISABLED = (0x00000000) +NV2080_CTRL_BUS_INFO_MSI_STATUS_ENABLED = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_PCIPM_L1_2_SUPPORTED = (0, 0) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_PCIPM_L1_2_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_PCIPM_L1_2_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_PCIPM_L1_1_SUPPORTED = (1, 1) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_PCIPM_L1_1_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_PCIPM_L1_1_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_ASPM_L1_2_SUPPORTED = (2, 2) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_ASPM_L1_2_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_ASPM_L1_2_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_ASPM_L1_1_SUPPORTED = (3, 3) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_ASPM_L1_1_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_ASPM_L1_1_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_L1PM_SUPPORTED = (4, 4) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_L1PM_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_L1PM_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_RESERVED = (7, 5) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_PORT_RESTORE_TIME = (15, 8) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_T_POWER_ON_SCALE = (17, 16) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CAP_T_POWER_ON_VALUE = (23, 19) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_PCIPM_L1_2_ENABLED = (0, 0) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_PCIPM_L1_2_ENABLED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_PCIPM_L1_2_ENABLED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_PCIPM_L1_1_ENABLED = (1, 1) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_PCIPM_L1_1_ENABLED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_PCIPM_L1_1_ENABLED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_ASPM_L1_2_ENABLED = (2, 2) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_ASPM_L1_2_ENABLED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_ASPM_L1_2_ENABLED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_ASPM_L1_1_ENABLED = (3, 3) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_ASPM_L1_1_ENABLED_YES = (0x00000001) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_ASPM_L1_1_ENABLED_NO = (0x00000000) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_COMMON_MODE_RESTORE_TIME = (15, 8) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_LTR_L1_2_THRESHOLD_VALUE = (25, 16) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL1_LTR_L1_2_THRESHOLD_SCALE = (31, 29) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL2_T_POWER_ON_SCALE = (1, 0) +NV2080_CTRL_BUS_INFO_PCIE_L1_SS_CTRL2_T_POWER_ON_VALUE = (7, 3) +NV2080_CTRL_BUS_INFO_INDEX_SYSMEM_CONNECTION_TYPE_PCIE = (0x00000000) +NV2080_CTRL_BUS_INFO_INDEX_SYSMEM_CONNECTION_TYPE_NVLINK = (0x00000001) +NV2080_CTRL_BUS_INFO_INDEX_SYSMEM_CONNECTION_TYPE_C2C = (0x00000002) +NV2080_CTRL_CMD_BUS_GET_INFO = (0x20801802) +NV2080_CTRL_BUS_GET_INFO_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_BUS_GET_INFO_V2 = (0x20801823) +NV2080_CTRL_BUS_GET_INFO_V2_PARAMS_MESSAGE_ID = (0x23) +NV2080_CTRL_CMD_BUS_GET_PCI_BAR_INFO = (0x20801803) +NV2080_CTRL_BUS_MAX_PCI_BARS = (8) +NV2080_CTRL_BUS_GET_PCI_BAR_INFO_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_BUS_SET_PCIE_LINK_WIDTH = (0x20801804) +NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_ERROR_PSTATE = (0x00000001) +NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_ERROR_PCIE_CFG_ACCESS = (0x00000002) +NV2080_CTRL_BUS_SET_PCIE_LINK_WIDTH_ERROR_TRAINING = (0x00000004) +NV2080_CTRL_CMD_BUS_SET_PCIE_SPEED = (0x20801805) +NV2080_CTRL_BUS_SET_PCIE_SPEED_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_BUS_SET_PCIE_SPEED_2500MBPS = (0x00000001) +NV2080_CTRL_BUS_SET_PCIE_SPEED_5000MBPS = (0x00000002) +NV2080_CTRL_BUS_SET_PCIE_SPEED_8000MBPS = (0x00000003) +NV2080_CTRL_BUS_SET_PCIE_SPEED_16000MBPS = (0x00000004) +NV2080_CTRL_BUS_SET_PCIE_SPEED_32000MBPS = (0x00000005) +NV2080_CTRL_BUS_SET_PCIE_SPEED_64000MBPS = (0x00000006) +NV2080_CTRL_CMD_BUS_MAP_BAR2 = (0x20801809) +NV2080_CTRL_BUS_MAP_BAR2_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_BUS_UNMAP_BAR2 = (0x2080180a) +NV2080_CTRL_BUS_UNMAP_BAR2_PARAMS_MESSAGE_ID = (0xA) +NV2080_CTRL_CMD_BUS_VERIFY_BAR2 = (0x2080180b) +NV2080_CTRL_BUS_VERIFY_BAR2_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_CMD_BUS_SERVICE_GPU_MULTIFUNC_STATE = (0x20801812) +NV2080_CTRL_BUS_SERVICE_GPU_MULTIFUNC_STATE_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_BUS_ENABLE_GPU_MULTIFUNC_STATE = (0x00000000) +NV2080_CTRL_BUS_DISABLE_GPU_MULTIFUNC_STATE = (0x00000001) +NV2080_CTRL_BUS_GET_GPU_MULTIFUNC_STATE = (0x00000002) +NV2080_CTRL_CMD_BUS_GET_PEX_COUNTERS = (0x20801813) +NV2080_CTRL_PEX_MAX_COUNTER_TYPES = 31 +NV2080_CTRL_BUS_GET_PEX_COUNTERS_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_BUS_PEX_COUNTER_TYPE = 0x00000000 +NV2080_CTRL_BUS_PEX_COUNTER_RECEIVER_ERRORS = 0x00000001 +NV2080_CTRL_BUS_PEX_COUNTER_REPLAY_COUNT = 0x00000002 +NV2080_CTRL_BUS_PEX_COUNTER_REPLAY_ROLLOVER_COUNT = 0x00000004 +NV2080_CTRL_BUS_PEX_COUNTER_BAD_DLLP_COUNT = 0x00000008 +NV2080_CTRL_BUS_PEX_COUNTER_BAD_TLP_COUNT = 0x00000010 +NV2080_CTRL_BUS_PEX_COUNTER_8B10B_ERRORS_COUNT = 0x00000020 +NV2080_CTRL_BUS_PEX_COUNTER_SYNC_HEADER_ERRORS_COUNT = 0x00000040 +NV2080_CTRL_BUS_PEX_COUNTER_LCRC_ERRORS_COUNT = 0x00000080 +NV2080_CTRL_BUS_PEX_COUNTER_FAILED_L0S_EXITS_COUNT = 0x00000100 +NV2080_CTRL_BUS_PEX_COUNTER_NAKS_SENT_COUNT = 0x00000200 +NV2080_CTRL_BUS_PEX_COUNTER_NAKS_RCVD_COUNT = 0x00000400 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_ERRORS = 0x00000800 +NV2080_CTRL_BUS_PEX_COUNTER_L1_TO_RECOVERY_COUNT = 0x00001000 +NV2080_CTRL_BUS_PEX_COUNTER_L0_TO_RECOVERY_COUNT = 0x00002000 +NV2080_CTRL_BUS_PEX_COUNTER_RECOVERY_COUNT = 0x00004000 +NV2080_CTRL_BUS_PEX_COUNTER_CHIPSET_XMIT_L0S_ENTRY_COUNT = 0x00008000 +NV2080_CTRL_BUS_PEX_COUNTER_GPU_XMIT_L0S_ENTRY_COUNT = 0x00010000 +NV2080_CTRL_BUS_PEX_COUNTER_L1_ENTRY_COUNT = 0x00020000 +NV2080_CTRL_BUS_PEX_COUNTER_L1P_ENTRY_COUNT = 0x00040000 +NV2080_CTRL_BUS_PEX_COUNTER_DEEP_L1_ENTRY_COUNT = 0x00080000 +NV2080_CTRL_BUS_PEX_COUNTER_ASLM_COUNT = 0x00100000 +NV2080_CTRL_BUS_PEX_COUNTER_TOTAL_CORR_ERROR_COUNT = 0x00200000 +NV2080_CTRL_BUS_PEX_COUNTER_CORR_ERROR_COUNT = 0x00400000 +NV2080_CTRL_BUS_PEX_COUNTER_NON_FATAL_ERROR_COUNT = 0x00800000 +NV2080_CTRL_BUS_PEX_COUNTER_FATAL_ERROR_COUNT = 0x01000000 +NV2080_CTRL_BUS_PEX_COUNTER_UNSUPP_REQ_COUNT = 0x02000000 +NV2080_CTRL_BUS_PEX_COUNTER_L1_1_ENTRY_COUNT = 0x04000000 +NV2080_CTRL_BUS_PEX_COUNTER_L1_2_ENTRY_COUNT = 0x08000000 +NV2080_CTRL_BUS_PEX_COUNTER_L1_2_ABORT_COUNT = 0x10000000 +NV2080_CTRL_BUS_PEX_COUNTER_L1SS_TO_DEEP_L1_TIMEOUT_COUNT = 0x20000000 +NV2080_CTRL_BUS_PEX_COUNTER_L1_SHORT_DURATION_COUNT = 0x40000000 +NV2080_CTRL_CMD_BUS_CLEAR_PEX_COUNTERS = (0x20801814) +NV2080_CTRL_BUS_CLEAR_PEX_COUNTERS_PARAMS_MESSAGE_ID = (0x14) +NV2080_CTRL_CMD_BUS_FREEZE_PEX_COUNTERS = (0x20801815) +NV2080_CTRL_BUS_FREEZE_PEX_COUNTERS_PARAMS_MESSAGE_ID = (0x15) +NV2080_CTRL_CMD_BUS_GET_PEX_LANE_COUNTERS = (0x20801816) +NV2080_CTRL_PEX_MAX_LANES = 16 +NV2080_CTRL_CMD_BUS_GET_PEX_LANE_COUNTERS_PARAMS_MESSAGE_ID = (0x16) +NV2080_CTRL_BUS_PEX_COUNTER_LANE_TYPE = 0x00000000 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_SYNC_HDR_CODING_ERR = 0x00000001 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_SYNC_HDR_ORDER_ERR = 0x00000002 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_OS_DATA_SEQ_ERR = 0x00000004 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_TSX_DATA_SEQ_ERR = 0x00000008 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_SKPOS_LFSR_ERR = 0x00000010 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_RX_CLK_FIFO_OVERFLOW = 0x00000020 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_ELASTIC_FIFO_OVERFLOW = 0x00000040 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_RCVD_LINK_NUM_ERR = 0x00000080 +NV2080_CTRL_BUS_PEX_COUNTER_LANE_RCVD_LANE_NUM_ERR = 0x00000100 +NV2080_CTRL_CMD_BUS_GET_PCIE_LTR_LATENCY = (0x20801817) +NV2080_CTRL_CMD_BUS_GET_PCIE_LTR_LATENCY_PARAMS_MESSAGE_ID = (0x17) +NV2080_CTRL_CMD_BUS_SET_PCIE_LTR_LATENCY = (0x20801818) +NV2080_CTRL_CMD_BUS_SET_PCIE_LTR_LATENCY_PARAMS_MESSAGE_ID = (0x18) +NV2080_CTRL_BUS_PEX_UTIL_COUNTER_TX_BYTES = 0x00000001 +NV2080_CTRL_BUS_PEX_UTIL_COUNTER_RX_BYTES = 0x00000002 +NV2080_CTRL_BUS_PEX_UTIL_COUNTER_TX_L0 = 0x00000004 +NV2080_CTRL_BUS_PEX_UTIL_COUNTER_RX_L0 = 0x00000008 +NV2080_CTRL_BUS_PEX_UTIL_COUNTER_TX_L0S = 0x00000010 +NV2080_CTRL_BUS_PEX_UTIL_COUNTER_RX_L0S = 0x00000020 +NV2080_CTRL_BUS_PEX_UTIL_COUNTER_NON_L0_L0S = 0x00000040 +NV2080_CTRL_PEX_UTIL_MAX_COUNTER_TYPES = 7 +NV2080_CTRL_CMD_BUS_GET_PEX_UTIL_COUNTERS = (0x20801819) +NV2080_CTRL_BUS_GET_PEX_UTIL_COUNTERS_PARAMS_MESSAGE_ID = (0x19) +NV2080_CTRL_CMD_BUS_CLEAR_PEX_UTIL_COUNTERS = (0x20801820) +NV2080_CTRL_BUS_CLEAR_PEX_UTIL_COUNTERS_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_BUS_CONTROL_PUBLIC_ASPM_BITS = (0x20801824) +NV2080_CTRL_CMD_BUS_CONTROL_PUBLIC_ASPM_BITS_PARAMS_MESSAGE_ID = (0x24) +NV2080_CTRL_CMD_BUS_GET_NVLINK_PEER_ID_MASK = (0x20801825) +NV2080_CTRL_BUS_MAX_NUM_GPUS = 32 +NV2080_CTRL_BUS_GET_NVLINK_PEER_ID_MASK_PARAMS_MESSAGE_ID = (0x25) +NV2080_CTRL_CMD_BUS_SET_EOM_PARAMETERS = (0x20801826) +NV2080_CTRL_CMD_BUS_SET_EOM_PARAMETERS_PARAMS_MESSAGE_ID = (0x26) +NV2080_CTRL_CMD_BUS_GET_UPHY_DLN_CFG_SPACE = (0x20801827) +NV2080_CTRL_CMD_BUS_GET_UPHY_DLN_CFG_SPACE_PARAMS_MESSAGE_ID = (0x27) +NV2080_CTRL_CMD_BUS_GET_EOM_STATUS = (0x20801828) +NV2080_CTRL_BUS_MAX_NUM_LANES = 32 +NV2080_CTRL_BUS_GET_EOM_STATUS_PARAMS_MESSAGE_ID = (0x28) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS = (0x20801829) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_PARAMS_MESSAGE_ID = (0x29) +NV2080_CTRL_CMD_BUS_PCIE_ATOMICS_CAPTYPE_SYSMEM = 0x0 +NV2080_CTRL_CMD_BUS_PCIE_ATOMICS_CAPTYPE_GPU = 0x1 +NV2080_CTRL_CMD_BUS_PCIE_ATOMICS_CAPTYPE_P2P = 0x2 +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_FETCHADD_32 = (0, 0) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_FETCHADD_32_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_FETCHADD_32_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_FETCHADD_64 = (1, 1) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_FETCHADD_64_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_FETCHADD_64_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_SWAP_32 = (2, 2) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_SWAP_32_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_SWAP_32_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_SWAP_64 = (3, 3) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_SWAP_64_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_SWAP_64_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_32 = (4, 4) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_32_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_32_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_64 = (5, 5) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_64_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_64_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_128 = (6, 6) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_128_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_REQ_ATOMICS_CAPS_CAS_128_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS = (0x2080182a) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_IADD = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_IMIN = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_IMAX = 2 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_INC = 3 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_DEC = 4 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_IAND = 5 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_IOR = 6 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_IXOR = 7 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_EXCH = 8 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_CAS = 9 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_FADD = 10 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_FMIN = 11 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_FMAX = 12 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_OP_TYPE_COUNT = 13 +NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS_PARAMS_MESSAGE_ID = (0x2A) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SCALAR = (0, 0) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SCALAR_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SCALAR_NO = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_VECTOR = (1, 1) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_VECTOR_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_VECTOR_NO = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_REDUCTION = (2, 2) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_REDUCTION_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_REDUCTION_NO = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_32 = (3, 3) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_32_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_32_NO = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_64 = (4, 4) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_64_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_64_NO = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_128 = (5, 5) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_128_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIZE_128_NO = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIGNED = (6, 6) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIGNED_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_SIGNED_NO = 0 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_UNSIGNED = (7, 7) +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_UNSIGNED_YES = 1 +NV2080_CTRL_PCIE_SUPPORTED_GPU_ATOMICS_ATTRIB_UNSIGNED_NO = 0 +NV2080_CTRL_CMD_BUS_GET_C2C_INFO = (0x2080182b) +NV2080_CTRL_CMD_BUS_GET_C2C_INFO_MAX_C2C_INSTANCES = 2 +NV2080_CTRL_CMD_BUS_GET_C2C_INFO_MAX_C2C_LINKS_PER_INSTANCE = 7 +NV2080_CTRL_CMD_BUS_GET_C2C_INFO_MAX_C2C_LINKS = (0xe) +NV2080_CTRL_CMD_BUS_GET_C2C_INFO_PARAMS_MESSAGE_ID = (0x2B) +NV2080_CTRL_BUS_GET_C2C_INFO_REMOTE_TYPE_CPU = 1 +NV2080_CTRL_BUS_GET_C2C_INFO_REMOTE_TYPE_GPU = 2 +NV2080_CTRL_CMD_BUS_SYSMEM_ACCESS = (0x2080182c) +NV2080_CTRL_BUS_SYSMEM_ACCESS_PARAMS_MESSAGE_ID = (0x2C) +NV2080_CTRL_CMD_BUS_GET_C2C_ERR_INFO = (0x2080182d) +NV2080_CTRL_BUS_GET_C2C_ERR_INFO_PARAMS_MESSAGE_ID = (0x2D) +NV2080_CTRL_CMD_BUS_SET_P2P_MAPPING = (0x2080182e) +NV2080_CTRL_CMD_BUS_SET_P2P_MAPPING_CONNECTION_TYPE_INVALID = 0 +NV2080_CTRL_CMD_BUS_SET_P2P_MAPPING_CONNECTION_TYPE_NVLINK = 1 +NV2080_CTRL_CMD_BUS_SET_P2P_MAPPING_CONNECTION_TYPE_PCIE = 2 +NV2080_CTRL_CMD_BUS_SET_P2P_MAPPING_CONNECTION_TYPE_PCIE_BAR1 = 3 +NV2080_CTRL_CMD_BUS_SET_P2P_MAPPING_CONNECTION_TYPE_C2C = 4 +NV2080_SET_P2P_MAPPING_UUID_LEN = 16 +NV2080_CTRL_BUS_SET_P2P_MAPPING_PARAMS_MESSAGE_ID = (0x2E) +NV2080_CTRL_CMD_BUS_UNSET_P2P_MAPPING = (0x2080182f) +NV2080_CTRL_BUS_UNSET_P2P_MAPPING_PARAMS_MESSAGE_ID = (0x2F) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS = (0x20801830) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_PARAMS_MESSAGE_ID = (0x30) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_FETCHADD_32 = (0, 0) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_FETCHADD_32_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_FETCHADD_32_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_FETCHADD_64 = (1, 1) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_FETCHADD_64_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_FETCHADD_64_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_SWAP_32 = (2, 2) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_SWAP_32_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_SWAP_32_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_SWAP_64 = (3, 3) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_SWAP_64_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_SWAP_64_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_32 = (4, 4) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_32_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_32_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_64 = (5, 5) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_64_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_64_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_128 = (6, 6) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_128_YES = (0x00000001) +NV2080_CTRL_CMD_BUS_GET_PCIE_CPL_ATOMICS_CAPS_CAS_128_NO = (0x00000000) +NV2080_CTRL_CMD_BUS_GET_C2C_LPWR_STATS = (0x20801831) +NV2080_CTRL_CMD_BUS_GET_C2C_LPWR_STATS_PARAMS_MESSAGE_ID = (0x31) +NV2080_CTRL_CMD_BUS_GET_C2C_STATE_FULL_POWER = 0x0 +NV2080_CTRL_CMD_BUS_GET_C2C_STATE_CL3 = 0x1 +NV2080_CTRL_CMD_BUS_GET_C2C_STATE_CL4 = 0x2 +NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_STATE_VOTE = (0x20801832) +NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_STATE_VOTE_PARAMS_MESSAGE_ID = (0x32) +NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_IDLE_THRESHOLD = (0x20801836) +NV2080_CTRL_CMD_BUS_SET_C2C_LPWR_IDLE_THRESHOLD_PARAMS_MESSAGE_ID = (0x36) +NV2080_CTRL_CMD_BUS_GET_C2C_PACKET_COUNTERS = (0x20801837) +NV2080_CTRL_CMD_BUS_GET_C2C_PACKET_COUNTERS_PARAMS_MESSAGE_ID = (0x37) +NV2080_CTRL_CMD_CE_GET_CAPS = (0x20802a01) +NV2080_CTRL_CE_CAPS_TBL_SIZE = 2 +NV2080_CTRL_CE_GET_CAPS_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_CE_GET_CAPS_V2 = (0x20802a03) +NV2080_CTRL_CE_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_CE_GET_CE_PCE_MASK = (0x20802a02) +NV2080_CTRL_CE_GET_CE_PCE_MASK_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_CE_SET_PCE_LCE_CONFIG = (0x20802a04) +NV2080_CTRL_MAX_PCES = 64 +NV2080_CTRL_MAX_GRCES = 4 +NV2080_CTRL_CE_SET_PCE_LCE_CONFIG_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_CE_UPDATE_PCE_LCE_MAPPINGS = (0x20802a05) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_INVALID_LCE = 0xf +NV2080_CTRL_CMD_CE_UPDATE_CLASS_DB = (0x20802a06) +NV2080_CTRL_CE_UPDATE_CLASS_DB_PARAMS_MESSAGE_ID = (0x6) +NV2080_CTRL_CMD_CE_GET_PHYSICAL_CAPS = (0x20802a07) +NV2080_CTRL_CE_GET_PHYSICAL_CAPS_PARAMS_MESSAGE_ID = (0x7) +NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_CMD_CE_GET_FAULT_METHOD_BUFFER_SIZE = (0x20802a08) +NV2080_CTRL_CMD_CE_GET_HUB_PCE_MASK = (0x20802a09) +NV2080_CTRL_CE_MAX_HSHUBS = 32 +NV2080_CTRL_CE_GET_HUB_PCE_MASK_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_CE_GET_ALL_CAPS = (0x20802a0a) +NV2080_CTRL_MAX_CES = 64 +NV2080_CTRL_CE_GET_ALL_CAPS_PARAMS_MESSAGE_ID = (0xa) +NV2080_CTRL_CMD_CE_GET_ALL_PHYSICAL_CAPS = (0x20802a0b) +NV2080_CTRL_CE_GET_ALL_PHYSICAL_CAPS_PARAMS_MESSAGE_ID = (0xb) +NV2080_CTRL_CMD_CE_GET_LCE_SHIM_INFO = (0x20802a0c) +NV2080_CTRL_CE_GET_LCE_SHIM_INFO_PARAMS_MESSAGE_ID = (0xc) +NV2080_CTRL_CMD_CE_UPDATE_PCE_LCE_MAPPINGS_V2 = (0x20802a0d) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_V2_PARAMS_MESSAGE_ID = (0xd) +NV2080_CTRL_CMD_CE_GET_HUB_PCE_MASK_V2 = (0x20802a0e) +NV2080_CTRL_CE_GET_HUB_PCE_MASK_V2_PARAMS_MESSAGE_ID = (0xe) +NV2080_CTRL_CMD_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_TYPE = (0x20802a0f) +NV2080_CTRL_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_TYPE_PARAMS_MESSAGE_ID = (0xf) +NV2080_CTRL_CMD_CE_GET_DECOMP_LCE_MASK = (0x20802a11) +NV2080_CTRL_CE_GET_DECOMP_LCE_MASK_PARAMS_MESSAGE_ID = (0x11) +NV2080_CTRL_CMD_CE_IS_DECOMP_LCE_ENABLED = (0x20802a12) +NV2080_CTRL_CE_IS_DECOMP_LCE_ENABLED_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_CMD_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_MIG_GPU_INSTANCE = (0x20802a13) +NV2080_CTRL_CMD_INTERNAL_CE_GET_PCE_CONFIG_FOR_LCE_MIG_GPU_INSTANCE_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MIG_MAPPINGS = (0x20802a14) +NV2080_CTRL_CE_UPDATE_PCE_LCE_MIG_MAPPINGS_PARAMS_MESSAGE_ID = (0x14) +NV2080_CTRL_CLK_DOMAIN_TEGRA_UNDEFINED = (0x00000000) +NV2080_CTRL_CLK_DOMAIN_TEGRA_GPCCLK = (0x00000001) +NV2080_CTRL_CLK_DOMAIN_TEGRA_NVDCLK = (0x00000002) +NV2080_CTRL_CMD_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES = (0x20801037) +NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_SAMPLE_COUNT = (250) +NV2080_CTRL_CLK_PMUMON_CLK_DOMAINS_GET_SAMPLES_PARAMS_MESSAGE_ID = (0x37) +NV2080_CTRL_CMD_DMA_INVALIDATE_TLB = (0x20802502) +NV2080_CTRL_DMA_INVALIDATE_TLB_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_GRAPHICS = (0, 0) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_GRAPHICS_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_GRAPHICS_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_VIDEO = (1, 1) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_VIDEO_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_VIDEO_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_DISPLAY = (2, 2) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_DISPLAY_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_DISPLAY_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_CAPTURE = (3, 3) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_CAPTURE_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_CAPTURE_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_IFB = (4, 4) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_IFB_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_IFB_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_MV = (5, 5) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_MV_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_MV_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_MPEG = (6, 6) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_MPEG_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_MPEG_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_VLD = (7, 7) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_VLD_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_VLD_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_ENCRYPTION = (8, 8) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_ENCRYPTION_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_ENCRYPTION_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_PERFMON = (9, 9) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_PERFMON_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_PERFMON_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_POSTPROCESS = (10, 10) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_POSTPROCESS_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_POSTPROCESS_TRUE = (0x00000001) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_BAR = (11, 11) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_BAR_FALSE = (0x00000000) +NV2080_CTRL_DMA_INVALIDATE_TLB_ENGINE_BAR_TRUE = (0x00000001) +NV2080_CTRL_DMA_INFO_INDEX_SYSTEM_ADDRESS_SIZE = (0x000000000) +NV2080_CTRL_DMA_INFO_INDEX_MAX = NV2080_CTRL_DMA_INFO_INDEX_SYSTEM_ADDRESS_SIZE +NV2080_CTRL_CMD_DMA_GET_INFO = (0x20802503) +NV2080_CTRL_DMA_GET_INFO_MAX_ENTRIES = (256) +NV2080_CTRL_DMA_GET_INFO_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_DMABUF_EXPORT_OBJECTS_TO_FD = (0x20803a01) +NV2080_CTRL_DMABUF_MAX_HANDLES = 128 +NV2080_CTRL_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT = (0x00000000) +NV2080_CTRL_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE = (0x00000001) +NV2080_CTRL_DMABUF_EXPORT_MEM_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_ECC_GET_CLIENT_EXPOSED_COUNTERS = (0x20803400) +NV2080_CTRL_ECC_GET_CLIENT_EXPOSED_COUNTERS_PARAMS_MESSAGE_ID = (0x0) +NV2080_CTRL_CMD_ECC_GET_VOLATILE_COUNTS = (0x20803401) +NV2080_CTRL_ECC_GET_VOLATILE_COUNTS_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_ECC_SRAM_UNIQUE_UNCORR_COUNTS_MAX_COUNT = 600 +NV2080_CTRL_CMD_ECC_GET_SRAM_UNIQUE_UNCORR_COUNTS = (0x20803402) +NV2080_CTRL_ECC_GET_SRAM_UNIQUE_UNCORR_COUNTS_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_ECC_ERROR_TYPE_CORRECTED = 0 +NV2080_CTRL_ECC_ERROR_TYPE_UNCORRECTED = 1 +NV2080_CTRL_CMD_ECC_INJECT_ERROR = (0x20803403) +NV2080_CTRL_ECC_INJECT_ERROR_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_ECC_GET_REPAIR_STATUS = (0x20803404) +NV2080_CTRL_ECC_GET_REPAIR_STATUS_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_ECC_INJECTION_SUPPORTED = (0x20803405) +NV2080_CTRL_ECC_INJECTION_SUPPORTED_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_CMD_ECC_GET_UNREPAIRABLE_MEMORY_FLAG = (0x20803406) +NV2080_CTRL_ECC_GET_UNREPAIRABLE_MEMORY_FLAG_PARAMS_MESSAGE_ID = (0x6) +NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION = (0x20800301) +NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE = (0x00000000) +NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE = (0x00000001) +NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT = (0x00000002) +NV2080_EVENT_DSTATE_XUSB_D0 = (0x00000000) +NV2080_EVENT_DSTATE_XUSB_D3 = (0x00000003) +NV2080_EVENT_DSTATE_XUSB_INVALID = (0xFFFFFFFF) +NV2080_EVENT_DSTATE_PPC_D0 = (0x00000000) +NV2080_EVENT_DSTATE_PPC_D3 = (0x00000003) +NV2080_EVENT_DSTATE_PPC_INVALID = (0xFFFFFFFF) +NV2080_CTRL_CMD_EVENT_SET_TRIGGER = (0x20800302) +NV2080_CTRL_CMD_EVENT_SET_MEMORY_NOTIFIES = (0x20800303) +NV2080_CTRL_EVENT_SET_MEMORY_NOTIFIES_PARAMS_MESSAGE_ID = (0x3) +NV2080_EVENT_MEMORY_NOTIFIES_STATUS_NOTIFIED = 0 +NV2080_EVENT_MEMORY_NOTIFIES_STATUS_PENDING = 1 +NV2080_EVENT_MEMORY_NOTIFIES_STATUS_ERROR = 2 +NV2080_CTRL_CMD_EVENT_SET_SEMAPHORE_MEMORY = (0x20800304) +NV2080_CTRL_EVENT_SET_SEMAPHORE_MEMORY_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_EVENT_SET_GUEST_MSI = (0x20800305) +NV2080_CTRL_EVENT_SET_GUEST_MSI_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_CMD_EVENT_SET_SEMA_MEM_VALIDATION = (0x20800306) +NV2080_CTRL_EVENT_SET_SEMA_MEM_VALIDATION_PARAMS_MESSAGE_ID = (0x6) +NV2080_CTRL_CMD_EVENT_SET_TRIGGER_FIFO = (0x20800308) +NV2080_CTRL_EVENT_SET_TRIGGER_FIFO_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_CMD_EVENT_VIDEO_BIND_EVTBUF = (0x20800309) +NV2080_CTRL_EVENT_VIDEO_BIND_EVTBUF_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_EVENT_RATS_GSP_TRACE_BIND_EVTBUF = (0x2080030a) +NV2080_CTRL_EVENT_RATS_GSP_TRACE_BIND_EVTBUF_PARAMS_MESSAGE_ID = (0xA) +NV2080_CTRL_CMD_EVENT_NOCAT_BIND_EVTBUF = (0x2080030b) +NV2080_CTRL_EVENT_NOCAT_BIND_EVTBUF_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_FB_INFO_INDEX_TILE_REGION_COUNT = (0x00000000) +NV2080_CTRL_FB_INFO_INDEX_COMPRESSION_SIZE = (0x00000001) +NV2080_CTRL_FB_INFO_INDEX_DRAM_PAGE_STRIDE = (0x00000002) +NV2080_CTRL_FB_INFO_INDEX_TILE_REGION_FREE_COUNT = (0x00000003) +NV2080_CTRL_FB_INFO_INDEX_PARTITION_COUNT = (0x00000004) +NV2080_CTRL_FB_INFO_INDEX_BAR1_SIZE = (0x00000005) +NV2080_CTRL_FB_INFO_INDEX_BANK_SWIZZLE_ALIGNMENT = (0x00000006) +NV2080_CTRL_FB_INFO_INDEX_RAM_SIZE = (0x00000007) +NV2080_CTRL_FB_INFO_INDEX_TOTAL_RAM_SIZE = (0x00000008) +NV2080_CTRL_FB_INFO_INDEX_HEAP_SIZE = (0x00000009) +NV2080_CTRL_FB_INFO_INDEX_MAPPABLE_HEAP_SIZE = (0x0000000A) +NV2080_CTRL_FB_INFO_INDEX_BUS_WIDTH = (0x0000000B) +NV2080_CTRL_FB_INFO_INDEX_RAM_CFG = (0x0000000C) +NV2080_CTRL_FB_INFO_INDEX_RAM_TYPE = (0x0000000D) +NV2080_CTRL_FB_INFO_INDEX_BANK_COUNT = (0x0000000E) +NV2080_CTRL_FB_INFO_INDEX_OVERLAY_OFFSET_ADJUSTMENT = (0x0000000F) +NV2080_CTRL_FB_INFO_INDEX_GPU_VADDR_SPACE_SIZE_KB = (0x0000000F) +NV2080_CTRL_FB_INFO_INDEX_GPU_VADDR_HEAP_SIZE_KB = (0x0000000F) +NV2080_CTRL_FB_INFO_INDEX_GPU_VADDR_MAPPBLE_SIZE_KB = (0x0000000F) +NV2080_CTRL_FB_INFO_INDEX_EFFECTIVE_BW = (0x0000000F) +NV2080_CTRL_FB_INFO_INDEX_FB_TAX_SIZE_KB = (0x00000010) +NV2080_CTRL_FB_INFO_INDEX_HEAP_BASE_KB = (0x00000011) +NV2080_CTRL_FB_INFO_INDEX_LARGEST_FREE_REGION_SIZE_KB = (0x00000012) +NV2080_CTRL_FB_INFO_INDEX_LARGEST_FREE_REGION_BASE_KB = (0x00000013) +NV2080_CTRL_FB_INFO_INDEX_PARTITION_MASK = (0x00000014) +NV2080_CTRL_FB_INFO_INDEX_VISTA_RESERVED_HEAP_SIZE = (0x00000015) +NV2080_CTRL_FB_INFO_INDEX_HEAP_FREE = (0x00000016) +NV2080_CTRL_FB_INFO_INDEX_RAM_LOCATION = (0x00000017) +NV2080_CTRL_FB_INFO_INDEX_FB_IS_BROKEN = (0x00000018) +NV2080_CTRL_FB_INFO_INDEX_FBP_COUNT = (0x00000019) +NV2080_CTRL_FB_INFO_INDEX_FBP_MASK = (0x0000001A) +NV2080_CTRL_FB_INFO_INDEX_L2CACHE_SIZE = (0x0000001B) +NV2080_CTRL_FB_INFO_INDEX_MEMORYINFO_VENDOR_ID = (0x0000001C) +NV2080_CTRL_FB_INFO_INDEX_BAR1_AVAIL_SIZE = (0x0000001D) +NV2080_CTRL_FB_INFO_INDEX_HEAP_START = (0x0000001E) +NV2080_CTRL_FB_INFO_INDEX_BAR1_MAX_CONTIGUOUS_AVAIL_SIZE = (0x0000001F) +NV2080_CTRL_FB_INFO_INDEX_USABLE_RAM_SIZE = (0x00000020) +NV2080_CTRL_FB_INFO_INDEX_TRAINIG_2T = (0x00000021) +NV2080_CTRL_FB_INFO_INDEX_LTC_COUNT = (0x00000022) +NV2080_CTRL_FB_INFO_INDEX_LTS_COUNT = (0x00000023) +NV2080_CTRL_FB_INFO_INDEX_L2CACHE_ONLY_MODE = (0x00000024) +NV2080_CTRL_FB_INFO_INDEX_PSEUDO_CHANNEL_MODE = (0x00000025) +NV2080_CTRL_FB_INFO_INDEX_SMOOTHDISP_RSVD_BAR1_SIZE = (0x00000026) +NV2080_CTRL_FB_INFO_INDEX_HEAP_OFFLINE_SIZE = (0x00000027) +NV2080_CTRL_FB_INFO_INDEX_1TO1_COMPTAG_ENABLED = (0x00000028) +NV2080_CTRL_FB_INFO_INDEX_SUSPEND_RESUME_RSVD_SIZE = (0x00000029) +NV2080_CTRL_FB_INFO_INDEX_ALLOW_PAGE_RETIREMENT = (0x0000002A) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK = (0x0000002B) +NV2080_CTRL_FB_INFO_POISON_FUSE_ENABLED = (0x0000002C) +NV2080_CTRL_FB_INFO_FBPA_ECC_ENABLED = (0x0000002D) +NV2080_CTRL_FB_INFO_DYNAMIC_PAGE_OFFLINING_ENABLED = (0x0000002E) +NV2080_CTRL_FB_INFO_INDEX_FORCED_BAR1_64KB_MAPPING_ENABLED = (0x0000002F) +NV2080_CTRL_FB_INFO_INDEX_P2P_MAILBOX_SIZE = (0x00000030) +NV2080_CTRL_FB_INFO_INDEX_P2P_MAILBOX_ALIGNMENT = (0x00000031) +NV2080_CTRL_FB_INFO_INDEX_P2P_MAILBOX_BAR1_MAX_OFFSET_64KB = (0x00000032) +NV2080_CTRL_FB_INFO_INDEX_PROTECTED_MEM_SIZE_TOTAL_KB = (0x00000033) +NV2080_CTRL_FB_INFO_INDEX_PROTECTED_MEM_SIZE_FREE_KB = (0x00000034) +NV2080_CTRL_FB_INFO_INDEX_ECC_STATUS_SIZE = (0x00000035) +NV2080_CTRL_FB_INFO_INDEX_IS_ZERO_FB = (0x00000036) +NV2080_CTRL_FB_INFO_INDEX_PARTITION_MASK_0 = (NV2080_CTRL_FB_INFO_INDEX_PARTITION_MASK) +NV2080_CTRL_FB_INFO_INDEX_PARTITION_MASK_1 = (0x00000037) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_0 = (NV2080_CTRL_FB_INFO_INDEX_LTC_MASK) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_1 = (0x00000038) +NV2080_CTRL_FB_INFO_INDEX_ACCESS_COUNTER_BUFFER_COUNT = (0x00000039) +NV2080_CTRL_FB_INFO_INDEX_COHERENCE_INFO = (0x0000003A) +NV2080_CTRL_FB_INFO_INDEX_NUMA_NODE_ID = (0x0000003B) +NV2080_CTRL_FB_INFO_INDEX_PARTITION_MASK_2 = (0x0000003C) +NV2080_CTRL_FB_INFO_INDEX_PARTITION_MASK_3 = (0x0000003D) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_2 = (0x0000003E) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_3 = (0x0000003F) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_4 = (0x00000040) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_5 = (0x00000041) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_6 = (0x00000042) +NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_7 = (0x00000043) +NV2080_CTRL_FB_INFO_INDEX_HEAP_RECLAIMABLE = (0x00000044) +NV2080_CTRL_FB_INFO_INDEX_MAX = NV2080_CTRL_FB_INFO_INDEX_HEAP_RECLAIMABLE +NV2080_CTRL_FB_INFO_MAX_LIST_SIZE = (0x00000080) +NV2080_CTRL_FB_INFO_RAM_TYPE_UNKNOWN = (0x00000000) +NV2080_CTRL_FB_INFO_RAM_TYPE_SDRAM = (0x00000001) +NV2080_CTRL_FB_INFO_RAM_TYPE_DDR1 = (0x00000002) +NV2080_CTRL_FB_INFO_RAM_TYPE_SDDR2 = (0x00000003) +NV2080_CTRL_FB_INFO_RAM_TYPE_DDR2 = NV2080_CTRL_FB_INFO_RAM_TYPE_SDDR2 +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR2 = (0x00000004) +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR3 = (0x00000005) +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR4 = (0x00000006) +NV2080_CTRL_FB_INFO_RAM_TYPE_SDDR3 = (0x00000007) +NV2080_CTRL_FB_INFO_RAM_TYPE_DDR3 = NV2080_CTRL_FB_INFO_RAM_TYPE_SDDR3 +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR5 = (0x00000008) +NV2080_CTRL_FB_INFO_RAM_TYPE_LPDDR2 = (0x00000009) +NV2080_CTRL_FB_INFO_RAM_TYPE_SDDR4 = (0x0000000C) +NV2080_CTRL_FB_INFO_RAM_TYPE_LPDDR4 = (0x0000000D) +NV2080_CTRL_FB_INFO_RAM_TYPE_HBM1 = (0x0000000E) +NV2080_CTRL_FB_INFO_RAM_TYPE_HBM2 = (0x0000000F) +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR5X = (0x00000010) +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR6 = (0x00000011) +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR6X = (0x00000012) +NV2080_CTRL_FB_INFO_RAM_TYPE_LPDDR5 = (0x00000013) +NV2080_CTRL_FB_INFO_RAM_TYPE_HBM3 = (0x00000014) +NV2080_CTRL_FB_INFO_RAM_TYPE_GDDR7 = (0x00000015) +NV2080_CTRL_FB_INFO_RAM_TYPE_HBM4 = (0x00000016) +NV2080_CTRL_FB_INFO_RAM_LOCATION_GPU_DEDICATED = (0x00000000) +NV2080_CTRL_FB_INFO_RAM_LOCATION_SYS_SHARED = (0x00000001) +NV2080_CTRL_FB_INFO_RAM_LOCATION_SYS_DEDICATED = (0x00000002) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_SAMSUNG = (0x00000001) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_QIMONDA = (0x00000002) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_ELPIDA = (0x00000003) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_ETRON = (0x00000004) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_NANYA = (0x00000005) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_HYNIX = (0x00000006) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_MOSEL = (0x00000007) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_WINBOND = (0x00000008) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_ESMT = (0x00000009) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_MICRON = (0x0000000F) +NV2080_CTRL_FB_INFO_MEMORYINFO_VENDOR_ID_UNKNOWN = (0xFFFFFFFF) +NV2080_CTRL_FB_INFO_PSEUDO_CHANNEL_MODE_UNSUPPORTED = (0x00000000) +NV2080_CTRL_FB_INFO_PSEUDO_CHANNEL_MODE_DISABLED = (0x00000001) +NV2080_CTRL_FB_INFO_PSEUDO_CHANNEL_MODE_ENABLED = (0x00000002) +NV2080_CTRL_FB_INFO_INDEX_COHERENCE_INFO_NON_FULLY_COHERENT = (0x00000000) +NV2080_CTRL_FB_INFO_INDEX_COHERENCE_INFO_FULLY_COHERENT = (0x00000001) +NV2080_CTRL_CMD_FB_GET_INFO = (0x20801301) +NV2080_CTRL_FB_GET_INFO_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_FB_GET_INFO_V2 = (0x20801303) +NV2080_CTRL_FB_GET_INFO_V2_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_FB_GET_BAR1_OFFSET = (0x20801310) +NV2080_CTRL_FB_GET_BAR1_OFFSET_PARAMS_MESSAGE_ID = (0x10) +NV2080_CTRL_CMD_FB_GET_CALIBRATION_LOCK_FAILED = (0x2080130c) +NV2080_CTRL_FB_GET_CALIBRATION_LOCK_FAILED_PARAMS_MESSAGE_ID = (0xC) +NV2080_CTRL_CMD_FB_GET_CAL_FLAG_NONE = (0x00000000) +NV2080_CTRL_CMD_FB_GET_CAL_FLAG_RESET = (0x00000001) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL = (0x2080130d) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_PARAMS_MESSAGE_ID = (0xD) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_WRITE_BACK = (0, 0) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_WRITE_BACK_NO = (0x00000000) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_WRITE_BACK_YES = (0x00000001) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_INVALIDATE = (1, 1) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_INVALIDATE_NO = (0x00000000) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_INVALIDATE_YES = (0x00000001) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_FB_FLUSH = (2, 2) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_FB_FLUSH_NO = (0x00000000) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE_IRQL_FLAGS_FB_FLUSH_YES = (0x00000001) +NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE = (0x2080130e) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_MAX_ADDRESSES = 500 +NV2080_CTRL_FB_FLUSH_GPU_CACHE_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_APERTURE = (2, 0) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_APERTURE_VIDEO_MEMORY = (0x00000000) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_APERTURE_SYSTEM_MEMORY = (0x00000001) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_APERTURE_PEER_MEMORY = (0x00000002) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_APERTURE_DIRTY_MEMORY = (0x00000003) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_APERTURE_COMPTAG_MEMORY = (0x00000004) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_WRITE_BACK = (3, 3) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_WRITE_BACK_NO = (0x00000000) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_WRITE_BACK_YES = (0x00000001) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_INVALIDATE = (4, 4) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_INVALIDATE_NO = (0x00000000) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_INVALIDATE_YES = (0x00000001) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FLUSH_MODE = (5, 5) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FLUSH_MODE_ADDRESS_ARRAY = (0x00000000) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FLUSH_MODE_FULL_CACHE = (0x00000001) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FB_FLUSH = (6, 6) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FB_FLUSH_NO = (0x00000000) +NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FB_FLUSH_YES = (0x00000001) +NV2080_CTRL_CMD_FB_IS_KIND = (0x20801313) +NV2080_CTRL_FB_IS_KIND_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_FB_IS_KIND_OPERATION_SUPPORTED = (0x00000000) +NV2080_CTRL_FB_IS_KIND_OPERATION_COMPRESSIBLE = (0x00000001) +NV2080_CTRL_FB_IS_KIND_OPERATION_COMPRESSIBLE_1 = (0x00000002) +NV2080_CTRL_FB_IS_KIND_OPERATION_COMPRESSIBLE_2 = (0x00000003) +NV2080_CTRL_FB_IS_KIND_OPERATION_COMPRESSIBLE_4 = (0x00000004) +NV2080_CTRL_FB_IS_KIND_OPERATION_ZBC = (0x00000005) +NV2080_CTRL_FB_IS_KIND_OPERATION_ZBC_ALLOWS_1 = (0x00000006) +NV2080_CTRL_FB_IS_KIND_OPERATION_ZBC_ALLOWS_2 = (0x00000007) +NV2080_CTRL_FB_IS_KIND_OPERATION_ZBC_ALLOWS_4 = (0x00000008) +NV2080_CTRL_CMD_FB_GET_GPU_CACHE_INFO = (0x20801315) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_PARAMS_MESSAGE_ID = (0x15) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_POWER_STATE_ENABLED = (0x00000000) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_POWER_STATE_DISABLED = (0x00000001) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_WRITE_MODE_WRITETHROUGH = (0x00000000) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_WRITE_MODE_WRITEBACK = (0x00000001) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_BYPASS_MODE_DISABLED = (0x00000000) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_BYPASS_MODE_ENABLED = (0x00000001) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_RCM_STATE_FULL = (0x00000000) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_RCM_STATE_TRANSITIONING = (0x00000001) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_RCM_STATE_REDUCED = (0x00000002) +NV2080_CTRL_FB_GET_GPU_CACHE_INFO_RCM_STATE_ZERO_CACHE = (0x00000003) +NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO = (0x20801320) +NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_MEM_TYPES = 18 +NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_MAX_ENTRIES = 16 +NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_FB_OFFLINE_PAGES = (0x20801321) +NV2080_CTRL_FB_OFFLINED_PAGES_MAX_PAGES = (0x00000040) +NV2080_CTRL_FB_OFFLINED_PAGES_INVALID_ADDRESS = (0xffffffffffffffff) +NV2080_CTRL_FB_OFFLINED_PAGES_PAGE_SIZE_4K = (0x00000000) +NV2080_CTRL_FB_OFFLINED_PAGES_PAGE_SIZE_64K = (0x00000001) +NV2080_CTRL_FB_OFFLINED_PAGES_PAGE_SIZE_128K = (0x00000002) +NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_MULTIPLE_SBE = (0x00000002) +NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_DBE = (0x00000004) +NV2080_CTRL_FB_OFFLINED_PAGES_STATUS_OK = (0x00000000) +NV2080_CTRL_FB_OFFLINED_PAGES_STATUS_PENDING_RETIREMENT = (0x00000001) +NV2080_CTRL_FB_OFFLINED_PAGES_STATUS_BLACKLISTING_FAILED = (0x00000002) +NV2080_CTRL_FB_OFFLINED_PAGES_STATUS_TABLE_FULL = (0x00000003) +NV2080_CTRL_FB_OFFLINED_PAGES_STATUS_INTERNAL_ERROR = (0x00000004) +NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_MULTIPLE_SBE = NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_MULTIPLE_SBE +NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DBE = NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_DBE +NV2080_CTRL_FB_OFFLINE_PAGES_PARAMS_MESSAGE_ID = (0x21) +NV2080_CTRL_CMD_FB_GET_OFFLINED_PAGES = (0x20801322) +NV2080_CTRL_FB_GET_OFFLINED_PAGES_RETIREMENT_PENDING_SBE = (0, 0) +NV2080_CTRL_FB_GET_OFFLINED_PAGES_RETIREMENT_PENDING_SBE_FALSE = 0 +NV2080_CTRL_FB_GET_OFFLINED_PAGES_RETIREMENT_PENDING_SBE_TRUE = 1 +NV2080_CTRL_FB_GET_OFFLINED_PAGES_RETIREMENT_PENDING_DBE = (1, 1) +NV2080_CTRL_FB_GET_OFFLINED_PAGES_RETIREMENT_PENDING_DBE_FALSE = 0 +NV2080_CTRL_FB_GET_OFFLINED_PAGES_RETIREMENT_PENDING_DBE_TRUE = 1 +NV2080_CTRL_FB_GET_OFFLINED_PAGES_PARAMS_MESSAGE_ID = (0x22) +NV2080_CTRL_CMD_FB_QUERY_ACR_REGION = (0x20801325) +NV2080_CTRL_CMD_FB_ACR_CLIENT_ID = 2 +NV2080_CTRL_CMD_FB_QUERY_ACR_REGION_PARAMS_MESSAGE_ID = (0x25) +NV2080_CTRL_CMD_FB_CLEAR_OFFLINED_PAGES = (0x20801326) +NV2080_CTRL_FB_CLEAR_OFFLINED_PAGES_PARAMS_MESSAGE_ID = (0x26) +NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_INFO = (0x20801327) +NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_INFO_PARAMS_MESSAGE_ID = (0x27) +NV2080_CTRL_CMD_FB_GET_LTC_INFO_FOR_FBP = (0x20801328) +NV2080_CTRL_FB_GET_LTC_INFO_FOR_FBP_PARAMS_MESSAGE_ID = (0x28) +NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_CONTEXT = (0x20801329) +NV2080_CTRL_CMD_FB_COMPBITCOPY_GET_COMPBITS = (0x2080132a) +NV2080_CTRL_CMD_FB_COMPBITCOPY_PUT_COMPBITS = (0x2080132b) +NV2080_CTRL_CMD_FB_COMPBITCOPY_READ_COMPBITS64KB = (0x2080132c) +NV2080_CTRL_CMD_FB_COMPBITCOPY_WRITE_COMPBITS64KB = (0x2080132d) +NV2080_CTRL_CMD_FB_COMPBITCOPY_SET_FORCE_BAR1 = (0x20801335) +NV2080_CTRL_CMD_FB_GET_AMAP_CONF = (0x20801336) +NV2080_CTRL_CMD_FB_GET_AMAP_CONF_PARAMS_MESSAGE_ID = (0x36) +NV2080_CTRL_CMD_FB_CBC_OP = (0x20801337) +NV2080_CTRL_CMD_FB_CBC_OP_PARAMS_MESSAGE_ID = (0x37) +NV2080_CTRL_CMD_FB_GET_CTAGS_FOR_CBC_EVICTION = (0x20801338) +NV2080_MAX_CTAGS_FOR_CBC_EVICTION = 0x7F +NV2080_CTRL_FB_GET_CTAGS_FOR_CBC_EVICTION_PARAMS_MESSAGE_ID = (0x38) +NV2080_CTRL_CMD_FB_ALLOC_COMP_RESOURCE = (0x20801339) +NV2080_CTRL_CMD_FB_ALLOC_COMP_RESOURCE_PARAMS_MESSAGE_ID = (0x39) +NV2080_CTRL_CMD_FB_FREE_TILE = (0x2080133a) +NV2080_CTRL_CMD_FB_FREE_TILE_PARAMS_MESSAGE_ID = (0x3A) +NV2080_CTRL_CMD_FB_SETUP_VPR_REGION = (0x2080133b) +NV2080_CTRL_CMD_FB_SETUP_VPR_REGION_PARAMS_MESSAGE_ID = (0x3B) +NV2080_CTRL_CMD_FB_GET_CLI_MANAGED_OFFLINED_PAGES = (0x2080133c) +NV2080_CTRL_FB_GET_CLI_MANAGED_OFFLINED_PAGES_PARAMS_MESSAGE_ID = (0x3C) +NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_CONSTRUCT_INFO = (0x2080133d) +NV2080_CTRL_CMD_FB_GET_COMPBITCOPY_CONSTRUCT_INFO_PARAMS_MESSAGE_ID = (0x3D) +NV2080_CTRL_CMD_FB_SET_RRD = (0x2080133e) +NV2080_CTRL_FB_SET_RRD_PARAMS_MESSAGE_ID = (0x3E) +NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_RESET_VALUE = (0xff) +NV2080_CTRL_CMD_FB_SET_READ_LIMIT = (0x2080133f) +NV2080_CTRL_FB_SET_READ_LIMIT_RESET_VALUE = NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_RESET_VALUE +NV2080_CTRL_FB_SET_READ_LIMIT_PARAMS_MESSAGE_ID = (0x3F) +NV2080_CTRL_CMD_FB_SET_WRITE_LIMIT = (0x20801340) +NV2080_CTRL_FB_SET_WRITE_LIMIT_RESET_VALUE = NV2080_CTRL_FB_SET_READ_WRITE_LIMIT_RESET_VALUE +NV2080_CTRL_FB_SET_WRITE_LIMIT_PARAMS_MESSAGE_ID = (0x40) +NV2080_CTRL_CMD_FB_PATCH_PBR_FOR_MINING = (0x20801341) +NV2080_CTRL_FB_PATCH_PBR_FOR_MINING_PARAMS_MESSAGE_ID = (0x41) +NV2080_CTRL_CMD_FB_GET_MEM_ALIGNMENT = (0x20801342) +NV2080_CTRL_FB_GET_MEM_ALIGNMENT_PARAMS_MESSAGE_ID = (0x42) +NV2080_CTRL_CMD_FB_GET_CBC_BASE_ADDR = (0x20801343) +NV2080_CTRL_CMD_FB_GET_CBC_BASE_ADDR_PARAMS_MESSAGE_ID = (0x43) +NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_PENDING = (0, 0) +NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_PENDING_FALSE = 0 +NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_PENDING_TRUE = 1 +NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_ACTIVE = (1, 1) +NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_ACTIVE_FALSE = 0 +NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_ACTIVE_TRUE = 1 +NV2080_CTRL_FB_REMAPPED_ROW_SOURCE_SBE_FIELD = (0x00000002) +NV2080_CTRL_FB_REMAPPED_ROW_SOURCE_DBE_FIELD = (0x00000003) +NV2080_CTRL_FB_REMAPPED_ROWS_MAX_ROWS = (0x00000200) +NV2080_CTRL_CMD_FB_GET_REMAPPED_ROWS = (0x20801344) +NV2080_CTRL_FB_GET_REMAPPED_ROWS_FLAGS_PENDING = NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_PENDING +NV2080_CTRL_FB_GET_REMAPPED_ROWS_FLAGS_PENDING_FALSE = NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_PENDING_FALSE +NV2080_CTRL_FB_GET_REMAPPED_ROWS_FLAGS_PENDING_TRUE = NV2080_CTRL_FB_REMAP_ENTRY_FLAGS_PENDING_TRUE +NV2080_CTRL_FB_GET_REMAPPED_ROWS_FLAGS_FAILURE = (1, 1) +NV2080_CTRL_FB_GET_REMAPPED_ROWS_FLAGS_FAILURE_FALSE = 0 +NV2080_CTRL_FB_GET_REMAPPED_ROWS_FLAGS_FAILURE_TRUE = 1 +NV2080_CTRL_FB_GET_REMAPPED_ROWS_PARAMS_MESSAGE_ID = (0x44) +NV2080_CTRL_FB_FS_INFO_MAX_QUERY_SIZE = 24 +NV2080_CTRL_FB_FS_INFO_INVALID_QUERY = 0x0 +NV2080_CTRL_FB_FS_INFO_FBP_MASK = 0x1 +NV2080_CTRL_FB_FS_INFO_LTC_MASK = 0x2 +NV2080_CTRL_FB_FS_INFO_LTS_MASK = 0x3 +NV2080_CTRL_FB_FS_INFO_FBPA_MASK = 0x4 +NV2080_CTRL_FB_FS_INFO_ROP_MASK = 0x5 +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTC_MASK = 0x6 +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LTS_MASK = 0x7 +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_MASK = 0x8 +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_ROP_MASK = 0x9 +NV2080_CTRL_FB_FS_INFO_FBPA_SUBP_MASK = 0xA +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_FBPA_SUBP_MASK = 0xB +NV2080_CTRL_FB_FS_INFO_FBP_LOGICAL_MAP = 0xC +NV2080_CTRL_SYSL2_FS_INFO_SYSLTC_MASK = 0xD +NV2080_CTRL_FB_FS_INFO_PAC_MASK = 0xE +NV2080_CTRL_FB_FS_INFO_LOGICAL_LTC_MASK = 0xF +NV2080_CTRL_FB_FS_INFO_PROFILER_MON_LOGICAL_LTC_MASK = 0x10 +NV2080_CTRL_SYSL2_FS_INFO_SYSLTS_MASK = 0x11 +NV2080_CTRL_FB_FS_INFO_SYS_MASK = 0x12 +NV2080_CTRL_FB_FS_INFO_MAX_QUERIES = 120 +NV2080_CTRL_FB_GET_FS_INFO_PARAMS_MESSAGE_ID = (0x46) +NV2080_CTRL_CMD_FB_GET_FS_INFO = (0x20801346) +NV2080_CTRL_FB_HISTOGRAM_IDX_NO_REMAPPED_ROWS = (0x0) +NV2080_CTRL_FB_HISTOGRAM_IDX_SINGLE_REMAPPED_ROW = (0x1) +NV2080_CTRL_FB_HISTOGRAM_IDX_MIXED_REMAPPED_REMAINING_ROWS = (0x2) +NV2080_CTRL_FB_HISTOGRAM_IDX_SINGLE_REMAINING_ROW = (0x3) +NV2080_CTRL_FB_HISTOGRAM_IDX_MAX_REMAPPED_ROWS = (0x4) +NV2080_CTRL_FB_GET_ROW_REMAPPER_HISTOGRAM_PARAMS_MESSAGE_ID = (0x47) +NV2080_CTRL_CMD_FB_GET_ROW_REMAPPER_HISTOGRAM = (0x20801347) +NV2080_CTRL_CMD_FB_GET_DYNAMIC_OFFLINED_PAGES = (0x20801348) +NV2080_CTRL_FB_DYNAMIC_BLACKLIST_MAX_PAGES = 512 +NV2080_CTRL_FB_DYNAMIC_BLACKLIST_MAX_ENTRIES = 64 +NV2080_CTRL_FB_GET_DYNAMIC_OFFLINED_PAGES_PARAMS_MESSAGE_ID = (0x48) +NV2080_CTRL_FB_DYNAMIC_BLACKLISTED_PAGES_SOURCE_INVALID = (0x00000000) +NV2080_CTRL_FB_DYNAMIC_BLACKLISTED_PAGES_SOURCE_DPR_DBE = (0x00000001) +NV2080_CTRL_CMD_FB_GET_CLIENT_ALLOCATION_INFO = (0x20801349) +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_TYPE = (4, 0) +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_TYPE_SYSMEM = 0 +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_TYPE_VIDMEM = 1 +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_SHARED = (5, 5) +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_SHARED_FALSE = 0 +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_SHARED_TRUE = 1 +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_OWNER = (6, 6) +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_OWNER_FALSE = 0 +NV2080_CTRL_CMD_FB_ALLOCATION_FLAGS_OWNER_TRUE = 1 +NV2080_CTRL_CMD_FB_GET_CLIENT_ALLOCATION_INFO_PARAMS_MESSAGE_ID = (0x49) +NV2080_CTRL_CMD_FB_UPDATE_NUMA_STATUS = (0x20801350) +NV2080_CTRL_FB_UPDATE_NUMA_STATUS_PARAMS_MESSAGE_ID = (0x50) +NV2080_CTRL_CMD_FB_GET_NUMA_INFO = (0x20801351) +NV2080_CTRL_FB_NUMA_INFO_MAX_OFFLINE_ADDRESSES = 64 +NV2080_CTRL_FB_GET_NUMA_INFO_PARAMS_MESSAGE_ID = (0x51) +NV2080_CTRL_CMD_FB_GET_SEMAPHORE_SURFACE_LAYOUT = (0x20801352) +NV2080_CTRL_FB_GET_SEMAPHORE_SURFACE_LAYOUT_CAPS_MONITORED_FENCE_SUPPORTED = (0x00000001) +NV2080_CTRL_FB_GET_SEMAPHORE_SURFACE_LAYOUT_CAPS_64BIT_SEMAPHORES_SUPPORTED = (0x00000002) +NV2080_CTRL_FB_GET_SEMAPHORE_SURFACE_LAYOUT_PARAMS_MESSAGE_ID = (0x52) +NV2080_CTRL_CMD_GMMU_COMMIT_TLB_INVALIDATE = (0x20801353) +NV2080_CTRL_GMMU_COMMIT_TLB_INVALIDATE_PARAMS_MESSAGE_ID = (0x53) +NV2080_CTRL_CMD_FB_STATS_MAX_OWNER = 201 +NV2080_CTRL_CMD_RM_FB_STATS_ENTRY_MESSAGE_ID = (0x2A) +NV2080_CTRL_CMD_FB_STATS_GET = (0x2080132f) +NV2080_CTRL_CMD_FB_STATS_GET_PARAMS_MESSAGE_ID = (0x2F) +NV2080_CTRL_CMD_GSP_FB_STATS_GET = (0x2080132a) +NV2080_CTRL_CMD_FB_GET_STATIC_BAR1_INFO = (0x20801354) +NV2080_CTRL_FB_GET_STATIC_BAR1_INFO_PARAMS_MESSAGE_ID = (0x54) +NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION = (0x20801355) +NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION_DISABLED = (0x00000000) +NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION_ENABLED = (0x00000001) +NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_PENDING_CONFIGURATION_PARAMS_MESSAGE_ID = (0x55) +NV2080_CTRL_CMD_FB_SET_DRAM_ENCRYPTION_CONFIGURATION = (0x20801356) +NV2080_CTRL_FB_SET_DRAM_ENCRYPTION_CONFIGURATION_DISABLE = (0x00000000) +NV2080_CTRL_FB_SET_DRAM_ENCRYPTION_CONFIGURATION_ENABLE = (0x00000001) +NV2080_CTRL_FB_SET_DRAM_ENCRYPTION_CONFIGURATION_PARAMS_MESSAGE_ID = (0x56) +NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT = (0x20801357) +NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT_DISABLED = (0x00000000) +NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_INFOROM_SUPPORT_ENABLED = (0x00000001) +NV2080_CTRL_FB_DRAM_ENCRYPTION_INFOROM_SUPPORT_PARAMS_MESSAGE_ID = (0x57) +NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS = (0x20801358) +NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS_DISABLED = (0x00000000) +NV2080_CTRL_CMD_FB_QUERY_DRAM_ENCRYPTION_STATUS_ENABLED = (0x00000001) +NV2080_CTRL_FB_QUERY_DRAM_ENCRYPTION_STATUS_PARAMS_MESSAGE_ID = (0x58) +NV2080_CTRL_CMD_FB_GET_MEMORY_BOOT_TRAINING_FLAGS = (0x20801359) +NV2080_CTRL_FB_GET_MEMORY_BOOT_TRAINING_FLAGS_PARAMS_MESSAGE_ID = (0x59) +NV2080_CTRL_CMD_FB_GET_CARVEOUT_REGION_INFO = (0x20801360) +NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO_MAX_ENTRIES = 8 +NV2080_CTRL_FB_GET_CARVEOUT_REGION_INFO_PARAMS_MESSAGE_ID = (0x60) +NV2080_CTRL_FB_REMAPPED_BANK_SOURCE_FIELD = (0x00000002) +NV2080_CTRL_FB_REMAPPED_BANKS_MAX_BANKS = (0x00000200) +NV2080_CTRL_CMD_FB_GET_REMAPPED_BANKS = (0x20801361) +NV2080_CTRL_FB_GET_REMAPPED_BANKS_FLAGS_PENDING = (0, 0) +NV2080_CTRL_FB_GET_REMAPPED_BANKS_FLAGS_PENDING_FALSE = 0 +NV2080_CTRL_FB_GET_REMAPPED_BANKS_FLAGS_PENDING_TRUE = 1 +NV2080_CTRL_FB_GET_REMAPPED_BANKS_PARAMS_MESSAGE_ID = (0x61) +NV2080_CTRL_CMD_FB_GET_UGPU_MEMORY_INFO = (0x20801362) +NV2080_CTRL_FB_GET_UGPU_MEMORY_INFO_MAX_UGPUS = 2 +NV2080_CTRL_FB_GET_UGPU_MEMORY_INFO_PARAMS_MESSAGE_ID = (0x62) +NV2080_CTRL_CMD_FB_GET_CPU_COHERENT_RANGE = (0x20801363) +NV2080_CTRL_FB_GET_CPU_COHERENT_RANGE_PARAMS_MESSAGE_ID = (0x63) +NV2080_CTRL_CMD_FB_GET_WPR_REGION_INFO = (0x20801364) +NV2080_CTRL_FB_GET_WPR_REGION_INFO_PARAMS_MESSAGE_ID = (0x64) +NV2080_CTRL_CMD_SET_GPFIFO = (0x20801102) +NV2080_CTRL_CMD_SET_GPFIFO_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_FIFO_GET_PHYSICAL_CHANNEL_COUNT = (0x20801108) +NV2080_CTRL_FIFO_GET_PHYSICAL_CHANNEL_COUNT_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_FIFO_INFO_INDEX_INSTANCE_TOTAL = (0x000000000) +NV2080_CTRL_FIFO_INFO_INDEX_MAX_CHANNEL_GROUPS = (0x000000001) +NV2080_CTRL_FIFO_INFO_INDEX_MAX_CHANNELS_PER_GROUP = (0x000000002) +NV2080_CTRL_FIFO_INFO_INDEX_MAX_SUBCONTEXT_PER_GROUP = (0x000000003) +NV2080_CTRL_FIFO_INFO_INDEX_BAR1_USERD_START_OFFSET = (0x000000004) +NV2080_CTRL_FIFO_INFO_INDEX_DEFAULT_CHANNEL_TIMESLICE = (0x000000005) +NV2080_CTRL_FIFO_INFO_INDEX_CHANNEL_GROUPS_IN_USE = (0x000000006) +NV2080_CTRL_FIFO_INFO_INDEX_IS_PER_RUNLIST_CHANNEL_RAM_SUPPORTED = (0x000000007) +NV2080_CTRL_FIFO_INFO_INDEX_MAX_CHANNEL_GROUPS_PER_ENGINE = (0x000000008) +NV2080_CTRL_FIFO_INFO_INDEX_CHANNEL_GROUPS_IN_USE_PER_ENGINE = (0x000000009) +NV2080_CTRL_FIFO_INFO_INDEX_MAX_LOWER_SUBCONTEXT = (0x00000000a) +NV2080_CTRL_FIFO_INFO_INDEX_MAX = NV2080_CTRL_FIFO_INFO_INDEX_MAX_LOWER_SUBCONTEXT +NV2080_CTRL_FIFO_GET_INFO_USERD_OFFSET_SHIFT = (12) +NV2080_CTRL_CMD_FIFO_GET_INFO = (0x20801109) +NV2080_CTRL_FIFO_GET_INFO_MAX_ENTRIES = (256) +NV2080_CTRL_FIFO_GET_INFO_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_FIFO_DISABLE_CHANNELS = (0x2080110b) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES = (64) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_FIFO_DISABLE_CHANNEL_FALSE = (0x00000000) +NV2080_CTRL_FIFO_DISABLE_CHANNEL_TRUE = (0x00000001) +NV2080_CTRL_FIFO_ONLY_DISABLE_SCHEDULING_FALSE = (0x00000000) +NV2080_CTRL_FIFO_ONLY_DISABLE_SCHEDULING_TRUE = (0x00000001) +NV2080_CTRL_FIFO_GET_CHANNEL_MEM_INFO_MAX_COUNT = 0x2 +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO = (0x2080110c) +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS_MESSAGE_ID = (0xC) +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_INVALID = 0x00000000 +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_VIDMEM = 0x00000001 +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_COH = 0x00000002 +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_NCOH = 0x00000003 +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION = (0x2080110d) +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_PARAMS_MESSAGE_ID = (0xD) +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_APERTURE_VIDMEM = 0x00000000 +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_APERTURE_SYSMEM = 0x00000001 +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_ATTRIBUTE_CACHED = 0x00000000 +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_ATTRIBUTE_UNCACHED = 0X00000001 +NV2080_CTRL_CMD_FIFO_GET_USERD_LOCATION_ATTRIBUTE_WRITECOMBINED = 0X00000002 +NV2080_CTRL_CMD_VGPU_SCHEDULER_POLICY_UNKNOWN = 0 +NV2080_CTRL_CMD_VGPU_SCHEDULER_POLICY_OTHER = 1 +NV2080_CTRL_CMD_VGPU_SCHEDULER_POLICY_BEST_EFFORT = 2 +NV2080_CTRL_CMD_VGPU_SCHEDULER_POLICY_EQUAL_SHARE = 3 +NV2080_CTRL_CMD_VGPU_SCHEDULER_POLICY_FIXED_SHARE = 4 +NV2080_CTRL_CMD_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT = 3 +NV2080_CTRL_CMD_VGPU_SCHEDULER_ARR_DEFAULT = 0 +NV2080_CTRL_CMD_VGPU_SCHEDULER_ARR_DISABLE = 1 +NV2080_CTRL_CMD_VGPU_SCHEDULER_ARR_ENABLE = 2 +NV2080_CTRL_CMD_FIFO_OBJSCHED_SW_GET_LOG = (0x2080110e) +NV2080_CTRL_FIFO_OBJSCHED_SW_COUNT = 32 +NV2080_CTRL_FIFO_OBJSCHED_SW_NCOUNTERS = 8 +NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_ENTRIES = 200 +NV2080_CTRL_FIFO_OBJSCHED_SW_GET_LOG_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_CMD_FIFO_CONFIG_CTXSW_TIMEOUT = (0x20801110) +NV2080_CTRL_FIFO_CONFIG_CTXSW_TIMEOUT_PARAMS_MESSAGE_ID = (0x10) +NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE = (0x20801112) +NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_DEVICES = 256 +NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_ENTRIES = 32 +NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_ENGINE_DATA_TYPES = 17 +NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_ENGINE_MAX_PBDMA = 2 +NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_ENGINE_MAX_NAME_LEN = 16 +NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_CMD_FIFO_CLEAR_FAULTED_BIT = (0x20801113) +NV2080_CTRL_FIFO_CLEAR_FAULTED_BIT_FAULT_TYPE_ENGINE = 0x00000001 +NV2080_CTRL_FIFO_CLEAR_FAULTED_BIT_FAULT_TYPE_PBDMA = 0x00000002 +NV2080_CTRL_CMD_FIFO_CLEAR_FAULTED_BIT_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_CMD_FIFO_RUNLIST_SET_SCHED_POLICY = (0x20801115) +NV2080_CTRL_FIFO_RUNLIST_SCHED_POLICY_DEFAULT = 0x0 +NV2080_CTRL_FIFO_RUNLIST_SCHED_POLICY_CHANNEL_INTERLEAVED = 0x1 +NV2080_CTRL_FIFO_RUNLIST_SCHED_POLICY_CHANNEL_INTERLEAVED_WDDM = 0x2 +NV2080_CTRL_CMD_FIFO_RUNLIST_SET_SCHED_POLICY_FLAGS_RESTORE = (0, 0) +NV2080_CTRL_CMD_FIFO_RUNLIST_SET_SCHED_POLICY_FLAGS_RESTORE_FALSE = (0x00000000) +NV2080_CTRL_CMD_FIFO_RUNLIST_SET_SCHED_POLICY_FLAGS_RESTORE_TRUE = (0x00000001) +NV2080_CTRL_FIFO_RUNLIST_SET_SCHED_POLICY_PARAMS_MESSAGE_ID = (0x15) +NV2080_CTRL_CMD_FIFO_UPDATE_CHANNEL_INFO = (0x20801116) +NV2080_CTRL_FIFO_UPDATE_CHANNEL_INFO_PARAMS_MESSAGE_ID = (0x16) +NV2080_CTRL_CMD_FIFO_DISABLE_USERMODE_CHANNELS = (0x20801117) +NV2080_CTRL_FIFO_DISABLE_USERMODE_CHANNELS_PARAMS_MESSAGE_ID = (0x17) +NV2080_CTRL_CMD_FIFO_SETUP_VF_ZOMBIE_SUBCTX_PDB = (0x20801118) +NV2080_CTRL_FIFO_SETUP_VF_ZOMBIE_SUBCTX_PDB_PARAMS_MESSAGE_ID = (0x18) +NV2080_CTRL_CMD_FIFO_GET_ALLOCATED_CHANNELS = (0x20801119) +NV2080_CTRL_FIFO_GET_ALLOCATED_CHANNELS_MAX_CHANNELS = 4096 +NV2080_CTRL_FIFO_GET_ALLOCATED_CHANNELS_PARAMS_MESSAGE_ID = (0x19) +NV2080_CTRL_CMD_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION = (0x2080111a) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_MAX_ENTRIES = (64) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_PARAMS_MESSAGE_ID = (0x1A) +NV2080_CTRL_CMD_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_V2 = (0x2080111b) +NV2080_CTRL_FIFO_DISABLE_CHANNELS_FOR_KEY_ROTATION_V2_PARAMS_MESSAGE_ID = (0x1B) +NV2080_CTRL_CMD_FIFO_ROTATE_KEYS = (0x2080111c) +NV2080_CTRL_CMD_FIFO_ROTATE_KEYS_PARAMS_MESSAGE_ID = (0x1C) +NV2080_CTRL_CMD_FIFO_OBJSCHED_GET_STATE = (0x20801120) +NV2080_CTRL_FIFO_OBJSCHED_GET_STATE_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_FIFO_OBJSCHED_SET_STATE = (0x20801121) +NV2080_CTRL_FIFO_OBJSCHED_SET_STATE_PARAMS_MESSAGE_ID = (0x21) +NV2080_CTRL_CMD_FIFO_OBJSCHED_GET_CAPS = (0x20801122) +NV2080_CTRL_FIFO_OBJSCHED_GET_CAPS_PARAMS_MESSAGE_ID = (0x22) +NV2080_CTRL_CMD_FIFO_MAX_CHANNELS_PER_TSG = 128 +NV2080_CTRL_CMD_FIFO_GET_CHANNEL_GROUP_UNIQUE_ID_INFO = (0x20801123) +NV2080_CTRL_FIFO_GET_CHANNEL_GROUP_UNIQUE_ID_INFO_PARAMS_MESSAGE_ID = (0x23) +NV2080_CTRL_CMD_FIFO_QUERY_CHANNEL_UNIQUE_ID = (0x20801124) +NV2080_CTRL_FIFO_QUERY_CHANNEL_UNIQUE_ID_PARAMS_MESSAGE_ID = (0x24) +NV2080_CTRL_CMD_FIFO_VGPU_SWRUNLIST_SUBMIT = (0x20801125) +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_MAX_CHANNEL_ENTRIES = 1024 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_SUBMIT_PARAMS_MESSAGE_ID = (0x25) +NV2080_CTRL_CMD_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE = (0x20801126) +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_OP_ADD = 0 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_OP_REMOVE = 1 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_OP_CLEAR_ALL = 2 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_OP_UPDATE_TSG_PRIORITY = 3 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_LEVEL_REALTIME = 0 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_LEVEL_FOCUS = 1 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_LEVEL_NORMAL = 2 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_LEVEL_IDLE = 3 +NV2080_CTRL_FIFO_VGPU_SWRUNLIST_CHANNEL_UPDATE_PARAMS_MESSAGE_ID = (0x26) +NV2080_CTRL_CMD_FLA_RANGE = (0x20803501) +NV2080_CTRL_FLA_RANGE_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_FLA_RANGE_PARAMS_MODE_NONE = 0x00000000 +NV2080_CTRL_FLA_RANGE_PARAMS_MODE_INITIALIZE = NVBIT(0) +NV2080_CTRL_FLA_RANGE_PARAMS_MODE_DESTROY = NVBIT(1) +NV2080_CTRL_FLA_RANGE_PARAMS_MODE_HOST_MANAGED_VAS_INITIALIZE = NVBIT(2) +NV2080_CTRL_FLA_RANGE_PARAMS_MODE_HOST_MANAGED_VAS_DESTROY = NVBIT(3) +NV2080_CTRL_CMD_FLA_SETUP_INSTANCE_MEM_BLOCK = (0x20803502) +NV2080_CTRL_FLA_SETUP_INSTANCE_MEM_BLOCK_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_FLA_GET_RANGE = (0x20803503) +NV2080_CTRL_FLA_GET_RANGE_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_FLA_GET_FABRIC_MEM_STATS = (0x20803504) +NV2080_CTRL_FLA_GET_FABRIC_MEM_STATS_PARAMS_MESSAGE_ID = (0x4) +FALCON_ID_PMU = (NV2080_ENGINE_TYPE_PMU) +FALCON_ID_DPU = (NV2080_ENGINE_TYPE_DPU) +FALCON_ID_SEC2 = (NV2080_ENGINE_TYPE_SEC2) +FALCON_ID_FBFLCN = (NV2080_ENGINE_TYPE_FBFLCN) +NV2080_CTRL_CMD_FLCN_GET_DMEM_USAGE = (0x20803101) +NV2080_CTRL_FLCN_GET_DMEM_USAGE_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_FLCN_NVOS_INST_EVT_RSVD_DO_NOT_USE = 0x00 +NV2080_CTRL_FLCN_NVOS_INST_EVT_CTXSW_END = 0x01 +NV2080_CTRL_FLCN_NVOS_INST_EVT_HW_IRQ_BEGIN = 0x02 +NV2080_CTRL_FLCN_NVOS_INST_EVT_HW_IRQ_END = 0x03 +NV2080_CTRL_FLCN_NVOS_INST_EVT_TIMER_TICK = 0x04 +NV2080_CTRL_FLCN_NVOS_INST_EVT_TASK_EVENT_BEGIN = 0x05 +NV2080_CTRL_FLCN_NVOS_INST_EVT_TASK_EVENT_END = 0x06 +NV2080_CTRL_FLCN_NVOS_INST_EVT_RM_QUEUE_LATENCY = 0x07 +NV2080_CTRL_FLCN_NVOS_INST_EVT_TASK_SPECIAL_EVENT = 0x08 +NV2080_CTRL_FLCN_NVOS_INST_EVT_UNUSED_0 = 0x09 +NV2080_CTRL_FLCN_NVOS_INST_EVT_DMA_END = 0x0A +NV2080_CTRL_FLCN_NVOS_INST_EVT_GENERIC_BEGIN = 0x0B +NV2080_CTRL_FLCN_NVOS_INST_EVT_GENERIC_END = 0x0C +NV2080_CTRL_FLCN_NVOS_INST_EVT_TASK_EVENT_LATENCY = 0x0D +NV2080_CTRL_FLCN_NVOS_INST_INVALID_TASK_ID = 0xFF +NV2080_CTRL_CMD_FLCN_GET_ENGINE_ARCH = (0x20803118) +NV2080_CTRL_FLCN_GET_ENGINE_ARCH_PARAMS_MESSAGE_ID = (0x18) +NV2080_CTRL_FLCN_GET_ENGINE_ARCH_DEFAULT = 0x0 +NV2080_CTRL_FLCN_GET_ENGINE_ARCH_FALCON = 0x1 +NV2080_CTRL_FLCN_GET_ENGINE_ARCH_RISCV = 0x2 +NV2080_CTRL_FLCN_GET_ENGINE_ARCH_RISCV_EB = 0x3 +NV2080_CTRL_FLCN_USTREAMER_EVENT_COMM_FLAG = (31, 31) +NV2080_CTRL_FLCN_USTREAMER_EVENT_COMM_HEAD = (30, 30) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_VARIABLE = (29, 29) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_EXTEND = (28, 28) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_EVENTID_DRF_EXTENT = (27) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_EVENTID_DRF_BASE = (20) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_EVENTIDCOMPACT_DRF_EXTENT = (28) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_EVENTIDCOMPACT_DRF_BASE = (24) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_LENGTH = (19, 8) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOAD = (7, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT = (23, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_HEAD_TIME = (29, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_DATA_PAYLOAD = (30, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_TASK_ID = (7, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_REASON = (10, 8) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_REASON_YIELD = 0x0 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_REASON_INT0 = 0x1 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_REASON_TIMER_TICK = 0x2 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_REASON_QUEUE_BLOCK = 0x3 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_REASON_DMA_SUSPENDED = 0x4 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_CTXSW_END_ODP_MISS_COUNT = (23, 11) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TIMER_TICK_TIME_SLIP = (23, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_BEGIN_TASK_ID = (7, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_BEGIN_UNIT_ID = (15, 8) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_BEGIN_EVENT_TYPE = (23, 16) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_END_TASK_ID = (7, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_END_CALLBACK_ID = (15, 8) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_END_RPC_FUNC = (15, 8) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_END_RPC_FUNC_BOBJ_CMD_BASE = 0xF0 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_END_CLASS_ID = (23, 16) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_RM_QUEUE_LATENCY_SHIFT = 10 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_SPECIAL_EVENT_TASK_ID = (7, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_SPECIAL_EVENT_ID = (23, 8) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_SPECIAL_EVENT_ID_RESERVED = 0x000000 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_SPECIAL_EVENT_ID_CB_ENQUEUE_FAIL = 0x000001 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_TASK_EVENT_LATENCY_SHIFT = 6 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_GENERIC_ID = (11, 0) +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_GENERIC_ID_INVALID = 0x000 +NV2080_CTRL_FLCN_USTREAMER_EVENT_TAIL_PAYLOADCOMPACT_GENERIC_ID_VF_SWITCH_TOTAL = 0x001 +NV2080_CTRL_FLCN_USTREAMER_FEATURE_DEFAULT = 0 +NV2080_CTRL_FLCN_USTREAMER_FEATURE__COUNT = 1 +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_IDLE_FLUSH = (0, 0) +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_IDLE_FLUSH_DISABLED = 0 +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_IDLE_FLUSH_ENABLED = 1 +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_FULL_FLUSH = (1, 1) +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_FULL_FLUSH_DISABLED = 0 +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_FULL_FLUSH_ENABLED = 1 +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_IMMEDIATE_FLUSH = (2, 2) +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_IMMEDIATE_FLUSH_DISABLED = 0 +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_IMMEDIATE_FLUSH_ENABLED = 1 +NV2080_CTRL_FLCN_USTREAMER_QUEUE_POLICY_IDLE_THRESHOLD = (31, 8) +NV2080_CTRL_FLCN_USTREAMER_NUM_EVT_TYPES_COMPACT = (0x20) +NV2080_CTRL_FLCN_USTREAMER_NUM_EVT_TYPES = (0x120) +NV2080_CTRL_FLCN_USTREAMER_MASK_SIZE_BYTES = (0x24) +NV2080_CTRL_CMD_FLCN_USTREAMER_QUEUE_INFO = (0x20803120) +NV2080_CTRL_FLCN_USTREAMER_QUEUE_INFO_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_FLCN_USTREAMER_CONTROL_GET = (0x20803122) +NV2080_CTRL_CMD_FLCN_USTREAMER_CONTROL_SET = (0x20803123) +NV2080_CTRL_FLCN_USTREAMER_CONTROL_GET_PARAMS_MESSAGE_ID = (0x22) +NV2080_CTRL_FLCN_USTREAMER_CONTROL_SET_PARAMS_MESSAGE_ID = (0x23) +NV2080_CTRL_CMD_FLCN_GET_CTX_BUFFER_INFO = (0x20803124) +NV2080_CTRL_FLCN_GET_CTX_BUFFER_INFO_PARAMS_MESSAGE_ID = (0x24) +NV2080_CTRL_CMD_FLCN_GET_CTX_BUFFER_SIZE = (0x20803125) +NV2080_CTRL_FLCN_GET_CTX_BUFFER_SIZE_PARAMS_MESSAGE_ID = (0x25) +NV2080_CTRL_CMD_INTERNAL_GPIO_PROGRAM_DIRECTION = (0x20802300) +NV2080_CTRL_INTERNAL_GPIO_PROGRAM_DIRECTION_PARAMS_MESSAGE_ID = (0x00) +NV2080_CTRL_CMD_INTERNAL_GPIO_PROGRAM_OUTPUT = (0x20802301) +NV2080_CTRL_INTERNAL_GPIO_PROGRAM_OUTPUT_PARAMS_MESSAGE_ID = (0x01) +NV2080_CTRL_CMD_INTERNAL_GPIO_READ_INPUT = (0x20802302) +NV2080_CTRL_INTERNAL_GPIO_READ_INPUT_PARAMS_MESSAGE_ID = (0x02) +NV2080_CTRL_CMD_INTERNAL_GPIO_ACTIVATE_HW_FUNCTION = (0x20802303) +NV2080_CTRL_INTERNAL_GPIO_ACTIVATE_HW_FUNCTION_PARAMS_MESSAGE_ID = (0x03) +NV_GRID_LICENSE_INFO_MAX_LENGTH = (128) +NV_GRID_LICENSE_FEATURE_VPC_EDITION = "GRID-Virtual-PC,2.0;Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0" +NV_GRID_LICENSE_FEATURE_VAPPS_EDITION = "GRID-Virtual-Apps,3.0" +NV_GRID_LICENSE_FEATURE_VIRTUAL_WORKSTATION_EDITION = "Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0" +NV_GRID_LICENSE_FEATURE_GAMING_EDITION = "GRID-vGaming,8.0" +NV_GRID_LICENSE_FEATURE_COMPUTE_EDITION = "NVIDIA-vComputeServer,9.0" +NV_GRID_LICENSE_FEATURE_VGAMEDEV_EDITION = "NVIDIA-vGameDev,21.0" +NV_GRID_LICENSED_PRODUCT_VWS = "NVIDIA RTX Virtual Workstation" +NV_GRID_LICENSED_PRODUCT_GAMING = "NVIDIA Cloud Gaming" +NV_GRID_LICENSED_PRODUCT_VPC = "NVIDIA Virtual PC" +NV_GRID_LICENSED_PRODUCT_VAPPS = "NVIDIA Virtual Applications" +NV_GRID_LICENSED_PRODUCT_VGPU_FOR_COMPUTE = "NVIDIA vGPU for Compute" +NV_GRID_LICENSED_PRODUCT_VGAMEDEV = "NVIDIA RTX Virtual Game Dev" +NV2080_CTRL_GPU_INFO_INDEX_INDEX = (23, 0) +NV2080_CTRL_GPU_INFO_INDEX_ECID_LO32 = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_ECID_HI32 = (0x00000002) +NV2080_CTRL_GPU_INFO_INDEX_MINOR_REVISION_EXT = (0x00000004) +NV2080_CTRL_GPU_INFO_INDEX_NETLIST_REV0 = (0x00000012) +NV2080_CTRL_GPU_INFO_INDEX_NETLIST_REV1 = (0x00000013) +NV2080_CTRL_GPU_INFO_INDEX_ECID_EXTENDED = (0x0000001b) +NV2080_CTRL_GPU_INFO_INDEX_SYSMEM_ACCESS = (0x0000001f) +NV2080_CTRL_GPU_INFO_INDEX_GEMINI_BOARD = (0x00000022) +NV2080_CTRL_GPU_INFO_INDEX_SURPRISE_REMOVAL_POSSIBLE = (0x00000025) +NV2080_CTRL_GPU_INFO_INDEX_GLOBAL_POISON_FUSE_ENABLED = (0x00000027) +NV2080_CTRL_GPU_INFO_INDEX_NVSWITCH_PROXY_DETECTED = (0x00000028) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SR_SUPPORT = (0x00000029) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SMC_MODE = (0x0000002a) +NV2080_CTRL_GPU_INFO_INDEX_SPLIT_VAS_MGMT_SERVER_CLIENT_RM = (0x0000002b) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SM_VERSION = (0x0000002c) +NV2080_CTRL_GPU_INFO_INDEX_GPU_FLA_CAPABILITY = (0x0000002d) +NV2080_CTRL_GPU_INFO_INDEX_PER_RUNLIST_CHANNEL_RAM = (0x0000002f) +NV2080_CTRL_GPU_INFO_INDEX_GPU_ATS_CAPABILITY = (0x00000030) +NV2080_CTRL_GPU_INFO_INDEX_NVENC_STATS_REPORTING_STATE = (0x00000031) +NV2080_CTRL_GPU_INFO_INDEX_DISPLAY_ENABLED = (0x00000034) +NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED = (0x00000035) +NV2080_CTRL_GPU_INFO_INDEX_GPU_PROFILING_CAPABILITY = (0x00000036) +NV2080_CTRL_GPU_INFO_INDEX_GPU_DEBUGGING_CAPABILITY = (0x00000037) +NV2080_CTRL_GPU_INFO_INDEX_GPU_LOCAL_EGM_CAPABILITY = (0x0000003a) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SELF_HOSTED_CAPABILITY = (0x0000003b) +NV2080_CTRL_GPU_INFO_INDEX_CMP_SKU = (0x0000003c) +NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY = (0x0000003d) +NV2080_CTRL_GPU_INFO_INDEX_IS_RESETLESS_MIG_SUPPORTED = (0x0000003f) +NV2080_CTRL_GPU_INFO_INDEX_IS_LOCALIZATION_SUPPORTED = (0x00000041) +NV2080_CTRL_GPU_INFO_INDEX_GPU_NON_PASID_ATS_CAPABILITY = (0x00000042) +NV2080_CTRL_GPU_INFO_INDEX_GPU_FLA_SYSMEM_CAPABILITY = (0x00000043) +NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE = (0x00000044) +NV2080_CTRL_GPU_INFO_INDEX_COMPR_BIT_BACKING_COPY_TYPE = (0x00000045) +NV2080_CTRL_GPU_INFO_INDEX_CPU_COHERENT_CACHING_GPU_MEMORY_CAPABILITY = (0x00000046) +NV2080_CTRL_GPU_INFO_INDEX_GPU_COHERENT_CACHING_CPU_MEMORY_CAPABILITY = (0x00000047) +NV2080_CTRL_GPU_INFO_MAX_LIST_SIZE = (0x00000048) +NV2080_CTRL_GPU_INFO_INDEX_TYPE = (30, 28) +NV2080_CTRL_GPU_INFO_INDEX_TYPE_SPECIFIC_ID = (27, 24) +NV2080_CTRL_GPU_INFO_INDEX_RESERVED = (31, 31) +NV2080_CTRL_GPU_INFO_MINOR_REVISION_EXT_NONE = (0x00000000) +NV2080_CTRL_GPU_INFO_MINOR_REVISION_EXT_P = (0x00000001) +NV2080_CTRL_GPU_INFO_MINOR_REVISION_EXT_V = (0x00000002) +NV2080_CTRL_GPU_INFO_MINOR_REVISION_EXT_PV = (0x00000003) +NV2080_CTRL_GPU_INFO_SYSMEM_ACCESS_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_SYSMEM_ACCESS_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GEMINI_BOARD_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GEMINI_BOARD_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_SURPRISE_REMOVAL_POSSIBLE_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_SURPRISE_REMOVAL_POSSIBLE_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GLOBAL_POISON_FUSE_ENABLED_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GLOBAL_POISON_FUSE_ENABLED_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_NVSWITCH_PROXY_DETECTED_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_NVSWITCH_PROXY_DETECTED_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SR_SUPPORT_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SR_SUPPORT_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_GPU_SMC_MODE_UNSUPPORTED = (0x00000000) +NV2080_CTRL_GPU_INFO_GPU_SMC_MODE_ENABLED = (0x00000001) +NV2080_CTRL_GPU_INFO_GPU_SMC_MODE_DISABLED = (0x00000002) +NV2080_CTRL_GPU_INFO_GPU_SMC_MODE_ENABLE_PENDING = (0x00000003) +NV2080_CTRL_GPU_INFO_GPU_SMC_MODE_DISABLE_PENDING = (0x00000004) +NV2080_CTRL_GPU_INFO_SPLIT_VAS_MGMT_SERVER_CLIENT_RM_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_SPLIT_VAS_MGMT_SERVER_CLIENT_RM_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_FLA_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_FLA_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_PER_RUNLIST_CHANNEL_RAM_DISABLED = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_PER_RUNLIST_CHANNEL_RAM_ENABLED = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_ATS_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_ATS_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_DISABLED = (0x00000000) +NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_ENABLED = (0x00000001) +NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_NOT_SUPPORTED = (0x00000002) +NV2080_CTRL_GPU_INFO_DISPLAY_ENABLED_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_DISPLAY_ENABLED_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_PROFILING_CAPABILITY_DISABLED = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_PROFILING_CAPABILITY_ENABLED = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_DEBUGGING_CAPABILITY_DISABLED = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_DEBUGGING_CAPABILITY_ENABLED = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_LOCAL_EGM_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_LOCAL_EGM_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_LOCAL_EGM_PEERID = (31, 1) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SELF_HOSTED_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_SELF_HOSTED_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_CMP_SKU_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_CMP_SKU_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_IS_RESETLESS_MIG_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_IS_RESETLESS_MIG_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_IS_LOCALIZATION_SUPPORTED_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_IS_LOCALIZATION_SUPPORTED_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_NON_PASID_ATS_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_NON_PASID_ATS_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_FLA_SYSMEM_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_FLA_SYSMEM_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NONE = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_NUMA = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_COHERENT_GPU_MEMORY_MODE_DRIVER = (0x00000002) +NV2080_CTRL_GPU_INFO_INDEX_COMP_BIT_BACKING_COPY_TYPE_PHYSICAL = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_COMP_BIT_BACKING_COPY_TYPE_VIRTUAL = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_CPU_COHERENT_CACHING_GPU_MEMORY_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_CPU_COHERENT_CACHING_GPU_MEMORY_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_GPU_INFO_INDEX_GPU_COHERENT_CACHING_CPU_MEMORY_CAPABILITY_NO = (0x00000000) +NV2080_CTRL_GPU_INFO_INDEX_GPU_COHERENT_CACHING_CPU_MEMORY_CAPABILITY_YES = (0x00000001) +NV2080_CTRL_CMD_GPU_GET_INFO = (0x20800101) +NV2080_CTRL_GPU_GET_INFO_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_GPU_GET_INFO_V2 = (0x20800102) +NV2080_CTRL_GPU_GET_INFO_V2_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_GPU_GET_NAME_STRING = (0x20800110) +NV2080_GPU_MAX_NAME_STRING_LENGTH = (0x0000040) +NV2080_CTRL_GPU_GET_NAME_STRING_FLAGS_TYPE = (31, 0) +NV2080_CTRL_GPU_GET_NAME_STRING_FLAGS_TYPE_ASCII = (0x00000000) +NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS_MESSAGE_ID = (0x10) +NV2080_CTRL_CMD_GPU_GET_SHORT_NAME_STRING = (0x20800111) +NV2080_CTRL_GPU_GET_SHORT_NAME_STRING_PARAMS_MESSAGE_ID = (0x11) +NV2080_CTRL_CMD_GPU_SET_POWER = (0x20800112) +NV2080_CTRL_GPU_SET_POWER_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0 = (0x00000000) +NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_1 = (0x00000001) +NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_2 = (0x00000002) +NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3 = (0x00000003) +NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_4 = (0x00000004) +NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_7 = (0x00000007) +NV2080_CTRL_CMD_GPU_GET_SDM = (0x20800118) +NV2080_CTRL_GPU_GET_SDM_PARAMS_MESSAGE_ID = (0x18) +NV2080_CTRL_CMD_GPU_SET_SDM = (0x20800120) +NV2080_CTRL_GPU_SET_SDM_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_GPU_GET_SIMULATION_INFO = (0x20800119) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_PARAMS_MESSAGE_ID = (0x19) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_NONE = (0x00000000) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_MODS_AMODEL = (0x00000001) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_LIVE_AMODEL = (0x00000002) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_FMODEL = (0x00000003) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_RTL = (0x00000004) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_EMU = (0x00000005) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_EMU_LOW_POWER = (0x00000006) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_DFPGA = (0x00000007) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_DFPGA_RTL = (0x00000008) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_DFPGA_FMODEL = (0x00000009) +NV2080_CTRL_GPU_GET_SIMULATION_INFO_TYPE_UNKNOWN = (0xFFFFFFFF) +NV2080_CTRL_GPU_REG_OP_READ_32 = (0x00000000) +NV2080_CTRL_GPU_REG_OP_WRITE_32 = (0x00000001) +NV2080_CTRL_GPU_REG_OP_READ_64 = (0x00000002) +NV2080_CTRL_GPU_REG_OP_WRITE_64 = (0x00000003) +NV2080_CTRL_GPU_REG_OP_READ_08 = (0x00000004) +NV2080_CTRL_GPU_REG_OP_WRITE_08 = (0x00000005) +NV2080_CTRL_GPU_REG_OP_TYPE_GLOBAL = (0x00000000) +NV2080_CTRL_GPU_REG_OP_TYPE_GR_CTX = (0x00000001) +NV2080_CTRL_GPU_REG_OP_TYPE_GR_CTX_TPC = (0x00000002) +NV2080_CTRL_GPU_REG_OP_TYPE_GR_CTX_SM = (0x00000004) +NV2080_CTRL_GPU_REG_OP_TYPE_GR_CTX_CROP = (0x00000008) +NV2080_CTRL_GPU_REG_OP_TYPE_GR_CTX_ZROP = (0x00000010) +NV2080_CTRL_GPU_REG_OP_TYPE_FB = (0x00000020) +NV2080_CTRL_GPU_REG_OP_TYPE_GR_CTX_QUAD = (0x00000040) +NV2080_CTRL_GPU_REG_OP_TYPE_DEVICE = (0x00000080) +NV2080_CTRL_GPU_REG_OP_STATUS_SUCCESS = (0x00) +NV2080_CTRL_GPU_REG_OP_STATUS_INVALID_OP = (0x01) +NV2080_CTRL_GPU_REG_OP_STATUS_INVALID_TYPE = (0x02) +NV2080_CTRL_GPU_REG_OP_STATUS_INVALID_OFFSET = (0x04) +NV2080_CTRL_GPU_REG_OP_STATUS_UNSUPPORTED_OP = (0x08) +NV2080_CTRL_GPU_REG_OP_STATUS_INVALID_MASK = (0x10) +NV2080_CTRL_GPU_REG_OP_STATUS_NOACCESS = (0x20) +NV2080_CTRL_CMD_GPU_EXEC_REG_OPS = (0x20800122) +NV2080_CTRL_GPU_EXEC_REG_OPS_PARAMS_MESSAGE_ID = (0x22) +NV2080_CTRL_CMD_GPU_GET_ENGINES = (0x20800123) +NV2080_CTRL_GPU_GET_ENGINES_PARAMS_MESSAGE_ID = (0x23) +NV2080_CTRL_CMD_GPU_GET_ENGINES_V2 = (0x20800170) +NV2080_GPU_MAX_ENGINES_LIST_SIZE = 0x54 +NV2080_CTRL_GPU_GET_ENGINES_V2_PARAMS_MESSAGE_ID = (0x70) +NV2080_CTRL_CMD_GPU_GET_ENGINE_CLASSLIST = (0x20800124) +NV2080_CTRL_GPU_GET_ENGINE_CLASSLIST_PARAMS_MESSAGE_ID = (0x24) +NV2080_CTRL_CMD_GPU_GET_ENGINE_FAULT_INFO = (0x20800125) +NV2080_CTRL_GPU_GET_ENGINE_FAULT_INFO_PARAMS_MESSAGE_ID = (0x25) +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN = 0 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM = 1 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH = 2 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB = 3 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL = 4 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB = 5 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL = 6 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL = 7 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK = 8 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT = 9 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP = 10 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP = 11 +NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP = 12 +NV2080_CTRL_GPU_PROMOTE_CONTEXT_MAX_ENTRIES = 16 +NV2080_CTRL_CMD_GPU_PROMOTE_CTX = (0x2080012b) +NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS_MESSAGE_ID = (0x2B) +NV2080_CTRL_CMD_GPU_EVICT_CTX = (0x2080012c) +NV2080_CTRL_GPU_EVICT_CTX_PARAMS_MESSAGE_ID = (0x2C) +NV2080_CTRL_CMD_GPU_INITIALIZE_CTX = (0x2080012d) +NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS_MESSAGE_ID = (0x2D) +NV2080_CTRL_GPU_INITIALIZE_CTX_APERTURE = (1, 0) +NV2080_CTRL_GPU_INITIALIZE_CTX_APERTURE_VIDMEM = (0x00000000) +NV2080_CTRL_GPU_INITIALIZE_CTX_APERTURE_COH_SYS = (0x00000001) +NV2080_CTRL_GPU_INITIALIZE_CTX_APERTURE_NCOH_SYS = (0x00000002) +NV2080_CTRL_GPU_INITIALIZE_CTX_GPU_CACHEABLE = (2, 2) +NV2080_CTRL_GPU_INITIALIZE_CTX_GPU_CACHEABLE_YES = (0x00000000) +NV2080_CTRL_GPU_INITIALIZE_CTX_GPU_CACHEABLE_NO = (0x00000001) +NV2080_CTRL_GPU_INITIALIZE_CTX_PRESERVE_CTX = (3, 3) +NV2080_CTRL_GPU_INITIALIZE_CTX_PRESERVE_CTX_NO = (0x00000000) +NV2080_CTRL_GPU_INITIALIZE_CTX_PRESERVE_CTX_YES = (0x00000001) +NV2080_CTRL_CMD_GPU_QUERY_ECC_INTR = (0x2080012e) +NV2080_CTRL_CMD_GPU_QUERY_ECC_STATUS = (0x2080012f) +NV2080_CTRL_GPU_ECC_UNIT_GSP = (0x0000001D) +NV2080_CTRL_GPU_ECC_UNIT_COUNT = (0x00000029) +NV2080_CTRL_GPU_QUERY_ECC_STATUS_FLAGS_TYPE = (0, 0) +NV2080_CTRL_GPU_QUERY_ECC_STATUS_FLAGS_TYPE_FILTERED = (0x00000000) +NV2080_CTRL_GPU_QUERY_ECC_STATUS_FLAGS_TYPE_RAW = (0x00000001) +NV2080_CTRL_GPU_QUERY_ECC_STATUS_UNC_ERR_FALSE = 0 +NV2080_CTRL_GPU_QUERY_ECC_STATUS_UNC_ERR_TRUE = 1 +NV2080_CTRL_GPU_QUERY_ECC_STATUS_UNC_ERR_INDETERMINATE = 2 +NV2080_CTRL_GPU_QUERY_ECC_STATUS_PARAMS_MESSAGE_ID = (0x2F) +NV2080_CTRL_CMD_GPU_SET_COMPUTE_MODE_RULES = (0x20800130) +NV2080_CTRL_GPU_COMPUTE_MODE_RULES_NONE = (0x00000000) +NV2080_CTRL_GPU_COMPUTE_MODE_RULES_EXCLUSIVE_COMPUTE = (0x00000001) +NV2080_CTRL_GPU_COMPUTE_MODE_RULES_COMPUTE_PROHIBITED = (0x00000002) +NV2080_CTRL_GPU_COMPUTE_MODE_RULES_EXCLUSIVE_COMPUTE_PROCESS = (0x00000003) +NV2080_CTRL_GPU_SET_COMPUTE_MODE_RULES_PARAMS_MESSAGE_ID = (0x30) +NV2080_CTRL_CMD_GPU_QUERY_COMPUTE_MODE_RULES = (0x20800131) +NV2080_CTRL_GPU_QUERY_COMPUTE_MODE_RULES_PARAMS_MESSAGE_ID = (0x31) +NV2080_CTRL_CMD_GPU_QUERY_ECC_CONFIGURATION = (0x20800133) +NV2080_CTRL_GPU_ECC_CONFIGURATION_DISABLED = (0x00000000) +NV2080_CTRL_GPU_ECC_CONFIGURATION_ENABLED = (0x00000001) +NV2080_CTRL_GPU_QUERY_ECC_CONFIGURATION_PARAMS_MESSAGE_ID = (0x33) +NV2080_CTRL_CMD_GPU_SET_ECC_CONFIGURATION = (0x20800134) +NV2080_CTRL_GPU_ECC_CONFIGURATION_DISABLE = (0x00000000) +NV2080_CTRL_GPU_ECC_CONFIGURATION_ENABLE = (0x00000001) +NV2080_CTRL_GPU_SET_ECC_CONFIGURATION_PARAMS_MESSAGE_ID = (0x34) +NV2080_CTRL_CMD_GPU_RESET_ECC_ERROR_STATUS = (0x20800136) +NV2080_CTRL_GPU_ECC_ERROR_STATUS_NONE = (0x00000000) +NV2080_CTRL_GPU_ECC_ERROR_STATUS_VOLATILE = (0x00000001) +NV2080_CTRL_GPU_ECC_ERROR_STATUS_AGGREGATE = (0x00000002) +NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_FLAGS_FORCE_PURGE = (0, 0) +NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_FLAGS_FORCE_PURGE_FALSE = 0 +NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_FLAGS_FORCE_PURGE_TRUE = 1 +NV2080_CTRL_GPU_RESET_ECC_ERROR_STATUS_PARAMS_MESSAGE_ID = (0x36) +NV2080_CTRL_CMD_GPU_GET_FERMI_GPC_INFO = (0x20800137) +NV2080_CTRL_GPU_GET_FERMI_GPC_INFO_PARAMS_MESSAGE_ID = (0x37) +NV2080_CTRL_CMD_GPU_GET_FERMI_TPC_INFO = (0x20800138) +NV2080_CTRL_GPU_GET_FERMI_TPC_INFO_PARAMS_MESSAGE_ID = (0x38) +NV2080_CTRL_CMD_GPU_GET_FERMI_ZCULL_INFO = (0x20800139) +NV2080_CTRL_GPU_GET_FERMI_ZCULL_INFO_PARAMS_MESSAGE_ID = (0x39) +NV2080_CTRL_CMD_GPU_GET_OEM_BOARD_INFO = (0x2080013f) +NV2080_GPU_MAX_MARKETING_NAME_LENGTH = (0x00000018) +NV2080_GPU_MAX_SERIAL_NUMBER_LENGTH = (0x00000010) +NV2080_GPU_MAX_MEMORY_PART_ID_LENGTH = (0x00000014) +NV2080_GPU_MAX_MEMORY_DATE_CODE_LENGTH = (0x00000006) +NV2080_GPU_MAX_PRODUCT_PART_NUMBER_LENGTH = (0x00000014) +NV2080_CTRL_GPU_GET_OEM_BOARD_INFO_PARAMS_MESSAGE_ID = (0x3F) +NV2080_CTRL_CMD_GPU_GET_ID = (0x20800142) +NV2080_CTRL_GPU_GET_ID_PARAMS_MESSAGE_ID = (0x42) +NV2080_CTRL_CMD_GPU_SET_GPU_DEBUG_MODE = (0x20800143) +NV2080_CTRL_GPU_SET_GPU_DEBUG_MODE_PARAMS_MESSAGE_ID = (0x43) +NV2080_CTRL_GPU_DEBUG_MODE_ENABLED = (0x00000001) +NV2080_CTRL_GPU_DEBUG_MODE_DISABLED = (0x00000002) +NV2080_CTRL_CMD_GPU_GET_GPU_DEBUG_MODE = (0x20800144) +NV2080_CTRL_GPU_GET_GPU_DEBUG_MODE_PARAMS_MESSAGE_ID = (0x44) +NV2080_CTRL_CMD_GPU_GET_ENGINE_PARTNERLIST = (0x20800147) +NV2080_CTRL_GPU_MAX_ENGINE_PARTNERS = (0x00000020) +NV2080_CTRL_GPU_GET_ENGINE_PARTNERLIST_PARAMS_MESSAGE_ID = (0x47) +NV2080_CTRL_CMD_GPU_GET_GID_INFO = (0x2080014a) +NV2080_GPU_MAX_GID_LENGTH = (0x000000100) +NV2080_GPU_MAX_SHA1_BINARY_GID_LENGTH = (0x000000010) +NV2080_CTRL_GPU_GET_GID_INFO_PARAMS_MESSAGE_ID = (0x4A) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_FORMAT = (1, 0) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_FORMAT_ASCII = (0x00000000) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_FORMAT_BINARY = (0x00000002) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_TYPE = (2, 2) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_TYPE_SHA1 = (0x00000000) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_MODE = (5, 3) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_MODE_GPU = (0x00000000) +NV2080_GPU_CMD_GPU_GET_GID_FLAGS_MODE_UGPU = (0x00000001) +NV2080_CTRL_CMD_GPU_GET_INFOROM_OBJECT_VERSION = (0x2080014b) +NV2080_CTRL_GPU_INFOROM_OBJ_TYPE_LEN = 3 +NV2080_CTRL_GPU_GET_INFOROM_OBJECT_VERSION_PARAMS_MESSAGE_ID = (0x4B) +NV2080_CTRL_CMD_SET_GPU_OPTIMUS_INFO = (0x2080014c) +NV2080_CTRL_GPU_OPTIMUS_INFO_PARAMS_MESSAGE_ID = (0x4C) +NV2080_CTRL_CMD_GPU_GET_IP_VERSION = (0x2080014d) +NV2080_CTRL_GPU_GET_IP_VERSION_PARAMS_MESSAGE_ID = (0x4D) +NV2080_CTRL_GPU_GET_IP_VERSION_DISPLAY = (0x00000001) +NV2080_CTRL_GPU_GET_IP_VERSION_HDACODEC = (0x00000002) +NV2080_CTRL_GPU_GET_IP_VERSION_PMGR = (0x00000003) +NV2080_CTRL_GPU_GET_IP_VERSION_PPWR_PMU = (0x00000004) +NV2080_CTRL_GPU_GET_IP_VERSION_DISP_FALCON = (0x00000005) +NV2080_CTRL_GPU_ILLUM_ATTRIB_LOGO_BRIGHTNESS = 0 +NV2080_CTRL_GPU_ILLUM_ATTRIB_SLI_BRIGHTNESS = 1 +NV2080_CTRL_CMD_GPU_QUERY_ILLUM_SUPPORT = (0x20800153) +NV2080_CTRL_CMD_GPU_QUERY_ILLUM_SUPPORT_PARAMS_MESSAGE_ID = (0x53) +NV2080_CTRL_CMD_GPU_GET_ILLUM = (0x20800154) +NV2080_CTRL_GPU_GET_ILLUM_PARAMS_MESSAGE_ID = (0x54) +NV2080_CTRL_CMD_GPU_SET_ILLUM = (0x20800155) +NV2080_CTRL_GPU_SET_ILLUM_PARAMS_MESSAGE_ID = (0x55) +NV2080_CTRL_CMD_GPU_GET_INFOROM_IMAGE_VERSION = (0x20800156) +NV2080_CTRL_GPU_INFOROM_IMAGE_VERSION_LEN = 16 +NV2080_CTRL_GPU_GET_INFOROM_IMAGE_VERSION_PARAMS_MESSAGE_ID = (0x56) +NV2080_CTRL_CMD_GPU_QUERY_INFOROM_ECC_SUPPORT = (0x20800157) +NV2080_CTRL_CMD_GPU_GET_PHYSICAL_BRIDGE_VERSION_INFO = (0x2080015a) +NV2080_CTRL_MAX_PHYSICAL_BRIDGE = (100) +NV2080_CTRL_GPU_GET_PHYSICAL_BRIDGE_VERSION_INFO_PARAMS_MESSAGE_ID = (0x5A) +NV2080_CTRL_CMD_GPU_GET_ALL_BRIDGES_UPSTREAM_OF_GPU = (0x2080015b) +NV2080_CTRL_GPU_GET_ALL_BRIDGES_UPSTREAM_OF_GPU_PARAMS_MESSAGE_ID = (0x5B) +NV2080_CTRL_CMD_GPU_QUERY_SCRUBBER_STATUS = (0x2080015f) +NV2080_CTRL_GPU_QUERY_SCRUBBER_STATUS_PARAMS_MESSAGE_ID = (0x5F) +NV2080_CTRL_GPU_QUERY_SCRUBBER_STATUS_SCRUBBER_RUNNING = (0x00000000) +NV2080_CTRL_GPU_QUERY_SCRUBBER_STATUS_SCRUBBER_IDLE = (0x00000001) +NV2080_CTRL_CMD_GPU_GET_VPR_CAPS = (0x20800160) +NV2080_CTRL_GPU_GET_VPR_CAPS_PARAMS_MESSAGE_ID = (0x60) +NV2080_CTRL_CMD_GPU_HANDLE_GPU_SR = (0x20800167) +NV2080_CTRL_CMD_GPU_GET_PES_INFO = (0x20800168) +NV2080_CTRL_CMD_GPU_GET_PES_INFO_MAX_TPC_PER_GPC_COUNT = 15 +NV2080_CTRL_GPU_GET_PES_INFO_PARAMS_MESSAGE_ID = (0x68) +NV2080_CTRL_CMD_GPU_GET_OEM_INFO = (0x20800169) +NV2080_GPU_MAX_OEM_INFO_LENGTH = (0x000001F8) +NV2080_CTRL_GPU_GET_OEM_INFO_PARAMS_MESSAGE_ID = (0x69) +NV2080_CTRL_CMD_GPU_GET_VPR_INFO = (0x2080016b) +NV2080_CTRL_GPU_GET_VPR_INFO_PARAMS_MESSAGE_ID = (0x6B) +NV2080_CTRL_CMD_GPU_GET_ENCODER_CAPACITY = (0x2080016c) +NV2080_CTRL_GPU_GET_ENCODER_CAPACITY_PARAMS_MESSAGE_ID = (0x6C) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_STATS = (0x2080016d) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_STATS_PARAMS_MESSAGE_ID = (0x6D) +NV2080_CTRL_GPU_NVENC_SESSION_INFO_MAX_COPYOUT_ENTRIES = 0x200 +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_PARAMS_MESSAGE_ID = (0x6E) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO = (0x2080016e) +NV2080_CTRL_GPU_SET_FABRIC_BASE_ADDR_PARAMS_MESSAGE_ID = (0x6F) +NV2080_CTRL_CMD_GPU_SET_FABRIC_BASE_ADDR = (0x2080016f) +NV2080_CTRL_GPU_VIRTUAL_INTERRUPT_PARAMS_MESSAGE_ID = (0x72) +NV2080_CTRL_CMD_GPU_VIRTUAL_INTERRUPT = (0x20800172) +NV2080_CTRL_CMD_GPU_QUERY_FUNCTION_STATUS_PARAMS_MESSAGE_ID = (0x73) +NV2080_CTRL_CMD_GPU_QUERY_FUNCTION_STATUS = (0x20800173) +NV_GI_UUID_LEN = 16 +NV2080_CTRL_GPU_PARTITION_ID_INVALID = 0xFFFFFFFF +NV2080_CTRL_GPU_MAX_PARTITIONS = 0x00000008 +NV2080_CTRL_GPU_MAX_PARTITION_IDS = 0x00000009 +NV2080_CTRL_GPU_MAX_SMC_IDS = 0x00000008 +NV2080_CTRL_GPU_MAX_GPC_PER_SMC = 0x00000040 +NV2080_CTRL_GPU_MAX_CE_PER_SMC = 0x00000008 +NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE = (1, 0) +NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE_FULL = 0x00000000 +NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE_HALF = 0x00000001 +NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE_QUARTER = 0x00000002 +NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE_EIGHTH = 0x00000003 +NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE__SIZE = 4 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE = (4, 2) +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_FULL = 0x00000000 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_HALF = 0x00000001 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_MINI_HALF = 0x00000002 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_QUARTER = 0x00000003 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_MINI_QUARTER = 0x00000004 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_EIGHTH = 0x00000005 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_RESERVED_INTERNAL_06 = 0x00000006 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE_RESERVED_INTERNAL_07 = 0x00000007 +NV2080_CTRL_GPU_PARTITION_FLAG_COMPUTE_SIZE__SIZE = 8 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE = (7, 5) +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_FULL = 0x00000001 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_HALF = 0x00000002 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_MINI_HALF = 0x00000003 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_QUARTER = 0x00000004 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_EIGHTH = 0x00000005 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_RESERVED_INTERNAL_06 = 0x00000006 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_RESERVED_INTERNAL_07 = 0x00000007 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE_NONE = 0x00000000 +NV2080_CTRL_GPU_PARTITION_FLAG_GFX_SIZE__SIZE = 8 +NV2080_CTRL_GPU_PARTITION_MAX_TYPES = 90 +NV2080_CTRL_GPU_PARTITION_FLAG_REQ_ALL_MEDIA = (29, 28) +NV2080_CTRL_GPU_PARTITION_FLAG_REQ_ALL_MEDIA_DEFAULT = 0 +NV2080_CTRL_GPU_PARTITION_FLAG_REQ_ALL_MEDIA_DISABLE = 1 +NV2080_CTRL_GPU_PARTITION_FLAG_REQ_ALL_MEDIA_ENABLE = 2 +NV2080_CTRL_GPU_PARTITION_FLAG_REQ_DEC_JPG_OFA = (30, 30) +NV2080_CTRL_GPU_PARTITION_FLAG_REQ_DEC_JPG_OFA_DISABLE = 0 +NV2080_CTRL_GPU_PARTITION_FLAG_REQ_DEC_JPG_OFA_ENABLE = 1 +NV2080_CTRL_GPU_PARTITION_FLAG_PLACE_AT_SPAN = (31, 31) +NV2080_CTRL_GPU_PARTITION_FLAG_PLACE_AT_SPAN_DISABLE = 0 +NV2080_CTRL_GPU_PARTITION_FLAG_PLACE_AT_SPAN_ENABLE = 1 +NV2080_CTRL_GPU_SET_PARTITIONS_PARAMS_MESSAGE_ID = (0x74) +NV2080_CTRL_CMD_GPU_SET_PARTITIONS = (0x20800174) +NV2080_CTRL_GPU_GET_PARTITIONS_PARAMS_MESSAGE_ID = (0x75) +NV2080_CTRL_CMD_GPU_GET_PARTITIONS = (0x20800175) +NV2080_CTRL_CMD_GPU_CONFIGURE_PARTITION = (0x20800176) +NV2080_CTRL_GPU_CONFIGURE_PARTITION_PARAMS_MESSAGE_ID = (0x76) +NV2080_CTRL_GPU_FAULT_PACKET_SIZE = 32 +NV2080_CTRL_CMD_GPU_REPORT_NON_REPLAYABLE_FAULT = (0x20800177) +NV2080_CTRL_GPU_REPORT_NON_REPLAYABLE_FAULT_PARAMS_MESSAGE_ID = (0x77) +NV2080_CTRL_CMD_GPU_EXEC_REG_OPS_VGPU = (0x20800178) +NV2080_CTRL_GPU_EXEC_REG_OPS_VGPU_PARAMS_MESSAGE_ID = (0x78) +NV2080_CTRL_CMD_GPU_GET_ENGINE_RUNLIST_PRI_BASE = (0x20800179) +NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_PRI_BASE_PARAMS_MESSAGE_ID = (0x79) +NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_PRI_BASE_NULL = (0xFFFFFFFF) +NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_PRI_BASE_ERROR = (0xFFFFFFFB) +NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_INVALID = (0xFFFFFFFF) +NV2080_CTRL_GPU_GET_ENGINE_RUNLIST_ERROR = (0xFFFFFFFB) +NV2080_CTRL_CMD_GPU_GET_HW_ENGINE_ID = (0x2080017a) +NV2080_CTRL_GPU_GET_HW_ENGINE_ID_PARAMS_MESSAGE_ID = (0x7A) +NV2080_CTRL_GPU_GET_HW_ENGINE_ID_NULL = (0xFFFFFFFF) +NV2080_CTRL_GPU_GET_HW_ENGINE_ID_ERROR = (0xFFFFFFFB) +NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_STATS = (0x2080017b) +NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_STATS_PARAMS_MESSAGE_ID = (0x7B) +NV2080_CTRL_NVFBC_SESSION_FLAG_DIFFMAP_ENABLED = 0x00000001 +NV2080_CTRL_NVFBC_SESSION_FLAG_CLASSIFICATIONMAP_ENABLED = 0x00000002 +NV2080_CTRL_NVFBC_SESSION_FLAG_CAPTURE_WITH_WAIT_NO_WAIT = 0x00000004 +NV2080_CTRL_NVFBC_SESSION_FLAG_CAPTURE_WITH_WAIT_INFINITE = 0x00000008 +NV2080_CTRL_NVFBC_SESSION_FLAG_CAPTURE_WITH_WAIT_TIMEOUT = 0x00000010 +NV2080_GPU_NVFBC_MAX_SESSION_COUNT = 256 +NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_INFO_PARAMS_MESSAGE_ID = (0x7C) +NV2080_CTRL_GPU_GET_NVFBC_SW_SESSION_INFO = (0x2080017c) +NV2080_CTRL_GPU_GET_FIRST_ASYNC_CE_IDX_PARAMS_MESSAGE_ID = (0xe6) +NV2080_CTRL_CMD_GPU_GET_FIRST_ASYNC_CE_IDX = (0x208001e6) +NV2080_CTRL_CMD_GPU_GET_VMMU_SEGMENT_SIZE = (0x2080017e) +NV2080_CTRL_GPU_GET_VMMU_SEGMENT_SIZE_PARAMS_MESSAGE_ID = (0x7E) +NV2080_CTRL_GPU_VMMU_SEGMENT_SIZE_32MB = 0x02000000 +NV2080_CTRL_GPU_VMMU_SEGMENT_SIZE_64MB = 0x04000000 +NV2080_CTRL_GPU_VMMU_SEGMENT_SIZE_128MB = 0x08000000 +NV2080_CTRL_GPU_VMMU_SEGMENT_SIZE_256MB = 0x10000000 +NV2080_CTRL_GPU_VMMU_SEGMENT_SIZE_512MB = 0x20000000 +NV2080_CTRL_GPU_VMMU_SEGMENT_SIZE_1024MB = 0x40000000 +NV2080_CTRL_CMD_GPU_GET_PARTITION_CAPACITY = (0x20800181) +NV2080_CTRL_GPU_GET_PARTITION_CAPACITY_PARAMS_MESSAGE_ID = (0x81) +NV2080_CTRL_CMD_GPU_GET_CACHED_INFO = (0x20800182) +NV2080_CTRL_GPU_GET_CACHED_INFO_PARAMS_MESSAGE_ID = (0x82) +NV2080_CTRL_CMD_GPU_SET_PARTITIONING_MODE = (0x20800183) +NV2080_CTRL_GPU_SET_PARTITIONING_MODE_REPARTITIONING = (1, 0) +NV2080_CTRL_GPU_SET_PARTITIONING_MODE_REPARTITIONING_LEGACY = 0 +NV2080_CTRL_GPU_SET_PARTITIONING_MODE_REPARTITIONING_MAX_PERF = 1 +NV2080_CTRL_GPU_SET_PARTITIONING_MODE_REPARTITIONING_FAST_RECONFIG = 2 +NV2080_CTRL_GPU_SET_PARTITIONING_MODE_PARAMS_MESSAGE_ID = (0x83) +NV2080_CTRL_GPU_DESCRIBE_PARTITIONS_PARAMS_MESSAGE_ID = (0x85) +NV2080_CTRL_CMD_GPU_DESCRIBE_PARTITIONS = (0x20800185) +NV2080_CTRL_CMD_GPU_GET_MAX_SUPPORTED_PAGE_SIZE = (0x20800188) +NV2080_CTRL_GPU_GET_MAX_SUPPORTED_PAGE_SIZE_PARAMS_MESSAGE_ID = (0x88) +NV2080_CTRL_GPU_GET_NUM_MMUS_PER_GPC_PARAMS_MESSAGE_ID = (0x8A) +NV2080_CTRL_CMD_GPU_GET_NUM_MMUS_PER_GPC = (0x2080018a) +NV2080_CTRL_GPU_GET_ACTIVE_PARTITION_IDS_PARAMS_MESSAGE_ID = (0x8B) +NV2080_CTRL_CMD_GPU_GET_ACTIVE_PARTITION_IDS = (0x2080018b) +NV2080_CTRL_CMD_GPU_GET_PIDS = (0x2080018d) +NV2080_CTRL_GPU_GET_PIDS_MAX_COUNT = 950 +NV2080_CTRL_GPU_GET_PIDS_PARAMS_MESSAGE_ID = (0x8D) +NV2080_CTRL_GPU_GET_PIDS_ID_TYPE_CLASS = (0x00000000) +NV2080_CTRL_GPU_GET_PIDS_ID_TYPE_VGPU_GUEST = (0x00000001) +NV2080_CTRL_GPU_PID_INFO_INDEX_VIDEO_MEMORY_USAGE = (0x00000000) +NV2080_CTRL_GPU_PID_INFO_INDEX_MAX = NV2080_CTRL_GPU_PID_INFO_INDEX_VIDEO_MEMORY_USAGE +NV2080_CTRL_CMD_GPU_GET_PID_INFO = (0x2080018e) +NV2080_CTRL_GPU_GET_PID_INFO_MAX_COUNT = 200 +NV2080_CTRL_GPU_GET_PID_INFO_PARAMS_MESSAGE_ID = (0x8E) +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT = (0x20800192) +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_TYPE_INVALID = 0 +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_TYPE_BAR1 = 1 +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_TYPE_BAR2 = 2 +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_TYPE_PHYSICAL = 3 +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_TYPE_UNBOUND_INSTANCE = 4 +NV2080_CTRL_CMD_GPU_HANDLE_VF_PRI_FAULT_PARAMS_MESSAGE_ID = (0x92) +NV2080_CTRL_GPU_COMPUTE_POLICY_TIMESLICE = 0 +NV2080_CTRL_GPU_COMPUTE_POLICY_MAX = 1 +NV2080_CTRL_GPU_SET_COMPUTE_POLICY_CONFIG_PARAMS_MESSAGE_ID = (0x94) +NV2080_CTRL_CMD_GPU_GET_COMPUTE_POLICY_CONFIG = (0x20800195) +NV2080_CTRL_GPU_COMPUTE_POLICY_CONFIG_LIST_MAX = 32 +NV2080_CTRL_GPU_GET_COMPUTE_POLICY_CONFIG_PARAMS_MESSAGE_ID = (0x95) +NV2080_CTRL_CMD_GPU_GET_GFID = (0x20800196) +NV2080_CTRL_GPU_GET_GFID_PARAMS_MESSAGE_ID = (0x96) +NV2080_CTRL_CMD_GPU_UPDATE_GFID_P2P_CAPABILITY = (0x20800197) +NV2080_CTRL_CMD_GPU_UPDATE_GFID_P2P_CAPABILITY_PARAMS_MESSAGE_ID = (0x97) +NV2080_CTRL_CMD_GPU_VALIDATE_MEM_MAP_REQUEST = (0x20800198) +NV2080_CTRL_GPU_VALIDATE_MEM_MAP_REQUEST_PARAMS_MESSAGE_ID = (0x98) +NV2080_CTRL_CMD_GPU_GET_ENGINE_LOAD_TIMES = (0x2080019b) +NV2080_CTRL_GPU_MAX_ENGINE_OBJECTS = 0xC9 +NV2080_CTRL_GPU_GET_ENGINE_LOAD_TIMES_PARAMS_MESSAGE_ID = (0x9B) +NV2080_CTRL_CMD_GPU_GET_ID_NAME_MAPPING = (0x2080019c) +NV2080_CTRL_GPU_GET_ID_NAME_MAPPING_PARAMS_MESSAGE_ID = (0x9C) +NV2080_CTRL_CMD_GPU_EXEC_REG_OPS_NOPTRS = (0x2080019d) +NV2080_CTRL_REG_OPS_ARRAY_MAX = 100 +NV2080_CTRL_GPU_EXEC_REG_OPS_NOPTRS_PARAMS_MESSAGE_ID = (0x9D) +NV2080_CTRL_GPU_SKYLINE_INFO_MAX_SKYLINES = 9 +NV2080_CTRL_GPU_SKYLINE_INFO_MAX_NON_SINGLETON_VGPCS = 64 +NV2080_CTRL_GPU_GET_SKYLINE_INFO_PARAMS_MESSAGE_ID = (0x9F) +NV2080_CTRL_CMD_GPU_GET_SKYLINE_INFO = (0x2080019f) +NV2080_GET_P2P_CAPS_UUID_LEN = 16 +NV2080_CTRL_CMD_GET_P2P_CAPS = (0x208001a0) +NV2080_CTRL_GET_P2P_CAPS_PARAMS_MESSAGE_ID = (0xA0) +NV2080_CTRL_GPU_COMPUTE_PROFILE_FLAG_MULTIDIE = (0, 0) +NV2080_CTRL_GPU_COMPUTE_PROFILE_FLAG_MULTIDIE_FALSE = 0 +NV2080_CTRL_GPU_COMPUTE_PROFILE_FLAG_MULTIDIE_TRUE = 1 +NV2080_CTRL_GPU_GET_COMPUTE_PROFILES_PARAMS_MESSAGE_ID = (0xA2) +NV2080_CTRL_CMD_GPU_GET_COMPUTE_PROFILES = (0x208001a2) +NV2080_CTRL_GPU_FABRIC_PROBE_STATE_UNSUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_PROBE_STATE_NOT_STARTED = 1 +NV2080_CTRL_GPU_FABRIC_PROBE_STATE_IN_PROGRESS = 2 +NV2080_CTRL_GPU_FABRIC_PROBE_STATE_COMPLETE = 3 +NV2080_GPU_FABRIC_CLUSTER_UUID_LEN = 16 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW = (1, 0) +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_TRUE = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ROUTE_UPDATE = (3, 2) +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ROUTE_UPDATE_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ROUTE_UPDATE_TRUE = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ROUTE_UPDATE_FALSE = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_CONNECTION_UNHEALTHY = (5, 4) +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_CONNECTION_UNHEALTHY_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_CONNECTION_UNHEALTHY_TRUE = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_CONNECTION_UNHEALTHY_FALSE = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY = (7, 6) +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_TRUE = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_FALSE = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION = (11, 8) +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NONE = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INCORRECT_SYSGUID = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INCORRECT_CHASSIS_SN = 3 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NO_PARTITION = 4 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INSUFFICIENT_NVLINKS = 5 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INCOMPATIBLE_GPU_FW = 6 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INVALID_LOCATION = 7 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_GPU_STATE_INVALID = 8 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_PARTITION_ASSIGNED = (13, 12) +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_PARTITION_ASSIGNED_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_PARTITION_ASSIGNED_TRUE = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_PARTITION_ASSIGNED_FALSE = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_GFM_STATE = (15, 14) +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_GFM_STATE_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_GFM_STATE_CONNECTED = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_MASK_GFM_STATE_DISCONNECTED = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_SUMMARY_NOT_SUPPORTED = 0 +NV2080_CTRL_GPU_FABRIC_HEALTH_SUMMARY_HEALTHY = 1 +NV2080_CTRL_GPU_FABRIC_HEALTH_SUMMARY_UNHEALTHY = 2 +NV2080_CTRL_GPU_FABRIC_HEALTH_SUMMARY_LIMITED_CAPACITY = 3 +NV_FABRIC_CLIQUE_TYPE_UNICAST_POINTER = 0 +NV_FABRIC_CLIQUE_TYPE_MULTICAST_POINTER = 1 +NV_FABRIC_CLIQUE_TYPE_UNICAST_HANDLE = 2 +NV_FABRIC_CLIQUE_TYPE_MULTICAST_HANDLE = 3 +NV2080_CTRL_GPU_FABRIC_CLIQUE_MAX_SIZE = 32 +NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO_PARAMS_MESSAGE_ID = (0xA3) +NV2080_CTRL_CMD_GET_GPU_FABRIC_PROBE_INFO = (0x208001a3) +NV2080_CTRL_CMD_GPU_GET_CHIP_DETAILS = (0x208001a4) +GPU_PART_NUMBER_FMT = "%4X-%s-%X%X" +NV2080_MAX_CHIP_SKU_LENGTH = 0x00000004 +NV2080_CTRL_GPU_GET_CHIP_DETAILS_PARAMS_MESSAGE_ID = (0xA4) +NV2080_CTRL_CMD_GPU_MOVE_RUNLISTS_ALLOCATION_TO_SUBHEAP = (0x208001a5) +NV2080_CTRL_GPU_MOVE_RUNLISTS_ALLOCATION_TO_SUBHEAP_PARAMS_MESSAGE_ID = (0xA5) +NV2080_CTRL_CMD_GPU_MIGRATABLE_OPS = (0x208001a6) +NV2080_CTRL_CMD_GPU_MIGRATABLE_OPS_GSP = (0x208001a7) +NV2080_CTRL_CMD_GPU_MIGRATABLE_OPS_VGPU = (0x208001a8) +NV2080_CTRL_MIGRATABLE_OPS_ARRAY_MAX = 50 +NV2080_CTRL_GPU_MIGRATABLE_OPS_PARAMS_MESSAGE_ID = (0xA6) +NV2080_CTRL_GPU_MIGRATABLE_OPS_GSP_PARAMS_MESSAGE_ID = (0xA7) +NV2080_CTRL_GPU_MIGRATABLE_OPS_VGPU_PARAMS_MESSAGE_ID = (0xA8) +NV2080_CTRL_CMD_GPU_MARK_DEVICE_FOR_RESET = (0x208001a9) +NV2080_CTRL_CMD_GPU_UNMARK_DEVICE_FOR_RESET = (0x208001aa) +NV2080_CTRL_CMD_GPU_GET_RESET_STATUS = (0x208001ab) +NV2080_CTRL_GPU_GET_RESET_STATUS_PARAMS_MESSAGE_ID = (0xAB) +NV2080_CTRL_CMD_GPU_MARK_DEVICE_FOR_DRAIN_AND_RESET = (0x208001ac) +NV2080_CTRL_CMD_GPU_UNMARK_DEVICE_FOR_DRAIN_AND_RESET = (0x208001ad) +NV2080_CTRL_CMD_GPU_GET_DRAIN_AND_RESET_STATUS = (0x208001ae) +NV2080_CTRL_GPU_GET_DRAIN_AND_RESET_STATUS_PARAMS_MESSAGE_ID = (0xAE) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_V2_PARAMS_MESSAGE_ID = (0xAF) +NV2080_CTRL_GPU_GET_NVENC_SW_SESSION_INFO_V2 = (0x208001af) +NV2080_CTRL_GPU_MAX_CONSTRUCTED_FALCONS = 0x40 +NV2080_CTRL_CMD_GPU_GET_CONSTRUCTED_FALCON_INFO = (0x208001b0) +NV2080_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO_PARAMS_MESSAGE_ID = (0xB0) +NV2080_CTRL_CMD_GPU_GET_VF_CAPS = (0x208001b1) +NV2080_CTRL_GPU_GET_VF_CAPS_PARAMS_MESSAGE_ID = (0xB1) +NV2080_CTRL_CMD_GPU_GET_RECOVERY_ACTION = (0x208001b2) +NV2080_CTRL_GPU_RECOVERY_ACTION_GPU_RESET = NV2080_CTRL_GPU_RECOVERY_ACTION_GPU_PF_FLR +NV2080_CTRL_GPU_RECOVERY_ACTION_NODE_REBOOT = NV2080_CTRL_GPU_RECOVERY_ACTION_OS_REBOOT +NV2080_CTRL_GPU_GET_RECOVERY_ACTION_PARAMS_MESSAGE_ID = (0xB2) +NV2080_CTRL_GPU_GET_FIPS_STATUS_PARAMS_MESSAGE_ID = (0xe4) +NV2080_CTRL_GPU_GET_FIPS_STATUS = (0x208001e4) +NV2080_CTRL_GPU_RAFTS_NUM_MAX_UGPU = 0x2 +NV2080_CTRL_GPU_RAFTS_NUM_MAX_GPC_PER_UGPU = 0xC +NV2080_CTRL_GPU_RAFTS_NUM_MAX_NUM_GPC = (0x18) +NV2080_CTRL_GPU_RAFTS_NUM_MAX_FS_UNIT = (0x1a) +NV2080_CTRL_GPU_GET_RAFTS_FS_MASK_PARAMS_MESSAGE_ID = (0xB3) +NV2080_CTRL_GPU_GET_RAFTS_FS_MASK = (0x208001b3) +NV2080_CTRL_CMD_GPU_GET_COMPUTE_PROFILE_CAPACITY = (0x208001e5) +NV2080_CTRL_GPU_GET_COMPUTE_PROFILE_CAPACITY_PARAMS_MESSAGE_ID = (0xe5) +NV2080_CTRL_CMD_GPU_GET_TPC_RECONFIG_MASK = (0x208001e7) +NV2080_CTRL_GPU_GET_TPC_RECONFIG_MASK_PARAMS_MESSAGE_ID = (0xe7) +NV2080_CTRL_CMD_GPU_RPC_GSP_TEST = (0x208001e8) +NV2080_CTRL_GPU_RPC_GSP_TEST_PARAMS_MESSAGE_ID = (0xe8) +NV2080_CTRL_GPU_RPC_GSP_TEST_SERIALIZED_INTEGRITY = 0x1 +NV2080_CTRL_GPU_RPC_GSP_TEST_UNSERIALIZED = 0x2 +NV2080_CTRL_GPU_RPC_GSP_TEST_SERIALIZED_NOP = 0x3 +NV2080_CTRL_GPU_RPC_GSP_TEST_TIMEOUT = 0x4 +NV2080_CTRL_CMD_GPU_RPC_GSP_QUERY_SIZES = (0x208001e9) +NV2080_CTRL_GPU_RPC_GSP_QUERY_SIZES_PARAMS_MESSAGE_ID = (0xe9) +RUSD_FEATURE_NON_POLLING = 0x1 +RUSD_FEATURE_POLLING = 0x2 +NV2080_CTRL_CMD_RUSD_GET_SUPPORTED_FEATURES = (0x208001ea) +NV2080_CTRL_RUSD_GET_SUPPORTED_FEATURES_PARAMS_MESSAGE_ID = (0xea) +NV2080_CTRL_CMD_GPU_RUSD_SET_FEATURES = (0x208001eb) +NV2080_CTRL_GPU_RUSD_SET_FEATURES_PARAMS_MESSAGE_ID = (0xeb) +NV2080_CTRL_CMD_GPU_FORCE_GSP_UNLOAD = (0x208001ec) +NV2080_CTRL_GPU_FORCE_GSP_UNLOAD_PARAMS_MESSAGE_ID = (0xec) +NV2080_CTRL_CMD_GPU_VGA_SR_READ_WRITE = (0x208001ed) +NV2080_CTRL_GPU_VGA_SR_READ_WRITE_PARAMS_MESSAGE_ID = (0xed) +NV2080_CTRL_CMD_GPU_ERROR_INJECTION_CONTROL = (0x208001ef) +NV2080_CTRL_GPU_ERROR_INJECTION_CONTROL_PARAMS_MESSAGE_ID = (0xef) +NV2080_CTRL_CMD_GPU_CHECK_MEM_SUBSYS_ERROR = (0x208001f0) +NV2080_CTRL_GPU_CHECK_MEM_SUBSYS_ERROR_PARAMS_MESSAGE_ID = (0xf0) +NV2080_CTRL_CMD_GPU_GET_DEFAULT_TIMEOUT = (0x208001f1) +NV2080_CTRL_GPU_GET_DEFAULT_TIMEOUT_PARAMS_MESSAGE_ID = (0xf1) +NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL = (0x208001f2) +NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_RSA_EXPONENT_SIZE = 4 +NV2080_CTRL_CMD_GSP_CRYPTO_CONTROL_NONCE_SIZE = 48 +NV2080_CTRL_GSP_CRYPTO_CONTROL_PARAMS_MESSAGE_ID = (0xf2) +NV2080_CTRL_CMD_GPU_IS_RESET_COUPLED = (0x208081f3) +NV2080_CTRL_CMD_GPU_IS_RESET_COUPLED_PARAMS_MESSAGE_ID = (0xf3) +NV2080_CTRL_GPU_GET_DIELET_INFO_MAX_SIZE = (256) +NV2080_CTRL_CMD_GPU_GET_DIELET_INFO = (0x208081f4) +NV2080_CTRL_GPU_GET_DIELET_INFO_PARAMS_MESSAGE_ID = (0xf4) +NV2080_CTRL_CMD_GPU_GET_UNIT_FS_INFO_FROM_DIELET = (0x208081f5) +NV2080_CTRL_GPU_GET_UNIT_FS_INFO_FROM_DIELET_PARAMS_MESSAGE_ID = (0xf5) +NV2080_CTRL_CMD_GPU_SET_MIGRATION_BLOCK = (0x208001fa) +NV2080_CTRL_GPU_SET_MIGRATION_BLOCK_PARAMS_MESSAGE_ID = (0xfa) +NV2080_CTRL_CMD_GPU_GET_EGM_INFO = (0x208001fb) +NV2080_CTRL_GPU_GET_EGM_INFO_PARAMS_MESSAGE_ID = (0xfb) +NV2080_CTRL_GPUMON_SAMPLE_TYPE_PWR_MONITOR_STATUS = 0x00000001 +NV2080_CTRL_GPUMON_SAMPLE_TYPE_PERFMON_UTIL = 0x00000002 +NV2080_GPUMON_PID_INVALID = ((NvU32)(~0)) +NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE = (1, 0) +NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE = 0x0 +NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_ENGID = 0x1 +NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_CHANNEL = 0x2 +NV2080_CTRL_GR_ROUTE_INFO_DATA_CHANNEL_HANDLE = (31, 0) +NV2080_CTRL_GR_ROUTE_INFO_DATA_ENGID = (31, 0) +NV2080_CTRL_GR_INFO_INDEX_MAXCLIPS = NV0080_CTRL_GR_INFO_INDEX_MAXCLIPS +NV2080_CTRL_GR_INFO_INDEX_MIN_ATTRS_BUG_261894 = NV0080_CTRL_GR_INFO_INDEX_MIN_ATTRS_BUG_261894 +NV2080_CTRL_GR_INFO_XBUF_MAX_PSETS_PER_BANK = NV0080_CTRL_GR_INFO_XBUF_MAX_PSETS_PER_BANK +NV2080_CTRL_GR_INFO_INDEX_BUFFER_ALIGNMENT = NV0080_CTRL_GR_INFO_INDEX_BUFFER_ALIGNMENT +NV2080_CTRL_GR_INFO_INDEX_SWIZZLE_ALIGNMENT = NV0080_CTRL_GR_INFO_INDEX_SWIZZLE_ALIGNMENT +NV2080_CTRL_GR_INFO_INDEX_VERTEX_CACHE_SIZE = NV0080_CTRL_GR_INFO_INDEX_VERTEX_CACHE_SIZE +NV2080_CTRL_GR_INFO_INDEX_VPE_COUNT = NV0080_CTRL_GR_INFO_INDEX_VPE_COUNT +NV2080_CTRL_GR_INFO_INDEX_SHADER_PIPE_COUNT = NV0080_CTRL_GR_INFO_INDEX_SHADER_PIPE_COUNT +NV2080_CTRL_GR_INFO_INDEX_THREAD_STACK_SCALING_FACTOR = NV0080_CTRL_GR_INFO_INDEX_THREAD_STACK_SCALING_FACTOR +NV2080_CTRL_GR_INFO_INDEX_SHADER_PIPE_SUB_COUNT = NV0080_CTRL_GR_INFO_INDEX_SHADER_PIPE_SUB_COUNT +NV2080_CTRL_GR_INFO_INDEX_SM_REG_BANK_COUNT = NV0080_CTRL_GR_INFO_INDEX_SM_REG_BANK_COUNT +NV2080_CTRL_GR_INFO_INDEX_SM_REG_BANK_REG_COUNT = NV0080_CTRL_GR_INFO_INDEX_SM_REG_BANK_REG_COUNT +NV2080_CTRL_GR_INFO_INDEX_SM_VERSION = NV0080_CTRL_GR_INFO_INDEX_SM_VERSION +NV2080_CTRL_GR_INFO_INDEX_MAX_WARPS_PER_SM = NV0080_CTRL_GR_INFO_INDEX_MAX_WARPS_PER_SM +NV2080_CTRL_GR_INFO_INDEX_MAX_THREADS_PER_WARP = NV0080_CTRL_GR_INFO_INDEX_MAX_THREADS_PER_WARP +NV2080_CTRL_GR_INFO_INDEX_GEOM_GS_OBUF_ENTRIES = NV0080_CTRL_GR_INFO_INDEX_GEOM_GS_OBUF_ENTRIES +NV2080_CTRL_GR_INFO_INDEX_GEOM_XBUF_ENTRIES = NV0080_CTRL_GR_INFO_INDEX_GEOM_XBUF_ENTRIES +NV2080_CTRL_GR_INFO_INDEX_FB_MEMORY_REQUEST_GRANULARITY = NV0080_CTRL_GR_INFO_INDEX_FB_MEMORY_REQUEST_GRANULARITY +NV2080_CTRL_GR_INFO_INDEX_HOST_MEMORY_REQUEST_GRANULARITY = NV0080_CTRL_GR_INFO_INDEX_HOST_MEMORY_REQUEST_GRANULARITY +NV2080_CTRL_GR_INFO_INDEX_MAX_SP_PER_SM = NV0080_CTRL_GR_INFO_INDEX_MAX_SP_PER_SM +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_ZCULL_BANKS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_ZCULL_BANKS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_TPC_PER_GPC = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_TPC_PER_GPC +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_MIN_FBPS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_MIN_FBPS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_MXBAR_FBP_PORTS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_MXBAR_FBP_PORTS +NV2080_CTRL_GR_INFO_INDEX_TIMESLICE_ENABLED = NV0080_CTRL_GR_INFO_INDEX_TIMESLICE_ENABLED +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPAS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPAS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_PES_PER_GPC = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_PES_PER_GPC +NV2080_CTRL_GR_INFO_INDEX_GPU_CORE_COUNT = NV0080_CTRL_GR_INFO_INDEX_GPU_CORE_COUNT +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_TPCS_PER_PES = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_TPCS_PER_PES +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_MXBAR_HUB_PORTS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_MXBAR_HUB_PORTS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_SM_PER_TPC = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_SM_PER_TPC +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_HSHUB_FBP_PORTS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_HSHUB_FBP_PORTS +NV2080_CTRL_GR_INFO_INDEX_RT_CORE_COUNT = NV0080_CTRL_GR_INFO_INDEX_RT_CORE_COUNT +NV2080_CTRL_GR_INFO_INDEX_TENSOR_CORE_COUNT = NV0080_CTRL_GR_INFO_INDEX_TENSOR_CORE_COUNT +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GRS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GRS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTCS = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTCS +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTC_SLICES = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTC_SLICES +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCMMU_PER_GPC = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCMMU_PER_GPC +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTC_PER_FBP = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_LTC_PER_FBP +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_ROP_PER_GPC = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_ROP_PER_GPC +NV2080_CTRL_GR_INFO_INDEX_FAMILY_MAX_TPC_PER_GPC = NV0080_CTRL_GR_INFO_INDEX_FAMILY_MAX_TPC_PER_GPC +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPA_PER_FBP = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_FBPA_PER_FBP +NV2080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT = NV0080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT +NV2080_CTRL_GR_INFO_INDEX_MAX_LEGACY_SUBCONTEXT_COUNT = NV0080_CTRL_GR_INFO_INDEX_MAX_LEGACY_SUBCONTEXT_COUNT +NV2080_CTRL_GR_INFO_INDEX_MAX_PER_ENGINE_SUBCONTEXT_COUNT = NV0080_CTRL_GR_INFO_INDEX_MAX_PER_ENGINE_SUBCONTEXT_COUNT +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_SLICES_PER_LTC = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_SLICES_PER_LTC +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GFXC_SMC_ENGINES = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GFXC_SMC_ENGINES +NV2080_CTRL_GR_INFO_INDEX_RESERVED = NV0080_CTRL_GR_INFO_INDEX_RESERVED +NV2080_CTRL_GR_INFO_INDEX_GFX_CAPABILITIES = NV0080_CTRL_GR_INFO_INDEX_GFX_CAPABILITIES +NV2080_CTRL_GR_INFO_INDEX_MAX_MIG_ENGINES = NV0080_CTRL_GR_INFO_INDEX_MAX_MIG_ENGINES +NV2080_CTRL_GR_INFO_INDEX_MAX_PARTITIONABLE_GPCS = NV0080_CTRL_GR_INFO_INDEX_MAX_PARTITIONABLE_GPCS +NV2080_CTRL_GR_INFO_INDEX_LITTER_MIN_SUBCTX_PER_SMC_ENG = NV0080_CTRL_GR_INFO_INDEX_LITTER_MIN_SUBCTX_PER_SMC_ENG +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS_PER_DIELET = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS_PER_DIELET +NV2080_CTRL_GR_INFO_INDEX_LITTER_MAX_NUM_SMC_ENGINES_PER_DIELET = NV0080_CTRL_GR_INFO_INDEX_LITTER_MAX_NUM_SMC_ENGINES_PER_DIELET +NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_CPC_PER_GPC = NV0080_CTRL_GR_INFO_INDEX_LITTER_NUM_CPC_PER_GPC +NV2080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_NVLINK_MASK = NV0080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_NVLINK_MASK +NV2080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_C2C_MASK = NV0080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_C2C_MASK +NV2080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_PCIE_MASK = NV0080_CTRL_GR_INFO_INDEX_LITTER_HSHUB_PCIE_MASK +NV2080_CTRL_GR_INFO_INDEX_RESERVED1 = NV0080_CTRL_GR_INFO_INDEX_RESERVED1 +NV2080_CTRL_GR_INFO_INDEX_MAX = NV0080_CTRL_GR_INFO_INDEX_MAX +NV2080_CTRL_GR_INFO_MAX_SIZE = NV0080_CTRL_GR_INFO_MAX_SIZE +NV2080_CTRL_GR_INFO_SM_VERSION_NONE = (0x00000000) +NV2080_CTRL_GR_INFO_SM_VERSION_1_05 = (0x00000105) +NV2080_CTRL_GR_INFO_SM_VERSION_1_1 = (0x00000110) +NV2080_CTRL_GR_INFO_SM_VERSION_1_2 = (0x00000120) +NV2080_CTRL_GR_INFO_SM_VERSION_1_3 = (0x00000130) +NV2080_CTRL_GR_INFO_SM_VERSION_1_4 = (0x00000140) +NV2080_CTRL_GR_INFO_SM_VERSION_1_5 = (0x00000150) +NV2080_CTRL_GR_INFO_SM_VERSION_2_0 = (0x00000200) +NV2080_CTRL_GR_INFO_SM_VERSION_2_1 = (0x00000210) +NV2080_CTRL_GR_INFO_SM_VERSION_2_2 = (0x00000220) +NV2080_CTRL_GR_INFO_SM_VERSION_3_0 = (0x00000300) +NV2080_CTRL_GR_INFO_SM_VERSION_3_1 = (0x00000310) +NV2080_CTRL_GR_INFO_SM_VERSION_3_2 = (0x00000320) +NV2080_CTRL_GR_INFO_SM_VERSION_3_3 = (0x00000330) +NV2080_CTRL_GR_INFO_SM_VERSION_3_5 = (0x00000350) +NV2080_CTRL_GR_INFO_SM_VERSION_3_6 = (0x00000360) +NV2080_CTRL_GR_INFO_SM_VERSION_3_8 = (0x00000380) +NV2080_CTRL_GR_INFO_SM_VERSION_3_9 = (0x00000390) +NV2080_CTRL_GR_INFO_SM_VERSION_4_0 = (0x00000400) +NV2080_CTRL_GR_INFO_SM_VERSION_5_0 = (0x00000500) +NV2080_CTRL_GR_INFO_SM_VERSION_5_02 = (0x00000502) +NV2080_CTRL_GR_INFO_SM_VERSION_5_03 = (0x00000503) +NV2080_CTRL_GR_INFO_SM_VERSION_6_0 = (0x00000600) +NV2080_CTRL_GR_INFO_SM_VERSION_6_01 = (0x00000601) +NV2080_CTRL_GR_INFO_SM_VERSION_6_02 = (0x00000602) +NV2080_CTRL_GR_INFO_SM_VERSION_7_0 = (0x00000700) +NV2080_CTRL_GR_INFO_SM_VERSION_7_01 = (0x00000701) +NV2080_CTRL_GR_INFO_SM_VERSION_7_02 = (0x00000702) +NV2080_CTRL_GR_INFO_SM_VERSION_7_03 = (0x00000703) +NV2080_CTRL_GR_INFO_SM_VERSION_7_05 = (0x00000705) +NV2080_CTRL_GR_INFO_SM_VERSION_8_02 = (0x00000802) +NV2080_CTRL_GR_INFO_SM_VERSION_8_06 = (0x00000806) +NV2080_CTRL_GR_INFO_SM_VERSION_8_07 = (0x00000807) +NV2080_CTRL_GR_INFO_SM_VERSION_8_08 = (0x00000808) +NV2080_CTRL_GR_INFO_SM_VERSION_8_09 = (0x00000809) +NV2080_CTRL_GR_INFO_SM_VERSION_9_00 = (0x00000900) +NV2080_CTRL_GR_INFO_SM_VERSION_10_00 = (0x00000A00) +NV2080_CTRL_GR_INFO_SM_VERSION_10_01 = (0x00000A01) +NV2080_CTRL_GR_INFO_SM_VERSION_10_03 = (0x00000A03) +NV2080_CTRL_GR_INFO_SM_VERSION_10_04 = (0x00000A04) +NV2080_CTRL_GR_INFO_SM_VERSION_12_00 = (0x00000C00) +NV2080_CTRL_GR_INFO_SM_VERSION_12_01 = (0x00000C01) +NV2080_CTRL_GR_INFO_SM_VERSION_10_05 = (0x00000A05) +NV2080_CTRL_GR_INFO_SM_VERSION_5_2 = (NV2080_CTRL_GR_INFO_SM_VERSION_5_02) +NV2080_CTRL_GR_INFO_SM_VERSION_5_3 = (NV2080_CTRL_GR_INFO_SM_VERSION_5_03) +NV2080_CTRL_GR_INFO_SM_VERSION_6_1 = (NV2080_CTRL_GR_INFO_SM_VERSION_6_01) +NV2080_CTRL_GR_INFO_SM_VERSION_6_2 = (NV2080_CTRL_GR_INFO_SM_VERSION_6_02) +NV2080_CTRL_GR_INFO_SM_VERSION_7_1 = (NV2080_CTRL_GR_INFO_SM_VERSION_7_01) +NV2080_CTRL_GR_INFO_SM_VERSION_7_2 = (NV2080_CTRL_GR_INFO_SM_VERSION_7_02) +NV2080_CTRL_GR_INFO_SM_VERSION_7_3 = (NV2080_CTRL_GR_INFO_SM_VERSION_7_03) +NV2080_CTRL_GR_INFO_SM_VERSION_7_5 = (NV2080_CTRL_GR_INFO_SM_VERSION_7_05) +NV2080_CTRL_GR_INFO_SM_VERSION_8_2 = (NV2080_CTRL_GR_INFO_SM_VERSION_8_02) +NV2080_CTRL_GR_INFO_SM_VERSION_8_6 = (NV2080_CTRL_GR_INFO_SM_VERSION_8_06) +NV2080_CTRL_GR_INFO_SM_VERSION_8_7 = (NV2080_CTRL_GR_INFO_SM_VERSION_8_07) +NV2080_CTRL_GR_INFO_SM_VERSION_8_8 = (NV2080_CTRL_GR_INFO_SM_VERSION_8_08) +NV2080_CTRL_GR_INFO_SM_VERSION_8_9 = (NV2080_CTRL_GR_INFO_SM_VERSION_8_09) +NV2080_CTRL_GR_INFO_SM_VERSION_9_0 = (NV2080_CTRL_GR_INFO_SM_VERSION_9_00) +NV2080_CTRL_GR_INFO_SM_VERSION_10_0 = (NV2080_CTRL_GR_INFO_SM_VERSION_10_00) +NV2080_CTRL_GR_INFO_SM_VERSION_10_1 = (NV2080_CTRL_GR_INFO_SM_VERSION_10_01) +NV2080_CTRL_GR_INFO_SM_VERSION_10_3 = (NV2080_CTRL_GR_INFO_SM_VERSION_10_03) +NV2080_CTRL_GR_INFO_SM_VERSION_10_4 = (NV2080_CTRL_GR_INFO_SM_VERSION_10_04) +NV2080_CTRL_GR_INFO_SM_VERSION_10_5 = (NV2080_CTRL_GR_INFO_SM_VERSION_10_05) +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_2D = (0, 0) +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_2D_FALSE = 0x0 +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_2D_TRUE = 0x1 +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_3D = (1, 1) +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_3D_FALSE = 0x0 +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_3D_TRUE = 0x1 +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_COMPUTE = (2, 2) +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_COMPUTE_FALSE = 0x0 +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_COMPUTE_TRUE = 0x1 +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_I2M = (3, 3) +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_I2M_FALSE = 0x0 +NV2080_CTRL_GR_INFO_GFX_CAPABILITIES_I2M_TRUE = 0x1 +NV2080_CTRL_CMD_GR_GET_INFO = (0x20801201) +NV2080_CTRL_GR_GET_INFO_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_GR_CTXSW_ZCULL_MODE = (0x20801205) +NV2080_CTRL_GR_CTXSW_ZCULL_MODE_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_CTXSW_ZCULL_MODE_GLOBAL = (0x00000000) +NV2080_CTRL_CTXSW_ZCULL_MODE_NO_CTXSW = (0x00000001) +NV2080_CTRL_CTXSW_ZCULL_MODE_SEPARATE_BUFFER = (0x00000002) +NV2080_CTRL_CMD_GR_GET_ZCULL_INFO = (0x20801206) +NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS_MESSAGE_ID = (0x6) +NV2080_CTRL_CMD_GR_CTXSW_PM_MODE = (0x20801207) +NV2080_CTRL_GR_CTXSW_PM_MODE_PARAMS_MESSAGE_ID = (0x7) +NV2080_CTRL_CTXSW_PM_MODE_NO_CTXSW = (0x00000000) +NV2080_CTRL_CTXSW_PM_MODE_CTXSW = (0x00000001) +NV2080_CTRL_CTXSW_PM_MODE_STREAM_OUT_CTXSW = (0x00000002) +NV2080_CTRL_CMD_GR_CTXSW_ZCULL_BIND = (0x20801208) +NV2080_CTRL_GR_CTXSW_ZCULL_BIND_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_CMD_GR_CTXSW_PM_BIND = (0x20801209) +NV2080_CTRL_GR_CTXSW_PM_BIND_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_GR_CTXSW_SETUP_BIND = (0x2080123a) +NV2080_CTRL_GR_CTXSW_SETUP_BIND_PARAMS_MESSAGE_ID = (0x3A) +NV2080_CTRL_GR_SET_GPC_TILE_MAP_MAX_VALUES = 128 +NV2080_CTRL_CMD_GR_SET_GPC_TILE_MAP = (0x2080120a) +NV2080_CTRL_GR_SET_GPC_TILE_MAP_PARAMS_MESSAGE_ID = (0xA) +NV2080_CTRL_CMD_GR_CTXSW_SMPC_MODE = (0x2080120e) +NV2080_CTRL_GR_CTXSW_SMPC_MODE_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_CTXSW_SMPC_MODE_NO_CTXSW = (0x00000000) +NV2080_CTRL_CTXSW_SMPC_MODE_CTXSW = (0x00000001) +NV2080_CTRL_CMD_GR_GET_SM_TO_GPC_TPC_MAPPINGS = (0x2080120f) +NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_MAX_SM_COUNT = 512 +NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS_MESSAGE_ID = (0xF) +NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE = (0x20801210) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_PARAMS_MESSAGE_ID = (0x10) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_FLAGS_CILP = (0, 0) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_FLAGS_CILP_IGNORE = (0x00000000) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_FLAGS_CILP_SET = (0x00000001) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_FLAGS_GFXP = (1, 1) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_FLAGS_GFXP_IGNORE = (0x00000000) +NV2080_CTRL_GR_SET_CTXSW_PREEMPTION_MODE_FLAGS_GFXP_SET = (0x00000001) +NV2080_CTRL_SET_CTXSW_PREEMPTION_MODE_GFX_WFI = (0x00000000) +NV2080_CTRL_SET_CTXSW_PREEMPTION_MODE_GFX_GFXP = (0x00000001) +NV2080_CTRL_SET_CTXSW_PREEMPTION_MODE_GFX_GFXP_POOL = (0x00000002) +NV2080_CTRL_SET_CTXSW_PREEMPTION_MODE_COMPUTE_WFI = (0x00000000) +NV2080_CTRL_SET_CTXSW_PREEMPTION_MODE_COMPUTE_CTA = (0x00000001) +NV2080_CTRL_SET_CTXSW_PREEMPTION_MODE_COMPUTE_CILP = (0x00000002) +NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND = (0x20801211) +NV2080_CTRL_GR_CTXSW_PREEMPTION_BIND_PARAMS_MESSAGE_ID = (0x11) +NV2080_CTRL_CMD_GR_PC_SAMPLING_MODE = (0x20801212) +NV2080_CTRL_GR_PC_SAMPLING_MODE_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_PC_SAMPLING_MODE_DISABLED = (0x00000000) +NV2080_CTRL_PC_SAMPLING_MODE_ENABLED = (0x00000001) +NV2080_CTRL_CMD_GR_GET_ROP_INFO = (0x20801213) +NV2080_CTRL_GR_GET_ROP_INFO_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_CMD_GR_GET_CTXSW_STATS = (0x20801215) +NV2080_CTRL_GR_GET_CTXSW_STATS_PARAMS_MESSAGE_ID = (0x15) +NV2080_CTRL_GR_GET_CTXSW_STATS_FLAGS_RESET = (0, 0) +NV2080_CTRL_GR_GET_CTXSW_STATS_FLAGS_RESET_FALSE = (0x00000000) +NV2080_CTRL_GR_GET_CTXSW_STATS_FLAGS_RESET_TRUE = (0x00000001) +NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_SIZE = (0x20801218) +NV2080_CTRL_GR_GET_CTX_BUFFER_SIZE_PARAMS_MESSAGE_ID = (0x18) +NV2080_CTRL_GR_MAX_CTX_BUFFER_COUNT = 64 +NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_INFO = (0x20801219) +NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS_MESSAGE_ID = (0x19) +NV2080_CTRL_GR_CTX_BUFFER_INFO_APERTURE_UNKNOWN = 0 +NV2080_CTRL_GR_CTX_BUFFER_INFO_APERTURE_SYSMEM = 1 +NV2080_CTRL_GR_CTX_BUFFER_INFO_APERTURE_FBMEM = 2 +NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER = (0x2080121b) +NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER_MAX_SM_COUNT = 512 +NV2080_CTRL_GR_DISABLED_SM_VGPC_ID = 0xFF +NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS_MESSAGE_ID = (0x1B) +NV2080_CTRL_CMD_GR_GET_CURRENT_RESIDENT_CHANNEL = (0x2080121c) +NV2080_CTRL_GR_GET_CURRENT_RESIDENT_CHANNEL_PARAMS_MESSAGE_ID = (0x1C) +NV2080_CTRL_CMD_GR_GET_ATTRIBUTE_BUFFER_SIZE = (0x2080121e) +NV2080_CTRL_GR_GET_ATTRIBUTE_BUFFER_SIZE_PARAMS_MESSAGE_ID = (0x1E) +NV2080_CTRL_CMD_GR_GFX_POOL_QUERY_SIZE = (0x2080121f) +NV2080_CTRL_GR_GFX_POOL_QUERY_SIZE_PARAMS_MESSAGE_ID = (0x1F) +NV2080_CTRL_CMD_GR_GFX_POOL_INITIALIZE = (0x20801220) +NV2080_CTRL_GR_GFX_POOL_INITIALIZE_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_GR_GFX_POOL_MAX_SLOTS = 64 +NV2080_CTRL_CMD_GR_GFX_POOL_ADD_SLOTS = (0x20801221) +NV2080_CTRL_GR_GFX_POOL_ADD_SLOTS_PARAMS_MESSAGE_ID = (0x21) +NV2080_CTRL_CMD_GR_GFX_POOL_REMOVE_SLOTS = (0x20801222) +NV2080_CTRL_GR_GFX_POOL_REMOVE_SLOTS_PARAMS_MESSAGE_ID = (0x22) +NV2080_CTRL_CMD_GR_GET_CAPS_V2 = (0x20801227) +NV2080_CTRL_GR_GET_CAPS_V2_PARAMS_MESSAGE_ID = (0x27) +NV2080_CTRL_CMD_GR_GET_INFO_V2 = (0x20801228) +NV2080_CTRL_GR_GET_INFO_V2_PARAMS_MESSAGE_ID = (0x28) +NV2080_CTRL_CMD_GR_GET_GPC_MASK = (0x2080122a) +NV2080_CTRL_GR_GET_GPC_MASK_PARAMS_MESSAGE_ID = (0x2A) +NV2080_CTRL_CMD_GR_GET_TPC_MASK = (0x2080122b) +NV2080_CTRL_GR_GET_TPC_MASK_PARAMS_MESSAGE_ID = (0x2B) +NV2080_CTRL_CMD_GR_SET_TPC_PARTITION_MODE = (0x2080122c) +NV2080_CTRL_GR_SET_TPC_PARTITION_MODE_PARAMS_MESSAGE_ID = (0x2C) +NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES = (0x2080122d) +NV2080_CTRL_GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS_MESSAGE_ID = (0x2D) +NV2080_CTRL_CMD_GR_GET_SM_ISSUE_RATE_MODIFIER = (0x20801230) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA0_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA0_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA0_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA0_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA0_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA0_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA0_REDUCED_SPEED_1_64 = (0x6) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA16_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA16_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA16_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA16_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA16_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA16_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_DP_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_DP_REDUCED_SPEED = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA32_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA32_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA32_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA32_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA32_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FMLA32_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FFMA_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FFMA_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FFMA_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FFMA_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FFMA_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_FFMA_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA1_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA1_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA1_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA1_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA1_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA1_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA1_REDUCED_SPEED_1_64 = (0x6) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA2_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA2_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA2_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA2_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA2_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA2_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA2_REDUCED_SPEED_1_64 = (0x6) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA3_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA3_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA3_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA3_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA3_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA3_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA3_REDUCED_SPEED_1_64 = (0x6) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA4_FULL_SPEED = (0x0) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA4_REDUCED_SPEED_1_2 = (0x1) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA4_REDUCED_SPEED_1_4 = (0x2) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA4_REDUCED_SPEED_1_8 = (0x3) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA4_REDUCED_SPEED_1_16 = (0x4) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA4_REDUCED_SPEED_1_32 = (0x5) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_IMLA4_REDUCED_SPEED_1_64 = (0x6) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS_MESSAGE_ID = (0x30) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_MAX_LIST_SIZE = (0xFF) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_FMLA16 = (0x0) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_DP = (0x1) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_FMLA32 = (0x2) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_FFMA = (0x3) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_IMLA0 = (0x4) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_IMLA1 = (0x5) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_IMLA2 = (0x6) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_IMLA3 = (0x7) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_IMLA4 = (0x8) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_FP16 = (0x9) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_FP32 = (0xA) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_DFMA = (0xB) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_DMLA = (0xC) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_CTRL0 = (0xD) +NV2080_CTRL_GR_SM_ISSUE_RATE_MODIFIER_V2_CTRL1 = (0xE) +NV2080_CTRL_CMD_GR_GET_SM_ISSUE_RATE_MODIFIER_V2 = (0x2080123c) +NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS_MESSAGE_ID = (0x3C) +NV2080_CTRL_GR_SM_ISSUE_THROTTLE_CTRL_MAX_LIST_SIZE = (0xFF) +NV2080_CTRL_GR_SM_ISSUE_THROTTLE_CTRL_MASK = (0x0) +NV2080_CTRL_GR_SM_ISSUE_THROTTLE_CTRL_CREDIT = (0x1) +NV2080_CTRL_CMD_GR_GET_SM_ISSUE_THROTTLE_CTRL = (0x2080123d) +NV2080_CTRL_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS_MESSAGE_ID = (0x3D) +NV2080_CTRL_CMD_GR_FECS_BIND_EVTBUF_FOR_UID = (0x20801231) +NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_PARAMS_MESSAGE_ID = (0x31) +NV2080_CTRL_CMD_GR_GET_PHYS_GPC_MASK = (0x20801232) +NV2080_CTRL_GR_GET_PHYS_GPC_MASK_PARAMS_MESSAGE_ID = (0x32) +NV2080_CTRL_CMD_GR_GET_PPC_MASK = (0x20801233) +NV2080_CTRL_GR_GET_PPC_MASK_PARAMS_MESSAGE_ID = (0x33) +NV2080_CTRL_CMD_GR_GET_NUM_TPCS_FOR_GPC = (0x20801234) +NV2080_CTRL_GR_GET_NUM_TPCS_FOR_GPC_PARAMS_MESSAGE_ID = (0x34) +NV2080_CTRL_CMD_GR_GET_CTXSW_MODES = (0x20801235) +NV2080_CTRL_GR_GET_CTXSW_MODES_PARAMS_MESSAGE_ID = (0x35) +NV2080_CTRL_CMD_GR_GET_GPC_TILE_MAP = (0x20801236) +NV2080_CTRL_GR_GET_GPC_TILE_MAP_PARAMS_MESSAGE_ID = (0x36) +NV2080_CTRL_CMD_GR_GET_ZCULL_MASK = (0x20801237) +NV2080_CTRL_GR_GET_ZCULL_MASK_PARAMS_MESSAGE_ID = (0x37) +NV2080_CTRL_CMD_GR_FECS_BIND_EVTBUF_FOR_UID_V2 = (0x20801238) +NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_V2_PARAMS_MESSAGE_ID = (0x38) +NV2080_CTRL_CMD_GR_GET_GFX_GPC_AND_TPC_INFO = (0x20801239) +NV2080_CTRL_GR_GET_GFX_GPC_AND_TPC_INFO_PARAMS_MESSAGE_ID = (0x39) +NV2080_CTRL_CMD_GR_GET_TPC_RECONFIG_MASK = (0x2080123b) +NV2080_CTRL_GR_GET_TPC_RECONFIG_MASK_PARAMS_MESSAGE_ID = (0x3b) +NV2080_CTRL_CMD_GR_TEST_CTXSW_ERROR_LOGS = (0x2080123e) +NV2080_CTRL_GR_TEST_CTXSW_ERROR_LOGS_PARAMS_MESSAGE_ID = (0x3E) +NV2080_CTRL_CMD_GRMGR_GET_GR_FS_INFO = (0x20803801) +NV2080_CTRL_GRMGR_GR_FS_INFO_MAX_QUERIES = 96 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_MAX_SIZE = 32 +NV2080_CTRL_GRMGR_MAX_SMC_IDS = 8 +NV2080_CTRL_GRMGR_GET_GR_FS_INFO_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_INVALID = 0 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_GPC_COUNT = 1 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_CHIPLET_GPC_MAP = 2 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_TPC_MASK = 3 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PPC_MASK = 4 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARTITION_CHIPLET_GPC_MAP = 5 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_CHIPLET_SYSPIPE_MASK = 6 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARTITION_CHIPLET_SYSPIPE_IDS = 7 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PROFILER_MON_GPC_MASK = 8 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_PARTITION_SYSPIPE_ID = 9 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_ROP_MASK = 10 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_CHIPLET_GRAPHICS_SYSPIPE_MASK = 11 +NV2080_CTRL_GRMGR_GR_FS_INFO_QUERY_GFX_CAPABLE_GPC_MASK = 12 +NV2080_CTRL_CMD_GSP_GET_FEATURES = (0x20803601) +NV2080_GSP_MAX_BUILD_VERSION_LENGTH = (0x0000040) +NV2080_CTRL_GSP_GET_FEATURES_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_GSP_GET_FEATURES_UVM_ENABLED = (0, 0) +NV2080_CTRL_GSP_GET_FEATURES_UVM_ENABLED_FALSE = (0x00000000) +NV2080_CTRL_GSP_GET_FEATURES_UVM_ENABLED_TRUE = (0x00000001) +NV2080_CTRL_GSP_GET_FEATURES_VGPU_GSP_MIG_REFACTORING_ENABLED = (1, 1) +NV2080_CTRL_GSP_GET_FEATURES_VGPU_GSP_MIG_REFACTORING_ENABLED_FALSE = (0x00000000) +NV2080_CTRL_GSP_GET_FEATURES_VGPU_GSP_MIG_REFACTORING_ENABLED_TRUE = (0x00000001) +NV2080_CTRL_CMD_GSP_GET_RM_HEAP_STATS = (0x20803602) +NV2080_CTRL_GSP_GET_RM_HEAP_STATS_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_GSP_GET_VGPU_HEAP_STATS = (0x20803603) +NV2080_CTRL_CMD_GSP_GET_VGPU_HEAP_STATS_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_GSP_GET_LIBOS_HEAP_STATS = (0x20803604) +NV2080_CTRL_GSP_LIBOS_POOL_COUNT_MAX = 64 +NV2080_CTRL_CMD_GSP_GET_LIBOS_HEAP_STATS_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_GSP_GDMA_FUZZ_TEST = (0x20803605) +NV2080_CTRL_CMD_GSP_GDMA_FUZZ_TEST_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_CMD_GSP_CORE_TEST = (0x20803606) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_INVAL_L1_LIBOS = (0, 0) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_INVAL_L1_LIBOS_YES = (0x00000001) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_INVAL_L1_LIBOS_NO = (0x00000000) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_EVICT_L2 = (1, 1) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_EVICT_L2_YES = (0x00000001) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_EVICT_L2_NO = (0x00000000) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_INVAL_L2 = (2, 2) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_INVAL_L2_YES = (0x00000001) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_INVAL_L2_NO = (0x00000000) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_UNCACHED = (3, 3) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_UNCACHED_YES = (0x00000001) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_UNCACHED_NO = (0x00000000) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_FENCE = (4, 4) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_FENCE_YES = (0x00000001) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_FENCE_NO = (0x00000000) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_POSTED = (5, 5) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_POSTED_YES = (0x00000001) +NV2080_CTRL_GSP_CORE_TEST_FLAGS_POSTED_NO = (0x00000000) +NV2080_CTRL_GSP_CORE_TEST_SIZE_MAX = (0x400000) +NV2080_CTRL_GSP_CORE_TEST_ITERS_MAX = (10000) +NV2080_CTRL_CMD_GSP_CORE_TEST_PARAMS_MESSAGE_ID = (0x6) +NV2080_CTRL_CMD_HSHUB_GET_AVAILABLE_MASK_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_HSHUB_GET_AVAILABLE_MASK = (0x20804101) +NV2080_CTRL_CMD_HSHUB_SET_EC_THROTTLE_MODE_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_HSHUB_SET_EC_THROTTLE_MODE = (0x20804102) +NV2080_CTRL_I2C_VERSION_0 = 0x00 +NV2080_CTRL_I2C_MAX_ENTRIES = 256 +NV2080_CTRL_I2C_MAX_REG_LEN = 8 +NV2080_CTRL_I2C_MAX_ADDR_ENTRIES = 20 +NV2080_CTRL_I2C_FLAGS_NONSTD_SI1930UC = (0x00000001) +NV2080_CTRL_I2C_FLAGS_PRIVILEGE = (0x00000002) +NV2080_CTRL_I2C_FLAGS_DATA_ENCRYPTED = (0x00000004) +NV2080_CTRL_I2C_FLAGS_PX3540 = (0x00000010) +NV2080_CTRL_I2C_FLAGS_ADDR_AUTO_INC_NOT_SUPPORTED = (0x00000008) +NV2080_CTRL_I2C_READ_BUFFER_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_I2C_READ_BUFFER = (0x20800601) +NV2080_CTRL_I2C_WRITE_BUFFER_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_I2C_WRITE_BUFFER = (0x20800602) +NV2080_CTRL_CMD_I2C_READ_REG = (0x20800603) +NV2080_CTRL_I2C_READ_REG_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_I2C_WRITE_REG = (0x20800604) +NV2080_CTRL_I2C_WRITE_REG_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_I2C_ACCESS = (0x20800610) +NV2080_CTRL_I2C_ACCESS_PARAMS_MESSAGE_ID = (0x10) +NV2080_CTRL_I2C_ACCESS_CMD_ACQUIRE = 0x1 +NV2080_CTRL_I2C_ACCESS_CMD_RELEASE = 0x2 +NV2080_CTRL_I2C_ACCESS_CMD_WRITE_BYTE = 0x3 +NV2080_CTRL_I2C_ACCESS_CMD_READ_BYTE = 0x4 +NV2080_CTRL_I2C_ACCESS_CMD_NULL = 0x5 +NV2080_CTRL_I2C_ACCESS_CMD_RESET = 0x6 +NV2080_CTRL_I2C_ACCESS_CMD_TEST_PORT = 0x11 +NV2080_CTRL_I2C_ACCESS_CMD_SET_FAST_MODE = 0x12 +NV2080_CTRL_I2C_ACCESS_CMD_SET_NORMAL_MODE = 0x13 +NV2080_CTRL_I2C_ACCESS_CMD_WRITE_BUFFER = 0x14 +NV2080_CTRL_I2C_ACCESS_CMD_READ_BUFFER = 0x15 +NV2080_CTRL_I2C_ACCESS_CMD_START = 0x17 +NV2080_CTRL_I2C_ACCESS_CMD_STOP = 0x18 +NV2080_CTRL_I2C_ACCESS_CMD_SET_SLOW_MODE = 0x20 +NV2080_CTRL_I2C_ACCESS_FLAG_START = 0x1 +NV2080_CTRL_I2C_ACCESS_FLAG_STOP = 0x2 +NV2080_CTRL_I2C_ACCESS_FLAG_ACK = 0x4 +NV2080_CTRL_I2C_ACCESS_FLAG_RAB = 0x8 +NV2080_CTRL_I2C_ACCESS_FLAG_ADDR_10BITS = 0x10 +NV2080_CTRL_I2C_ACCESS_FLAG_PRIVILEGE = 0x20 +NV2080_CTRL_I2C_ACCESS_FLAG_DATA_ENCRYPTED = 0x40 +NV2080_CTRL_I2C_ACCESS_FLAG_RESTART = 0x80 +NV2080_CTRL_I2C_ACCESS_FLAG_SLOW_MODE_33_33PCT = 0x100 +NV2080_CTRL_I2C_ACCESS_FLAG_SLOW_MODE_33PCT = 0x200 +NV2080_CTRL_I2C_ACCESS_FLAG_SLOW_MODE_10PCT = 0x400 +NV2080_CTRL_I2C_ACCESS_FLAG_SLOW_MODE_3_33PCT = 0x800 +NV2080_CTRL_I2C_ACCESS_FLAG_SLOW_MODE_3PCT = 0x1000 +NV2080_CTRL_I2C_ACCESS_PORT_DYNAMIC = 0x0 +NV2080_CTRL_I2C_ACCESS_PORT_PRIMARY = 0x1 +NV2080_CTRL_I2C_ACCESS_PORT_SECONDARY = 0x2 +NV2080_CTRL_I2C_ACCESS_PORT_TERTIARY = 0x3 +NV2080_CTRL_I2C_ACCESS_PORT_QUARTIARY = 0x4 +NV2080_CTRL_I2C_ACCESS_PORT_1 = 0x1 +NV2080_CTRL_I2C_ACCESS_PORT_2 = 0x2 +NV2080_CTRL_I2C_ACCESS_PORT_3 = 0x3 +NV2080_CTRL_I2C_ACCESS_PORT_4 = 0x4 +NV2080_CTRL_I2C_ACCESS_PORT_5 = 0x5 +NV2080_CTRL_I2C_ACCESS_PORT_6 = 0x6 +NV2080_CTRL_I2C_ACCESS_PORT_7 = 0x7 +NV2080_CTRL_I2C_ACCESS_PORT_8 = 0x8 +NV2080_CTRL_I2C_ACCESS_PORT_9 = 0x9 +NV2080_CTRL_I2C_ACCESS_PORT_10 = 0x10 +NV2080_CTRL_I2C_ACCESS_NUM_PORTS = NV2080_CTRL_I2C_ACCESS_PORT_10 +NV2080_CTRL_I2C_ACCESS_STATUS_SUCCESS = 0x0 +NV2080_CTRL_I2C_ACCESS_STATUS_ERROR = 0x1 +NV2080_CTRL_I2C_ACCESS_STATUS_PROTOCOL_ERROR = 0x2 +NV2080_CTRL_I2C_ACCESS_STATUS_DEVICE_BUSY = 0x3 +NV2080_CTRL_I2C_ACCESS_STATUS_NACK_AFTER_SEND = 0x4 +NV2080_CTRL_I2C_ACCESS_STATUS_DP2TMDS_DONGLE_MISSING = 0x5 +NV2080_CTRL_CMD_I2C_ENABLE_MONITOR_3D_MODE = (0x20800620) +NV2080_CTRL_I2C_ENABLE_MONITOR_3D_MODE_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_GET_STATIC_INFO = (0x20800a01) +NV2080_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS_MESSAGE_ID = (0x1C) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_STATIC_CONFIG = (0x20800a1c) +NV2080_CTRL_CMD_INTERNAL_UVM_REGISTER_ACCESS_CNTR_BUFFER = (0x20800a1d) +NV2080_CTRL_INTERNAL_UVM_ACCESS_CNTR_BUFFER_MAX_PAGES = 64 +NV2080_CTRL_INTERNAL_UVM_REGISTER_ACCESS_CNTR_BUFFER_PARAMS_MESSAGE_ID = (0x1D) +NV2080_CTRL_CMD_INTERNAL_UVM_UNREGISTER_ACCESS_CNTR_BUFFER = (0x20800a1e) +NV2080_CTRL_INTERNAL_UVM_UNREGISTER_ACCESS_CNTR_BUFFER_PARAMS_MESSAGE_ID = (0x1E) +NV2080_CTRL_INTERNAL_GR_MAX_ENGINES = 8 +NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CAPS = (0x20800a1f) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_CAPS_PARAMS_MESSAGE_ID = (0x1F) +NV2080_CTRL_INTERNAL_GR_MAX_SM = 512 +NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS_MESSAGE_ID = (0x23) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_GLOBAL_SM_ORDER = (0x20800a22) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_GLOBAL_SM_ORDER_PARAMS_MESSAGE_ID = (0x22) +NV2080_CTRL_CMD_INTERNAL_MAX_NVDECS = 8 +NV2080_CTRL_CMD_INTERNAL_MAX_BSPS = NV2080_CTRL_CMD_INTERNAL_MAX_NVDECS +NV2080_CTRL_INTERNAL_BSP_CAPS = NV2080_CTRL_INTERNAL_NVDEC_CAPS +NV2080_CTRL_INTERNAL_BSP_GET_CAPS_PARAMS = NV2080_CTRL_INTERNAL_NVDEC_GET_CAPS_PARAMS +NV2080_CTRL_CMD_INTERNAL_MAX_NVENCS = 8 +NV2080_CTRL_CMD_INTERNAL_MAX_MSENCS = NV2080_CTRL_CMD_INTERNAL_MAX_NVENCS +NV2080_CTRL_INTERNAL_MSENC_CAPS = NV2080_CTRL_INTERNAL_NVENC_CAPS +NV2080_CTRL_INTERNAL_MSENC_GET_CAPS_PARAMS = NV2080_CTRL_INTERNAL_NVENC_GET_CAPS_PARAMS +NV2080_CTRL_INTERNAL_GR_MAX_GPC = 64 +NV2080_CTRL_INTERNAL_MAX_TPC_PER_GPC_COUNT = 15 +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS_MESSAGE_ID = (0x27) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FLOORSWEEPING_MASKS = (0x20800a26) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_FLOORSWEEPING_MASKS_PARAMS_MESSAGE_ID = (0x26) +NV2080_CTRL_KGR_MAX_BUFFER_PTES = 128 +NV2080_CTRL_CMD_KGR_GET_CTX_BUFFER_PTES = (0x20800a28) +NV2080_CTRL_KGR_GET_CTX_BUFFER_PTES_PARAMS_MESSAGE_ID = (0x28) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS_MESSAGE_ID = (0x2B) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_INFO = (0x20800a2a) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_INFO_PARAMS_MESSAGE_ID = (0x2A) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS_MESSAGE_ID = (0x2D) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_ZCULL_INFO = (0x20800a2c) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_ZCULL_INFO_PARAMS_MESSAGE_ID = (0x2C) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS_MESSAGE_ID = (0x2F) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_ROP_INFO = (0x20800a2e) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_ROP_INFO_PARAMS_MESSAGE_ID = (0x2E) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS_MESSAGE_ID = (0x31) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_PPC_MASKS = (0x20800a30) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_PPC_MASKS_PARAMS_MESSAGE_ID = (0x30) +NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT = 0x1a +NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS_MESSAGE_ID = (0x33) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO = (0x20800a32) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO_PARAMS_MESSAGE_ID = (0x32) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS_MESSAGE_ID = (0x35) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER = (0x20800a34) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS_MESSAGE_ID = (0x34) +NV2080_CTRL_CMD_INTERNAL_GPU_SYNC_FABRIC_PROBE_INFO_PARAMS_MESSAGE_ID = (0x36) +NV2080_CTRL_CMD_INTERNAL_GPU_SYNC_FABRIC_PROBE_INFO = (0x20800a36) +NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE = (0x20800a37) +NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE = (0x20800a38) +NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS_MESSAGE_ID = (0x37) +NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS_MESSAGE_ID = (0x38) +NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET = (0x20800a39) +NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET = (0x20800a3a) +NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET = (0x20800a3b) +NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS_MESSAGE_ID = (0x39) +NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET_PARAMS_MESSAGE_ID = (0x3A) +NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET_PARAMS_MESSAGE_ID = (0x3B) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_RECORD_SIZE = (0x20800a3d) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS_MESSAGE_ID = (0x3C) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_RECORD_SIZE_PARAMS_MESSAGE_ID = (0x3D) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS_MESSAGE_ID = (0x3E) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_TRACE_DEFINES = (0x20800a3f) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_FECS_TRACE_DEFINES_PARAMS_MESSAGE_ID = (0x3F) +NV2080_CTRL_CMD_INTERNAL_DEVICE_INFO_MAX_ENTRIES = 512 +NV2080_CTRL_CMD_INTERNAL_GET_DEVICE_INFO_TABLE = (0x20800a40) +NV2080_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS_MESSAGE_ID = (0x40) +NV2080_CTRL_CMD_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP = (0x20800a41) +NV2080_CTRL_INTERNAL_GPU_USER_REGISTER_ACCESS_MAP_MAX_COMPRESSED_SIZE = 16384 +NV2080_CTRL_INTERNAL_GPU_USER_REGISTER_ACCESS_MAP_MAX_PROFILING_RANGES = 4096 +NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS_MESSAGE_ID = (0x41) +NV2080_CTRL_INTERNAL_MIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS_MESSAGE_ID = (0x43) +NV2080_CTRL_CMD_INTERNAL_KMIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE = (0x20800a44) +NV2080_CTRL_INTERNAL_KMIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE_PARAMS_MESSAGE_ID = (0x44) +NV2080_CTRL_CMD_INTERNAL_MIGMGR_PROMOTE_GPU_INSTANCE_MEM_RANGE = (0x20800a43) +NV2080_CTRL_CMD_INTERNAL_GPU_CHECK_CTS_ID_VALID = (0x20800a46) +NV2080_CTRL_INTERNAL_GPU_CHECK_CTS_ID_VALID_PARAMS_MESSAGE_ID = (0x46) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS_MESSAGE_ID = (0x47) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_PDB_PROPERTIES = (0x20800a48) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_PDB_PROPERTIES_PARAMS_MESSAGE_ID = (0x48) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_WRITE_INST_MEM = (0x20800a49) +NV2080_CTRL_INTERNAL_DISPLAY_WRITE_INST_MEM_PARAMS_MESSAGE_ID = (0x49) +NV2080_CTRL_CMD_INTERNAL_RECOVER_ALL_COMPUTE_CONTEXTS = (0x20800a4a) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_GET_IP_VERSION = (0x20800a4b) +NV2080_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION_PARAMS_MESSAGE_ID = (0x4B) +NV2080_CTRL_CMD_INTERNAL_GPU_GET_SMC_MODE = (0x20800a4c) +NV2080_CTRL_INTERNAL_GPU_GET_SMC_MODE_PARAMS_MESSAGE_ID = (0x4C) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_SETUP_RG_LINE_INTR = (0x20800a4d) +NV2080_CTRL_INTERNAL_DISPLAY_SETUP_RG_LINE_INTR_PARAMS_MESSAGE_ID = (0x4D) +NV2080_CTRL_INTERNAL_GRMGR_PARTITION_MAX_TYPES = 90 +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PROFILES_PARAMS_MESSAGE_ID = (0x4F) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM = (0x20800a51) +NV2080_CTRL_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM_PARAMS_MESSAGE_ID = (0x51) +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PARTITIONABLE_ENGINES_MAX_ENGINES_MASK_SIZE = 4 +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS_MESSAGE_ID = (0x52) +NV2080_CTRL_INTERNAL_FIFO_MAX_RUNLIST_BUFFERS = 2 +NV2080_CTRL_INTERNAL_FIFO_MAX_RUNLIST_ID = 64 +NV2080_CTRL_CMD_INTERNAL_FIFO_PROMOTE_RUNLIST_BUFFERS = (0x20800a53) +NV2080_CTRL_INTERNAL_FIFO_PROMOTE_RUNLIST_BUFFERS_PARAMS_MESSAGE_ID = (0x53) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_SET_IMP_INIT_INFO = (0x20800a54) +NV2080_CTRL_INTERNAL_DISPLAY_SET_IMP_INIT_INFO_PARAMS_MESSAGE_ID = (0x54) +NV2080_CTRL_CMD_INTERNAL_GET_EGPU_BRIDGE_INFO = (0x20800a55) +NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS_MESSAGE_ID = (0x55) +NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_NONE = (0x00000000) +NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_CUSTOM = (0x00000001) +NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_TB2 = (0x00000002) +NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_TB3 = (0x00000003) +NV2080_CTRL_CMD_INTERNAL_BUS_FLUSH_WITH_SYSMEMBAR = (0x20800a70) +NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL = (0x20800a71) +NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS_MESSAGE_ID = (0x71) +NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE = (0x20800a72) +NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS_MESSAGE_ID = (0x72) +NV2080_CTRL_CMD_INTERNAL_BUS_DESTROY_P2P_MAILBOX = (0x20800a73) +NV2080_CTRL_INTERNAL_BUS_DESTROY_P2P_MAILBOX_PARAMS_MESSAGE_ID = (0x73) +NV2080_CTRL_CMD_INTERNAL_BUS_CREATE_C2C_PEER_MAPPING = (0x20800a74) +NV2080_CTRL_INTERNAL_BUS_CREATE_C2C_PEER_MAPPING_PARAMS_MESSAGE_ID = (0x74) +NV2080_CTRL_CMD_INTERNAL_BUS_REMOVE_C2C_PEER_MAPPING = (0x20800a75) +NV2080_CTRL_INTERNAL_BUS_REMOVE_C2C_PEER_MAPPING_PARAMS_MESSAGE_ID = (0x75) +NV2080_CTRL_CMD_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES = (0x20800a57) +NV2080_CTRL_INTERNAL_VMMU_MAX_SPA_FOR_GPA_ENTRIES = 128 +NV2080_CTRL_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES_PARAMS_MESSAGE_ID = (0x57) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER = (0x20800a58) +NV2080_CTRL_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER_PARAMS_MESSAGE_ID = (0x58) +NV2080_CTRL_CMD_INTERNAL_GMMU_GET_STATIC_INFO = (0x20800a59) +NV2080_CTRL_INTERNAL_GMMU_GET_STATIC_INFO_PARAMS_MESSAGE_ID = (0x59) +NV2080_CTRL_CMD_INTERNAL_GR_GET_CTXSW_MODES = (0x20800a5a) +NV2080_CTRL_INTERNAL_GR_GET_CTXSW_MODES_PARAMS_MESSAGE_ID = (0x5A) +NV2080_CTRL_CMD_INTERNAL_FB_GET_HEAP_RESERVATION_SIZE = (0x20800a5b) +NV2080_CTRL_INTERNAL_FB_GET_HEAP_RESERVATION_SIZE_PARAMS_MESSAGE_ID = (0x5B) +NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE = (0x20800a5c) +NV2080_CTRL_INTERNAL_INTR_MAX_TABLE_SIZE = 128 +NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS_MESSAGE_ID = (0x5C) +NV2080_CTRL_INTERNAL_FB_GET_HEAP_RESERVATION_SIZE_GR = (0x00000000) +NV2080_CTRL_CMD_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_CHECK = (0x20800a98) +NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_CHECK_PARAMS_MESSAGE_ID = (0x98) +NV2080_CTRL_CMD_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_SET = (0x20800a99) +NV2080_CTRL_INTERNAL_PERF_PERFMON_CLIENT_RESERVATION_SET_PARAMS_MESSAGE_ID = (0x99) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES = (0x20800a5d) +NV2080_CTRL_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES_PARAMS_MESSAGE_ID = (0x5D) +NV2080_CTRL_CMD_INTERNAL_STATIC_MIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES = (0x20800a60) +NV2080_CTRL_INTERNAL_MAX_SWIZZ_ID = 15 +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS_MESSAGE_ID = (0x60) +NV2080_CTRL_CMD_INTERNAL_FIFO_GET_NUM_CHANNELS = (0x20800a61) +NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS_MESSAGE_ID = (0x61) +NV2080_CTRL_CMD_INTERNAL_STATIC_KMIGMGR_GET_PROFILES = (0x20800a63) +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_PROFILES_PARAMS_MESSAGE_ID = (0x63) +NV2080_CTRL_CMD_INTERNAL_STATIC_KMIGMGR_GET_PARTITIONABLE_ENGINES = (0x20800a65) +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_PARTITIONABLE_ENGINES_PARAMS_MESSAGE_ID = (0x65) +NV2080_CTRL_CMD_INTERNAL_STATIC_KMIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES = (0x20800a66) +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_SWIZZ_ID_FB_MEM_PAGE_RANGES_PARAMS_MESSAGE_ID = (0x66) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG = (0x20800a68) +NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS_MESSAGE_ID = (0x68) +NV2080_CTRL_CMD_INTERNAL_KMEMSYS_GET_MIG_MEMORY_CONFIG = (0x20800a67) +NV2080_CTRL_INTERNAL_KMEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS_MESSAGE_ID = (0x67) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE = (0x20800a6b) +NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE_SIZE = 8 +NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE_PARAMS_MESSAGE_ID = (0x6B) +NV2080_CTRL_CMD_INTERNAL_RC_WATCHDOG_TIMEOUT = (0x20800a6a) +NV2080_CTRL_CMD_INTERNAL_PERF_GET_AUX_POWER_STATE = (0x20800a81) +NV2080_CTRL_INTERNAL_PERF_GET_AUX_POWER_STATE_PARAMS_MESSAGE_ID = (0x81) +NV2080_CTRL_CMD_INTERNAL_PERF_BOOST_SET_2X = (0x20800a9a) +NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_2X_MESSAGE_ID = (0x9A) +NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_PSTATE = 0 +NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_GPCCLK = 1 +NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_LAST = NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_GPCCLK +NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM = (0x2) +NV2080_CTRL_CMD_INTERNAL_PERF_GPU_BOOST_SYNC_SET_CONTROL = (0x20800a7e) +NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_CONTROL_PARAMS_MESSAGE_ID = (0x7E) +NV2080_CTRL_CMD_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS = (0x20800a7f) +NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS_MESSAGE_ID = (0x7F) +NV2080_CTRL_CMD_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO = (0x20800a80) +NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO_PARAMS_MESSAGE_ID = (0x80) +NV2080_CTRL_CMD_INTERNAL_GMMU_REGISTER_FAULT_BUFFER = (0x20800a9b) +NV2080_CTRL_INTERNAL_GMMU_FAULT_BUFFER_MAX_PAGES = 256 +NV2080_CTRL_INTERNAL_GMMU_REGISTER_FAULT_BUFFER_PARAMS_MESSAGE_ID = (0x9B) +NV2080_CTRL_CMD_INTERNAL_GMMU_UNREGISTER_FAULT_BUFFER = (0x20800a9c) +NV2080_CTRL_FAULT_BUFFER_NON_REPLAYABLE = (0x00000000) +NV2080_CTRL_FAULT_BUFFER_REPLAYABLE = (0x00000001) +NV2080_CTRL_CMD_INTERNAL_GMMU_REGISTER_CLIENT_SHADOW_FAULT_BUFFER = (0x20800a9d) +NV2080_CTRL_INTERNAL_GMMU_CLIENT_SHADOW_FAULT_BUFFER_MAX_PAGES = 3000 +NV2080_CTRL_INTERNAL_GMMU_REGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS_MESSAGE_ID = (0x9D) +NV2080_CTRL_CMD_INTERNAL_GMMU_UNREGISTER_CLIENT_SHADOW_FAULT_BUFFER = (0x20800a9e) +NV2080_CTRL_INTERNAL_GMMU_UNREGISTER_CLIENT_SHADOW_FAULT_BUFFER_PARAMS_MESSAGE_ID = (0x9E) +NV2080_CTRL_CMD_INTERNAL_GMMU_COPY_RESERVED_SPLIT_GVASPACE_PDES_TO_SERVER = (0x20800a9f) +NV2080_CTRL_INTERNAL_GMMU_COPY_RESERVED_SPLIT_GVASPACE_PDES_TO_SERVER_PARAMS_MESSAGE_ID = (0x9F) +NV2080_CTRL_CMD_INTERNAL_PERF_BOOST_SET_3X = (0x20800aa0) +NV2080_CTRL_INTERNAL_PERF_BOOST_SET_PARAMS_3X_MESSAGE_ID = (0xA0) +NV2080_CTRL_CMD_INTERNAL_PERF_BOOST_CLEAR_3X = (0x20800aa1) +NV2080_CTRL_INTERNAL_PERF_BOOST_CLEAR_PARAMS_3X_MESSAGE_ID = (0xA1) +NV2080_CTRL_CMD_INTERNAL_STATIC_GRMGR_GET_SKYLINE_INFO = (0x20800aa2) +NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO_MAX_SKYLINES = 9 +NV2080_CTRL_INTERNAL_GRMGR_SKYLINE_INFO_MAX_NON_SINGLETON_VGPCS = 64 +NV2080_CTRL_INTERNAL_STATIC_GRMGR_GET_SKYLINE_INFO_PARAMS_MESSAGE_ID = (0xA2) +NV2080_CTRL_CMD_INTERNAL_MIGMGR_SET_PARTITIONING_MODE = (0x20800aa3) +NV2080_CTRL_INTERNAL_MIGMGR_SET_PARTITIONING_MODE_PARAMS_MESSAGE_ID = (0xA3) +NV2080_CTRL_CMD_INTERNAL_MIGMGR_CONFIGURE_GPU_INSTANCE = (0x20800aa4) +NV2080_CTRL_INTERNAL_MIGMGR_CONFIGURE_GPU_INSTANCE_PARAMS_MESSAGE_ID = (0xA4) +NV2080_CTRL_CMD_INTERNAL_MIGMGR_SET_GPU_INSTANCES = (0x20800aa5) +NV2080_CTRL_INTERNAL_MIGMGR_SET_GPU_INSTANCES_PARAMS_MESSAGE_ID = (0xA5) +NV2080_CTRL_CMD_INTERNAL_MIGMGR_GET_GPU_INSTANCES = (0x20800aa6) +NV2080_CTRL_INTERNAL_MIGMGR_GET_GPU_INSTANCES_PARAMS_MESSAGE_ID = (0xA6) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_SET_ZBC_REFERENCED = (0x20800a69) +NV2080_CTRL_INTERNAL_MEMSYS_SET_ZBC_REFERENCED_PARAMS_MESSAGE_ID = (0x69) +NV2080_CTRL_CMD_INTERNAL_KMIGMGR_EXPORT_GPU_INSTANCE = (0x20800aa7) +NV2080_CTRL_CMD_INTERNAL_MIGMGR_EXPORT_GPU_INSTANCE = (0x20800aa8) +NV2080_CTRL_CMD_INTERNAL_KMIGMGR_IMPORT_GPU_INSTANCE = (0x20800aa9) +NV2080_CTRL_CMD_INTERNAL_MIGMGR_IMPORT_GPU_INSTANCE = (0x20800aaa) +NV2080_CTRL_INTERNAL_KMIGMGR_EXPORTED_GPU_INSTANCE_MAX_ENGINES_MASK_SIZE = 4 +NV2080_CTRL_INTERNAL_KMIGMGR_EXPORT_GPU_INSTANCE_PARAMS_MESSAGE_ID = (0xA7) +NV2080_CTRL_INTERNAL_MIGMGR_EXPORT_GPU_INSTANCE_PARAMS_MESSAGE_ID = (0xA8) +NV2080_CTRL_INTERNAL_KMIGMGR_IMPORT_GPU_INSTANCE_PARAMS_MESSAGE_ID = (0xA9) +NV2080_CTRL_INTERNAL_MIGMGR_IMPORT_GPU_INSTANCE_PARAMS_MESSAGE_ID = (0xAA) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT = (0x20800a6c) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_PARAMS_MESSAGE_ID = (0x6c) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_ALL = (0x00000001) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_FIRST = (0x00000002) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_LAST = (0x00000004) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_NORMAL = (0x00000008) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_CLEAN = (0x00000010) +NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_WAIT_FB_PULL = (0x00000020) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_FLUSH_L2_ALL_RAMS_AND_CACHES = (0x20800a6d) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_CLEAN_LTC_PROBE_FILTER = (0x20800ab0) +NV2080_CTRL_CMD_INTERNAL_BIF_GET_STATIC_INFO = (0x20800aac) +NV2080_CTRL_INTERNAL_BIF_GET_STATIC_INFO_PARAMS_MESSAGE_ID = (0xac) +NV2080_CTRL_CMD_INTERNAL_BIF_GET_DATA = (0x20800a00) +NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG = (0x20800a88) +NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS_MESSAGE_ID = (0x88) +NV2080_CTRL_CMD_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS = (0x20800a8a) +NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_TABLE_SIZE = 32 +NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS_MESSAGE_ID = (0x8a) +NV2080_CTRL_CMD_INTERNAL_HSHUB_GET_NUM_UNITS = (0x20800a8b) +NV2080_CTRL_INTERNAL_HSHUB_GET_NUM_UNITS_PARAMS_MESSAGE_ID = (0x8b) +NV2080_CTRL_CMD_INTERNAL_HSHUB_NEXT_HSHUB_ID = (0x20800a8c) +NV2080_CTRL_INTERNAL_HSHUB_NEXT_HSHUB_ID_PARAMS_MESSAGE_ID = (0x8c) +NV2080_CTRL_CMD_INTERNAL_HSHUB_EGM_CONFIG = (0x20800a8d) +NV2080_CTRL_INTERNAL_HSHUB_EGM_CONFIG_PARAMS_MESSAGE_ID = (0x8d) +NV2080_CTRL_CMD_INTERNAL_NVLINK_ENABLE_COMPUTE_PEER_ADDR = (0x20800aad) +NV2080_CTRL_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR_PARAMS_MESSAGE_ID = (0xae) +NV2080_CTRL_CMD_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR = (0x20800aae) +NV2080_CTRL_CMD_INTERNAL_PERF_CF_CONTROLLERS_SET_MAX_VGPU_VM_COUNT = (0x20800ab1) +NV2080_CTRL_CMD_INTERNAL_PERF_CF_CONTROLLERS_MAX_ACTIVE_VGPU_VM_COUNT_MAX_VALUE = 32 +NV2080_CTRL_INTERNAL_PERF_CF_CONTROLLERS_SET_MAX_VGPU_VM_COUNT_PARAMS_MESSAGE_ID = (0xB1) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_DISABLE_NVLINK_PEERS = (0x20800a6e) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE = (0x20800a6f) +NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS_MESSAGE_ID = (0x6f) +NV2080_CTRL_CMD_INTERNAL_CCU_GET_SAMPLE_INFO = (0x20800ab2) +NV2080_CTRL_CMD_INTERNAL_CCU_MAP = (0x20800ab3) +NV2080_CTRL_INTERNAL_CCU_DEV_SHRBUF_COUNT_MAX = 1 +NV2080_CTRL_INTERNAL_CCU_MAP_INFO_PARAMS_MESSAGE_ID = (0xB3) +NV2080_CTRL_CMD_INTERNAL_CCU_UNMAP = (0x20800ab4) +NV2080_CTRL_INTERNAL_CCU_UNMAP_INFO_PARAMS_MESSAGE_ID = (0xB4) +NV2080_CTRL_CMD_INTERNAL_SET_P2P_CAPS = (0x20800ab5) +NV2080_CTRL_INTERNAL_SET_P2P_CAPS_PARAMS_MESSAGE_ID = (0xB5) +NV2080_CTRL_CMD_INTERNAL_REMOVE_P2P_CAPS = (0x20800ab6) +NV2080_CTRL_INTERNAL_REMOVE_P2P_CAPS_PARAMS_MESSAGE_ID = (0xB6) +NV2080_CTRL_CMD_INTERNAL_GET_PCIE_P2P_CAPS = (0x20800ab8) +NV2080_CTRL_INTERNAL_GET_PCIE_P2P_CAPS_PARAMS_MESSAGE_ID = (0xB8) +NV2080_CTRL_CMD_INTERNAL_BIF_SET_PCIE_RO = (0x20800ab9) +NV2080_CTRL_INTERNAL_BIF_SET_PCIE_RO_PARAMS_MESSAGE_ID = (0xB9) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE = (0x20800a76) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS_MESSAGE_ID = (0x76) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE = (0x20800a77) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS_MESSAGE_ID = (0x77) +NV2080_CTRL_INTERNAL_STATIC_MIGMGR_GET_COMPUTE_PROFILES_PARAMS_MESSAGE_ID = (0xBB) +NV2080_CTRL_CMD_INTERNAL_STATIC_KMIGMGR_GET_COMPUTE_PROFILES = (0x20800aba) +NV2080_CTRL_INTERNAL_STATIC_KMIGMGR_GET_COMPUTE_PROFILES_PARAMS_MESSAGE_ID = (0xBA) +NV2080_CTRL_CMD_INTERNAL_CCU_SET_STREAM_STATE = (0x20800abd) +NV2080_CTRL_INTERNAL_CCU_STREAM_STATE_PARAMS_MESSAGE_ID = (0xBD) +NV2080_CTRL_CMD_INTERNAL_GSYNC_ATTACH_AND_INIT = (0x20800abe) +NV2080_CTRL_INTERNAL_GSYNC_ATTACH_AND_INIT_PARAMS_MESSAGE_ID = (0xBE) +NV2080_CTRL_CMD_INTERNAL_GSYNC_OPTIMIZE_TIMING_PARAMETERS = (0x20800abf) +NV2080_CTRL_INTERNAL_GSYNC_OPTIMIZE_TIMING_PARAMETERS_PARAMS_MESSAGE_ID = (0xBF) +NV2080_CTRL_CMD_INTERNAL_GSYNC_GET_DISPLAY_IDS = (0x20800ac0) +NV2080_MAX_NUM_HEADS = 4 +NV2080_CTRL_INTERNAL_GSYNC_GET_DISPLAY_IDS_PARAMS_MESSAGE_ID = (0xC0) +NV2080_CTRL_CMD_INTERNAL_GSYNC_SET_STREO_SYNC = (0x20800ac1) +NV2080_CTRL_INTERNAL_GSYNC_SET_STREO_SYNC_PARAMS_MESSAGE_ID = (0xC1) +NV2080_CTRL_CMD_INTERNAL_GSYNC_GET_VERTICAL_ACTIVE_LINES = (0x20800ac4) +NV2080_CTRL_INTERNAL_GSYNC_GET_VERTICAL_ACTIVE_LINES_PARAMS_MESSAGE_ID = (0xC4) +NV2080_CTRL_CMD_INTERNAL_GSYNC_IS_DISPLAYID_VALID = (0x20800ac9) +NV2080_CTRL_INTERNAL_GSYNC_IS_DISPLAYID_VALID_PARAMS_MESSAGE_ID = (0xC9) +NV2080_CTRL_CMD_INTERNAL_GSYNC_SET_OR_RESTORE_RASTER_SYNC = (0x20800aca) +NV2080_CTRL_INTERNAL_GSYNC_SET_OR_RESTORE_RASTER_SYNC_PARAMS_MESSAGE_ID = (0xCA) +NV2080_CTRL_CMD_INTERNAL_FBSR_INIT = (0x20800ac2) +NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS_MESSAGE_ID = (0xC2) +NV2080_CTRL_CMD_INTERNAL_FIFO_TOGGLE_ACTIVE_CHANNEL_SCHEDULING = (0x20800ac3) +NV2080_CTRL_CMD_INTERNAL_FIFO_TOGGLE_ACTIVE_CHANNEL_SCHEDULING_PARAMS_MESSAGE_ID = (0xC3) +NV2080_CTRL_CMD_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB = (0x20800ac5) +NV2080_CTRL_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB_PARAMS_MESSAGE_ID = (0xC5) +NV2080_CTRL_CMD_INTERNAL_INIT_BRIGHTC_STATE_LOAD = (0x20800ac6) +NV2080_CTRL_ACPI_DSM_READ_SIZE = (0x1000) +NV2080_CTRL_INTERNAL_INIT_BRIGHTC_STATE_LOAD_PARAMS_MESSAGE_ID = (0xC6) +NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL_PARAMS_MESSAGE_ID = (0xC7) +NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL = (0x20800ac7) +NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL_PARAMS_MESSAGE_ID = (0xC8) +NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL = (0x20800ac8) +NV2080_CTRL_CMD_INTERNAL_SMBPBI_PFM_REQ_HNDLR_CAP_UPDATE = (0x20800acb) +NV2080_CTRL_INTERNAL_SMBPBI_PFM_REQ_HNDLR_CAP_UPDATE_PARAMS_MESSAGE_ID = (0xCB) +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_TYPE_PMGR = 0x00 +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_TYPE_THERM = 0x01 +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_TYPE_SMBPBI = 0x02 +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_FLAGS_PMGR_LOAD = 0x00 +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_FLAGS_THERM_INIT = 0x01 +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_FLAGS_SMBPBI_OP_CLEAR = 0x02 +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_FLAGS_SMBPBI_OP_SET = 0x03 +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_STATE_LOAD_SYNC = (0x20800acc) +NV2080_CTRL_INTERNAL_PMGR_PFM_REQ_HNDLR_STATE_LOAD_SYNC_PARAMS_MESSAGE_ID = (0xCC) +NV2080_CTRL_CMD_INTERNAL_THERM_PFM_REQ_HNDLR_STATE_INIT_SYNC = (0x20800acd) +NV2080_CTRL_INTERNAL_THERM_PFM_REQ_HNDLR_STATE_INIT_SYNC_PARAMS_MESSAGE_ID = (0xCD) +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_GET_PM1_FORCED_OFF_STATUS = 0x00 +NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_GET_PM1_STATUS = 0x01 +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_PM1_STATE = (0x20800ace) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_PM1_STATE_PARAMS_MESSAGE_ID = (0xCE) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_PM1_STATE = (0x20800acf) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_PM1_STATE_PARAMS_MESSAGE_ID = (0xCF) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_UPDATE_EDPP_LIMIT = (0x20800ad0) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_UPDATE_EDPP_LIMIT_PARAMS_MESSAGE_ID = (0xD0) +NV2080_CTRL_CMD_INTERNAL_THERM_PFM_REQ_HNDLR_UPDATE_TGPU_LIMIT = (0x20800ad1) +NV2080_CTRL_CMD_INTERNAL_THERM_PFM_REQ_HNDLR_UPDATE_TGPU_LIMIT_PARAMS_MESSAGE_ID = (0xD1) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TGP_MODE = (0x20800ad2) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TGP_MODE_PARAMS_MESSAGE_ID = (0xD2) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TURBO_V2 = (0x20800ad3) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_CONFIGURE_TURBO_V2_PARAMS_MESSAGE_ID = (0xD3) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_INFO = (0x20800ad4) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_INFO_PARAMS_MESSAGE_ID = (0xD4) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_MAPPING = (0x20800ad5) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_GET_VPSTATE_MAPPING_PARAMS_MESSAGE_ID = (0xD5) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_VPSTATE = (0x20800ad6) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_SET_VPSTATE_PARAMS_MESSAGE_ID = (0xD6) +NV2080_CTRL_CMD_INTERNAL_GCX_ENTRY_PREREQUISITE = (0x2080a7d7) +NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS_MESSAGE_ID = (0xD7) +NV2080_CTRL_CMD_INTERNAL_PMGR_UNSET_DYNAMIC_BOOST_LIMIT = (0x20800a7b) +NV2080_CTRL_CMD_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS = (0x20800ad8) +NV2080_CTRL_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS_PARAMS_MESSAGE_ID = (0xD8) +NV2080_CTRL_CMD_INTERNAL_PERF_PFM_REQ_HNDLR_DEPENDENCY_CHECK = (0x20800a45) +NV2080_CTRL_CMD_INTERNAL_BIF_DISABLE_SYSTEM_MEMORY_ACCESS = (0x20800adb) +NV2080_CTRL_INTERNAL_BIF_DISABLE_SYSTEM_MEMORY_ACCESS_PARAMS_MESSAGE_ID = (0xDB) +NV2080_CTRL_CMD_INTERNAL_DISP_PINSETS_TO_LOCKPINS = (0x20800adc) +NV2080_CTRL_INTERNAL_DISP_PINSETS_TO_LOCKPINS_PARAMS_MESSAGE_ID = (0xDC) +NV2080_CTRL_CMD_INTERNAL_DETECT_HS_VIDEO_BRIDGE = (0x20800add) +NV2080_CTRL_CMD_INTERNAL_DISP_SET_SLI_LINK_GPIO_SW_CONTROL = (0x20800ade) +NV2080_CTRL_INTERNAL_DISP_SET_SLI_LINK_GPIO_SW_CONTROL_PARAMS_MESSAGE_ID = (0xDE) +NV2080_CTRL_CMD_INTERNAL_SET_STATIC_EDID_DATA = (0x20800adf) +MAX_EDID_SIZE_FROM_SBIOS = 512 +NV2080_CTRL_CMD_INTERNAL_SET_STATIC_EDID_DATA_PARAMS_MESSAGE_ID = (0xDF) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_ACPI_SUBSYSTEM_ACTIVATED = (0x20800af0) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_MODESET = (0x20800af1) +NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_MODESET = (0x20800af2) +NV2080_CTRL_CMD_INTERNAL_GET_GPU_FABRIC_PROBE_INFO_PARAMS_MESSAGE_ID = (0xF4) +NV2080_CTRL_CMD_INTERNAL_GPU_GET_FABRIC_PROBE_INFO = (0x208001f4) +NV2080_CTRL_CMD_INTERNAL_START_GPU_FABRIC_PROBE_INFO_PARAMS_MESSAGE_ID = (0xF5) +NV2080_CTRL_CMD_INTERNAL_GPU_START_FABRIC_PROBE = (0x208001f5) +NV2080_CTRL_CMD_INTERNAL_GPU_STOP_FABRIC_PROBE = (0x208001f6) +NV2080_CTRL_CMD_INTERNAL_GPU_SUSPEND_FABRIC_PROBE = (0x208001f7) +NV2080_CTRL_CMD_INTERNAL_RESUME_GPU_FABRIC_PROBE_INFO_PARAMS_MESSAGE_ID = (0xF8) +NV2080_CTRL_CMD_INTERNAL_GPU_RESUME_FABRIC_PROBE = (0x208001f8) +NV2080_CTRL_CMD_INTERNAL_GPU_INVALIDATE_FABRIC_PROBE = (0x208001f9) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_GET_STATIC_INFO = (0x20800af3) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_GET_STATIC_INFO_PARAMS_MESSAGE_ID = (0xF3) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_DERIVE_SWL_KEYS = (0x20800ae1) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK_SIZE = 3 +NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK_SWL_KERNEL = 0 +NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK_SWL_USER = 1 +NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK_SWL_SCRUBBER = 2 +NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK_SWL_COUNT = 3 +NV2080_CTRL_INTERNAL_CONF_COMPUTE_IVMASK_LCE_COUNT = 6 +NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_SWL_KEYS_PARAMS_MESSAGE_ID = (0xE1) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_DERIVE_LCE_KEYS = (0x20800ae2) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_DERIVE_LCE_KEYS_PARAMS_MESSAGE_ID = (0xE2) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_ROTATE_KEYS = (0x20800ae5) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_ROTATE_KEYS_PARAMS_MESSAGE_ID = (0xE5) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_RC_CHANNELS_FOR_KEY_ROTATION = (0x20800ae6) +NV2080_CTRL_INTERNAL_CONF_COMPUTE_RC_CHANNELS_FOR_KEY_ROTATION_PARAMS_MESSAGE_ID = (0xE6) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE = (0x20800ae7) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_GPU_STATE_PARAMS_MESSAGE_ID = (0xE7) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY = (0x20800ae8) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_SET_SECURITY_POLICY_PARAMS_MESSAGE_ID = (0xE8) +NV2080_CTRL_CMD_INTERNAL_FIFO_GET_LOGICAL_UPROC_ID = (0x20800aef) +NV2080_CTRL_INTERNAL_FIFO_GET_LOGICAL_UPROC_ID_PARAMS_MESSAGE_ID = (0xEF) +NV2080_CTRL_CMD_INTERNAL_MEMMGR_MEMORY_TRANSFER_WITH_GSP = (0x20800afa) +CC_AES_256_GCM_AUTH_TAG_SIZE_BYTES = (0x10) +NV2080_CTRL_INTERNAL_MEMMGR_MEMORY_TRANSFER_WITH_GSP_PARAMS_MESSAGE_ID = (0xFA) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG = (0x20800afb) +NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS_MESSAGE_ID = (0xFB) +NV2080_CTRL_CMD_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG = (0x20800afc) +NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS_MESSAGE_ID = (0xFC) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_GET_EDPP_LIMIT_INFO = (0x20800afd) +NV2080_CTRL_CMD_INTERNAL_PMGR_PFM_REQ_HNDLR_GET_EDPP_LIMIT_INFO_PARAMS_MESSAGE_ID = (0xFD) +NV2080_CTRL_INTERNAL_INIT_USER_SHARED_DATA_PARAMS_MESSAGE_ID = (0xFE) +NV2080_CTRL_CMD_INTERNAL_INIT_USER_SHARED_DATA = (0x20800afe) +NV2080_CTRL_INTERNAL_USER_SHARED_DATA_SET_DATA_POLL_PARAMS_MESSAGE_ID = (0xFF) +NV2080_CTRL_CMD_INTERNAL_USER_SHARED_DATA_SET_DATA_POLL = (0x20800aff) +NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_PARAMS_MESSAGE_ID = (0xE3) +NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_FLAG_START_KEEP_OLDEST = 0x00 +NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_FLAG_START_KEEP_NEWEST = 0x01 +NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE_FLAG_STOP = 0x02 +NV2080_CTRL_CMD_INTERNAL_CONTROL_GSP_TRACE = (0x208001e3) +NV2080_CTRL_CMD_INTERNAL_GET_ENABLED_SEC2_CLASSES_PARAMS_MESSAGE_ID = (0xAF) +NV2080_CTRL_CMD_INTERNAL_GET_ENABLED_SEC2_CLASSES = (0x20800aaf) +NV2080_CTRL_CMD_INTERNAL_GR_CTXSW_SETUP_BIND = (0x20800ae4) +NV2080_CTRL_INTERNAL_GR_CTXSW_SETUP_BIND_PARAMS_MESSAGE_ID = (0xE4) +NV2080_CTRL_INTERNAL_GPU_CLIENT_LOW_POWER_MODE_ENTER_PARAMS_MESSAGE_ID = (0xE9) +NV2080_CTRL_INTERNAL_GPU_CLIENT_LOW_POWER_MODE_ENTER = (0x20800ae9) +NV2080_CTRL_CMD_INTERNAL_NVLINK_POST_FATAL_ERROR_RECOVERY = (0x20800aea) +NV2080_CTRL_INTERNAL_NVLINK_POST_FATAL_ERROR_RECOVERY_PARAMS_MESSAGE_ID = (0xEA) +NV2080_CTRL_CMD_INTERNAL_NVLINK_LFM_RM_ACTION = (0x20800af6) +NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_QUIESCE_TRAFFIC = 0 +NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_RESUME_TRAFFIC = 1 +NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_QUIESCE_TRAFFIC_DONE = 2 +NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_RESUME_TRAFFIC_DONE = 3 +NV2080_CTRL_INTERNAL_NVLINK_LFM_RM_ACTION_PARAMS_MESSAGE_ID = (0xF6) +NV2080_CTRL_INTERNAL_GPU_GET_GSP_RM_FREE_HEAP_PARAMS_MESSAGE_ID = (0xEB) +NV2080_CTRL_CMD_INTERNAL_GPU_GET_GSP_RM_FREE_HEAP = (0x20800aeb) +NV2080_CTRL_INTERNAL_GPU_SET_ILLUM_PARAMS_MESSAGE_ID = (0xEC) +NV2080_CTRL_CMD_INTERNAL_GPU_SET_ILLUM = (0x20800aec) +NV2080_CTRL_INTERNAL_GSYNC_APPLY_STEREO_PIN_ALWAYS_HI_WAR_PARAMS_MESSAGE_ID = (0xED) +NV2080_CTRL_CMD_INTERNAL_GSYNC_APPLY_STEREO_PIN_ALWAYS_HI_WAR = (0x20800aed) +NV2080_CTRL_CMD_INTERNAL_HSHUB_GET_MAX_HSHUBS_PER_SHIM = (0x20800a79) +NV2080_CTRL_INTERNAL_HSHUB_GET_MAX_HSHUBS_PER_SHIM_PARAMS_MESSAGE_ID = (0x79) +NV2080_CTRL_INTERNAL_GSYNC_GET_RASTER_SYNC_DECODE_MODE_PARAMS_MESSAGE_ID = (0x14) +NV2080_CTRL_CMD_INTERNAL_GSYNC_GET_RASTER_SYNC_DECODE_MODE = (0x20800a14) +NV2080_CTRL_INTERNAL_GPU_GET_PF_BAR1_SPA_PARAMS_MESSAGE_ID = (0xEE) +NV2080_CTRL_CMD_INTERNAL_GPU_GET_PF_BAR1_SPA = (0x20800aee) +NV2080_CTRL_CMD_INTERNAL_NVLINK_ENABLE_NVLINK_PEER = (0x20800a21) +NV2080_CTRL_INTERNAL_NVLINK_ENABLE_NVLINK_PEER_PARAMS_MESSAGE_ID = (0x21) +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_OFF = 0x00 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_HS = 0x01 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_SAFE = 0x02 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_FAULT = 0x03 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_RECOVERY = 0x04 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_FAIL = 0x05 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_DETECT = 0x06 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_RESET = 0x07 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_ENABLE_PM = 0x08 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_DISABLE_PM = 0x09 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_SLEEP = 0x0A +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_SAVE_STATE = 0x0B +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_RESTORE_STATE = 0x0C +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_PRE_HS = 0x0E +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_DISABLE_ERR_DETECT = 0x0F +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_LANE_DISABLE = 0x10 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_LANE_SHUTDOWN = 0x11 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_TRAFFIC_SETUP = 0x12 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_INITPHASE1 = 0x13 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_INITNEGOTIATE = 0x14 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_POST_INITNEGOTIATE = 0x15 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_INITOPTIMIZE = 0x16 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_POST_INITOPTIMIZE = 0x17 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_DISABLE_HEARTBEAT = 0x18 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_CONTAIN = 0x19 +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_INITTL = 0x1A +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_INITPHASE5 = 0x1B +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_ALI = 0x1C +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_ACTIVE_PENDING = 0x1D +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_PHYSICAL_UP_NO_TRAFFIC = 0x1F +NV2080_INTERNAL_NVLINK_CORE_LINK_STATE_INVALID = 0xFF +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_HS = 0x00 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_SINGLE_LANE = 0x04 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_LOW_POWER = 0x04 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_TRAIN = 0x05 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_SAFE = 0x06 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_OFF = 0x07 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_COMMON_MODE = 0x08 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_COMMON_MODE_DISABLE = 0x09 +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_DATA_READY = 0x0A +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_EQ = 0x0B +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_PRBS_EN = 0x0C +NV2080_NVLINK_CORE_SUBLINK_STATE_TX_POST_HS = 0x0D +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_HS = 0x00 +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_SINGLE_LANE = 0x04 +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_LOW_POWER = 0x04 +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_TRAIN = 0x05 +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_SAFE = 0x06 +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_OFF = 0x07 +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_RXCAL = 0x08 +NV2080_NVLINK_CORE_SUBLINK_STATE_RX_INIT_TERM = 0x09 +NV2080_CTRL_INTERNAL_NVLINK_MAX_SEED_NUM = 6 +NV2080_CTRL_INTERNAL_NVLINK_MAX_SEED_BUFFER_SIZE = (0x7) +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_GET_DL_LINK_MODE = 0x00 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_SET_DL_LINK_MODE = 0x01 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_GET_TL_LINK_MODE = 0x02 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_SET_TL_LINK_MODE = 0x03 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_GET_TX_SUBLINK_MODE = 0x04 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_SET_TX_SUBLINK_MODE = 0x05 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_GET_RX_SUBLINK_MODE = 0x06 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_SET_RX_SUBLINK_MODE = 0x07 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_GET_RX_SUBLINK_DETECT = 0x08 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_SET_RX_SUBLINK_DETECT = 0x09 +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_WRITE_DISCOVERY_TOKEN = 0x0A +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_READ_DISCOVERY_TOKEN = 0x0B +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_TRAINING_COMPLETE = 0x0C +NV2080_CTRL_INTERNAL_NVLINK_CALLBACK_TYPE_GET_UPHY_LOAD = 0x0D +NV2080_CTRL_INTERNAL_NVLINK_CORE_CALLBACK_PARAMS_MESSAGE_ID = (0x24) +NV2080_CTRL_CMD_INTERNAL_NVLINK_CORE_CALLBACK = (0x20800a24) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_REMOTE_LOCAL_SID_PARAMS_MESSAGE_ID = (0x25) +NV2080_CTRL_CMD_INTERNAL_NVLINK_UPDATE_REMOTE_LOCAL_SID = (0x20800a25) +NV2080_CTRL_INTERNAL_NVLINK_GET_ALI_ENABLED_PARAMS_MESSAGE_ID = (0x29) +NV2080_CTRL_CMD_INTERNAL_NVLINK_GET_ALI_ENABLED = (0x20800a29) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_HSHUB_MUX_TYPE_PROGRAM = 0x0 +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_HSHUB_MUX_TYPE_RESET = 0x1 +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_HSHUB_MUX_PARAMS_MESSAGE_ID = (0x42) +NV2080_CTRL_CMD_INTERNAL_NVLINK_UPDATE_HSHUB_MUX = (0x20800a42) +NV2080_CTRL_INTERNAL_NVLINK_PRE_SETUP_NVLINK_PEER_PARAMS_MESSAGE_ID = (0x4E) +NV2080_CTRL_CMD_INTERNAL_NVLINK_PRE_SETUP_NVLINK_PEER = (0x20800a4e) +NV2080_CTRL_INTERNAL_NVLINK_POST_SETUP_NVLINK_PEER_PARAMS_MESSAGE_ID = (0x50) +NV2080_CTRL_CMD_INTERNAL_NVLINK_POST_SETUP_NVLINK_PEER = (0x20800a50) +NV2080_CTRL_CMD_INTERNAL_LOG_OOB_XID = (0x20800a56) +NV2080_INTERNAL_OOB_XID_MESSAGE_BUFFER_SIZE = (81) +NV2080_CTRL_INTERNAL_LOG_OOB_XID_PARAMS_MESSAGE_ID = (0x56) +NV2080_CTRL_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING_TYPE_SYSMEM = 0x1 +NV2080_CTRL_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING_TYPE_PEER = 0x2 +NV2080_CTRL_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING_PARAMS_MESSAGE_ID = (0x5F) +NV2080_CTRL_CMD_INTERNAL_NVLINK_REMOVE_NVLINK_MAPPING = (0x20800a5f) +NV2080_CTRL_INTERNAL_NVLINK_SAVE_RESTORE_HSHUB_STATE_PARAMS_MESSAGE_ID = (0x62) +NV2080_CTRL_CMD_INTERNAL_NVLINK_SAVE_RESTORE_HSHUB_STATE = (0x20800a62) +NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_FLAGS_SET = (0x00000000) +NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_FLAGS_SAVE = (0x00000001) +NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_FLAGS_RESTORE = (0x00000002) +NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_BUFFERREADY_PARAMS_MESSAGE_ID = (0x64) +NV2080_CTRL_CMD_INTERNAL_NVLINK_PROGRAM_BUFFERREADY = (0x20800a64) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_CURRENT_CONFIG_PARAMS_MESSAGE_ID = (0x78) +NV2080_CTRL_CMD_INTERNAL_NVLINK_UPDATE_CURRENT_CONFIG = (0x20800a78) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_PEER_LINK_MASK_PARAMS_MESSAGE_ID = (0x7D) +NV2080_CTRL_CMD_INTERNAL_NVLINK_UPDATE_PEER_LINK_MASK = (0x20800a7d) +NV2080_CTRL_INTERNAL_NVLINK_UPDATE_LINK_CONNECTION_PARAMS_MESSAGE_ID = (0x82) +NV2080_CTRL_CMD_INTERNAL_NVLINK_UPDATE_LINK_CONNECTION = (0x20800a82) +NV2080_CTRL_INTERNAL_NVLINK_ENABLE_LINKS_POST_TOPOLOGY_PARAMS_MESSAGE_ID = (0x83) +NV2080_CTRL_CMD_INTERNAL_NVLINK_ENABLE_LINKS_POST_TOPOLOGY = (0x20800a83) +NV2080_CTRL_INTERNAL_NVLINK_PRE_LINK_TRAIN_ALI_PARAMS_MESSAGE_ID = (0x84) +NV2080_CTRL_CMD_INTERNAL_NVLINK_PRE_LINK_TRAIN_ALI = (0x20800a84) +NV2080_CTRL_INTERNAL_NVLINK_MAX_ARR_SIZE = 64 +NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_MASK_POST_RX_DET_PARAMS_MESSAGE_ID = (0x85) +NV2080_CTRL_CMD_INTERNAL_NVLINK_GET_LINK_MASK_POST_RX_DET = (0x20800a85) +NV2080_CTRL_INTERNAL_NVLINK_LINK_TRAIN_ALI_PARAMS_MESSAGE_ID = (0x86) +NV2080_CTRL_CMD_INTERNAL_NVLINK_LINK_TRAIN_ALI = (0x20800a86) +NV2080_CTRL_INTERNAL_NVLINK_GET_NVLINK_DEVICE_INFO_INVALID_TIME_MS = 0 +NV2080_CTRL_INTERNAL_NVLINK_GET_NVLINK_DEVICE_INFO_PARAMS_MESSAGE_ID = (0x87) +NV2080_CTRL_CMD_INTERNAL_NVLINK_GET_NVLINK_DEVICE_INFO = (0x20800a87) +NV2080_CTRL_INTERNAL_NVLINK_MAX_LINKS_PER_IOCTRL_SW = 6 +NV2080_CTRL_INTERNAL_NVLINK_GET_IOCTRL_DEVICE_INFO_PARAMS_MESSAGE_ID = (0x8E) +NV2080_CTRL_CMD_INTERNAL_NVLINK_GET_IOCTRL_DEVICE_INFO = (0x20800a8e) +NV2080_CTRL_INTERNAL_NVLINK_PROGRAM_LINK_SPEED_PARAMS_MESSAGE_ID = (0x8F) +NV2080_CTRL_CMD_INTERNAL_NVLINK_PROGRAM_LINK_SPEED = (0x20800a8f) +NV2080_CTRL_INTERNAL_NVLINK_ARE_LINKS_TRAINED_PARAMS_MESSAGE_ID = (0x90) +NV2080_CTRL_CMD_INTERNAL_NVLINK_ARE_LINKS_TRAINED = (0x20800a90) +NV2080_CTRL_INTERNAL_NVLINK_RESET_FLAGS_ASSERT = (0x00000000) +NV2080_CTRL_INTERNAL_NVLINK_RESET_FLAGS_DEASSERT = (0x00000001) +NV2080_CTRL_INTERNAL_NVLINK_RESET_FLAGS_TOGGLE = (0x00000002) +NV2080_CTRL_INTERNAL_NVLINK_RESET_LINKS_PARAMS_MESSAGE_ID = (0x91) +NV2080_CTRL_CMD_INTERNAL_NVLINK_RESET_LINKS = (0x20800a91) +NV2080_CTRL_INTERNAL_NVLINK_DISABLE_DL_INTERRUPTS_PARAMS_MESSAGE_ID = (0x92) +NV2080_CTRL_CMD_INTERNAL_NVLINK_DISABLE_DL_INTERRUPTS = (0x20800a92) +NV2080_CTRL_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_INFO_PARAMS_MESSAGE_ID = (0x93) +NV2080_CTRL_CMD_INTERNAL_NVLINK_GET_LINK_AND_CLOCK_INFO = (0x20800a93) +NV2080_CTRL_INTERNAL_NVLINK_SETUP_NVLINK_SYSMEM_PARAMS_MESSAGE_ID = (0x94) +NV2080_CTRL_CMD_INTERNAL_NVLINK_SETUP_NVLINK_SYSMEM = (0x20800a94) +NV2080_CTRL_INTERNAL_NVLINK_PROCESS_FORCED_CONFIGS_PARAMS_MESSAGE_ID = (0x95) +NV2080_CTRL_CMD_INTERNAL_NVLINK_PROCESS_FORCED_CONFIGS = (0x20800a95) +NV2080_CTRL_INTERNAL_NVLINK_FORCE_DEBUG_STALL_AND_UPDATE_AMAP_PARAMS_MESSAGE_ID = (0x9C) +NV2080_CTRL_CMD_INTERNAL_NVLINK_FORCE_DEBUG_STALL_AND_UPDATE_AMAP = (0x20800b9c) +NV2080_CTRL_INTERNAL_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS_PARAMS_MESSAGE_ID = (0x96) +NV2080_CTRL_CMD_INTERNAL_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS = (0x20800a96) +NV2080_CTRL_INTERNAL_NVLINK_ENABLE_SYSMEM_NVLINK_ATS_PARAMS_MESSAGE_ID = (0x97) +NV2080_CTRL_CMD_INTERNAL_NVLINK_ENABLE_SYSMEM_NVLINK_ATS = (0x20800a97) +NV2080_CTRL_INTERNAL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK_PARAMS_MESSAGE_ID = (0xAB) +NV2080_CTRL_CMD_INTERNAL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK = (0x20800aab) +NV2080_CTRL_INTERNAL_SEND_CMC_LIBOS_BUFFER_INFO_PARAMS_MESSAGE_ID = (0x89) +NV2080_CTRL_CMD_INTERNAL_SEND_CMC_LIBOS_BUFFER_INFO = (0x20800a89) +NV2080_CTRL_INTERNAL_HFRP_INSTANCE_SIZE = 5 +NV2080_CTRL_INTERNAL_GPU_GET_HFRP_INFO_PARAMS_MESSAGE_ID = (0x7A) +NV2080_CTRL_CMD_INTERNAL_GPU_GET_HFRP_INFO = (0x20800a7a) +NV2080_CTRL_INTERNAL_SEND_CMC_UMD_API_OP_PARAMS_MESSAGE_ID = (0x7C) +NV2080_INTERNAL_CMC_UMD_API_TASK_CONSTRUCT = 0x0 +NV2080_INTERNAL_CMC_UMD_API_TASK_DESTROY = 0x1 +NV2080_INTERNAL_CMC_UMD_API_TASK_ADD_QUEUE = 0x2 +NV2080_CTRL_CMD_INTERNAL_SEND_CMC_UMD_API_OP = (0x20800a7c) +NV2080_CTRL_CMD_INTERNAL_NVLINK_REPLAY_SUPPRESSED_ERRORS = (0x20800b01) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS_MESSAGE_ID = (0x02) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER_V2 = (0x20800b03) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER_V2_PARAMS_MESSAGE_ID = (0x03) +NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS_MESSAGE_ID = (0x04) +NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_SM_ISSUE_THROTTLE_CTRL = (0x20800b05) +NV2080_CTRL_INTERNAL_STATIC_KGR_GET_SM_ISSUE_THROTTLE_CTRL_PARAMS_MESSAGE_ID = (0x05) +NV2080_CTRL_CMD_INTERNAL_FLCN_SET_VIDEO_EVENT_BUFFER_MEMORY = (0x20800b11) +NV2080_CTRL_INTERNAL_FLCN_SET_VIDEO_EVENT_BUFFER_MEMORY_PARAMS_MESSAGE_ID = (0x11) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_ROTATE_PER_CHANNEL_KEYS = (0x20800b07) +NV2080_CTRL_CMD_INTERNAL_CONF_COMPUTE_ROTATE_PER_CHANNEL_KEYS_PARAMS_MESSAGE_ID = (0x07) +NV2080_CTRL_CMD_INTERNAL_NVLINK_RC_USER_MODE_CHANNELS = (0x20800b08) +NV2080_CTRL_INTERNAL_NVLINK_RESUME_DISABLE_CHANNELS_PARAMS_MESSAGE_ID = (0xAD) +NV2080_CTRL_CMD_INTERNAL_NVLINK_RESUME_DISABLE_CHANNELS = (0x20800bad) +NV2080_CTRL_CMD_INTERNAL_GSYNC_GET_POSSIBLE_REFRESH_RATES = (0x20800b06) +NV2080_CTRL_INTERNAL_GSYNC_GET_POSSIBLE_REFRESH_RATES_PARAMS_MESSAGE_ID = (0x06) +NV2080_CTRL_CMD_INTERNAL_UCODE_INSTRUMENTATION_GET_STATE = (0x20800b12) +NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_STATE_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_CMD_INTERNAL_UCODE_INSTRUMENTATION_SET_STATE = (0x20800b13) +NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_SET_STATE_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_MAX_DATA_SIZE = 65536 +NV2080_CTRL_CMD_INTERNAL_UCODE_INSTRUMENTATION_GET_DATA = (0x20800b14) +NV2080_CTRL_INTERNAL_UCODE_INSTRUMENTATION_GET_DATA_PARAMS_MESSAGE_ID = (0x14) +NV2080_CTRL_CMD_INTERNAL_PERF_CLIENT_PERF_MODES_SET_CONTROL = (0x20800b20) +NV2080_CTRL_INTERNAL_PERF_CLIENT_PERF_MODES_SET_CONTROL_PARAMS_MESSAGE_ID = (0x20) +NV2080_CTRL_CMD_LPWR_DIFR_CTRL = (0x20802801) +NV2080_CTRL_LPWR_DIFR_CTRL_DISABLE = (0x00000001) +NV2080_CTRL_LPWR_DIFR_CTRL_ENABLE = (0x00000002) +NV2080_CTRL_LPWR_DIFR_CTRL_SUPPORT_STATUS = (0x00000003) +NV2080_CTRL_CMD_LPWR_DIFR_CTRL_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_LPWR_DIFR_SUPPORTED = (0x00000001) +NV2080_CTRL_LPWR_DIFR_NOT_SUPPORTED = (0x00000002) +NV2080_CTRL_CMD_LPWR_DIFR_PREFETCH_RESPONSE = (0x20802802) +NV2080_CTRL_LPWR_DIFR_PREFETCH_SUCCESS = (0x00000001) +NV2080_CTRL_LPWR_DIFR_PREFETCH_FAIL_OS_FLIPS_ENABLED = (0x00000002) +NV2080_CTRL_LPWR_DIFR_PREFETCH_FAIL_INSUFFICIENT_L2_SIZE = (0x00000003) +NV2080_CTRL_LPWR_DIFR_PREFETCH_FAIL_CE_HW_ERROR = (0x00000004) +NV2080_CTRL_CMD_LPWR_DIFR_PREFETCH_RESPONSE_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_MC_GET_ARCH_INFO = (0x20801701) +NV2080_CTRL_MC_GET_ARCH_INFO_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_T23X = (0xE0000023) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100 = (0x00000160) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100 = (0x00000170) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100 = (0x00000180) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100 = (0x00000190) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100 = (0x000001A0) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB200 = (0x000001B0) +NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GR100 = (0x000001C0) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_T234 = (0x00000004) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_T234D = (0x00000005) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_TU100 = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_TU102 = (0x00000002) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_TU104 = (0x00000004) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_TU106 = (0x00000006) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_TU116 = (0x00000008) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_TU117 = (0x00000007) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GA100 = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GA102 = (0x00000002) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GA103 = (0x00000003) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GA104 = (0x00000004) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GA106 = (0x00000006) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GA107 = (0x00000007) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GA10B = (0x0000000B) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GH100 = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GH100_SOC = (0x00000001) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD100 = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD000 = (0x00000001) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD101 = (0x00000001) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD102 = (0x00000002) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD103 = (0x00000003) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD104 = (0x00000004) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD106 = (0x00000006) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_AD107 = (0x00000007) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB100 = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB102 = (0x00000002) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB110 = (0x00000003) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB112 = (0x00000004) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB10C = (0x0000000C) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB200 = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB202 = (0x00000002) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB203 = (0x00000003) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB205 = (0x00000005) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB206 = (0x00000006) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB207 = (0x00000007) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB20B = (0x0000000B) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB20C = (0x0000000C) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GR100 = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GR102 = (0x00000002) +NV2080_CTRL_MC_ARCH_INFO_SUBREVISION_NO_SUBREVISION = (0x00000000) +NV2080_CTRL_MC_ARCH_INFO_SUBREVISION_P = (0x00000001) +NV2080_CTRL_MC_ARCH_INFO_SUBREVISION_Q = (0x00000002) +NV2080_CTRL_MC_ARCH_INFO_SUBREVISION_R = (0x00000003) +NV2080_CTRL_CMD_MC_SERVICE_INTERRUPTS = (0x20801702) +NV2080_CTRL_MC_ENGINE_ID_GRAPHICS = 0x00000001 +NV2080_CTRL_MC_ENGINE_ID_ALL = 0xFFFFFFFF +NV2080_CTRL_MC_SERVICE_INTERRUPTS_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_MC_GET_MANUFACTURER = (0x20801703) +NV2080_CTRL_MC_GET_MANUFACTURER_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_MC_CHANGE_REPLAYABLE_FAULT_OWNERSHIP = (0x2080170c) +NV2080_CTRL_MC_CHANGE_REPLAYABLE_FAULT_OWNERSHIP_PARAMS_MESSAGE_ID = (0xC) +NV2080_CTRL_CMD_MC_GET_ENGINE_NOTIFICATION_INTR_VECTORS = (0x2080170d) +NV2080_CTRL_MC_GET_ENGINE_NOTIFICATION_INTR_VECTORS_MAX_ENGINES = 256 +NV2080_CTRL_MC_GET_ENGINE_NOTIFICATION_INTR_VECTORS_PARAMS_MESSAGE_ID = (0xD) +NV2080_CTRL_CMD_MC_GET_STATIC_INTR_TABLE = (0x2080170e) +NV2080_CTRL_MC_GET_STATIC_INTR_TABLE_MAX = 32 +NV2080_INTR_TYPE_NULL = (0x00000000) +NV2080_INTR_TYPE_NON_REPLAYABLE_FAULT = (0x00000001) +NV2080_INTR_TYPE_NON_REPLAYABLE_FAULT_ERROR = (0x00000002) +NV2080_INTR_TYPE_INFO_FAULT = (0x00000003) +NV2080_INTR_TYPE_REPLAYABLE_FAULT = (0x00000004) +NV2080_INTR_TYPE_REPLAYABLE_FAULT_ERROR = (0x00000005) +NV2080_INTR_TYPE_ACCESS_CNTR = (0x00000006) +NV2080_INTR_TYPE_TMR = (0x00000007) +NV2080_INTR_TYPE_CPU_DOORBELL = (0x00000008) +NV2080_INTR_TYPE_GR0_FECS_LOG = (0x00000009) +NV2080_INTR_TYPE_GR1_FECS_LOG = (0x0000000A) +NV2080_INTR_TYPE_GR2_FECS_LOG = (0x0000000B) +NV2080_INTR_TYPE_GR3_FECS_LOG = (0x0000000C) +NV2080_INTR_TYPE_GR4_FECS_LOG = (0x0000000D) +NV2080_INTR_TYPE_GR5_FECS_LOG = (0x0000000E) +NV2080_INTR_TYPE_GR6_FECS_LOG = (0x0000000F) +NV2080_INTR_TYPE_GR7_FECS_LOG = (0x00000010) +NV2080_CTRL_MC_GET_STATIC_INTR_TABLE_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_CMD_MC_GET_INTR_CATEGORY_SUBTREE_MAP = (0x2080170f) +NV2080_CTRL_MC_GET_INTR_CATEGORY_SUBTREE_MAP_PARAMS_MESSAGE_ID = (0xF) +NV2080_CTRL_CMD_NVD_GET_DUMP_SIZE = (0x20802401) +NV2080_CTRL_NVD_GET_DUMP_SIZE_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_NVD_GET_DUMP = (0x20802402) +NV2080_CTRL_NVD_GET_DUMP_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_NVD_GET_NOCAT_JOURNAL = (0x20802409) +NV2080_NOCAT_JOURNAL_MAX_DIAG_BUFFER = 1024 +NV2080_NOCAT_JOURNAL_MAX_STR_LEN = 65 +NV2080_NOCAT_JOURNAL_MAX_JOURNAL_RECORDS = 10 +NV2080_NOCAT_JOURNAL_MAX_ASSERT_RECORDS = 32 +NV2080_NOCAT_JOURNAL_REC_TYPE_UNKNOWN = 0 +NV2080_NOCAT_JOURNAL_REC_TYPE_BUGCHECK = 1 +NV2080_NOCAT_JOURNAL_REC_TYPE_ENGINE = 2 +NV2080_NOCAT_JOURNAL_REC_TYPE_TDR = 3 +NV2080_NOCAT_JOURNAL_REC_TYPE_RC = 4 +NV2080_NOCAT_JOURNAL_REC_TYPE_ASSERT = 5 +NV2080_NOCAT_JOURNAL_REC_TYPE_ANY = 6 +NV2080_NOCAT_JOURNAL_REC_TYPE_COUNT = (0x7) +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_COLLECT_REQ_IDX = 0 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_GRANDFATHERED_RECORD_IDX = 1 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_ALLOCATED_IDX = 2 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_COLLECTED_IDX = 3 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_NOTIFICATIONS_IDX = 4 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_NOTIFICATION_FAIL_IDX = 5 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_ALLOC_FAILED_IDX = 6 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_COLLECT_FAILED_IDX = 7 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_COLLECT_LOCKED_OUT_IDX = 8 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_CTRL_INSERT_RECORDS_IDX = 9 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_RPC_INSERT_RECORDS_IDX = 10 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_JOURNAL_LOCKED_IDX = 11 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_JOURNAL_LOCK_UPDATED_IDX = 12 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_JOURNAL_UNLOCKED_IDX = 13 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_NO_RECORDS_IDX = 14 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_BAD_BUFFER_IDX = 15 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_MATCH_FOUND_IDX = 16 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_NO_MATCH_IDX = 17 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_CLOSEST_FOUND_IDX = 18 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_REQUESTED_IDX = 19 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_REPORTED_IDX = 20 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_DROPPED_IDX = 21 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_UPDATE_REQ_IDX = 22 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_UPDATED_IDX = 23 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_UPDATE_FAILED_IDX = 24 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_BUSY_IDX = 25 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_BAD_PARAM_IDX = 26 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_BAD_TYPE_IDX = 27 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_RES4_IDX = 28 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_RES3_IDX = 29 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_RES2_IDX = 30 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_RES1_IDX = 31 +NV2080_NOCAT_JOURNAL_REPORT_ACTIVITY_COUNTER_COUNT = (0x20) +NV2080_CTRL_NOCAT_GET_COUNTERS_ONLY = (0, 0) +NV2080_CTRL_NOCAT_GET_COUNTERS_ONLY_YES = 1 +NV2080_CTRL_NOCAT_GET_COUNTERS_ONLY_NO = 0 +NV2080_CTRL_NOCAT_GET_RESET_COUNTERS = (1, 1) +NV2080_CTRL_NOCAT_GET_RESET_COUNTERS_YES = 1 +NV2080_CTRL_NOCAT_GET_RESET_COUNTERS_NO = 0 +NV2080_CTRL_NVD_GET_NOCAT_JOURNAL_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_NVD_SET_NOCAT_JOURNAL_DATA = (0x2080240b) +NV2080_CTRL_NOCAT_JOURNAL_DATA_TYPE_EMPTY = 0 +NV2080_CTRL_NOCAT_JOURNAL_DATA_TYPE_TDR_REASON = 1 +NV2080_CTRL_NOCAT_JOURNAL_DATA_TYPE_SET_TAG = 2 +NV2080_CTRL_NOCAT_JOURNAL_DATA_TYPE_RCLOG = 3 +NV2080_CTRL_NOCAT_TDR_TYPE_NONE = 0 +NV2080_CTRL_NOCAT_TDR_TYPE_LEGACY = 1 +NV2080_CTRL_NOCAT_TDR_TYPE_FULLCHIP = 2 +NV2080_CTRL_NOCAT_TDR_TYPE_BUSRESET = 3 +NV2080_CTRL_NOCAT_TDR_TYPE_GC6_RESET = 4 +NV2080_CTRL_NOCAT_TDR_TYPE_SURPRISE_REMOVAL = 5 +NV2080_CTRL_NOCAT_TDR_TYPE_UCODE_RESET = 6 +NV2080_CTRL_NOCAT_TDR_TYPE_GPU_RC_RESET = 7 +NV2080_CTRL_NOCAT_TDR_TYPE_TEST = 8 +NV2080_CTRL_NOCAT_TAG_CLEAR = (0, 0) +NV2080_CTRL_NOCAT_TAG_CLEAR_YES = 1 +NV2080_CTRL_NOCAT_TAG_CLEAR_NO = 0 +NV2080_CTRL_NVD_SET_NOCAT_JOURNAL_DATA_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_CMD_NVD_INSERT_NOCAT_JOURNAL_RECORD = (0x2080240c) +NV2080_CTRL_NOCAT_INSERT_ALLOW_NULL_STR = (0, 0) +NV2080_CTRL_NOCAT_INSERT_ALLOW_NULL_STR_YES = 1 +NV2080_CTRL_NOCAT_INSERT_ALLOW_NULL_STR_NO = 0 +NV2080_CTRL_NOCAT_INSERT_ALLOW_0_LEN_BUFFER = (1, 1) +NV2080_CTRL_NOCAT_INSERT_ALLOW_0_LEN_BUFFER_YES = 1 +NV2080_CTRL_NOCAT_INSERT_ALLOW_0_LEN_BUFFER_NO = 0 +NV2080_CTRL_CMD_NVD_INSERT_NOCAT_JOURNAL_RECORD_PARAMS_MESSAGE_ID = (0xC) +NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_NVLINK_CAPS_TBL_SIZE = 2 +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_INVALID = (0x00000000) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_1_0 = (0x00000001) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_2_0 = (0x00000002) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_2_2 = (0x00000004) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_3_0 = (0x00000005) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_3_1 = (0x00000006) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_4_0 = (0x00000007) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_5_0 = (0x00000008) +NV2080_CTRL_NVLINK_CAPS_NVLINK_VERSION_6_0 = (0x00000009) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_INVALID = (0x00000000) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_1_0 = (0x00000001) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_2_0 = (0x00000002) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_2_2 = (0x00000004) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_3_0 = (0x00000005) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_3_1 = (0x00000006) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_4_0 = (0x00000007) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_5_0 = (0x00000008) +NV2080_CTRL_NVLINK_CAPS_NCI_VERSION_6_0 = (0x00000009) +NV2080_CTRL_CMD_NVLINK_GET_NVLINK_CAPS = (0x20803001) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_ID_FLAGS = (31, 0) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_ID_FLAGS_NONE = (0x00000000) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_ID_FLAGS_PCI = (0x00000001) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_ID_FLAGS_UUID = (0x00000002) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_EBRIDGE = (0x00000000) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_NPU = (0x00000001) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_GPU = (0x00000002) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_SWITCH = (0x00000003) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_TEGRA = (0x00000004) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_NONE = (0x000000FF) +NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_UUID_INVALID = (0xFFFFFFFF) +NV2080_CTRL_NVLINK_DEVICE_INFO_FABRIC_RECOVERY_STATUS_MASK_GPU_DEGRADED = (0, 0) +NV2080_CTRL_NVLINK_DEVICE_INFO_FABRIC_RECOVERY_STATUS_MASK_GPU_DEGRADED_FALSE = (0x00000000) +NV2080_CTRL_NVLINK_DEVICE_INFO_FABRIC_RECOVERY_STATUS_MASK_GPU_DEGRADED_TRUE = (0x00000001) +NV2080_CTRL_NVLINK_DEVICE_INFO_FABRIC_RECOVERY_STATUS_MASK_UNCONTAINED_ERROR_RECOVERY = (1, 1) +NV2080_CTRL_NVLINK_DEVICE_INFO_FABRIC_RECOVERY_STATUS_MASK_UNCONTAINED_ERROR_RECOVERY_INACTIVE = (0x00000000) +NV2080_CTRL_NVLINK_DEVICE_INFO_FABRIC_RECOVERY_STATUS_MASK_UNCONTAINED_ERROR_RECOVERY_ACTIVE = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_INIT = (0x00000000) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_HWCFG = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_SWCFG = (0x00000002) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_ACTIVE = (0x00000003) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_FAULT = (0x00000004) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_SLEEP = (0x00000005) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_RECOVERY = (0x00000006) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_RECOVERY_AC = (0x00000008) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_RECOVERY_RX = (0x0000000a) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_PHYSICAL_UP_NO_TRAFFIC = (0x0000000b) +NV2080_CTRL_NVLINK_STATUS_LINK_STATE_INVALID = (0xFFFFFFFF) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_HIGH_SPEED_1 = (0x00000000) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_SINGLE_LANE = (0x00000004) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_LOW_POWER = (0x00000004) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_TRAINING = (0x00000005) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_SAFE_MODE = (0x00000006) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_OFF = (0x00000007) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_TEST = (0x00000008) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_FAULT = (0x0000000e) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_RX_STATE_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_HIGH_SPEED_1 = (0x00000000) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_SINGLE_LANE = (0x00000004) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_LOW_POWER = (0x00000004) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_TRAINING = (0x00000005) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_SAFE_MODE = (0x00000006) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_OFF = (0x00000007) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_TEST = (0x00000008) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_FAULT = (0x0000000e) +NV2080_CTRL_NVLINK_STATUS_SUBLINK_TX_STATE_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_STATUS_PHY_NVHS = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_PHY_GRS = (0x00000002) +NV2080_CTRL_NVLINK_STATUS_PHY_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_1_0 = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_2_0 = (0x00000002) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_2_2 = (0x00000004) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_3_0 = (0x00000005) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_3_1 = (0x00000006) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_4_0 = (0x00000007) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_5_0 = (0x00000008) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_6_0 = (0x00000009) +NV2080_CTRL_NVLINK_STATUS_NVLINK_VERSION_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_1_0 = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_2_0 = (0x00000002) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_2_2 = (0x00000004) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_3_0 = (0x00000005) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_3_1 = (0x00000006) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_4_0 = (0x00000007) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_5_0 = (0x00000008) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_6_0 = (0x00000009) +NV2080_CTRL_NVLINK_STATUS_NCI_VERSION_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_STATUS_NVHS_VERSION_1_0 = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_NVHS_VERSION_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_STATUS_GRS_VERSION_1_0 = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_GRS_VERSION_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_STATUS_CONNECTED_TRUE = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_CONNECTED_FALSE = (0x00000000) +NV2080_CTRL_NVLINK_STATUS_LOOP_PROPERTY_LOOPBACK = (0x00000001) +NV2080_CTRL_NVLINK_STATUS_LOOP_PROPERTY_LOOPOUT = (0x00000002) +NV2080_CTRL_NVLINK_STATUS_LOOP_PROPERTY_NONE = (0x00000000) +NV2080_CTRL_NVLINK_STATUS_REMOTE_LINK_NUMBER_INVALID = (0x000000FF) +NV2080_CTRL_NVLINK_REFCLK_TYPE_INVALID = (0x00) +NV2080_CTRL_NVLINK_REFCLK_TYPE_NVHS = (0x01) +NV2080_CTRL_NVLINK_REFCLK_TYPE_PEX = (0x02) +NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS = (0x20803002) +NV2080_CTRL_NVLINK_GET_TL_ERRLOG_BIT = lambda intr,i: (((1 << i) & (intr)) >> i) # type: ignore +NV2080_CTRL_NVLINK_GET_TL_INTEN_BIT = lambda intr,i: NV2080_CTRL_NVLINK_GET_TL_ERRLOG_BIT(intr, i) # type: ignore +NV2080_CTRL_NVLINK_TL_ERRLOG_TRUE = (0x00000001) +NV2080_CTRL_NVLINK_TL_ERRLOG_FALSE = (0x00000000) +NV2080_CTRL_NVLINK_TL_INTEN_TRUE = (0x00000001) +NV2080_CTRL_NVLINK_TL_INTEN_FALSE = (0x00000000) +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXDLDATAPARITYEN = 0 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXDLCTRLPARITYEN = 1 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXPROTOCOLEN = 2 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXOVERFLOWEN = 3 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXRAMDATAPARITYEN = 4 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXRAMHDRPARITYEN = 5 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXRESPEN = 6 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_RXPOISONEN = 7 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_TXRAMDATAPARITYEN = 8 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_TXRAMHDRPARITYEN = 9 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_DLFLOWPARITYEN = 10 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_DLHDRPARITYEN = 12 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_TXCREDITEN = 13 +NV2080_CTRL_NVLINK_TL_INTEN_IDX_MAX = 14 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXDLDATAPARITYERR = 0 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXDLCTRLPARITYERR = 1 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXPROTOCOLERR = 2 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXOVERFLOWERR = 3 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXRAMDATAPARITYERR = 4 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXRAMHDRPARITYERR = 5 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXRESPERR = 6 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_RXPOISONERR = 7 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_TXRAMDATAPARITYERR = 8 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_TXRAMHDRPARITYERR = 9 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_DLFLOWPARITYERR = 10 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_DLHDRPARITYERR = 12 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_TXCREDITERR = 13 +NV2080_CTRL_NVLINK_TL_ERRLOG_IDX_MAX = 14 +NV2080_CTRL_NVLINK_SL0_SLSM_STATUS_TX_PRIMARY_STATE_HS = (0x00000000) +NV2080_CTRL_NVLINK_SL0_SLSM_STATUS_TX_PRIMARY_STATE_SINGLE_LANE = (0x00000004) +NV2080_CTRL_NVLINK_SL0_SLSM_STATUS_TX_PRIMARY_STATE_TRAIN = (0x00000005) +NV2080_CTRL_NVLINK_SL0_SLSM_STATUS_TX_PRIMARY_STATE_SAFE = (0x00000006) +NV2080_CTRL_NVLINK_SL0_SLSM_STATUS_TX_PRIMARY_STATE_OFF = (0x00000007) +NV2080_CTRL_NVLINK_SL1_SLSM_STATUS_RX_PRIMARY_STATE_HS = (0x00000000) +NV2080_CTRL_NVLINK_SL1_SLSM_STATUS_RX_PRIMARY_STATE_SINGLE_LANE = (0x00000004) +NV2080_CTRL_NVLINK_SL1_SLSM_STATUS_RX_PRIMARY_STATE_TRAIN = (0x00000005) +NV2080_CTRL_NVLINK_SL1_SLSM_STATUS_RX_PRIMARY_STATE_SAFE = (0x00000006) +NV2080_CTRL_NVLINK_SL1_SLSM_STATUS_RX_PRIMARY_STATE_OFF = (0x00000007) +NV2080_CTRL_NVLINK_ERR_INFO_FLAGS_DEFAULT = (0x0) +NV2080_CTRL_NVLINK_ERR_INFO_FLAGS_INTR_STATUS = (0x1) +NV2080_CTRL_NVLINK_ERR_INFO_FLAGS_ALI_STATUS = (0x2) +NV2080_CTRL_NVLINK_MAX_IOCTRLS = 3 +NV2080_CTRL_NVLINK_GET_ERR_INFO_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_NVLINK_GET_ERR_INFO = (0x20803003) +NV2080_CTRL_NVLINK_COUNTER_INVALID = 0x00000000 +NV2080_CTRL_NVLINK_COUNTER_TL_TX0 = 0x00000001 +NV2080_CTRL_NVLINK_COUNTER_TL_TX1 = 0x00000002 +NV2080_CTRL_NVLINK_COUNTER_TL_RX0 = 0x00000004 +NV2080_CTRL_NVLINK_COUNTER_TL_RX1 = 0x00000008 +NV2080_CTRL_NVLINK_LP_COUNTERS_DL = 0x00000010 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_ECC_LANE_L = lambda i: (1 << (i + 8)) # type: ignore +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_ECC_LANE__SIZE = 4 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_ECC_LANE_L0 = 0x00000100 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_ECC_LANE_L1 = 0x00000200 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_ECC_LANE_L2 = 0x00000400 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_ECC_LANE_L3 = 0x00000800 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_FLIT = 0x00010000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L = lambda i: (1 << (i + 17)) # type: ignore +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE__SIZE = 8 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L0 = 0x00020000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L1 = 0x00040000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L2 = 0x00080000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L3 = 0x00100000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L4 = 0x00200000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L5 = 0x00400000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L6 = 0x00800000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_LANE_L7 = 0x01000000 +NV2080_CTRL_NVLINK_COUNTER_DL_TX_ERR_REPLAY = 0x02000000 +NV2080_CTRL_NVLINK_COUNTER_DL_TX_ERR_RECOVERY = 0x04000000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_REPLAY = 0x08000000 +NV2080_CTRL_NVLINK_COUNTER_DL_RX_ERR_CRC_MASKED = 0x10000000 +NV2080_CTRL_NVLINK_COUNTER_MAX_TYPES = 32 +NV2080_CTRL_NVLINK_GET_COUNTERS_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_NVLINK_GET_COUNTERS = (0x20803004) +NV2080_CTRL_CMD_NVLINK_CLEAR_COUNTERS = (0x20803005) +NV2080_CTRL_NVLINK_CLEAR_COUNTERS_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_NVLINK_COUNTER_TP_TL_TX0 = 0 +NV2080_CTRL_NVLINK_COUNTER_TP_TL_TX1 = 1 +NV2080_CTRL_NVLINK_COUNTER_TP_TL_RX0 = 2 +NV2080_CTRL_NVLINK_COUNTER_TP_TL_RX1 = 3 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_ECC_LANE_SIZE = 4 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_ECC_LANE_L0 = 4 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_ECC_LANE_L1 = 5 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_ECC_LANE_L2 = 6 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_ECC_LANE_L3 = 7 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_SIZE = 8 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L0 = 8 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L1 = 9 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L2 = 10 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L3 = 11 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L4 = 12 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L5 = 13 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L6 = 14 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_LANE_L7 = 15 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_TX_ERR_RECOVERY = 16 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_TX_ERR_REPLAY = 17 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_REPLAY = 18 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_MASKED = 19 +NV2080_CTRL_NVLINK_COUNTER_ERR_DL_RX_ERR_CRC_FLIT = 20 +NV2080_CTRL_NVLINK_COUNTER_LP_DL = 21 +NV2080_CTRL_NVLINK_COUNTER_V1_MAX_COUNTER = NV2080_CTRL_NVLINK_COUNTER_LP_DL +NV2080_CTRL_NVLINK_COUNTER_XMIT_PACKETS = 22 +NV2080_CTRL_NVLINK_COUNTER_XMIT_BYTES = 23 +NV2080_CTRL_NVLINK_COUNTER_RCV_PACKETS = 24 +NV2080_CTRL_NVLINK_COUNTER_RCV_BYTES = 25 +NV2080_CTRL_NVLINK_COUNTER_LINK_ERROR_RECOVERY_COUNTER = 26 +NV2080_CTRL_NVLINK_COUNTER_LINK_DOWNED_COUNTER = 27 +NV2080_CTRL_NVLINK_COUNTER_LINK_RECOVERY_SUCCESSFUL_COUNTER = 28 +NV2080_CTRL_NVLINK_COUNTER_RCV_ERRORS = 29 +NV2080_CTRL_NVLINK_COUNTER_RCV_REMOTE_ERRORS = 30 +NV2080_CTRL_NVLINK_COUNTER_RCV_GENERAL_ERRORS = 31 +NV2080_CTRL_NVLINK_COUNTER_RCV_MALFORMED_PKT_ERROR = 32 +NV2080_CTRL_NVLINK_COUNTER_RCV_BUFFER_OVERRUN_ERROR = 33 +NV2080_CTRL_NVLINK_COUNTER_RCV_VL15DROPPED_ERROR = 34 +NV2080_CTRL_NVLINK_COUNTER_LINK_INTEGRITY_ERRORS = 35 +NV2080_CTRL_NVLINK_COUNTER_BUFFER_OVERRUN_ERRORS = 36 +NV2080_CTRL_NVLINK_COUNTER_XMIT_WAIT_TIME = 37 +NV2080_CTRL_NVLINK_COUNTER_XMIT_ERRORS = 38 +NV2080_CTRL_NVLINK_COUNTER_SINGLE_ERROR_BLOCKS = 39 +NV2080_CTRL_NVLINK_COUNTER_CORRECTED_BLOCKS = 40 +NV2080_CTRL_NVLINK_COUNTER_UNCORRECTED_BLOCKS = 41 +NV2080_CTRL_NVLINK_COUNTER_CORRECTED_SYMBOLS_LANE_0 = 42 +NV2080_CTRL_NVLINK_COUNTER_CORRECTED_SYMBOLS_LANE_1 = 43 +NV2080_CTRL_NVLINK_COUNTER_CORRECTED_SYMBOLS_TOTAL = 44 +NV2080_CTRL_NVLINK_COUNTER_RAW_ERRORS_LANE_0 = 45 +NV2080_CTRL_NVLINK_COUNTER_RAW_ERRORS_LANE_1 = 46 +NV2080_CTRL_NVLINK_COUNTER_CORRECTED_BITS = 47 +NV2080_CTRL_NVLINK_COUNTER_RAW_BER_LANE_0 = 48 +NV2080_CTRL_NVLINK_COUNTER_RAW_BER_LANE_1 = 49 +NV2080_CTRL_NVLINK_COUNTER_RAW_BER_TOTAL = 50 +NV2080_CTRL_NVLINK_COUNTER_NO_ERROR_BLOCKS = 51 +NV2080_CTRL_NVLINK_COUNTER_EFFECTIVE_ERRORS = 52 +NV2080_CTRL_NVLINK_COUNTER_EFFECTIVE_BER = 53 +NV2080_CTRL_NVLINK_COUNTER_SYMBOL_ERRORS = 54 +NV2080_CTRL_NVLINK_COUNTER_SYMBOL_BER = 55 +NV2080_CTRL_NVLINK_COUNTER_RECEIVED_BITS = 56 +NV2080_CTRL_NVLINK_COUNTER_SYNC_HEADER_ERRORS = 57 +NV2080_CTRL_NVLINK_COUNTER_TIME_SINCE_LAST_CLEAR = 58 +NV2080_CTRL_NVLINK_COUNTER_PLR_RCV_BLOCKS = 59 +NV2080_CTRL_NVLINK_COUNTER_PLR_RCV_BLOCKS_WITH_UNCORRECTABLE_ERRORS = 60 +NV2080_CTRL_NVLINK_COUNTER_PLR_RCV_BLOCKS_WITH_ERRORS = 61 +NV2080_CTRL_NVLINK_COUNTER_PLR_XMIT_BLOCKS = 62 +NV2080_CTRL_NVLINK_COUNTER_PLR_XMIT_RETRY_BLOCKS = 63 +NV2080_CTRL_NVLINK_COUNTER_PLR_XMIT_RETRY_EVENTS = 64 +NV2080_CTRL_NVLINK_COUNTER_PLR_BW_LOSS = 65 +NV2080_CTRL_NVLINK_COUNTER_NVLE_RX_GOOD = 66 +NV2080_CTRL_NVLINK_COUNTER_NVLE_RX_ERROR = 67 +NV2080_CTRL_NVLINK_COUNTER_NVLE_RX_AUTH = 68 +NV2080_CTRL_NVLINK_COUNTER_NVLE_TX_GOOD = 69 +NV2080_CTRL_NVLINK_COUNTER_NVLE_TX_ERROR = 70 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_0 = 71 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_1 = 72 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_2 = 73 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_3 = 74 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_4 = 75 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_5 = 76 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_6 = 77 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_7 = 78 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_8 = 79 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_9 = 80 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_10 = 81 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_11 = 82 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_12 = 83 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_13 = 84 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_14 = 85 +NV2080_CTRL_NVLINK_COUNTER_HISTORY_15 = 86 +NV2080_CTRL_NVLINK_COUNTER_TP_RX_DATA = 87 +NV2080_CTRL_NVLINK_COUNTER_TP_TX_DATA = 88 +NV2080_CTRL_NVLINK_COUNTER_TP_RX_RAW = 89 +NV2080_CTRL_NVLINK_COUNTER_TP_TX_RAW = 90 +NV2080_CTRL_NVLINK_COUNTER_L1_ENTRY = 91 +NV2080_CTRL_NVLINK_COUNTER_L1_ENTRY_FORCE = 92 +NV2080_CTRL_NVLINK_COUNTER_L1_EXIT = 93 +NV2080_CTRL_NVLINK_COUNTER_L1_EXIT_RECAL = 94 +NV2080_CTRL_NVLINK_COUNTER_L1_EXIT_REMOTE = 95 +NV2080_CTRL_NVLINK_COUNTER_L1_LP_STEADY_STATE_TIME = 96 +NV2080_CTRL_NVLINK_COUNTER_L1_HIGH_SPEED_STEADY_STATE_TIME = 97 +NV2080_CTRL_NVLINK_COUNTER_L1_OTHER_STATE_TIME = 98 +NV2080_CTRL_NVLINK_COUNTER_LP_LOCAL_ENTRY_TIME = 99 +NV2080_CTRL_NVLINK_COUNTER_LP_LOCAL_EXIT_TIME = 100 +NV2080_CTRL_NVLINK_COUNTER_LP_LOCAL_FULL_BW_ENTRY_TIME = 101 +NV2080_CTRL_NVLINK_COUNTER_LP_LOCAL_FULL_BW_EXIT_TIME = 102 +NV2080_CTRL_NVLINK_COUNTER_LP_REMOTE_ENTRY_TIME = 103 +NV2080_CTRL_NVLINK_COUNTER_LP_REMOTE_EXIT_TIME = 104 +NV2080_CTRL_NVLINK_COUNTER_LP_REMOTE_FULL_BW_ENTRY_TIME = 105 +NV2080_CTRL_NVLINK_COUNTER_LP_REMOTE_FULL_BW_EXIT_TIME = 106 +NV2080_CTRL_NVLINK_COUNTER_PLR_LAST_RAW_BER = 107 +NV2080_CTRL_NVLINK_COUNTER_PLR_XMIT_RETRY_EVENTS_WITHIN_T_SEC_MAX_LOW = 108 +NV2080_CTRL_NVLINK_COUNTER_L0_TX_IDLE_ENTRY = 109 +NV2080_CTRL_NVLINK_COUNTER_L0_TX_IDLE_ENTRY_DUE_TO_L1 = 110 +NV2080_CTRL_NVLINK_COUNTER_L0_TX_IDLE_REQ_NOT_YET_ACTIVE = 111 +NV2080_CTRL_NVLINK_COUNTER_L0_TX_IDLE_ACTIVE_DUE_TO_L1_REQ = 112 +NV2080_CTRL_NVLINK_COUNTER_L0_TX_IDLE_ACTIVE = 113 +NV2080_CTRL_NVLINK_COUNTER_TOTAL_SUCCESSFUL_RECOVERY_EVENTS = 114 +NV2080_CTRL_NVLINK_COUNTER_UNINTENTIONAL_LINK_DOWN_EVENTS_PHY_REC = 115 +NV2080_CTRL_NVLINK_COUNTER_INTENTIONAL_LINK_DOWN_EVENTS_PHY_REC = 116 +NV2080_CTRL_NVLINK_COUNTER_TIME_IN_LAST_HOST_SERDES_FEQ_RECOVERY = 117 +NV2080_CTRL_NVLINK_COUNTER_TOTAL_TIME_IN_HOST_SERDES_FEQ_RECOVERY = 118 +NV2080_CTRL_NVLINK_COUNTER_TOTAL_HOST_SERDES_FEQ_RECOVERY_COUNT = 119 +NV2080_CTRL_NVLINK_COUNTER_TOTAL_HOST_SERDES_FEQ_SUCCESSFUL_RECOVERY_COUNT = 120 +NV2080_CTRL_NVLINK_COUNTER_LAST_HOST_SERDES_FEQ_ATTEMPTS_COUNT = 121 +NV2080_CTRL_NVLINK_COUNTER_TIME_SINCE_LAST_RECOVERY = 122 +NV2080_CTRL_NVLINK_COUNTER_TIME_BETWEEN_LAST_2_RECOVERIES = 123 +NV2080_CTRL_NVLINK_COUNTER_LAST_EFF_BER_LF_COEF = 124 +NV2080_CTRL_NVLINK_COUNTER_LAST_EFF_BER_LF_MAGNITUDE = 125 +NV2080_CTRL_NVLINK_COUNTER_MEAN_EFF_BER_LF_COEF = 126 +NV2080_CTRL_NVLINK_COUNTER_MEAN_EFF_BER_LF_MAGNITUDE = 127 +NV2080_CTRL_NVLINK_COUNTER_MAX_EFF_BER_LF_COEF = 128 +NV2080_CTRL_NVLINK_COUNTER_MAX_EFF_BER_LF_MAGNITUDE = 129 +NV2080_CTRL_NVLINK_COUNTER_CONS_BELOW_EFF_BER_LF_THR = 130 +NV2080_CTRL_NVLINK_COUNTER_NUM_OF_EFF_BER_LF_WARNINGS = 131 +NV2080_CTRL_NVLINK_COUNTER_NUM_OF_EFF_BER_LF_ALARMS = 132 +NV2080_CTRL_NVLINK_COUNTER_LAST_RECOVERY_BW_LOSS_LF = 133 +NV2080_CTRL_NVLINK_COUNTER_MEAN_RECOVERY_BW_LOSS_LF = 134 +NV2080_CTRL_NVLINK_COUNTER_MAX_RECOVERY_BW_LOSS_LF = 135 +NV2080_CTRL_NVLINK_COUNTER_CONS_BELOW_RECOVERY_BW_LOSS_LF_THR = 136 +NV2080_CTRL_NVLINK_COUNTER_NUM_OF_RECOVERY_BW_LOSS_LF_WARNINGS = 137 +NV2080_CTRL_NVLINK_COUNTER_NUM_OF_RECOVERY_BW_LOSS_LF_ALARMS = 138 +NV2080_CTRL_NVLINK_COUNTER_LAST_PLR_TX_BW_LOSS_LF = 139 +NV2080_CTRL_NVLINK_COUNTER_MEAN_PLR_TX_BW_LOSS_LF = 140 +NV2080_CTRL_NVLINK_COUNTER_MAX_PLR_TX_BW_LOSS_LF = 141 +NV2080_CTRL_NVLINK_COUNTER_CONS_BELOW_PLR_TX_BW_LOSS_LF_THR = 142 +NV2080_CTRL_NVLINK_COUNTER_NUM_OF_PLR_TX_BW_LOSS_LF_WARNINGS = 143 +NV2080_CTRL_NVLINK_COUNTER_NUM_OF_PLR_TX_BW_LOSS_LF_ALARMS = 144 +NV2080_CTRL_NVLINK_COUNTERS_MAX = 145 +NV2080_CTRL_NVLINK_COUNTER_MAX_GROUPS = 3 +NV2080_CTRL_NVLINK_COUNTER_MAX_COUNTERS_PER_LINK_IN_REQ = 28 +NV2080_CTRL_NVLINK_COUNTER_V2_GROUP = lambda i: ((i) / 64) # type: ignore +NV2080_CTRL_NVLINK_GET_COUNTERS_V2_PARAMS_MESSAGE_ID = (0x50) +NV2080_CTRL_CMD_NVLINK_GET_COUNTERS_V2 = (0x20803050) +NV2080_CTRL_NVLINK_CLEAR_COUNTERS_V2_PARAMS_MESSAGE_ID = (0x51) +NV2080_CTRL_CMD_NVLINK_CLEAR_COUNTERS_V2 = (0x20803051) +NV2080_CTRL_CMD_NVLINK_INJECT_ERROR = (0x20803006) +NV2080_CTRL_NVLINK_INJECT_ERROR_PARAMS_MESSAGE_ID = (0x6) +NV2080_CTRL_NVLINK_UNIT_DL = 0x01 +NV2080_CTRL_NVLINK_UNIT_TL = 0x02 +NV2080_CTRL_NVLINK_UNIT_TLC_RX_0 = 0x03 +NV2080_CTRL_NVLINK_UNIT_TLC_RX_1 = 0x04 +NV2080_CTRL_NVLINK_UNIT_TLC_TX_0 = 0x05 +NV2080_CTRL_NVLINK_UNIT_MIF_RX_0 = 0x06 +NV2080_CTRL_NVLINK_UNIT_MIF_TX_0 = 0x07 +NV2080_CTRL_NVLINK_UNIT_MINION = 0x08 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_TX_ERR_TYPE = (31, 28) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_TX_ERR_TYPE_NO_ERROR = 0x00000000 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_TX_ERR_TYPE_RAW_BER = 0x00000001 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_TX_ERR_TYPE_EFFECTIVE_BER = 0x00000002 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_TX_ERR_ERR_INJECT_DURATION = (27, 12) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_TX_ERR_BER_MANTISSA = (11, 8) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_TX_ERR_BER_EXPONENT = (7, 0) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_INJECT_COUNT = (15, 0) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_STOMP = (16, 16) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_STOMP_DIS = 0x00000000 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_STOMP_EN = 0x00000001 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_POISON = (17, 17) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_POISON_DIS = 0x00000000 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_POISON_EN = 0x00000001 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_PKT_ERR_CLEAR_COUNTERS = (18, 18) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_AUTH_TAG_ERR_PIPE_INDEX = (3, 0) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_AUTH_TAG_ERR_AUTH_ERR = (4, 4) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_AUTH_TAG_ERR_AUTH_ERR_DIS = 0x00000000 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_AUTH_TAG_ERR_AUTH_ERR_EN = 0x00000001 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_LINK_ERR_FORCE_LINK_DOWN = (0, 0) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_LINK_ERR_FORCE_LINK_DOWN_DIS = 0x00000000 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_LINK_ERR_FORCE_LINK_DOWN_EN = 0x00000001 +NV2080_CTRL_NVLINK_SET_HW_ERROR_INJECT_PARAMS_MESSAGE_ID = (0x81) +NV2080_CTRL_CMD_NVLINK_SET_HW_ERROR_INJECT = (0x20803081) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_STS_LINK_STATE = (1, 0) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_STS_LINK_STATE_UP = 0x00000000 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_STS_LINK_STATE_DOWN_BY_REQUEST = 0x00000001 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_STS_LINK_STATE_DOWN_BY_HW_ERR = 0x00000002 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_STS_OPER_STS = (0, 0) +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_STS_OPER_STS_NO_ERR_INJECT = 0x00000000 +NV2080_CTRL_NVLINK_HW_ERROR_INJECT_STS_OPER_STS_PERFORMING_ERR_INJECT = 0x00000001 +NV2080_CTRL_NVLINK_GET_HW_ERROR_INJECT_PARAMS_MESSAGE_ID = (0x82) +NV2080_CTRL_CMD_NVLINK_GET_HW_ERROR_INJECT = (0x20803082) +NV2080_CTRL_CMD_NVLINK_GET_ERROR_RECOVERIES = (0x20803007) +NV2080_CTRL_CMD_NVLINK_GET_ERROR_RECOVERIES_PARAMS_MESSAGE_ID = (0x7) +NV2080_CTRL_CMD_NVLINK_GET_LINK_LAST_ERROR_REMOTE_TYPE = (0x20803008) +NV2080_CTRL_NVLINK_GET_LINK_LAST_ERROR_REMOTE_TYPE_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_CMD_NVLINK_GET_LINK_FATAL_ERROR_COUNTS = (0x20803009) +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_DL_DATA_PARITY = 0 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_DL_CTRL_PARITY = 1 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_PROTOCOL = 2 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_OVERFLOW = 3 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_RAM_DATA_PARITY = 4 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_RAM_HDR_PARITY = 5 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_RESP = 6 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_RX_POISON = 7 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_TX_RAM_DATA_PARITY = 8 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_TX_RAM_HDR_PARITY = 9 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_TX_CREDIT = 10 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_DL_FLOW_CTRL_PARITY = 11 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TL_DL_HDR_PARITY = 12 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_TX_RECOVERY_LONG = 13 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_TX_FAULT_RAM = 14 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_TX_FAULT_INTERFACE = 15 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_TX_FAULT_SUBLINK_CHANGE = 16 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_RX_FAULT_SUBLINK_CHANGE = 17 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_RX_FAULT_DL_PROTOCOL = 18 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_LTSSM_FAULT = 19 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DL_DATA_PARITY = 0 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DL_CTRL_PARITY = 1 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RAM_DATA_PARITY = 4 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RAM_HDR_PARITY = 5 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DATA_POISONED_PKT_RCVD = 7 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_RAM_DATA_PARITY = 8 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_RAM_HDR_PARITY = 9 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_DL_FLOW_CONTROL_PARITY = 11 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DL_HDR_PARITY = 20 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_INVALID_AE_FLIT_RCVD = 21 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_INVALID_BE_FLIT_RCVD = 22 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_INVALID_ADDR_ALIGN = 23 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_PKT_LEN = 24 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RSVD_CMD_ENC = 25 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RSVD_DAT_LEN_ENC = 26 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RSVD_ADDR_TYPE = 27 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RSVD_RSP_STATUS = 28 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RSVD_PKT_STATUS = 29 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_REQ = 30 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RSVD_CACHE_ATTR_ENC_IN_PROBE_RESP = 31 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DAT_LEN_GT_ATOMIC_REQ_MAX_SIZE = 32 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DAT_LEN_GT_RMW_REQ_MAX_SIZE = 33 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DAT_LEN_LT_ATR_RESP_MIN_SIZE = 34 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_INVALID_PO_FOR_CACHE_ATTR = 35 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_INVALID_COMPRESSED_RESP = 36 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RESP_STATUS_TARGET = 37 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_RESP_STATUS_UNSUPPORTED_REQUEST = 38 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_HDR_OVERFLOW = 39 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_DATA_OVERFLOW = 40 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_STOMPED_PKT_RCVD = 41 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_CORRECTABLE_INTERNAL = 42 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_UNSUPPORTED_VC_OVERFLOW = 43 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_UNSUPPORTED_NVLINK_CREDIT_RELEASE = 44 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_UNSUPPORTED_NCISOC_CREDIT_RELEASE = 45 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_HDR_CREDIT_OVERFLOW = 46 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_DATA_CREDIT_OVERFLOW = 47 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_DL_REPLAY_CREDIT_OVERFLOW = 48 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_UNSUPPORTED_VC_OVERFLOW = 49 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_STOMPED_PKT_SENT = 50 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_DATA_POISONED_PKT_SENT = 51 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_RESP_STATUS_TARGET = 52 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_RESP_STATUS_UNSUPPORTED_REQUEST = 53 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_MIF_RX_RAM_DATA_PARITY = 54 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_MIF_RX_RAM_HDR_PARITY = 55 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_MIF_TX_RAM_DATA_PARITY = 56 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_MIF_TX_RAM_HDR_PARITY = 57 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_RX_INVALID_COLLAPSED_RESPONSE = 58 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_NCISOC_HDR_ECC_DBE = 59 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_TLC_TX_NCISOC_PARITY = 60 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_LTSSM_FAULT_UP = 61 +NV2080_CTRL_NVLINK_FATAL_ERROR_TYPE_DL_LTSSM_FAULT_DOWN = 62 +NV2080_CTRL_NVLINK_NUM_FATAL_ERROR_TYPES = 63 +NV2080_CTRL_NVLINK_IS_FATAL_ERROR_COUNT_VALID = lambda count,supportedCounts: ( not not ((supportedCounts) & NVBIT64(count))) # type: ignore +NV2080_CTRL_NVLINK_GET_LINK_FATAL_ERROR_COUNTS_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_NVLINK_GET_LINK_NONFATAL_ERROR_RATES = (0x2080300a) +NV2080_CTRL_NVLINK_NONFATAL_ERROR_RATE_ENTRIES = 5 +NV2080_CTRL_NVLINK_GET_LINK_NONFATAL_ERROR_RATES_PARAMS_MESSAGE_ID = (0xA) +NV2080_CTRL_CMD_NVLINK_SET_ERROR_INJECTION_MODE = (0x2080300b) +NV2080_CTRL_NVLINK_SET_ERROR_INJECTION_MODE_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_CMD_NVLINK_SETUP_EOM = (0x2080300c) +NV2080_CTRL_CMD_NVLINK_SETUP_EOM_PARAMS_MESSAGE_ID = (0xC) +NV2080_CTRL_CMD_NVLINK_SET_POWER_STATE = (0x2080300d) +NV2080_CTRL_NVLINK_SET_POWER_STATE_PARAMS_MESSAGE_ID = (0xD) +NV2080_CTRL_NVLINK_POWER_STATE_L0 = (0x00) +NV2080_CTRL_NVLINK_POWER_STATE_L1 = (0x01) +NV2080_CTRL_NVLINK_POWER_STATE_L2 = (0x02) +NV2080_CTRL_NVLINK_POWER_STATE_L3 = (0x03) +NV2080_CTRL_CMD_NVLINK_GET_POWER_STATE = (0x2080300e) +NV2080_CTRL_NVLINK_GET_POWER_STATE_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_CMD_NVLINK_INJECT_TLC_ERROR = (0x2080300f) +NV2080_CTRL_NVLINK_INJECT_TLC_ERROR_PARAMS_MESSAGE_ID = (0xF) +NV2080_CTRL_CMD_NVLINK_GET_LINK_FOM_VALUES = (0x20803011) +NV2080_CTRL_NVLINK_MAX_LANES = 4 +NV2080_CTRL_CMD_NVLINK_GET_LINK_FOM_VALUES_PARAMS_MESSAGE_ID = (0x11) +NV2080_CTRL_CMD_NVLINK_SET_NVLINK_PEER = (0x20803012) +NV2080_CTRL_NVLINK_SET_NVLINK_PEER_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_CMD_NVLINK_READ_UPHY_PAD_LANE_REG = (0x20803013) +NV2080_CTRL_NVLINK_READ_UPHY_PAD_LANE_REG_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_NVLINK_GET_NVLINK_ECC_ERRORS_PARAMS_MESSAGE_ID = (0x14) +NV2080_CTRL_CMD_NVLINK_GET_NVLINK_ECC_ERRORS = (0x20803014) +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_TYPE_DATA_TX = 0 +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_TYPE_DATA_RX = 1 +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_TYPE_RAW_TX = 2 +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_TYPE_RAW_RX = 3 +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_TYPE_MAX = 4 +NV2080_CTRL_NVLINK_READ_TP_COUNTERS_PARAMS_MESSAGE_ID = (0x15) +NV2080_CTRL_CMD_NVLINK_READ_TP_COUNTERS = (0x20803015) +NV2080_CTRL_CMD_NVLINK_LOCK_LINK_POWER_STATE = (0x20803016) +NV2080_CTRL_CMD_NVLINK_LOCK_LINK_POWER_STATE_PARAMS_MESSAGE_ID = (0x16) +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_COUNT_TX_NVHS = 0 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_COUNT_TX_EIGHTH = 1 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_COUNT_TX_OTHER = 2 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_NUM_TX_LP_ENTER = 3 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_NUM_TX_LP_EXIT = 4 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_COUNT_TX_SLEEP = 5 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_HS_TIME = 6 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_LOCAL_FULL_BW_EXIT_TIME = 7 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_LOCAL_LP_ENTRY_TIME = 8 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_LOCAL_LP_EXIT_TIME = 9 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_LOCAL_FULL_BW_ENTRY_TIME = 10 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_REMOTE_FULL_BW_EXIT_TIME = 11 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_REMOTE_LP_ENTRY_TIME = 12 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_REMOTE_LP_EXIT_TIME = 13 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_REMOTE_FULL_BW_ENTRY_TIME = 14 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_OTHER_STATE_TIME = 15 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_MAX_COUNTERS = 16 +NV2080_CTRL_NVLINK_GET_LP_COUNTERS_PARAMS_MESSAGE_ID = (0x18) +NV2080_CTRL_CMD_NVLINK_GET_LP_COUNTERS = (0x20803018) +NV2080_CTRL_CMD_NVLINK_CLEAR_LP_COUNTERS = (0x20803052) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_DEFAULT = (0x00000000) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_NEA = (0x00000001) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_NEDR = (0x00000002) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_NEDW = (0x00000003) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_PHY_REMOTE = (0x00000004) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_PHY_LOCAL = (0x00000005) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_EXT_LOCAL = (0x00000006) +NV2080_CTRL_NVLINK_SET_LOOPBACK_MODE_PARAMS_MESSAGE_ID = (0x23) +NV2080_CTRL_CMD_NVLINK_SET_LOOPBACK_MODE = (0x20803023) +NV2080_CTRL_NVLINK_MAX_LINK_COUNT = 32 +NV2080_CTRL_NVLINK_GET_REFRESH_COUNTERS_PARAMS_MESSAGE_ID = (0x28) +NV2080_CTRL_CMD_NVLINK_GET_REFRESH_COUNTERS = (0x20803028) +NV2080_CTRL_NVLINK_CLEAR_REFRESH_COUNTERS_PARAMS_MESSAGE_ID = (0x29) +NV2080_CTRL_CMD_NVLINK_CLEAR_REFRESH_COUNTERS = (0x20803029) +NV2080_CTRL_NVLINK_GET_SET_NVSWITCH_FLA_ADDR_PARAMS_MESSAGE_ID = (0x38) +NV2080_CTRL_CMD_NVLINK_GET_SET_NVSWITCH_FLA_ADDR = (0x20803038) +NV2080_CTRL_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO_PARAMS_MESSAGE_ID = (0x39) +NV2080_CTRL_CMD_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO = (0x20803039) +NV2080_CTRL_CMD_NVLINK_ENABLE_LINKS = (0x2080303a) +NV2080_CTRL_NVLINK_PROCESS_INIT_DISABLED_LINKS_PARAMS_MESSAGE_ID = (0x3b) +NV2080_CTRL_CMD_NVLINK_PROCESS_INIT_DISABLED_LINKS = (0x2080303b) +NV2080_CTRL_NVLINK_EOM_CONTROL_PARAMS_MESSAGE_ID = (0x3c) +NV2080_CTRL_CMD_NVLINK_EOM_CONTROL = (0x2080303c) +NV2080_CTRL_NVLINK_INBAND_MAX_DATA_SIZE = 5120 +NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS_MESSAGE_ID = (0x3d) +NV2080_CTRL_CMD_READ_NVLINK_INBAND_RESPONSE = (0x2080303d) +NV2080_CTRL_NVLINK_L1_THRESHOLD_VALUE_DEFAULT = (0xFFFFFFFF) +NV2080_CTRL_NVLINK_SET_L1_THRESHOLD_PARAMS_MESSAGE_ID = (0x3e) +NV2080_CTRL_CMD_NVLINK_SET_L1_THRESHOLD = (0x2080303e) +NV2080_CTRL_NVLINK_GET_L1_THRESHOLD_PARAMS_MESSAGE_ID = (0x3f) +NV2080_CTRL_CMD_NVLINK_GET_L1_THRESHOLD = (0x2080303f) +NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS_MESSAGE_ID = (0x40) +NV2080_CTRL_CMD_NVLINK_INBAND_SEND_DATA = (0x20803040) +NV2080_CTRL_NVLINK_IS_GPU_DEGRADED_PARAMS_MESSAGE_ID = (0x41) +NV2080_CTRL_CMD_NVLINK_IS_GPU_DEGRADED = (0x20803041) +NV2080_CTRL_NVLINK_DIRECT_CONNECT_CHECK_PARAMS_MESSAGE_ID = (0x42) +NV2080_CTRL_CMD_NVLINK_DIRECT_CONNECT_CHECK = (0x20803042) +NV2080_CTRL_NVLINK_POST_FAULT_UP_PARAMS_MESSAGE_ID = (0x43) +NV2080_CTRL_CMD_NVLINK_POST_FAULT_UP = (0x20803043) +NV2080_CTRL_NVLINK_PORT_EVENT_COUNT_SIZE = 64 +NV2080_CTRL_NVLINK_GET_PORT_EVENTS_PARAMS_MESSAGE_ID = (0x44) +NV2080_CTRL_CMD_NVLINK_GET_PORT_EVENTS = (0x20803044) +NV2080_CTRL_NVLINK_CYCLE_LINK_PARAMS_MESSAGE_ID = (0x45) +NV2080_CTRL_CMD_NVLINK_CYCLE_LINK = (0x20803045) +NV2080_CTRL_NVLINK_IS_REDUCED_CONFIG_PARAMS_MESSAGE_ID = (0x46) +NV2080_CTRL_CMD_NVLINK_IS_REDUCED_CONFIG = (0x20803046) +NV2080_CTRL_NVLINK_PRM_ACCESS_MAX_LENGTH = 496 +NV2080_CTRL_CMD_NVLINK_GET_SUPPORTED_COUNTERS = (0x20803074) +NV2080_CTRL_NVLINK_GET_SUPPORTED_COUNTERS_PARAMS_MESSAGE_ID = (0x74) +NV2080_CTRL_CMD_NVLINK_GET_PLATFORM_INFO = (0x20803083) +NV2080_CTRL_NVLINK_GET_PLATFORM_INFO_PARAMS_MESSAGE_ID = (0x83) +NV2080_CTRL_CMD_NVLINK_READ_UPHY_CLN = (0x20803084) +NV2080_CTRL_NVLINK_READ_UPHY_CLN_REG_PARAMS_MESSAGE_ID = (0x84) +NV2080_CTRL_NVLINK_SUPPORTED_MAX_BW_MODE_COUNT = 23 +NV2080_CTRL_CMD_NVLINK_GET_SUPPORTED_BW_MODE = (0x20803085) +NV2080_CTRL_NVLINK_GET_SUPPORTED_BW_MODE_PARAMS_MESSAGE_ID = (0x85) +NV2080_CTRL_CMD_NVLINK_SET_BW_MODE_POLL_TIMEOUT_MS_INVALID = 0 +NV2080_CTRL_CMD_NVLINK_SET_BW_MODE = (0x20803086) +NV2080_CTRL_NVLINK_SET_BW_MODE_PARAMS_MESSAGE_ID = (0x86) +NV2080_CTRL_CMD_NVLINK_SET_BW_MODE_ASYNC = (0x208030a1) +NV2080_CTRL_NVLINK_SET_BW_MODE_ASYNC_PARAMS_MESSAGE_ID = (0xA1) +NV2080_CTRL_CMD_NVLINK_GET_BW_MODE = (0x20803087) +NV2080_CTRL_NVLINK_GET_BW_MODE_PARAMS_MESSAGE_ID = (0x87) +NV2080_CTRL_CMD_NVLINK_GET_LOCAL_DEVICE_INFO = (0x20803088) +NV2080_CTRL_NVLINK_GET_LOCAL_DEVICE_INFO_PARAMS_MESSAGE_ID = (0x88) +NV2080_CTRL_CMD_NVLINK_INJECT_SW_ERROR = (0x20803089) +NV2080_CTRL_NVLINK_INJECT_SW_ERROR_PARAMS_MESSAGE_ID = (0x89) +NV2080_CTRL_NVLINK_UPDATE_NVLE_TOPOLOGY = (0x2080308c) +NV2080_CTRL_NVLINK_UPDATE_NVLE_TOPOLOGY_PARAMS_MESSAGE_ID = (0x8c) +NV2080_CTRL_NVLINK_MAX_REMAP_TABLE_ENTRIES = 32 +NV2080_CTRL_NVLINK_MAX_REMAP_TABLE_ENTRIES_V2 = 4096 +NV2080_CTRL_NVLINK_REMAP_TABLE_ENTRIES_CHUNK = 128 +NV2080_CTRL_NVLINK_MAX_ALID_CLID_TABLE_ENTRIES = 128 +NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS = (0x2080308d) +NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_PARAMS_MESSAGE_ID = (0x8d) +NV2080_CTRL_CMD_NVLINK_POST_LAZY_ERROR_RECOVERY = (0x2080308a) +NV2080_CTRL_CMD_NVLINK_CONFIGURE_L1_TOGGLE = (0x2080308e) +NV2080_CTRL_NVLINK_CONFIGURE_L1_TOGGLE_PARAMS_MESSAGE_ID = (0x8E) +NV2080_CTRL_CMD_NVLINK_GET_L1_TOGGLE = (0x2080308f) +NV2080_CTRL_NVLINK_GET_L1_TOGGLE_PARAMS_MESSAGE_ID = (0x8F) +NV2080_CTRL_CMD_NVLINK_GET_NVLE_ENCRYPT_EN_INFO = (0x2080308b) +NV2080_CTRL_NVLINK_GET_NVLE_ENCRYPT_EN_INFO_PARAMS_MESSAGE_ID = (0x8b) +NV2080_CTRL_NVLINK_FIRMWARE_VERSION_LENGTH = (0x10) +NV2080_CTRL_NVLINK_SEMANTIC_VERSION_UCODE_TYPE_MSE = (0x01) +NV2080_CTRL_NVLINK_SEMANTIC_VERSION_UCODE_TYPE_NETIR = (0x02) +NV2080_CTRL_NVLINK_SEMANTIC_VERSION_UCODE_TYPE_NETIR_UPHY = (0x03) +NV2080_CTRL_NVLINK_SEMANTIC_VERSION_UCODE_TYPE_NETIR_CLN = (0x04) +NV2080_CTRL_NVLINK_SEMANTIC_VERSION_UCODE_TYPE_NETIR_DLN = (0x05) +NV2080_CTRL_NVLINK_GET_FIRMWARE_VERSION_INFO = (0x20803091) +NV2080_CTRL_NVLINK_GET_FIRMWARE_VERSION_INFO_PARAMS_MESSAGE_ID = (0x91) +NV2080_CTRL_CMD_NVLINK_SET_NVLE_ENABLED_STATE = (0x20803092) +NV2080_CTRL_NVLINK_SET_NVLE_ENABLED_STATE_PARAMS_MESSAGE_ID = (0x92) +NV2080_CTRL_CMD_NVLINK_PRM_ACCESS = (0x20803095) +NV2080_CTRL_NVLINK_PRM_ACCESS_PARAMS_MESSAGE_ID = (0x95) +NV2080_CTRL_NVLINK_FIXED_POINT_HISTOGRAM_MAX_SIZE = 32 +NV2080_CTRL_CMD_NVLINK_GET_LINK_ACCUMULATIVE_METRIC_DATA = (0x20803097) +NV2080_CTRL_NVLINK_GET_LINK_ACCUMULATIVE_METRIC_DATA_PARAMS_MESSAGE_ID = (0x97) +NV2080_CTRL_CMD_NVLINK_GET_LINK_RECORD_METRIC_DATA = (0x20803098) +NV2080_CTRL_NVLINK_GET_LINK_RECORD_METRIC_DATA_PARAMS_MESSAGE_ID = (0x98) +NV2080_CTRL_CMD_NVLINK_GET_DEVICE_RECORD_METRIC_DATA = (0x20803099) +NV2080_CTRL_NVLINK_GET_DEVICE_RECORD_METRIC_DATA_PARAMS_MESSAGE_ID = (0x99) +NV2080_CTRL_NVLINK_SAVE_NODE_HOSTNAME_PARAMS_MESSAGE_ID = (0x9A) +NV2080_CTRL_CMD_NVLINK_SAVE_NODE_HOSTNAME = (0x2080309a) +NV2080_CTRL_NVLINK_GET_SAVED_NODE_HOSTNAME_PARAMS_MESSAGE_ID = (0x9B) +NV2080_CTRL_CMD_NVLINK_GET_SAVED_NODE_HOSTNAME = (0x2080309b) +NV2080_CTRL_NVLINK_UPDATE_CLID_PARAMS_MESSAGE_ID = (0x9C) +NV2080_CTRL_CMD_NVLINK_UPDATE_CLID = (0x2080309c) +NV2080_CTRL_NVLINK_LOCK_REMAP_TABLE_AND_MSE_PARAMS_MESSAGE_ID = (0x9D) +NV2080_CTRL_CMD_NVLINK_LOCK_REMAP_TABLE_AND_MSE = (0x2080309d) +NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_PARAMS_MESSAGE_ID = (0x9E) +NV2080_CTRL_CMD_NVLINK_GET_REMAP_TABLE_INFO = (0x2080309e) +NV2080_CTRL_NVLINK_GET_NVLE_PKT_COUNTERS_PARAMS_MESSAGE_ID = (0x9F) +NV2080_CTRL_CMD_NVLINK_GET_NVLE_PKT_COUNTERS = (0x2080309f) +NV2080_CTRL_NVLINK_NVLE_AES_256_GCM_KEY_SIZE_BYTES = 32 +NV2080_CTRL_NVLINK_SETUP_NVLE_ENCRYPTION_KEY_PARAMS_MESSAGE_ID = (0xA0) +NV2080_CTRL_CMD_NVLINK_SETUP_NVLE_ENCRYPTION_KEY = (0x208030a0) +NV2080_CTRL_NVLINK_GET_REMAP_TABLE_INFO_V2_PARAMS_MESSAGE_ID = (0xA2) +NV2080_CTRL_CMD_NVLINK_GET_REMAP_TABLE_INFO_V2 = (0x208030a2) +NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_V2_PARAMS_MESSAGE_ID = (0xA3) +NV2080_CTRL_NVLINK_GET_UPDATE_NVLE_LIDS_V2 = (0x208030a3) +NV2080_CTRL_NVLINK_GET_LINK_TRAINING_TIME_PARAMS_MESSAGE_ID = (0xA4) +NV2080_CTRL_CMD_NVLINK_GET_LINK_TRAINING_TIME = (0x208030a4) +NV2080_CTRL_NVLINK_MAX_LINKS = 64 +NV2080_CTRL_NVLINK_MAX_ARR_SIZE = 64 +NV2080_CTRL_LINKS_PER_MASK_INDEX = 64 +NV2080_CTRL_NVLINK_MAX_MASK_SIZE = (0x1) +NV_SUBPROC_NAME_MAX_LENGTH = 100 +NV2080_CTRL_PERF_BOOST_FLAGS_CMD = (1, 0) +NV2080_CTRL_PERF_BOOST_FLAGS_CMD_CLEAR = (0x00000000) +NV2080_CTRL_PERF_BOOST_FLAGS_CMD_BOOST_1LEVEL = (0x00000001) +NV2080_CTRL_PERF_BOOST_FLAGS_CMD_BOOST_TO_MAX = (0x00000002) +NV2080_CTRL_PERF_BOOST_FLAGS_CUDA = (4, 4) +NV2080_CTRL_PERF_BOOST_FLAGS_CUDA_NO = (0x00000000) +NV2080_CTRL_PERF_BOOST_FLAGS_CUDA_YES = (0x00000001) +NV2080_CTRL_PERF_BOOST_FLAGS_ASYNC = (5, 5) +NV2080_CTRL_PERF_BOOST_FLAGS_ASYNC_NO = (0x00000000) +NV2080_CTRL_PERF_BOOST_FLAGS_ASYNC_YES = (0x00000001) +NV2080_CTRL_PERF_BOOST_FLAGS_CUDA_PRIORITY = (6, 6) +NV2080_CTRL_PERF_BOOST_FLAGS_CUDA_PRIORITY_DEFAULT = (0x00000000) +NV2080_CTRL_PERF_BOOST_FLAGS_CUDA_PRIORITY_HIGH = (0x00000001) +NV2080_CTRL_PERF_BOOST_FLAGS_MULTIMEDIA = (7, 7) +NV2080_CTRL_PERF_BOOST_FLAGS_MULTIMEDIA_NO = (0x00000000) +NV2080_CTRL_PERF_BOOST_FLAGS_MULTIMEDIA_YES = (0x00000001) +NV2080_CTRL_PERF_BOOST_DURATION_MAX = 3600 +NV2080_CTRL_PERF_BOOST_DURATION_INFINITE = 0xffffffff +NV2080_CTRL_CMD_PERF_BOOST = (0x2080200a) +NV2080_CTRL_PERF_BOOST_PARAMS_MESSAGE_ID = (0xA) +NV2080_CTRL_CMD_PERF_RESERVE_PERFMON_HW = (0x20802093) +NV2080_CTRL_PERF_RESERVE_PERFMON_HW_PARAMS_MESSAGE_ID = (0x93) +NV2080_CTRL_PERF_POWER_SOURCE_AC = (0x00000000) +NV2080_CTRL_PERF_POWER_SOURCE_BATTERY = (0x00000001) +NV2080_CTRL_CMD_PERF_SET_POWERSTATE = (0x2080205b) +NV2080_CTRL_PERF_SET_POWERSTATE_PARAMS_MESSAGE_ID = (0x5B) +NV2080_CTRL_CMD_PERF_SET_AUX_POWER_STATE = (0x20802092) +NV2080_CTRL_PERF_SET_AUX_POWER_STATE_PARAMS_MESSAGE_ID = (0x92) +NV2080_CTRL_PERF_AUX_POWER_STATE_P0 = (0x00000000) +NV2080_CTRL_PERF_AUX_POWER_STATE_P1 = (0x00000001) +NV2080_CTRL_PERF_AUX_POWER_STATE_P2 = (0x00000002) +NV2080_CTRL_PERF_AUX_POWER_STATE_P3 = (0x00000003) +NV2080_CTRL_PERF_AUX_POWER_STATE_P4 = (0x00000004) +NV2080_CTRL_PERF_AUX_POWER_STATE_COUNT = (0x00000005) +NV2080_CTRL_PERF_RATED_TDP_STATUS_PARAMS_MESSAGE_ID = (0x6D) +NV2080_CTRL_CMD_PERF_RATED_TDP_GET_CONTROL = (0x2080206e) +NV2080_CTRL_PERF_RATED_TDP_GET_CONTROL_PARAMS_MESSAGE_ID = (0x6E) +NV2080_CTRL_CMD_PERF_RATED_TDP_SET_CONTROL = (0x2080206f) +NV2080_CTRL_PERF_RATED_TDP_SET_CONTROL_PARAMS_MESSAGE_ID = (0x6F) +NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_PARAM_MESSAGE_ID = (0x83) +NV2080_CTRL_PERF_GPUMON_SAMPLE_COUNT_PERFMON_UTIL = 72 +NV2080_CTRL_CMD_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2 = (0x20802096) +NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS_MESSAGE_ID = (0x96) +NV2080_CTRL_CMD_PERF_GPU_IS_IDLE = (0x20802089) +NV2080_CTRL_PERF_GPU_IS_IDLE_PARAMS_MESSAGE_ID = (0x89) +NV2080_CTRL_PERF_GPU_IS_IDLE_TRUE = (0x00000001) +NV2080_CTRL_PERF_GPU_IS_IDLE_FALSE = (0x00000002) +NV2080_CTRL_CMD_PERF_AGGRESSIVE_PSTATE_NOTIFY = (0x2080208f) +NV2080_CTRL_PERF_AGGRESSIVE_PSTATE_NOTIFY_PARAMS_MESSAGE_ID = (0x8F) +NV2080_CTRL_PERF_CLK_MAX_DOMAINS = 32 +NV2080_CTRL_CMD_PERF_GET_LEVEL_INFO = (0x20802002) +NV2080_CTRL_PERF_GET_LEVEL_INFO_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_PERF_GET_LEVEL_INFO_V2 = (0x2080200b) +NV2080_CTRL_PERF_GET_LEVEL_INFO_V2_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_PERF_GET_LEVEL_INFO_LEVEL_STANDARD = 0 +NV2080_CTRL_PERF_GET_LEVEL_INFO_LEVEL_CONSERVATIVE = 1 +NV2080_CTRL_PERF_GET_LEVEL_INFO_LEVEL_AGGRESSIVE = 2 +NV2080_CTRL_PERF_GET_LEVEL_INFO_FLAGS_TYPE = (0, 0) +NV2080_CTRL_PERF_GET_LEVEL_INFO_FLAGS_TYPE_DEFAULT = (0x00000000) +NV2080_CTRL_PERF_GET_LEVEL_INFO_FLAGS_TYPE_OVERCLOCK = (0x00000001) +NV2080_CTRL_PERF_GET_LEVEL_INFO_FLAGS_MODE = (2, 1) +NV2080_CTRL_PERF_GET_LEVEL_INFO_FLAGS_MODE_NONE = (0x00000000) +NV2080_CTRL_PERF_GET_LEVEL_INFO_FLAGS_MODE_DESKTOP = (0x00000001) +NV2080_CTRL_PERF_GET_LEVEL_INFO_FLAGS_MODE_MAXPERF = (0x00000002) +NV2080_CTRL_CMD_PERF_GET_VID_ENG_PERFMON_SAMPLE = (0x20802087) +NV2080_CTRL_PERF_GET_VID_ENG_PERFMON_SAMPLE_PARAMS_MESSAGE_ID = (0x87) +NV2080_CTRL_CMD_PERF_GET_POWERSTATE = (0x2080205a) +NV2080_CTRL_PERF_GET_POWERSTATE_PARAMS_MESSAGE_ID = (0x5A) +NV2080_CTRL_CMD_PERF_NOTIFY_VIDEOEVENT = (0x2080205d) +NV2080_CTRL_PERF_NOTIFY_VIDEOEVENT_PARAMS_MESSAGE_ID = (0x5D) +NV2080_CTRL_PERF_VIDEOEVENT_EVENT_MASK = (0x0000ffff) +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_HD_START = (0x00000001) +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_HD_STOP = (0x00000002) +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_START = NV2080_CTRL_PERF_VIDEOEVENT_STREAM_HD_START +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_STOP = NV2080_CTRL_PERF_VIDEOEVENT_STREAM_HD_STOP +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_SD_START = (0x00000003) +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_SD_STOP = (0x00000004) +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_4K_START = (0x00000005) +NV2080_CTRL_PERF_VIDEOEVENT_STREAM_4K_STOP = (0x00000006) +NV2080_CTRL_PERF_VIDEOEVENT_OFA_START = (0x00000007) +NV2080_CTRL_PERF_VIDEOEVENT_OFA_STOP = (0x00000008) +NV2080_CTRL_PERF_VIDEOEVENT_FLAG_LINEAR_MODE = (0x00010000) +NV2080_CTRL_PERF_PSTATES_UNDEFINED = (0x00000000) +NV2080_CTRL_PERF_PSTATES_CLEAR_FORCED = (0x00000000) +NV2080_CTRL_PERF_PSTATES_MIN = (0x00000001) +NV2080_CTRL_PERF_PSTATES_P0 = (0x00000001) +NV2080_CTRL_PERF_PSTATES_P1 = (0x00000002) +NV2080_CTRL_PERF_PSTATES_P2 = (0x00000004) +NV2080_CTRL_PERF_PSTATES_P3 = (0x00000008) +NV2080_CTRL_PERF_PSTATES_P4 = (0x00000010) +NV2080_CTRL_PERF_PSTATES_P5 = (0x00000020) +NV2080_CTRL_PERF_PSTATES_P6 = (0x00000040) +NV2080_CTRL_PERF_PSTATES_P7 = (0x00000080) +NV2080_CTRL_PERF_PSTATES_P8 = (0x00000100) +NV2080_CTRL_PERF_PSTATES_P9 = (0x00000200) +NV2080_CTRL_PERF_PSTATES_P10 = (0x00000400) +NV2080_CTRL_PERF_PSTATES_P11 = (0x00000800) +NV2080_CTRL_PERF_PSTATES_P12 = (0x00001000) +NV2080_CTRL_PERF_PSTATES_P13 = (0x00002000) +NV2080_CTRL_PERF_PSTATES_P14 = (0x00004000) +NV2080_CTRL_PERF_PSTATES_P15 = (0x00008000) +NV2080_CTRL_PERF_PSTATES_MAX = NV2080_CTRL_PERF_PSTATES_P15 +NV2080_CTRL_PERF_PSTATES_SKIP_ENTRY = (0x10000) +NV2080_CTRL_PERF_PSTATES_ALL = (0xffff) +NV2080_CTRL_CMD_PERF_GET_CURRENT_PSTATE = (0x20802068) +NV2080_CTRL_PERF_GET_CURRENT_PSTATE_PARAMS_MESSAGE_ID = (0x68) +NV2080_CTRL_CMD_PERF_GET_TEGRA_PERFMON_SAMPLE = (0x20802069) +NV2080_CTRL_PERF_GET_TEGRA_PERFMON_SAMPLE_PARAMS_MESSAGE_ID = (0x69) +NV2080_CTRL_PMGR_MODULE_INFO_NVSWITCH_NOT_SUPPORTED = 0 +NV2080_CTRL_PMGR_MODULE_INFO_NVSWITCH_SUPPORTED = 1 +NV2080_CTRL_PMGR_MODULE_INFO_NVSWITCH_NOT_APPLICABLE = 2 +NV2080_CTRL_PMGR_MODULE_INFO_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_PMGR_GET_MODULE_INFO = (0x20802609) +NV2080_CTRL_CMD_GC6_ENTRY = (0x2080270d) +NV2080_CTRL_GC6_ENTRY_PARAMS_MESSAGE_ID = (0xD) +NV2080_CTRL_CMD_GC6_EXIT = (0x2080270e) +NV2080_CTRL_GC6_EXIT_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_RC_READ_VIRTUAL_MEM_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_RC_READ_VIRTUAL_MEM = (0x20802204) +NV2080_CTRL_RC_GET_ERROR_COUNT_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_CMD_RC_GET_ERROR_COUNT = (0x20802205) +NV2080_CTRL_RC_ERROR_PARAMS_BUFFER_SIZE = (0x2000) +NV2080_CTRL_CMD_RC_GET_ERROR = (0x20802206) +NV2080_CTRL_RC_GET_ERROR_V2_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_CMD_RC_GET_ERROR_V2 = (0x20802213) +NV2080_CTRL_CMD_RC_SET_CLEAN_ERROR_HISTORY = (0x20802207) +NV2080_CTRL_RC_GET_WATCHDOG_INFO_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_RC_GET_WATCHDOG_INFO = (0x20802209) +NV2080_CTRL_RC_GET_WATCHDOG_INFO_FLAGS_NONE = (0x00000000) +NV2080_CTRL_RC_GET_WATCHDOG_INFO_FLAGS_DISABLED = (0x00000001) +NV2080_CTRL_RC_GET_WATCHDOG_INFO_FLAGS_RUNNING = (0x00000002) +NV2080_CTRL_RC_GET_WATCHDOG_INFO_FLAGS_INITIALIZED = (0x00000004) +NV2080_CTRL_RC_GET_WATCHDOG_INFO_FLAGS_PAUSED = (0x00000008) +NV2080_CTRL_CMD_RC_DISABLE_WATCHDOG = (0x2080220a) +NV2080_CTRL_CMD_RC_ENABLE_WATCHDOG = (0x2080220b) +NV2080_CTRL_CMD_RC_RELEASE_WATCHDOG_REQUESTS = (0x2080220c) +NV2080_CTRL_CMD_SET_RC_RECOVERY = (0x2080220d) +NV2080_CTRL_SET_RC_RECOVERY_PARAMS_MESSAGE_ID = (0xD) +NV2080_CTRL_CMD_GET_RC_RECOVERY = (0x2080220e) +NV2080_CTRL_GET_RC_RECOVERY_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_CMD_RC_RECOVERY_DISABLED = (0x00000000) +NV2080_CTRL_CMD_RC_RECOVERY_ENABLED = (0x00000001) +NV2080_CTRL_CMD_TDR_SET_TIMEOUT_STATE = (0x2080220f) +NV2080_CTRL_TDR_SET_TIMEOUT_STATE_PARAMS_MESSAGE_ID = (0xF) +NV2080_CTRL_TDR_SET_TIMEOUT_STATE_CMD_GPU_RESET_BEGIN = (0x00000000) +NV2080_CTRL_TDR_SET_TIMEOUT_STATE_CMD_GPU_RESET_END = (0x00000001) +NV2080_CTRL_TDR_SET_TIMEOUT_STATE_STATUS_SUCCESS = (0x00000000) +NV2080_CTRL_TDR_SET_TIMEOUT_STATE_STATUS_FAIL = (0x00000001) +NV2080_CTRL_CMD_RC_SOFT_DISABLE_WATCHDOG = (0x20802210) +NV2080_CTRL_CMD_SET_RC_INFO = (0x20802211) +NV2080_CTRL_SET_RC_INFO_PARAMS_MESSAGE_ID = (0x11) +NV2080_CTRL_CMD_GET_RC_INFO = (0x20802212) +NV2080_CTRL_GET_RC_INFO_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_CMD_RC_INFO_MODE_DISABLE = (0x00000000) +NV2080_CTRL_CMD_RC_INFO_MODE_ENABLE = (0x00000001) +NV2080_CTRL_CMD_RC_INFO_BREAK_DISABLE = (0x00000000) +NV2080_CTRL_CMD_RC_INFO_BREAK_ENABLE = (0x00000001) +NV2080_CTRL_CMD_SET_RC_WATCHDOG_INFO = (0x20802214) +NV2080_CTRL_SET_RC_WATCHDOG_INFO_PARAMS_MESSAGE_ID = (0x14) +NV2080_CTRL_CMD_GET_RC_WATCHDOG_INFO = (0x20802215) +NV2080_CTRL_GET_RC_WATCHDOG_INFO_PARAMS_MESSAGE_ID = (0x15) +NV2080_CTRL_CMD_RC_PAUSE_WATCHDOG = (0x20802216) +RM_GSP_SPDM_CMD_ID_CC_INIT = (0x1) +RM_GSP_SPDM_CMD_ID_CC_DEINIT = (0x2) +RM_GSP_SPDM_CMD_ID_CC_CTRL = (0x3) +RM_GSP_SPDM_CMD_ID_CC_INIT_RM_DATA = (0x4) +RM_GSP_SPDM_CMD_ID_CC_HEARTBEAT_CTRL = (0x5) +RM_GSP_SPDM_CMD_ID_FIPS_SELFTEST = (0x6) +RM_GSP_SPDM_CMD_ID_INVALID_COMMAND = (0xFF) +SPDM_SESSION_ESTABLISHMENT_TRANSCRIPT_BUFFER_SIZE = 0x2400 +RSVD7_SIZE = 16 +RSVD8_SIZE = 2 +CE_FIPS_SELF_TEST_DATA_SIZE = 16 +CE_FIPS_SELF_TEST_AUTH_TAG_SIZE = 16 +CE_FIPS_SELF_TEST_IV_SIZE = 12 +RM_GSP_SPDM_MSG_ID_CC_INIT = (0x1) +RM_GSP_SPDM_MSG_ID_CC_DEINIT = (0x2) +RM_GSP_SPDM_MSG_ID_CC_CTRL = (0x3) +RM_GSP_SPDM_MSG_ID_CC_INIT_RM_DATA = (0x4) +RM_GSP_SPDM_MSG_ID_CC_HEARTBEAT_CTRL = (0x5) +RM_GSP_SPDM_MSG_ID_FIPS_SELFTEST = (0x6) +RM_GSP_SPDM_MSG_ID_INVALID_COMMAND = (0xFF) +NV2080_CTRL_INTERNAL_SPDM_PARTITION = (0x20800ad9) +NV2080_CTRL_INTERNAL_SPDM_PARTITION_PARAMS_MESSAGE_ID = (0xD9) +NV2080_CTRL_INTERNAL_SPDM_RETRIEVE_TRANSCRIPT = (0x20800ada) +NV2080_CTRL_INTERNAL_SPDM_RETRIEVE_TRANSCRIPT_PARAMS_MESSAGE_ID = (0xDA) +THERMAL_SYSTEM_API_VER = 1 +THERMAL_SYSTEM_API_REV = 0 +NV2080_CTRL_THERMAL_SYSTEM_TARGET_NONE = (0x00000000) +NV2080_CTRL_THERMAL_SYSTEM_TARGET_GPU = (0x00000001) +NV2080_CTRL_THERMAL_SYSTEM_TARGET_MEMORY = (0x00000002) +NV2080_CTRL_THERMAL_SYSTEM_TARGET_POWER_SUPPLY = (0x00000004) +NV2080_CTRL_THERMAL_SYSTEM_TARGET_BOARD = (0x00000008) +NV2080_CTRL_THERMAL_SYSTEM_TARGET_UNKNOWN = (0xFFFFFFFF) +NV2080_CTRL_THERMAL_SYSTEM_EXECUTE_FLAGS_DEFAULT = (0x00000000) +NV2080_CTRL_THERMAL_SYSTEM_EXECUTE_FLAGS_IGNORE_FAIL = (0x00000001) +NV2080_CTRL_CMD_THERMAL_SYSTEM_EXECUTE_V2 = (0x20800513) +NV2080_CTRL_CMD_THERMAL_SYSTEM_EXECUTE_V2_PHYSICAL = (0x20808513) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGETS_AVAILABLE_OPCODE = (0x00000100) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_TARGET_TYPE_OPCODE = (0x00000101) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_PROVIDER_TYPE_OPCODE = (0x00000301) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSORS_AVAILABLE_OPCODE = (0x00000500) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_PROVIDER_OPCODE = (0x00000510) +NV2080_CTRL_THERMAL_SYSTEM_GET_STATUS_SENSOR_READING_OPCODE = (0x00001500) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_TARGET_OPCODE = (0x00000520) +NV2080_CTRL_THERMAL_SYSTEM_GET_INFO_SENSOR_READING_RANGE_OPCODE = (0x00000540) +NV2080_CTRL_THERMAL_SYSTEM_INSTRUCTION_MAX_COUNT = 0x20 +NV2080_CTRL_THERMAL_SYSTEM_EXECUTE_V2_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_CMD_TIMER_SCHEDULE = (0x20800401) +NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_TIMER_SCHEDULE_FLAGS_TIME = (0, 0) +NV2080_CTRL_TIMER_SCHEDULE_FLAGS_TIME_ABS = (0x00000000) +NV2080_CTRL_TIMER_SCHEDULE_FLAGS_TIME_REL = (0x00000001) +NV2080_CTRL_CMD_TIMER_CANCEL = (0x20800402) +NV2080_CTRL_CMD_TIMER_GET_TIME = (0x20800403) +NV2080_CTRL_TIMER_GET_TIME_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_TIMER_GET_REGISTER_OFFSET = (0x20800404) +NV2080_CTRL_TIMER_GET_REGISTER_OFFSET_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO = (0x20800406) +NV2080_CTRL_TIMER_GPU_CPU_TIME_MAX_SAMPLES = 16 +NV2080_CTRL_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO_PARAMS_MESSAGE_ID = (0x6) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_SOURCE = (3, 0) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_OSTIME = (0x00000001) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_TSC = (0x00000002) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_PLATFORM_API = (0x00000003) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_GSP_OS = (0x00000004) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_PROCESSOR = (7, 4) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_PROCESSOR_CPU = (0x00000000) +NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_PROCESSOR_GSP = (0x00000001) +NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ = (0x20800407) +NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ_PARAMS_MESSAGE_ID = (0x7) +NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT = (0x20803d01) +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC = (0x00000001) +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC = (0x00000002) +NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF = (0x20803d02) +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW = (0x00000001) +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW = (0x00000002) +NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER = (0x20803d03) +NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS = (0x20803d04) +NV2080_CTRL_OS_UNIX_INSTALL_PROFILER_HOOKS_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER = (0x20803d05) +NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER = (0x20803d06) +NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS = (0x20803d07) +NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS_MESSAGE_ID = (0x7) +NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS = (0x20803d08) +NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK = (0x20804001) +NV2080_CTRL_MAX_VMMU_SEGMENTS = 384 +NV2080_GPU_MAX_ENGINES = 0x54 +NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS_MESSAGE_ID = (0x1) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK = (0x20804002) +NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS_MESSAGE_ID = (0x2) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE = (0x20804003) +NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_GSP_BUILD_VERSION_LEN = 128 +NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS_MESSAGE_ID = (0x3) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU = (0x20804004) +NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS_MESSAGE_ID = (0x4) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO = (0x20804005) +NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS_MESSAGE_ID = (0x5) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE = (0x20804006) +NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS_MESSAGE_ID = (0x6) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY = (0x20804007) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS_MESSAGE_ID = (0x7) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_VGPU_PLUGIN_CLEANUP = (0x20804008) +NV2080_CTRL_VGPU_MGR_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS_MESSAGE_ID = (0x8) +NV2080_CTRL_MAX_NVU32_TO_CONVERTED_STR_LEN = 8 +NV2080_CTRL_MAX_GPC_COUNT = 32 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_PGPU_FS_ENCODING = (0x20804009) +NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_FS_ENCODING_PARAMS_MESSAGE_ID = (0x9) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_PGPU_MIGRATION_SUPPORT = (0x2080400a) +NV2080_CTRL_VGPU_MGR_INTERNAL_GET_PGPU_MIGRATION_SUPPORT_PARAMS_MESSAGE_ID = (0xA) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_MGR_CONFIG = (0x2080400b) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MGR_CONFIG_PARAMS_MESSAGE_ID = (0xB) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_FREE_STATES = (0x2080400c) +NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_STATES_PARAMS_MESSAGE_ID = (0xC) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_FRAME_RATE_LIMITER_STATUS = (0x2080400d) +NV2080_CTRL_VGPU_MGR_GET_FRAME_RATE_LIMITER_STATUS_PARAMS_MESSAGE_ID = (0xD) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_HETEROGENEOUS_MODE = (0x2080400e) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_HETEROGENEOUS_MODE_PARAMS_MESSAGE_ID = (0xE) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_MIG_TIMESLICE_MODE = (0x2080400f) +NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_MIG_TIMESLICE_MODE_PARAMS_MESSAGE_ID = (0xF) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_POWER_STATE = (0x20804010) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_POWER_STATE_PARAMS_MESSAGE_ID = (0x10) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SCRUB_GUEST_FB = (0x20804011) +NV2080_CTRL_VGPU_MGR_INTERNAL_SCRUB_GUEST_FB_PARAMS_MESSAGE_ID = (0x11) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CHECK_SCRUB_COMPLETE = (0x20804012) +NV2080_CTRL_VGPU_MGR_INTERNAL_CHECK_SCRUB_COMPLETE_PARAMS_MESSAGE_ID = (0x12) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ALLOC_GSP_CEUTILS = (0x20804013) +NV2080_CTRL_VGPU_MGR_INTERNAL_ALLOC_GSP_CEUTILS_PARAMS_MESSAGE_ID = (0x13) +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_FREE_GSP_CEUTILS = (0x20804014) +NV2080_CTRL_VGPU_MGR_INTERNAL_FREE_GSP_CEUTILS_PARAMS_MESSAGE_ID = (0x14) +NV83DE_CTRL_RESERVED = (0x00) +NV83DE_CTRL_GR = (0x01) +NV83DE_CTRL_FIFO = (0x02) +NV83DE_CTRL_DEBUG = (0x03) +NV83DE_CTRL_INTERNAL = (0x04) +NV83DE_CTRL_CMD_NULL = (0x83de0000) +NV83DE_CTRL_CMD_SM_DEBUG_MODE_ENABLE = (0x83de0301) +NV83DE_CTRL_CMD_SM_DEBUG_MODE_DISABLE = (0x83de0302) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG = (0x83de0307) +NV83DE_CTRL_DEBUG_SET_MODE_MMU_DEBUG_PARAMS_MESSAGE_ID = (0x7) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG_ENABLE = (0x00000001) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG_DISABLE = (0x00000002) +NV83DE_CTRL_CMD_DEBUG_RELEASE_MMU_DEBUG_REQUESTS = (0x00000003) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_MMU_DEBUG = (0x83de0308) +NV83DE_CTRL_DEBUG_GET_MODE_MMU_DEBUG_PARAMS_MESSAGE_ID = (0x8) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_MMU_DEBUG_ENABLED = (0x00000001) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_MMU_DEBUG_DISABLED = (0x00000002) +NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK = (0x83de0309) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PARAMS_MESSAGE_ID = (0x9) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_FATAL = (0x00000001) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_TRAP = (0x00000002) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_SINGLE_STEP = (0x00000004) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_INT = (0x00000008) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_CILP = (0x00000010) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PREEMPTION_STARTED = (0x00000020) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_NONE = (0x00000000) +NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_ALL = (0x0000FFFF) +NV83DE_CTRL_CMD_DEBUG_READ_SINGLE_SM_ERROR_STATE = (0x83de030b) +NV83DE_CTRL_DEBUG_READ_SINGLE_SM_ERROR_STATE_PARAMS_MESSAGE_ID = (0xB) +NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES = (0x83de030c) +NV83DE_CTRL_DEBUG_MAX_SMS_PER_CALL = 100 +NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS_MESSAGE_ID = (0xC) +NV83DE_CTRL_CMD_DEBUG_CLEAR_SINGLE_SM_ERROR_STATE = (0x83de030f) +NV83DE_CTRL_DEBUG_CLEAR_SINGLE_SM_ERROR_STATE_PARAMS_MESSAGE_ID = (0xF) +NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES = (0x83de0310) +NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS_MESSAGE_ID = (0x10) +NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_PARAMS_DEFINED = 1 +NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_HAS_RESIDENT_CHANNEL = 1 +NV83DE_CTRL_CMD_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE = (0x83de0313) +NV83DE_CTRL_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE_PARAMS_MESSAGE_ID = (0x13) +NV83DE_CTRL_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE_SINGLE_SM = (0x00000001) +NV83DE_CTRL_DEBUG_SET_NEXT_STOP_TRIGGER_TYPE_BROADCSAT = (0x00000002) +NV83DE_CTRL_CMD_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING = (0x83de0314) +NV83DE_CTRL_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING_PARAMS_MESSAGE_ID = (0x14) +NV83DE_CTRL_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING_NONPAUSING = (0x00000001) +NV83DE_CTRL_DEBUG_SET_SINGLE_STEP_INTERRUPT_HANDLING_PAUSING = (0x00000002) +NV83DE_CTRL_CMD_DEBUG_READ_MEMORY = (0x83de0315) +NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS_MESSAGE_ID = (0x15) +NV83DE_CTRL_CMD_DEBUG_WRITE_MEMORY = (0x83de0316) +NV83DE_CTRL_DEBUG_WRITE_MEMORY_PARAMS_MESSAGE_ID = (0x16) +NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT = (0x83de0317) +NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT_PARAMS_MESSAGE_ID = (0x17) +NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT = (0x83de0318) +NV83DE_CTRL_CMD_DEBUG_GET_HANDLES = (0x83de0319) +NV83DE_CTRL_CMD_READ_SURFACE = (0x83de031a) +NV83DE_CTRL_CMD_WRITE_SURFACE = (0x83de031b) +MAX_ACCESS_OPS = 64 +NV83DE_CTRL_CMD_GET_MAPPINGS = (0x83de031c) +MAX_GET_MAPPINGS_OPS = 64 +NV83DE_CTRL_CMD_DEBUG_EXEC_REG_OPS = (0x83de031d) +NV83DE_CTRL_GPU_EXEC_REG_OPS_MAX_OPS = 100 +NV83DE_CTRL_DEBUG_EXEC_REG_OPS_PARAMS_MESSAGE_ID = (0x1D) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG = (0x83de031f) +NV83DE_CTRL_DEBUG_SET_MODE_ERRBAR_DEBUG_PARAMS_MESSAGE_ID = (0x1F) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG_DISABLE = (0x00000000) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG_ENABLE = (0x00000001) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_ERRBAR_DEBUG = (0x83de0320) +NV83DE_CTRL_DEBUG_GET_MODE_ERRBAR_DEBUG_PARAMS_MESSAGE_ID = (0x20) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_ERRBAR_DEBUG_DISABLED = (0x00000000) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_ERRBAR_DEBUG_ENABLED = (0x00000001) +NV83DE_CTRL_CMD_DEBUG_SET_SINGLE_SM_SINGLE_STEP = (0x83de0321) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SINGLE_STEP_PARAMS_MESSAGE_ID = (0x21) +NV83DE_CTRL_CMD_DEBUG_SET_SINGLE_SM_STOP_TRIGGER = (0x83de0322) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_STOP_TRIGGER_PARAMS_MESSAGE_ID = (0x22) +NV83DE_CTRL_CMD_DEBUG_SET_SINGLE_SM_RUN_TRIGGER = (0x83de0323) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_RUN_TRIGGER_PARAMS_MESSAGE_ID = (0x23) +NV83DE_CTRL_CMD_DEBUG_SET_SINGLE_SM_SKIP_IDLE_WARP_DETECT = (0x83de0324) +NV83DE_CTRL_DEBUG_SET_SINGLE_SM_SKIP_IDLE_WARP_DETECT_PARAMS_MESSAGE_ID = (0x24) +NV83DE_CTRL_CMD_DEBUG_GET_SINGLE_SM_DEBUGGER_STATUS = (0x83de0325) +NV83DE_CTRL_DEBUG_GET_SINGLE_SM_DEBUGGER_STATUS_PARAMS_MESSAGE_ID = (0x25) +NV83DE_CTRL_CMD_DEBUG_READ_BATCH_MEMORY = (0x83de0326) +NV83DE_CTRL_CMD_DEBUG_WRITE_BATCH_MEMORY = (0x83de0327) +MAX_ACCESS_MEMORY_OPS = 150 +NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_MAX_ENTRIES = 4 +NV83DE_CTRL_CMD_DEBUG_READ_MMU_FAULT_INFO = (0x83de0328) +NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS_MESSAGE_ID = (0x28) +NV83DE_CTRL_CMD_DEBUG_SET_DROP_DEFERRED_RC = (0x83de0329) +NV83DE_CTRL_DEBUG_SET_DROP_DEFERRED_RC_PARAMS_MESSAGE_ID = (0x29) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_GCC_DEBUG = (0x83de032a) +NV83DE_CTRL_DEBUG_SET_MODE_MMU_GCC_DEBUG_PARAMS_MESSAGE_ID = (0x2A) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_GCC_DEBUG_ENABLE = (0x00000001) +NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_GCC_DEBUG_DISABLE = (0x00000002) +NV83DE_CTRL_CMD_DEBUG_RELEASE_MMU_GCC_DEBUG_REQUESTS = (0x00000003) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_MMU_GCC_DEBUG = (0x83de032b) +NV83DE_CTRL_DEBUG_GET_MODE_MMU_GCC_DEBUG_PARAMS_MESSAGE_ID = (0x2B) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_MMU_GCC_DEBUG_ENABLED = (0x00000001) +NV83DE_CTRL_CMD_DEBUG_GET_MODE_MMU_GCC_DEBUG_DISABLED = (0x00000002) +NV83DE_CTRL_CMD_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT = (0x83de022d) +NV83DE_CTRL_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT_PARAMS_MESSAGE_ID = (0x2D) +NV83DE_CTRL_CMD_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT_ACTIVE = (0x00000001) +NV83DE_CTRL_CMD_DEBUG_SET_NON_PREEMPTABLE_DEBUGGER_CONTEXT_INACTIVE = (0x00000000) +NV83DE_CTRL_CMD_DEBUG_SET_RECOVERY_SUPPRESSION_TIMEOUT = (0x83de022e) +NV83DE_CTRL_DEBUG_SET_RECOVERY_SUPPRESSION_TIMEOUT_PARAMS_MESSAGE_ID = (0x2E) +NVB0CC_CTRL_RESERVED = (0x00) +NVB0CC_CTRL_PROFILER = (0x01) +NVB0CC_CTRL_INTERNAL = (0x02) +NVB0CC_CTRL_CMD_NULL = (0xb0cc0000) +NVB0CC_CTRL_CMD_INTERNAL_ALLOC_PMA_STREAM = (0xb0cc0204) +NVB0CC_CTRL_CMD_INTERNAL_ALLOC_PMA_STREAM_FINN_PARAMS_MESSAGE_ID = (0x0) +NVB0CC_CTRL_CMD_INTERNAL_QUIESCE_PMA_CHANNEL = (0xb0cc0201) +NVB0CC_CTRL_INTERNAL_QUIESCE_PMA_CHANNEL_PARAMS_MESSAGE_ID = (0x1) +NVB0CC_CTRL_CMD_INTERNAL_SRIOV_PROMOTE_PMA_STREAM = (0xb0cc0202) +NVB0CC_CTRL_INTERNAL_SRIOV_PROMOTE_PMA_STREAM_PARAMS_MESSAGE_ID = (0x2) +NVB0CC_CTRL_CMD_INTERNAL_PERMISSIONS_INIT = (0xb0cc0203) +NVB0CC_CTRL_INTERNAL_PERMISSIONS_INIT_PARAMS_MESSAGE_ID = (0x3) +NVB0CC_CTRL_INTERNAL_ALLOC_PMA_STREAM_PARAMS_MESSAGE_ID = (0x4) +NVB0CC_CTRL_CMD_INTERNAL_FREE_PMA_STREAM = (0xb0cc0206) +NVB0CC_CTRL_INTERNAL_FREE_PMA_STREAM_PARAMS_MESSAGE_ID = (0x6) +NVB0CC_CTRL_CMD_INTERNAL_GET_MAX_PMAS = (0xb0cc0207) +NVB0CC_CTRL_INTERNAL_GET_MAX_PMAS_PARAMS_MESSAGE_ID = (0x7) +NVB0CC_CTRL_CMD_INTERNAL_BIND_PM_RESOURCES = (0xb0cc0208) +NVB0CC_CTRL_CMD_INTERNAL_UNBIND_PM_RESOURCES = (0xb0cc0209) +NVB0CC_CTRL_CMD_INTERNAL_RESERVE_HWPM_LEGACY = (0xb0cc020a) +NVB0CC_CTRL_INTERNAL_RESERVE_HWPM_LEGACY_PARAMS_MESSAGE_ID = (0xa) +NVB0CC_CTRL_CMD_POWER_REQUEST_FEATURES = (0xb0cc0301) +NVB0CC_CTRL_POWER_REQUEST_FEATURES_PARAMS_MESSAGE_ID = (0x1) +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG = (1, 0) +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG = (3, 2) +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG = (5, 4) +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG = (7, 6) +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN = (9, 8) +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT = (11, 10) +NVB0CC_CTRL_POWER_FEATURE_IGNORE = (0x00000000) +NVB0CC_CTRL_POWER_FEATURE_DISABLE = (0x00000001) +NVB0CC_CTRL_POWER_FEATURE_ENABLE = (0x00000002) +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_IGNORE = NVB0CC_CTRL_POWER_FEATURE_IGNORE +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_DISABLE = NVB0CC_CTRL_POWER_FEATURE_DISABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_ENABLE = NVB0CC_CTRL_POWER_FEATURE_ENABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_IGNORE = NVB0CC_CTRL_POWER_FEATURE_IGNORE +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_DISABLE = NVB0CC_CTRL_POWER_FEATURE_DISABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_ENABLE = NVB0CC_CTRL_POWER_FEATURE_ENABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_IGNORE = NVB0CC_CTRL_POWER_FEATURE_IGNORE +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_DISABLE = NVB0CC_CTRL_POWER_FEATURE_DISABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_ENABLE = NVB0CC_CTRL_POWER_FEATURE_ENABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_IGNORE = NVB0CC_CTRL_POWER_FEATURE_IGNORE +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_DISABLE = NVB0CC_CTRL_POWER_FEATURE_DISABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_ENABLE = NVB0CC_CTRL_POWER_FEATURE_ENABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_IGNORE = NVB0CC_CTRL_POWER_FEATURE_IGNORE +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_DISABLE = NVB0CC_CTRL_POWER_FEATURE_DISABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_ENABLE = NVB0CC_CTRL_POWER_FEATURE_ENABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_IGNORE = NVB0CC_CTRL_POWER_FEATURE_IGNORE +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_DISABLE = NVB0CC_CTRL_POWER_FEATURE_DISABLE +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_ENABLE = NVB0CC_CTRL_POWER_FEATURE_ENABLE +NVB0CC_CTRL_POWER_FEATURE_REQUEST_FULFILLED = (0x00000000) +NVB0CC_CTRL_POWER_FEATURE_REQUEST_REJECTED = (0x00000001) +NVB0CC_CTRL_POWER_FEATURE_REQUEST_NOT_SUPPORTED = (0x00000002) +NVB0CC_CTRL_POWER_FEATURE_REQUEST_FAILED = (0x00000003) +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_REQUEST_FULFILLED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FULFILLED +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_REQUEST_REJECTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_REJECTED +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_REQUEST_NOT_SUPPORTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_NOT_SUPPORTED +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_REQUEST_FAILED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FAILED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_REQUEST_FULFILLED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FULFILLED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_REQUEST_REJECTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_REJECTED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_REQUEST_NOT_SUPPORTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_NOT_SUPPORTED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_REQUEST_FAILED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FAILED +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_REQUEST_FULFILLED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FULFILLED +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_REQUEST_REJECTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_REJECTED +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_REQUEST_NOT_SUPPORTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_NOT_SUPPORTED +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_REQUEST_FAILED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FAILED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_REQUEST_FULFILLED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FULFILLED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_REQUEST_REJECTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_REJECTED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_REQUEST_NOT_SUPPORTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_NOT_SUPPORTED +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_REQUEST_FAILED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FAILED +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_REQUEST_FULFILLED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FULFILLED +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_REQUEST_REJECTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_REJECTED +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_REQUEST_NOT_SUPPORTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_NOT_SUPPORTED +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_REQUEST_FAILED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FAILED +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_REQUEST_FULFILLED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FULFILLED +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_REQUEST_REJECTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_REJECTED +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_REQUEST_NOT_SUPPORTED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_NOT_SUPPORTED +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_REQUEST_FAILED = NVB0CC_CTRL_POWER_FEATURE_REQUEST_FAILED +NVB0CC_CTRL_CMD_POWER_RELEASE_FEATURES = (0xb0cc0302) +NVB0CC_CTRL_POWER_RELEASE_FEATURES_PARAMS_MESSAGE_ID = (0x2) +NVB0CC_CTRL_POWER_FEATURE_RELEASE = (0x00000003) +NVB0CC_CTRL_POWER_FEATURE_MASK_ELCG_RELEASE = NVB0CC_CTRL_POWER_FEATURE_RELEASE +NVB0CC_CTRL_POWER_FEATURE_MASK_BLCG_RELEASE = NVB0CC_CTRL_POWER_FEATURE_RELEASE +NVB0CC_CTRL_POWER_FEATURE_MASK_SLCG_RELEASE = NVB0CC_CTRL_POWER_FEATURE_RELEASE +NVB0CC_CTRL_POWER_FEATURE_MASK_ELPG_RELEASE = NVB0CC_CTRL_POWER_FEATURE_RELEASE +NVB0CC_CTRL_POWER_FEATURE_MASK_IDLE_SLOWDOWN_RELEASE = NVB0CC_CTRL_POWER_FEATURE_RELEASE +NVB0CC_CTRL_POWER_FEATURE_MASK_VAT_RELEASE = NVB0CC_CTRL_POWER_FEATURE_RELEASE +NVB0CC_CTRL_CMD_RESERVE_HWPM_LEGACY = (0xb0cc0101) +NVB0CC_CTRL_RESERVE_HWPM_LEGACY_PARAMS_MESSAGE_ID = (0x1) +NVB0CC_CTRL_CMD_RELEASE_HWPM_LEGACY = (0xb0cc0102) +NVB0CC_CTRL_CMD_RESERVE_PM_AREA_SMPC = (0xb0cc0103) +NVB0CC_CTRL_RESERVE_PM_AREA_SMPC_PARAMS_MESSAGE_ID = (0x3) +NVB0CC_CTRL_CMD_RELEASE_PM_AREA_SMPC = (0xb0cc0104) +NVB0CC_CTRL_CMD_ALLOC_PMA_STREAM = (0xb0cc0105) +NVB0CC_PMA_BUFFER_SIZE_MAX = (0xffe00000) +NVB0CC_PMA_BYTES_AVAILABLE_SIZE = (0x1000) +NVB0CC_PMA_INSTANCE_BLOCK_SIZE = (0x1000) +NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS_MESSAGE_ID = (0x5) +NVB0CC_CTRL_CMD_FREE_PMA_STREAM = (0xb0cc0106) +NVB0CC_CTRL_FREE_PMA_STREAM_PARAMS_MESSAGE_ID = (0x6) +NVB0CC_CTRL_CMD_BIND_PM_RESOURCES = (0xb0cc0107) +NVB0CC_CTRL_CMD_UNBIND_PM_RESOURCES = (0xb0cc0108) +NVB0CC_CTRL_CMD_PMA_STREAM_UPDATE_GET_PUT = (0xb0cc0109) +NVB0CC_AVAILABLE_BYTES_DEFAULT_VALUE = 0xFFFFFFFF +NVB0CC_CTRL_PMA_STREAM_UPDATE_GET_PUT_PARAMS_MESSAGE_ID = (0x9) +NVB0CC_REGOPS_MAX_COUNT = (124) +NVB0CC_CTRL_CMD_EXEC_REG_OPS = (0xb0cc010a) +NVB0CC_CTRL_EXEC_REG_OPS_PARAMS_MESSAGE_ID = (0xA) +NVB0CC_CTRL_CMD_RESERVE_PM_AREA_PC_SAMPLER = (0xb0cc010b) +NVB0CC_CTRL_CMD_RELEASE_PM_AREA_PC_SAMPLER = (0xb0cc010c) +NVB0CC_CTRL_CMD_GET_TOTAL_HS_CREDITS = (0xb0cc010d) +NVB0CC_CTRL_GET_TOTAL_HS_CREDITS_PARAMS_MESSAGE_ID = (0xD) +NVB0CC_CTRL_CMD_SET_HS_CREDITS = (0xb0cc010e) +NVB0CC_MAX_CREDIT_INFO_ENTRIES = (63) +NVB0CC_CTRL_SET_HS_CREDITS_PARAMS_MESSAGE_ID = (0xE) +NVB0CC_CTRL_CMD_GET_HS_CREDITS = (0xb0cc010f) +NVB0CC_CTRL_GET_HS_CREDITS_PARAMS_MESSAGE_ID = (0xF) +NVB0CC_CTRL_HES_INVALID = (0x00000000) +NVB0CC_CTRL_HES_CWD = (0x00000001) +NVB0CC_CTRL_HES_TPC = (0x00000002) +NVB0CC_CTRL_CMD_RESERVE_HES = (0xb0cc0113) +NVB0CC_CTRL_RESERVE_HES_PARAMS_MESSAGE_ID = (0x13) +NVB0CC_CTRL_CMD_RELEASE_HES = (0xb0cc0114) +NVB0CC_CTRL_RELEASE_HES_PARAMS_MESSAGE_ID = (0x14) +NVB0CC_CREDIT_POOL_MAX_COUNT = 30 +NVB0CC_CTRL_CMD_GET_CHIPLET_HS_CREDIT_POOL = (0xb0cc0115) +NVB0CC_CTRL_GET_CHIPLET_HS_CREDIT_POOL_MESSAGE_ID = (0x15) +NVB0CC_CTRL_CMD_GET_HS_CREDITS_MAPPING = (0xb0cc0116) +NVB0CC_CTRL_GET_HS_CREDITS_POOL_MAPPING_PARAMS_MESSAGE_ID = (0x16) +NVB0CC_CTRL_CMD_DISABLE_DYNAMIC_MMA_BOOST = (0xb0cc0117) +NVB0CC_CTRL_DISABLE_DYNAMIC_MMA_BOOST_PARAMS_MESSAGE_ID = (0x17) +NVB0CC_CTRL_CMD_GET_DYNAMIC_MMA_BOOST_STATUS = (0xb0cc0118) +NVB0CC_CTRL_GET_DYNAMIC_MMA_BOOST_STATUS_PARAMS_MESSAGE_ID = (0x18) +NVB0CC_CTRL_CMD_RESERVE_CCU_PROF = (0xb0cc0119) +NVB0CC_CTRL_RESERVE_CCUPROF_PARAMS_MESSAGE_ID = (0x19) +NVB0CC_CTRL_CMD_RELEASE_CCU_PROF = (0xb0cc011a) +NV_STATUS_LEVEL_OK = 0 +NV_STATUS_LEVEL_WARN = 1 +NV_STATUS_LEVEL_ERR = 3 +NV01_ROOT = (0x00000000) +NV1_ROOT = (0x00000000) +NV01_NULL_OBJECT = (0x00000000) +NV1_NULL_OBJECT = (0x00000000) +NV01_ROOT_NON_PRIV = (0x00000001) +NV1_ROOT_NON_PRIV = (0x00000001) +NV01_ROOT_CLIENT = (0x00000041) +FABRIC_MANAGER_SESSION = (0x0000000f) +NV0020_GPU_MANAGEMENT = (0x00000020) +NV01_DEVICE_0 = (0x00000080) +NV20_SUBDEVICE_0 = (0x00002080) +NV2081_BINAPI = (0x00002081) +NV2082_BINAPI_PRIVILEGED = (0x00002082) +NV20_SUBDEVICE_DIAG = (0x0000208f) +NV01_CONTEXT_DMA = (0x00000002) +NV01_MEMORY_SYSTEM = (0x0000003e) +NV1_MEMORY_SYSTEM = (0x0000003e) +NV01_MEMORY_LOCAL_PRIVILEGED = (0x0000003f) +NV1_MEMORY_LOCAL_PRIVILEGED = (0x0000003f) +NV01_MEMORY_PRIVILEGED = (0x0000003f) +NV1_MEMORY_PRIVILEGED = (0x0000003f) +NV01_MEMORY_LOCAL_USER = (0x00000040) +NV1_MEMORY_LOCAL_USER = (0x00000040) +NV01_MEMORY_USER = (0x00000040) +NV1_MEMORY_USER = (0x00000040) +NV01_MEMORY_VIRTUAL = (0x00000070) +NV01_MEMORY_SYSTEM_DYNAMIC = (0x00000070) +NV1_MEMORY_SYSTEM_DYNAMIC = (0x00000070) +NV_MEMORY_MAPPER = (0x000000fe) +NV01_MEMORY_LOCAL_PHYSICAL = (0x000000c2) +NV01_MEMORY_SYNCPOINT = (0x000000c3) +NV01_MEMORY_SYSTEM_OS_DESCRIPTOR = (0x00000071) +NV01_MEMORY_DEVICELESS = (0x000090ce) +NV01_MEMORY_FRAMEBUFFER_CONSOLE = (0x00000076) +NV01_MEMORY_HW_RESOURCES = (0x000000b1) +NV01_MEMORY_LIST_SYSTEM = (0x00000081) +NV01_MEMORY_LIST_FBMEM = (0x00000082) +NV01_MEMORY_LIST_OBJECT = (0x00000083) +NV_IMEX_SESSION = (0x000000f1) +PROFILER_DEVICE_EVENT = (0x0000b0cd) +PROFILER_CONTEXT_EVENT = (0x0000b0ce) +TRACE_DEVICE_EVENT = (0x0000cdcd) +WPPS_CONFIG_SESSION = (0x0000d0cd) +NV_MEMORY_EXPORT = (0x000000e0) +NV_CE_UTILS = (0x00000050) +NV_MEMORY_FABRIC = (0x000000f8) +NV_MEMORY_FABRIC_IMPORT_V2 = (0x000000f9) +NV_MEMORY_FABRIC_IMPORTED_REF = (0x000000fb) +FABRIC_VASPACE_A = (0x000000fc) +NV_MEMORY_MULTICAST_FABRIC = (0x000000fd) +IO_VASPACE_A = (0x000000f2) +NV01_NULL = (0x00000030) +NV1_NULL = (0x00000030) +NV01_EVENT = (0x00000005) +NV1_EVENT = (0x00000005) +NV01_EVENT_KERNEL_CALLBACK = (0x00000078) +NV1_EVENT_KERNEL_CALLBACK = (0x00000078) +NV01_EVENT_OS_EVENT = (0x00000079) +NV1_EVENT_OS_EVENT = (0x00000079) +NV01_EVENT_WIN32_EVENT = (0x00000079) +NV1_EVENT_WIN32_EVENT = (0x00000079) +NV01_EVENT_KERNEL_CALLBACK_EX = (0x0000007e) +NV1_EVENT_KERNEL_CALLBACK_EX = (0x0000007e) +NV01_TIMER = (0x00000004) +NV1_TIMER = (0x00000004) +KERNEL_GRAPHICS_CONTEXT = (0x00000090) +KERNEL_WATCHDOG = (0x0000008f) +LOCK_STRESS_OBJECT = (0x00000100) +LOCK_TEST_RELAXED_DUP_OBJECT = (0x00000101) +NV50_CHANNEL_GPFIFO = (0x0000506f) +GF100_CHANNEL_GPFIFO = (0x0000906f) +KEPLER_CHANNEL_GPFIFO_A = (0x0000a06f) +UVM_CHANNEL_RETAINER = (0x0000c574) +KEPLER_CHANNEL_GPFIFO_B = (0x0000a16f) +MAXWELL_CHANNEL_GPFIFO_A = (0x0000b06f) +PASCAL_CHANNEL_GPFIFO_A = (0x0000c06f) +VOLTA_CHANNEL_GPFIFO_A = (0x0000c36f) +TURING_CHANNEL_GPFIFO_A = (0x0000c46f) +AMPERE_CHANNEL_GPFIFO_A = (0x0000c56f) +HOPPER_CHANNEL_GPFIFO_A = (0x0000c86f) +BLACKWELL_CHANNEL_GPFIFO_A = (0x0000c96f) +BLACKWELL_CHANNEL_GPFIFO_B = (0x0000ca6f) +NV04_SOFTWARE_TEST = (0x0000007d) +NV4_SOFTWARE_TEST = (0x0000007d) +NV30_GSYNC = (0x000030f1) +VOLTA_USERMODE_A = (0x0000c361) +TURING_USERMODE_A = (0x0000c461) +AMPERE_USERMODE_A = (0x0000c561) +HOPPER_USERMODE_A = (0x0000c661) +BLACKWELL_USERMODE_A = (0x0000c761) +NVC371_DISP_SF_USER = (0x0000c371) +NVC372_DISPLAY_SW = (0x0000c372) +NVC573_DISP_CAPABILITIES = (0x0000c573) +NVC673_DISP_CAPABILITIES = (0x0000c673) +NVC773_DISP_CAPABILITIES = (0x0000c773) +NVC873_DISP_CAPABILITIES = (0x0000c873) +NVC973_DISP_CAPABILITIES = (0x0000c973) +NVCA73_DISP_CAPABILITIES = (0x0000ca73) +NVCB73_DISP_CAPABILITIES = (0x0000cb73) +NVCC73_DISP_CAPABILITIES = (0x0000cc73) +NV04_DISPLAY_COMMON = (0x00000073) +NV50_DEFERRED_API_CLASS = (0x00005080) +MPS_COMPUTE = (0x0000900e) +NVC570_DISPLAY = (0x0000c570) +NVC57A_CURSOR_IMM_CHANNEL_PIO = (0x0000c57a) +NVC57B_WINDOW_IMM_CHANNEL_DMA = (0x0000c57b) +NVC57D_CORE_CHANNEL_DMA = (0x0000c57d) +NVC57E_WINDOW_CHANNEL_DMA = (0x0000c57e) +NVC670_DISPLAY = (0x0000c670) +NVC671_DISP_SF_USER = (0x0000c671) +NVC67A_CURSOR_IMM_CHANNEL_PIO = (0x0000c67a) +NVC67B_WINDOW_IMM_CHANNEL_DMA = (0x0000c67b) +NVC67D_CORE_CHANNEL_DMA = (0x0000c67d) +NVC67E_WINDOW_CHANNEL_DMA = (0x0000c67e) +NVC77F_ANY_CHANNEL_DMA = (0x0000c77f) +NVC770_DISPLAY = (0x0000c770) +NVC771_DISP_SF_USER = (0x0000c771) +NVC77D_CORE_CHANNEL_DMA = (0x0000c77d) +NVC870_DISPLAY = (0x0000c870) +NVC871_DISP_SF_USER = (0x0000c871) +NVC87A_CURSOR_IMM_CHANNEL_PIO = (0x0000c87a) +NVC87B_WINDOW_IMM_CHANNEL_DMA = (0x0000c87b) +NVC87D_CORE_CHANNEL_DMA = (0x0000c87d) +NVC87E_WINDOW_CHANNEL_DMA = (0x0000c87e) +NVC970_DISPLAY = (0x0000c970) +NVC971_DISP_SF_USER = (0x0000c971) +NVC97A_CURSOR_IMM_CHANNEL_PIO = (0x0000c97a) +NVC97B_WINDOW_IMM_CHANNEL_DMA = (0x0000c97b) +NVC97D_CORE_CHANNEL_DMA = (0x0000c97d) +NVC97E_WINDOW_CHANNEL_DMA = (0x0000c97e) +NVCA70_DISPLAY = (0x0000ca70) +NVCA71_DISP_SF_USER = (0x0000ca71) +NVCA7A_CURSOR_IMM_CHANNEL_PIO = (0x0000ca7a) +NVCA7B_WINDOW_IMM_CHANNEL_DMA = (0x0000ca7b) +NVCA7D_CORE_CHANNEL_DMA = (0x0000ca7d) +NVCA7E_WINDOW_CHANNEL_DMA = (0x0000ca7e) +NVCB70_DISPLAY = (0x0000cb70) +NVCB71_DISP_SF_USER = (0x0000cb71) +NVCB7A_CURSOR_IMM_CHANNEL_PIO = (0x0000cb7a) +NVCB7B_WINDOW_IMM_CHANNEL_DMA = (0x0000cb7b) +NVCB7D_CORE_CHANNEL_DMA = (0x0000cb7d) +NVCB7E_WINDOW_CHANNEL_DMA = (0x0000cb7e) +NVCC70_DISPLAY = (0x0000cc70) +NVCC71_DISP_SF_USER = (0x0000cc71) +NVCC7A_CURSOR_IMM_CHANNEL_PIO = (0x0000cc7a) +NVCC7B_WINDOW_IMM_CHANNEL_DMA = (0x0000cc7b) +NVCC7D_CORE_CHANNEL_DMA = (0x0000cc7d) +NVCC7E_WINDOW_CHANNEL_DMA = (0x0000cc7e) +NV9010_VBLANK_CALLBACK = (0x00009010) +GF100_PROFILER = (0x000090cc) +MAXWELL_PROFILER = (0x0000b0cc) +MAXWELL_PROFILER_CONTEXT = (0x0000b1cc) +MAXWELL_PROFILER_DEVICE = (0x0000b2cc) +GF100_SUBDEVICE_MASTER = (0x000090e6) +GF100_SUBDEVICE_INFOROM = (0x000090e7) +GF100_ZBC_CLEAR = (0x00009096) +GF100_DISP_SW = (0x00009072) +GF100_TIMED_SEMAPHORE_SW = (0x00009074) +G84_PERFBUFFER = (0x0000844c) +NV50_MEMORY_VIRTUAL = (0x000050a0) +NV50_P2P = (0x0000503b) +NV50_THIRD_PARTY_P2P = (0x0000503c) +FERMI_TWOD_A = (0x0000902d) +FERMI_VASPACE_A = (0x000090f1) +HOPPER_SEC2_WORK_LAUNCH_A = (0x0000cba2) +GF100_HDACODEC = (0x000090ec) +NVB8B0_VIDEO_DECODER = (0x0000b8b0) +NVC4B0_VIDEO_DECODER = (0x0000c4b0) +NVC6B0_VIDEO_DECODER = (0x0000c6b0) +NVC7B0_VIDEO_DECODER = (0x0000c7b0) +NVC9B0_VIDEO_DECODER = (0x0000c9b0) +NVCDB0_VIDEO_DECODER = (0x0000cdb0) +NVCEB0_VIDEO_DECODER = (0x0000ceb0) +NVCFB0_VIDEO_DECODER = (0x0000cfb0) +NVD1B0_VIDEO_DECODER = (0x0000d1b0) +NVD2B0_VIDEO_DECODER = (0x0000d2b0) +NVC4B7_VIDEO_ENCODER = (0x0000c4b7) +NVB4B7_VIDEO_ENCODER = (0x0000b4b7) +NVC7B7_VIDEO_ENCODER = (0x0000c7b7) +NVC9B7_VIDEO_ENCODER = (0x0000c9b7) +NVCEB7_VIDEO_ENCODER = (0x0000ceb7) +NVCFB7_VIDEO_ENCODER = (0x0000cfb7) +NVD1B7_VIDEO_ENCODER = (0x0000d1b7) +NVB8D1_VIDEO_NVJPG = (0x0000b8d1) +NVC4D1_VIDEO_NVJPG = (0x0000c4d1) +NVC9D1_VIDEO_NVJPG = (0x0000c9d1) +NVCED0_VIDEO_NVJPG = (0x0000ced0) +NVCDD1_VIDEO_NVJPG = (0x0000cdd1) +NVCFD1_VIDEO_NVJPG = (0x0000cfd1) +NVD2D1_VIDEO_NVJPG = (0x0000d2d1) +NVB8FA_VIDEO_OFA = (0x0000b8fa) +NVC6FA_VIDEO_OFA = (0x0000c6fa) +NVC7FA_VIDEO_OFA = (0x0000c7fa) +NVC9FA_VIDEO_OFA = (0x0000c9fa) +NVCDFA_VIDEO_OFA = (0x0000cdfa) +NVCEFA_VIDEO_OFA = (0x0000cefa) +NVCFFA_VIDEO_OFA = (0x0000cffa) +NVD1FA_VIDEO_OFA = (0x0000d1fa) +NVD2FA_VIDEO_OFA = (0x0000d2fa) +KEPLER_INLINE_TO_MEMORY_B = (0x0000a140) +FERMI_CONTEXT_SHARE_A = (0x00009067) +KEPLER_CHANNEL_GROUP_A = (0x0000a06c) +PASCAL_DMA_COPY_A = (0x0000c0b5) +TURING_DMA_COPY_A = (0x0000c5b5) +AMPERE_DMA_COPY_A = (0x0000c6b5) +AMPERE_DMA_COPY_B = (0x0000c7b5) +HOPPER_DMA_COPY_A = (0x0000c8b5) +BLACKWELL_DMA_COPY_A = (0x0000c9b5) +BLACKWELL_DMA_COPY_B = (0x0000cab5) +MAXWELL_DMA_COPY_A = (0x0000b0b5) +ACCESS_COUNTER_NOTIFY_BUFFER = (0x0000c365) +MMU_FAULT_BUFFER = (0x0000c369) +MMU_VIDMEM_ACCESS_BIT_BUFFER = (0x0000c763) +HOPPER_MMU_VIDMEM_ACCESS_BIT_BUFFER = (0x0000c863) +TURING_A = (0x0000c597) +TURING_COMPUTE_A = (0x0000c5c0) +AMPERE_A = (0x0000c697) +AMPERE_COMPUTE_A = (0x0000c6c0) +AMPERE_B = (0x0000c797) +AMPERE_COMPUTE_B = (0x0000c7c0) +ADA_A = (0x0000c997) +ADA_COMPUTE_A = (0x0000c9c0) +AMPERE_SMC_PARTITION_REF = (0x0000c637) +AMPERE_SMC_EXEC_PARTITION_REF = (0x0000c638) +AMPERE_SMC_CONFIG_SESSION = (0x0000c639) +NV0092_RG_LINE_CALLBACK = (0x00000092) +AMPERE_SMC_MONITOR_SESSION = (0x0000c640) +HOPPER_A = (0x0000cb97) +HOPPER_COMPUTE_A = (0x0000cbc0) +BLACKWELL_A = (0x0000cd97) +BLACKWELL_COMPUTE_A = (0x0000cdc0) +BLACKWELL_B = (0x0000ce97) +BLACKWELL_COMPUTE_B = (0x0000cec0) +BLACKWELL_INLINE_TO_MEMORY_A = (0x0000cd40) +NV40_DEBUG_BUFFER = (0x000000db) +RM_USER_SHARED_DATA = (0x000000de) +GT200_DEBUGGER = (0x000083de) +NV40_I2C = (0x0000402c) +KEPLER_DEVICE_VGPU = (0x0000a080) +NVA081_VGPU_CONFIG = (0x0000a081) +NVA084_KERNEL_HOST_VGPU_DEVICE = (0x0000a084) +NVA083_GRID_DISPLAYLESS = (0x0000a083) +NV0060_SYNC_GPU_BOOST = (0x00000060) +GP100_UVM_SW = (0x0000c076) +NVENC_SW_SESSION = (0x0000a0bc) +NV_EVENT_BUFFER = (0x000090cd) +NV_OPERATIONAL_EVENT_BUFFER_BIND = (0x000090d0) +NVFBC_SW_SESSION = (0x0000a0bd) +NV_CONFIDENTIAL_COMPUTE = (0x0000cb33) +NV_COUNTER_COLLECTION_UNIT = (0x0000cbca) +NV_SEMAPHORE_SURFACE = (0x000000da) \ No newline at end of file diff --git a/tinygrad_repo/tinygrad/runtime/autogen/webgpu.py b/tinygrad_repo/tinygrad/runtime/autogen/webgpu.py index 8ebc7c9230..209a6971e2 100644 --- a/tinygrad_repo/tinygrad/runtime/autogen/webgpu.py +++ b/tinygrad_repo/tinygrad/runtime/autogen/webgpu.py @@ -4,9 +4,9 @@ import ctypes from typing import Literal, TypeAlias from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR from tinygrad.runtime.support import c -from tinygrad.helpers import WIN, OSX +from tinygrad.helpers import OSX import sysconfig, os -dll = c.DLL('webgpu', os.path.join(sysconfig.get_paths()['purelib'], 'pydawn', 'lib', 'libwebgpu_dawn.dll') if WIN else 'webgpu_dawn') +dll = c.DLL('webgpu', 'webgpu_dawn') WGPUFlags: TypeAlias = ctypes.c_uint64 WGPUBool: TypeAlias = ctypes.c_uint32 class struct_WGPUAdapterImpl(c.Struct): pass diff --git a/tinygrad_repo/tinygrad/runtime/ops_amd.py b/tinygrad_repo/tinygrad/runtime/ops_amd.py index 78b92646b8..9ce0ebcf90 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_amd.py +++ b/tinygrad_repo/tinygrad/runtime/ops_amd.py @@ -1,33 +1,26 @@ from __future__ import annotations from typing import cast, Any -import os, ctypes, struct, functools, importlib, mmap, errno, contextlib, sys, itertools, collections, atexit -assert sys.platform != 'win32' +import os, ctypes, struct, functools, importlib, mmap, errno, contextlib, itertools, collections, atexit from dataclasses import dataclass, replace -from tinygrad.runtime.support.hcq2 import HWQueue, encode_cmdbuf, to_name, patch, unwrap_view, layout_args -from tinygrad.runtime.support.hcq2 import pack_args, make_program -from tinygrad.uop.ops import sint, UOp, ProgramInfo +from tinygrad.runtime.support.hcq2 import HWQueue, encode_cmdbuf, to_name, patch, unwrap_view, layout_args, pack_args, make_program, ccall +from tinygrad.uop.ops import sint, UOp, ProgramInfo, Ops, UPat, PatternMatcher, KernelInfo, uopfunc from tinygrad.device import BufferStorage, BufferSpec, Buffer, Device, Allocator, Compiled from tinygrad.dtype import dtypes, DType from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, ProfileEvent, lo32, hi32, prod, colored from tinygrad.helpers import ceildiv, unwrap, pluralize, ContextVar, VIZ, DEV from tinygrad.renderer.cstyle import HIPRenderer, HIPCCRenderer from tinygrad.renderer.llvmir import AMDLLVMRenderer -from tinygrad.runtime.autogen import kfd, hsa, amdgpu_kd, amdgpu_drm +from tinygrad.runtime.autogen import kfd, hsa, amdgpu_kd, amdgpu_drm, libc from tinygrad.runtime.autogen.am import am from tinygrad.runtime.support.elf import elf_loader -from tinygrad.runtime.support.memory import MMIOInterface -from tinygrad.runtime.support.system import FileIOInterface -from tinygrad.runtime.support.system import filter_visible_devices +from tinygrad.runtime.support.memory import MMIOInterface, AddrSpace +from tinygrad.runtime.support.system import FileIOInterface, filter_visible_devices, PCIIfaceBase, USBPCIDevice, MAP_FIXED, MAP_NORESERVE from tinygrad.runtime.support.am.amdev import AMDev, AMMemoryManager from tinygrad.runtime.support.amd import AMDReg, AMDIP, import_module, import_soc, import_pmc -from tinygrad.runtime.support.system import PCIIfaceBase, USBPCIDevice, MAP_FIXED, MAP_NORESERVE from tinygrad.runtime.support.usb import USB3, setup_usb_rules, usb_reset -from tinygrad.runtime.support.memory import AddrSpace +from tinygrad.engine.realize import get_call_kernel_args if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl # noqa: F401 # pylint: disable=unused-import -from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops -from tinygrad.uop.ops import Ops, UPat, PatternMatcher, uopfunc - SQTT = ContextVar("SQTT", abs(VIZ.value)>=2) SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE = \ ContextVar("SQTT_ITRACE_SE_MASK", 0b11), ContextVar("SQTT_LIMIT_SE", 0), ContextVar("SQTT_SIMD_SEL", 0), ContextVar("SQTT_TOKEN_EXCLUDE", 0) @@ -52,18 +45,17 @@ class ProfilePMCEvent(ProfileEvent): device:str; kern:int; sched:list[PMCSample] # ***************** # PM4 -def _queue_args(hq:HWQueue, q) -> list[UOp]: # the ring and its pointers, tagged {device}_{name}_{queue}. put is the host's copy of the write pointer +def _queue_args(hq:HWQueue, q) -> list[UOp]: # the ring and its pointers, named {device}_{name}_{queue}. put is the host's copy of the write pointer shapes = [("ring", (q.ring.nbytes // 4,), dtypes.uint32, hq.devs[0])] + [(n, (1,), dtypes.uint64, hq.devs[0]) for n in ("write_ptr", "doorbell")] shapes += [("put_value", (1,), dtypes.uint64, hq.dev.host)] - return [UOp.alloc(s, dt, 0, device=d).rtag(hq.dev.tag(n, hq.queue)) for n, s, dt, d in shapes] + return [UOp.alloc(s, dt, 0, device=d, name=to_name(hq.dev.device, n, hq.queue)) for n, s, dt, d in shapes] @uopfunc def amd_push(cmdbuf:UOp, words:UOp, ring:UOp, wptr:UOp, doorbell:UOp, put:UOp, unit:int=4, doorbell_lag:int=0) -> UOp: rs, n, p = ring.max_numel(), words.max_numel() // 4, put.index(0).load() # put counts units, the ring dwords first = (rs - (tail:=((p * (unit // 4)) % rs).cast(dtypes.int))).minimum(n) - for rid, (dst, src, count) in enumerate(((tail, 0, first), (0, first, n - first)), 10): - i = UOp.range(count.after(cmdbuf), rid, dtype=dtypes.int) - cmdbuf = ring.after(cmdbuf).index(dst + i).store(words.bitcast(dtypes.uint32).index(src + i).load()).end(i) + for dst, src, count in ((tail, 0, first), (0, first, n - first)): + cmdbuf = ccall(libc.memcpy, ring.after(cmdbuf).index(dst), words.bitcast(dtypes.uint32).index(src), (count * 4).cast(dtypes.uint64)) w = wptr.after(cmdbuf).index(0).store(nxt:=p + words.max_numel() // unit).barrier() return doorbell.after(put.after(w).index(0).store(nxt)).index(0).store(nxt - doorbell_lag).sink() @@ -76,10 +68,8 @@ def amd_sdma_submit(cmdbuf:UOp, ring:UOp, wptr:UOp, doorbell:UOp, put:UOp) -> UO tail = ((put_b % (rs * 4)) // 4).cast(dtypes.int) fits = (size_dw <= rs - tail).cast(dtypes.int) start_dw, zero_amt = fits * tail, (1 - fits) * (rs - tail) - zi = UOp.range(zero_amt.after(cmdbuf), 10, dtype=dtypes.int) - zero_tail = ring.index(tail + zi).store(UOp.const(0, dtypes.uint32)).end(zi) - i = UOp.range(UOp.const(size_dw, dtypes.int).after(cmdbuf), 11, dtype=dtypes.int) - copy = ring.after(zero_tail).index(start_dw + i).store(cmdbuf.bitcast(dtypes.uint32).index(i).load()).end(i) + zero_tail = ccall(libc.memset, ring.after(cmdbuf).index(tail), 0, (zero_amt * 4).cast(dtypes.uint64)) + copy = ccall(libc.memcpy, ring.after(zero_tail).index(start_dw), cmdbuf.bitcast(dtypes.uint32).index(0), UOp.const(size_dw * 4, dtypes.uint64)) next_put = put_b + ((zero_amt + size_dw) * 4).cast(put_b.dtype) w = wptr.after(copy).index(0).store(next_put).barrier() return doorbell.after(put.after(w).index(0).store(next_put)).index(0).store(next_put).sink() @@ -168,9 +158,8 @@ class AMDComputeQueue(HWQueue): self.pkt3(self.pm4.PACKET3_RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, addr_w, val_w, ctxid) def memory_barrier(self): - pf = '' if self.nbio.version[0] == 2 else '0' if self.nbio.version[:2] != (7, 11) else '1' - self.wait_reg_mem(reg=getattr(self.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_REQ').addr[0], - reg_done=getattr(self.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_DONE').addr[0], value=0xffffffff) + (req, done), mask = self.dev.hdp_flush, self.dev.hdp_flush[0].fields_mask('cp2') + self.wait_reg_mem(reg=req.addr[0], reg_done=done.addr[0], value=mask, mask=mask, op=WAIT_REG_MEM_FUNCTION_EQ) self.acquire_mem() def spi_config(self, tracing:bool): @@ -180,7 +169,7 @@ class AMDComputeQueue(HWQueue): ### profiling: a kernel's slot holds its counters and trace until a synchronize reads them back def prof_buf(self, name:str, dtype:DType) -> UOp: - return UOp.alloc((getattr(self.dev, name).nbytes // dtype.itemsize,), dtype, 0, device=self.devs[0]).rtag(self.dev.tag(name)) + return UOp.alloc((getattr(self.dev, name).nbytes // dtype.itemsize,), dtype, 0, device=self.devs[0], name=name) def prof_start(self, info:ProgramInfo, lib:UOp) -> UOp: log = self.prof_buf("prof_log", dtypes.uint64) @@ -192,6 +181,7 @@ class AMDComputeQueue(HWQueue): return slot def prof_stop(self, slot:UOp): + self.exec_barrier() if self.dev.pmc_enabled: self.pmc_read(slot) if self.dev.sqtt_enabled: self.sqtt_stop(slot) @@ -377,8 +367,7 @@ class AMDComputeQueue(HWQueue): def kernargs(self, call:UOp, prg:UOp, data:AMDProgramData) -> UOp: if not data.kernargs_segment_size and not data.enable_dispatch_ptr: return UOp.const(0, dtypes.uint64) # no kernargs, nothing to point at - args = [get_call_arg_uops(call)[gi].getaddr(self.devs) for gi in prg.arg.globals] + \ - [b.ccast(v.dtype) for v, b in zip(prg.arg.vars, get_call_var_uops(call, prg))] # a bound value is a bare const, the var has the width + args = get_call_kernel_args(call, prg, self.devs) words = pack_args(layout_args(args), data.kernargs_segment_size) + (dispatch_packet(data, prg.arg) if data.enable_dispatch_ptr else []) return UOp(Ops.LINEAR, src=tuple(words), arg="kernargs").getaddr(self.devs) @@ -387,7 +376,7 @@ class AMDComputeQueue(HWQueue): prog_addr = lib.getaddr(self.devs) + data.entry_point_offset args_addr = self.kernargs(call, prg, data) - scratch_addr = UOp.alloc((data.private_segment_size,), dtypes.uint8, 0, device=self.devs[0]).rtag(self.dev.tag("scratch")).getaddr(self.devs) + scratch_addr = UOp.alloc((data.private_segment_size,), dtypes.uint8, 0, device=self.devs[0], name="scratch").getaddr(self.devs) user_regs:list = [] if data.enable_private_segment_sgpr: user_regs = [scratch_addr | (1 << 63), 0xffffffff, 0x20c14000] @@ -396,6 +385,7 @@ class AMDComputeQueue(HWQueue): dispatch_init = self.gc.regCOMPUTE_DISPATCH_INITIATOR.encode( **({'cs_w32_en': int(data.wave32)} if self.target[0] != 9 else {}), force_start_at_000=1, compute_shader_en=1) + if data.private_segment_size: self.exec_barrier() # kernels share the scratch ring, each with its own TMPRING wave size self.acquire_mem(gli=0, gl2=0) if self.dev.prof_enabled: slot = self.prof_start(prg.arg, lib) self.wreg(self.gc.regCOMPUTE_PGM_LO, prog_addr >> 8) @@ -411,9 +401,11 @@ class AMDComputeQueue(HWQueue): self.wreg(self.gc.regCOMPUTE_START_X, 0, 0, 0, *prg.arg.local_size, 0, 0) self.pkt3(self.pm4.PACKET3_DISPATCH_DIRECT, *prg.arg.global_size, dispatch_init) if self.dev.sqtt_enabled: self.pkt3(self.pm4.PACKET3_EVENT_WRITE, self.pm4.EVENT_TYPE(self.soc.THREAD_TRACE_MARKER) | self.pm4.EVENT_INDEX(0)) - self.pkt3(self.pm4.PACKET3_EVENT_WRITE, self.pm4.EVENT_TYPE(self.soc.CS_PARTIAL_FLUSH) | self.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH)) if self.dev.prof_enabled: self.prof_stop(slot) + def exec_barrier(self): + self.pkt3(self.pm4.PACKET3_EVENT_WRITE, self.pm4.EVENT_TYPE(self.soc.CS_PARTIAL_FLUSH) | self.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH)) + def wait(self, signal:UOp, value:UOp): self.wait_reg_mem(value.cast(dtypes.uint32), mem=signal.getaddr(self.devs)) def timestamp(self, signal:UOp): @@ -429,7 +421,7 @@ class AMDComputeQueue(HWQueue): def submit(self, cmdbuf:UOp) -> UOp: # the ring gets an indirect buffer packet: 4 dwords, put stays aligned so it never wraps mid packet base, off = unwrap_view(cmdbuf) blob = struct.pack("IIII", self.pm4.PACKET3(self.pm4.PACKET3_INDIRECT_BUFFER, 2), 0, 0, cmdbuf.max_numel() // 4 | self.pm4.INDIRECT_BUFFER_VALID) - ib = UOp.alloc((16,), dtypes.uint8, device=self.dev.host).rtag(to_name("ib", self.queue)) + ib = UOp.alloc((16,), dtypes.uint8, device=self.dev.host, name=to_name("ib", self.queue)) return amd_push(self.prof_bump(cmdbuf), patch(ib, [(4, base.getaddr(self.devs) + off)], blob), *_queue_args(self, self.dev.compute_queue)) class AMDComputeAQLQueue(AMDComputeQueue): # the ring holds 64 byte aql packets: a dispatch per kernel, the pm4 between them wrapped as an ib @@ -451,6 +443,8 @@ class AMDComputeAQLQueue(AMDComputeQueue): # the ring holds 64 byte aql packets: self.close_run(len(self.blob)) # close pm4, so we can sync xccs. without that wait-signal can race super().signal(signal, value) + def exec_barrier(self): pass + def exec(self, call:UOp, prg:UOp): data, lib = amd_build_program(self.dev, prg, self.devs) self.dev.scratch_buffer(data.private_segment_size) # the queue descriptor holds the scratch @@ -484,6 +478,11 @@ class AMDSDMAQueue(HWQueue): self.q(self.sdma.SDMA_OP_COPY|self.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(self.sdma.SDMA_SUBOP_COPY_LINEAR), min(sz-off, self.max_copy_size)-1, 0, *(a + UOp.const(off, dtypes.uint64) if off else a for a in (src.getaddr(self.devs), dst.getaddr(self.devs)))) + def memory_barrier(self): + (req, done), op = self.dev.hdp_flush, self.sdma.SDMA_OP_POLL_REGMEM | self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) + self.q(op | self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(WAIT_REG_MEM_FUNCTION_EQ), done.addr[0] << 2, req.addr[0] << 2, + m:=req.fields_mask('sdma0'), m, self.sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10) | self.sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff)) + def wait(self, signal:UOp, value:UOp, eq:bool=False): func = WAIT_REG_MEM_FUNCTION_EQ if eq else WAIT_REG_MEM_FUNCTION_GEQ op = self.sdma.SDMA_OP_POLL_REGMEM | self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(func) | self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1) @@ -506,9 +505,6 @@ class AMDSDMAQueue(HWQueue): def encode(self) -> UOp: return self.submit(encode_cmdbuf(self, self.lin, device=self.dev.host)) -def amd_compute_queue(submit:UOp) -> HWQueue: - return (AMDComputeAQLQueue if cast(AMDDevice, Device[submit.src[1].without_after.arg[0][0]]).is_aql else AMDComputeQueue)(submit) - @dataclass(frozen=True) class AMDProgramData: desc_offset:int; entry_point_offset:int; rsrc1:int; rsrc2:int; rsrc3:int; wave32:bool # noqa: E702 @@ -820,10 +816,6 @@ class AMDDevice(Compiled): timestamp_divider = 100.0 # AMD GPU clock: ticks/us sleep_timeout_ms = 200 max_scratch_psize = 0 - pm_encode = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat.custom_function("submit_amd_compute"), UPat()), name="s"), lambda s: amd_compute_queue(s).encode()), - (UPat(Ops.CALL, src=(UPat.custom_function("submit_amd_copy"), UPat()), name="s"), lambda s: AMDSDMAQueue(s).encode()), - ]) ifaces = [KFDIface, PCIIface, USBIface, _mock(KFDIface, "MOCKIface"), _mock(KFDIface), _mock(PCIIface), _mock(USBIface)] @@ -835,7 +827,7 @@ class AMDDevice(Compiled): self.iface = self._select_iface(device) self.is_usb = isinstance(self.iface, USBIface) self.is_vf = self.is_am() and self.iface.dev_impl.is_vf - self.can_recover, self.rtalloc_size = self.is_am() and not self.is_vf, (4 if self.is_usb else 64)<<20 + self.can_recover = self.is_am() and not self.is_vf self.target:tuple[int, ...] = ((trgt:=self.iface.props['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100) self.arch = "gfx%d%x%x" % self.target @@ -857,6 +849,8 @@ class AMDDevice(Compiled): self.nbio = AMDIP('nbio' if self.target[0] < 12 else 'nbif', self.iface.ip_versions[am.NBIF_HWIP], bases={i: tuple(getattr(self.ip_off, f'NBIO_BASE__INST{i}_SEG{s}', 0) for s in range(9)) for i in range(6)}) + pf = '' if self.nbio.version[0] == 2 else '0' if self.nbio.version[:2] != (7, 11) else '1' + self.hdp_flush = getattr(self.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_REQ'), getattr(self.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_DONE') self.is_aql = getenv("AMD_AQL", int(self.xccs > 1)) self.max_copy_size = 0x40000000 if (4, 4, 2) <= (v:=self.iface.ip_versions[am.SDMA0_HWIP]) < (5, 0, 0) or v >= (5, 2, 0) else 0x400000 @@ -867,8 +861,11 @@ class AMDDevice(Compiled): # Scratch setup self.max_private_segment_size = 0 - Compiled.pm_bufferize += PatternMatcher([(UPat(Ops.ALLOC, tag=self.tag("scratch"), name="b"), lambda b, d=self: d.scratch_buffer(b.max_numel())), - (UPat(Ops.ALLOC, name="b"), lambda b, d=self: d.queue_buffer(b.tag))]) + # the ring and its pointers of every queue it can make, made on first use + queues = {"COMPUTE:0": lambda: self.compute_queue} | {f"COPY:{i}": functools.partial(self.sdma_queue, i) for i in range(8)} + names = ("ring", "write_ptr", "doorbell", "put_value") + Compiled.link_bufs |= {to_name(self.device, n, q): lambda b, n=n, f=f: getattr(f(), n) for q, f in queues.items() for n in names} + Compiled.link_bufs |= {to_name(self.device, "scratch"): lambda b: self.scratch_buffer(b.max_numel())} if self.is_usb: setup_usb_rules(self) # SQTT is disabled by default because of runtime overhead and big file sizes (~200mb to Tensor.full() two 4096x4096 tensors and matmul them) @@ -878,8 +875,7 @@ class AMDDevice(Compiled): self.prof_slots, self.prof_read = getenv("PROF_SLOTS", 32), 0 self.pmc_sched:list[PMCSample] = [] self.sqtt_ses, self.sqtt_win = self.se_cnt * self.xccs, (getenv("SQTT_BUFFER_SIZE", 256) << 20) // self.prof_slots # mb, per shader engine - Compiled.pm_bufferize += PatternMatcher([(UPat(Ops.ALLOC, tag=self.tag(n)), lambda n=n, d=self: getattr(d, n)) - for n in ("prof_log", "pmc_buf", "sqtt_buf", "sqtt_wptrs")]) + Compiled.link_bufs |= {to_name(self.device, n): lambda b, n=n: getattr(self, n) for n in ("prof_log", "pmc_buf", "sqtt_buf", "sqtt_wptrs")} if self.pmc_enabled: self.pmc_counters = import_pmc(self.target) # validate counters: SQ for SIMD busy/instruction counts, LDS stats, GRBM for GPU cycles, L2 cache hits/misses @@ -916,11 +912,6 @@ class AMDDevice(Compiled): wptr=getattr(hsa.amd_queue_t, 'write_dispatch_id').offset, eop_buffer=eop_buffer, cwsr_buffer=cwsr_buffer, ctx_save_restore_size=ctx_save_restore_size, ctl_stack_size=ctl_stack_size, idx=idx) - def queue_buffer(self, tag): - if not isinstance(tag, str) or not (t:=tag.removeprefix(self.tag(""))).startswith(("ring_", "write_ptr_", "doorbell_", "put_value_")): return None - name, queue, idx = t.rsplit('_', 2) - return getattr(self.compute_queue if queue == 'compute' else self.sdma_queue(int(idx)), name) - @functools.cached_property def compute_queue(self) -> AMDQueueDesc: # https://gitlab.freedesktop.org/agd5f/linux/-/blob/a1fc9f584c4aaf8bc1ebfa459fc57a3f26a290d8/drivers/gpu/drm/amd/amdkfd/kfd_queue.c#L391 @@ -1015,11 +1006,6 @@ class AMDDevice(Compiled): blob = bytes(self.sqtt_buf.host.view(fmt='B')[off:off + wptr]) return (struct.pack(' str: + check(fn(*args, 0, None, ctypes.byref(size:=ctypes.c_size_t()))) + return (check(fn(*args, size.value, buf:=ctypes.create_string_buffer(size.value), None)), buf.value.decode())[1] class CLCompiler(Compiler): - def __init__(self, dev:CLDevice, compile_key:str): + def __init__(self, dev:CLDevice): self.dev = dev - super().__init__(f"compile_cl_{compile_key}") + super().__init__(f"compile_cl_{hashlib.md5(dev.device_name.encode() + dev.driver_version.encode()).hexdigest()}") def compile(self, src:str) -> bytes: - program = checked(cl.clCreateProgramWithSource(self.dev.context, 1, to_char_p_p([src.encode()]), None, status := ctypes.c_int32()), status) - build_status: int = cl.clBuildProgram(program, 1, self.dev.cl_dev, None, BP_CB(), None) - if build_status != 0: - cl.clGetProgramBuildInfo(program, self.dev.cl_dev, cl.CL_PROGRAM_BUILD_LOG, 0, None, log_size := ctypes.c_size_t()) - cl.clGetProgramBuildInfo(program, self.dev.cl_dev, cl.CL_PROGRAM_BUILD_LOG, - log_size.value, mstr := ctypes.create_string_buffer(log_size.value), None) - raise CompileError(f"OpenCL Compile Error\n\n{mstr.value.decode()}") - check(cl.clGetProgramInfo(program, cl.CL_PROGRAM_BINARY_SIZES, ctypes.sizeof(ctypes.c_size_t), binary_sizes := (ctypes.c_size_t * 1)(), None)) - check(cl.clGetProgramInfo(program, cl.CL_PROGRAM_BINARIES, ctypes.sizeof(ctypes.c_void_p), - (ctypes.c_void_p * 1)(ctypes.addressof(binary := ctypes.create_string_buffer(binary_sizes[0]))), None)) - check(cl.clReleaseProgram(program)) - return bytes(binary) + program = checked(cl.clCreateProgramWithSource, self.dev.context, 1, to_char_p_p([src.encode()]), None) + if cl.clBuildProgram(program, 1, self.dev.cl_dev, None, BP_CB(), None) != 0: + raise CompileError(f"OpenCL Compile Error\n\n{info(cl.clGetProgramBuildInfo, program, self.dev.cl_dev, cl.CL_PROGRAM_BUILD_LOG)}") + check(cl.clGetProgramInfo(program, cl.CL_PROGRAM_BINARY_SIZES, 8, ctypes.byref(size:=ctypes.c_size_t()), None)) + check(cl.clGetProgramInfo(program, cl.CL_PROGRAM_BINARIES, 8, ctypes.byref(ctypes.pointer(lib:=ctypes.create_string_buffer(size.value))), None)) + return (check(cl.clReleaseProgram(program)), bytes(lib))[1] -class CLProgram(Program['CLDevice']): - def __init__(self, device:CLDevice, obj:TinyELF): - self.dev, self.lib, self.signature = device, device.cl_compiler.compile_cached(obj.lib.decode()), obj.signature - self.program = checked(cl.clCreateProgramWithBinary(device.context, 1, device.cl_dev, (ctypes.c_size_t * 1)(len(self.lib)), - to_char_p_p([self.lib], ctypes.c_ubyte), binary_status := ctypes.c_int32(), - errcode_ret := ctypes.c_int32()), errcode_ret) - check(binary_status.value) - check(cl.clBuildProgram(self.program, 1, device.cl_dev, None, BP_CB(), None)) # NOTE: OSX requires this - self.kernel = checked(cl.clCreateKernel(self.program, obj.name.encode(), status := ctypes.c_int32()), status) +# ***************** +# queue - def __del__(self): - try: check(cl.clReleaseKernel(self.kernel)) - except (TypeError, AttributeError): pass - try: check(cl.clReleaseProgram(self.program)) - except (TypeError, AttributeError): pass +@uopfunc +def cl_run(buf:UOp, cmds:tuple) -> UOp: # one in-order queue, every call after the last + cb = buf + for fn, *args in cmds: + h = ccall(fn, *[cb.index(a[0]) if isinstance(a, tuple) else a if isinstance(a, UOp) else cb.bitcast(dtypes.uint64).index(a).load() for a in args]) + if fn is cl.clCreateImage: h = buf.bitcast(dtypes.uint64).index(args[-1][0] // 8).store(h) # the image replaces its errcode, not chained + cb = cb.after(h) + return cb.sink() - def __call__(self, *bufs:cl.cl_mem, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), - wait=False, **kw) -> float|None: - for i, (_, slot, dt, shape) in enumerate(self.signature): - b = bufs[slot] if slot < len(bufs) else getattr(ctypes, f"c_int{dt.bitsize}")(vals[slot-len(bufs)]) - if is_image_shape(shape): - pitch = (round_up(shape[1], 256) if OSX else shape[1]) * 4 * dt.itemsize - fmt = cl.cl_image_format(cl.CL_RGBA, {2:cl.CL_HALF_FLOAT, 4:cl.CL_FLOAT}[dt.itemsize]) - desc = cl.cl_image_desc(cl.CL_MEM_OBJECT_IMAGE2D, shape[1], shape[0], image_row_pitch=pitch, buffer=b) - img = checked(cl.clCreateImage(self.dev.context, cl.CL_MEM_READ_WRITE, fmt, desc, None, status:=ctypes.c_int32()), status) - check(cl.clSetKernelArg(self.kernel, i, ctypes.sizeof(img), ctypes.byref(img))) - else: check(cl.clSetKernelArg(self.kernel, i, ctypes.sizeof(b), ctypes.byref(b))) - global_size = cast(tuple[int,int,int], tuple(int(g*l) for g,l in zip(global_size, local_size))) - event = cl.cl_event() if wait else None - check(cl.clEnqueueNDRangeKernel(self.dev.queue, self.kernel, len(global_size), None, (ctypes.c_size_t * len(global_size))(*global_size), - (ctypes.c_size_t * len(local_size))(*local_size), 0, None, event)) - if wait: - assert event is not None - check(cl.clWaitForEvents(1, event)) - check(cl.clGetEventProfilingInfo(event, cl.CL_PROFILING_COMMAND_START, 8, ctypes.byref(start := ctypes.c_uint64()), None)) - check(cl.clGetEventProfilingInfo(event, cl.CL_PROFILING_COMMAND_END, 8, ctypes.byref(end := ctypes.c_uint64()), None)) - return float(end.value-start.value) * OSX_TIMING_RATIO * 1e-9 - return None +class CLQueue(HWQueue): + dev:CLDevice + def q64(self, *vals) -> int: return self.q(*[UOp.const(v, dtypes.uint64) if isinstance(v, int) else v.cast(dtypes.uint64) for v in vals]) + def call(self, fn, *args): self.cmds.append((fn, *[UOp.const(a, dtypes.uint64) if isinstance(a, int) else a for a in args])) -class CLAllocator(Allocator['CLDevice']): + def exec(self, call:UOp, prg:UOp): + args, images = get_call_kernel_args(call, prg, self.devs), [] + kernel = (p:=make_program(prg, 8, self.devs[0])).after(p.store(prg)).bitcast(dtypes.u64).index(0).load() + for i, (_, _, dt, shape) in enumerate(prg.to_elf().signature): + if is_image_shape(shape): # [format][desc][errcode] + desc = (cl.CL_MEM_OBJECT_IMAGE2D, shape[1], shape[0], 0, 0, round_up(shape[1], 256 if OSX else 1) * 4 * dt.itemsize, 0, 0, args[i]) + end = self.q64(cl.CL_RGBA | {2:cl.CL_HALF_FLOAT, 4:cl.CL_FLOAT}[dt.itemsize] << 32, *desc, 0) + self.call(cl.clCreateImage, self.dev.handles[0], cl.CL_MEM_READ_WRITE, (end - 88,), (end - 80,), 0, (end - 8,)) + images.append((end - 8) // 8) + else: end = self.q64(args[i]) + self.call(cl.clSetKernelArg, kernel, i, args[i].dtype.itemsize, (end - 8,)) + end = self.q64(*[g * l for g, l in zip(prg.arg.global_size, prg.arg.local_size)], *prg.arg.local_size) # [global][local] + self.call(cl.clEnqueueNDRangeKernel, self.dev.handles[1], kernel, 3, 0, (end - 48,), (end - 24,), 0, 0, 0) + self.cmds += [(cl.clReleaseMemObject, w) for w in images] + + def wait(self, sig:UOp, val:UOp, eq=False): pass # the queue is in order + def signal(self, sig:UOp, val:UOp): self.call(libc.memcpy, sig.getaddr(self.devs), (self.q64(val) - 8,), 8) # synchronize finishes the queue + def timestamp(self, sig:UOp): self.call(cl.clEnqueueMarkerWithWaitList, self.dev.handles[1], 0, 0, sig.getaddr(self.devs)) # [event][time] + + def encode(self) -> UOp: + self.cmds:list[tuple] = [] + return cl_run(encode_cmdbuf(self, self.lin), tuple(self.cmds)) + +Compiled.pm_encode += PatternMatcher([(UPat.sink(arg=KernelInfo("cl_compute_submit")).call(UPat()).after(allow_any_len=True, name="s"), + lambda s: CLQueue(s).encode())]) + +# ***************** +# device + +class CLAllocator(HostAllocator): def _alloc(self, size:int, options:BufferSpec) -> BufferStorage: - return BufferStorage(checked(cl.clCreateBuffer(self.dev.context, cl.CL_MEM_READ_WRITE, size, None, status := ctypes.c_int32()), status)) + if options.host or options.cpu_access: return super()._alloc(size, options) + return BufferStorage(int.from_bytes(bytes(mem:=checked(cl.clCreateBuffer, self.dev.context, cl.CL_MEM_READ_WRITE, size, None)), "little"), mem) @suppress_finalizing - def _free(self, storage:BufferStorage, options:BufferSpec): check(cl.clReleaseMemObject(storage.buf)) - def _copyin(self, dest:cl.cl_mem, src:memoryview): - self.dev.pending_copyin.append(src:=memoryview(bytearray(src))) # NOTE: these can't be freed until the GPU actually executes this command - check(cl.clEnqueueWriteBuffer(self.dev.queue, dest, False, 0, len(src)*src.itemsize, from_mv(src), 0, None, None)) - def _copyout(self, dest:memoryview, src:cl.cl_mem): - check(cl.clEnqueueReadBuffer(self.dev.queue, src, False, 0, len(dest)*dest.itemsize, from_mv(dest), 0, None, None)) - self.dev.synchronize() + def _free(self, storage:BufferStorage, options:BufferSpec): + if storage.host is None: check(cl.clReleaseMemObject(storage.meta)) + def _copy(self, fn, name:str, mem:int, host, size:int): # blocking, so the host memory can go + with cpu_profile(name, f"{self.dev.device}:COPY"): check(fn(self.dev.queue, ctypes.cast(mem, cl.cl_mem), True, 0, size, host, 0, None, None)) + def _copyin(self, dest:int, src:memoryview): self._copy(cl.clEnqueueWriteBuffer, f"TINY -> {self.dev.device}", dest, bytes(src), src.nbytes) + def _copyout(self, dest:memoryview, src:int): self._copy(cl.clEnqueueReadBuffer, f"{self.dev.device} -> TINY", src, from_mv(dest), dest.nbytes) class CLDevice(Compiled): device_ids = None # this is global and only initted once + timestamp_divider = 3000 / 125 if OSX else 1000.0 + def __init__(self, device:str=""): if CLDevice.device_ids is None: - check(cl.clGetPlatformIDs(0, None, num_platforms := ctypes.c_uint32())) - check(cl.clGetPlatformIDs(num_platforms.value, platform_ids := (cl.cl_platform_id * num_platforms.value)(), None)) + check(cl.clGetPlatformIDs(1, platform:=cl.cl_platform_id(), num_platforms:=ctypes.c_uint32())) for device_type in [cl.CL_DEVICE_TYPE_GPU, cl.CL_DEVICE_TYPE_DEFAULT]: - err = cl.clGetDeviceIDs(platform_ids[0], device_type, 0, None, num_devices := ctypes.c_uint32()) - if err == 0 and num_devices.value != 0: break + if cl.clGetDeviceIDs(platform, device_type, 0, None, num_devices:=ctypes.c_uint32()) == 0 and num_devices.value != 0: break if DEBUG >= 1: print(f"CLDevice: got {num_platforms.value} platforms and {num_devices.value} devices") - CLDevice.device_ids = c.init_c_var((cl.cl_device_id * num_devices.value), - lambda x: check(cl.clGetDeviceIDs(platform_ids[0], device_type, num_devices, x, None))) + CLDevice.device_ids = (check(cl.clGetDeviceIDs(platform, device_type, num_devices, ids:=(cl.cl_device_id * num_devices.value)(), None)), ids)[1] self.cl_dev = CLDevice.device_ids[0 if ":" not in device else int(device.split(":")[1])] - self.device_name = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_NAME, 256, - buf:=ctypes.create_string_buffer(256), None), buf.value.decode())[1] - self.driver_version = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DRIVER_VERSION, 256, - buf:=ctypes.create_string_buffer(256), None), buf.value.decode())[1] + self.device_name, self.driver_version = [info(cl.clGetDeviceInfo, self.cl_dev, p) for p in (cl.CL_DEVICE_NAME, cl.CL_DRIVER_VERSION)] if DEBUG >= 1: print(f"CLDevice: opening {self.device_name} with version {self.driver_version}") - self.context = checked(cl.clCreateContext(None, 1, self.cl_dev, CC_CB(), None, status := ctypes.c_int32()), status) - self.queue = checked(cl.clCreateCommandQueue(self.context, self.cl_dev, cl.CL_QUEUE_PROFILING_ENABLE, status), status) - self.pending_copyin: list[memoryview] = [] - check(cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_EXTENSIONS, 0, None, ctypes.byref(exts_len:=ctypes.c_size_t()))) - self.device_exts = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_EXTENSIONS, exts_len.value, - ctypes.byref(buf := ctypes.create_string_buffer(exts_len.value)), None), - ctypes.string_at(buf).decode().split())[1] + self.context = checked(cl.clCreateContext, None, 1, self.cl_dev, CC_CB(), None) + self.queue = checked(cl.clCreateCommandQueue, self.context, self.cl_dev, cl.CL_QUEUE_PROFILING_ENABLE) + self.handles = [int.from_bytes(bytes(h), "little") for h in (self.context, self.queue)] - self.cl_compiler = CLCompiler(self, f"{hashlib.md5(self.device_name.encode() + self.driver_version.encode()).hexdigest()}") - - arch = ",".join(self.device_exts) - if "cl_khr_image2d_from_buffer" in self.device_exts: + arch = ",".join(exts:=info(cl.clGetDeviceInfo, self.cl_dev, cl.CL_DEVICE_EXTENSIONS).split()) + if "cl_khr_image2d_from_buffer" in exts: check(cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_IMAGE_PITCH_ALIGNMENT, 4, ctypes.byref(ipa := ctypes.c_uint32()), None)) arch += f",IMAGE_PITCH_ALIGNMENT={ipa.value}" - super().__init__(device, CLAllocator(self), [OpenCLRenderer], CLProgram, arch=arch) + super().__init__(device, CLAllocator(self), [OpenCLRenderer], None, arch=arch) + Compiled.pm_link += PatternMatcher([(UPat(Ops.ALLOC, name="p").store(UPat(Ops.PROGRAM, name="prg")), lambda p, prg, d=self: + p.store(UOp(Ops.BINARY, arg=d.function(prg.src[3].arg, prg.src[0].arg.function_name))) if p.device == d.device else None)]) + + @functools.cache + def function(self, src:bytes, name:str) -> bytes: + lib = CLCompiler(self).compile_cached(src.decode()) + program = checked(cl.clCreateProgramWithBinary, self.context, 1, self.cl_dev, ctypes.c_size_t(len(lib)), to_char_p_p([lib], ctypes.c_ubyte), None) + check(cl.clBuildProgram(program, 1, self.cl_dev, None, BP_CB(), None)) # NOTE: OSX requires this + return bytes(checked(cl.clCreateKernel, program, name.encode())) def count(self) -> int: return len(unwrap(self.device_ids)) - - def synchronize(self, timeout:int|None=None): + def _wait_signal(self, sig:MMIOInterface|memoryview, value:int, timeout:int|None=None): check(cl.clFinish(self.queue)) - self.pending_copyin.clear() + for s, i in [(s, i) for b, st in self.prof_ents for s in [b.host.view(fmt='Q')] for i in (st - 1, st + 1) if s[i]]: # [event, 0] + check(cl.clGetEventProfilingInfo(ev:=ctypes.cast(s[i], cl.cl_event), cl.CL_PROFILING_COMMAND_END, 8, s.addr + i * 8 + 8, None)) + s[i] = (check(cl.clReleaseEvent(ev)), 0)[1] diff --git a/tinygrad_repo/tinygrad/runtime/ops_cpu.py b/tinygrad_repo/tinygrad/runtime/ops_cpu.py index 978c59039a..986b7baa8f 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_cpu.py +++ b/tinygrad_repo/tinygrad/runtime/ops_cpu.py @@ -1,85 +1,62 @@ from __future__ import annotations -import platform, sys, ctypes, mmap, struct -from typing import cast, Any -from tinygrad.helpers import OSX, WIN, mv_address, suppress_finalizing, unwrap, data64_le, cpu_profile -from tinygrad.device import Compiled, TinyELF, Program, HostAllocator +import platform, ctypes, struct, json +from typing import Any +from tinygrad.helpers import OSX, unwrap, data64_le, cpu_profile +from tinygrad.device import Compiled, TinyELF, Program, HostAllocator, BufferSpec from tinygrad.runtime.support.c import DLL from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.renderer.llvmir import CPULLVMRenderer from tinygrad.renderer.nir import LVPRenderer -from tinygrad.renderer.isa.x86 import X86Renderer +from tinygrad.renderer.x86 import X86Renderer from tinygrad.runtime.support.elf import jit_loader from tinygrad.runtime.support.system import RemotePCIDevice, RemoteCmd from tinygrad.runtime.autogen import libc -# NOTE: MAP_JIT is added to mmap module in python 3.13 -MAP_JIT = 0x0800 - class CPUProgram(Program['CPUDevice']): - rt_lib, libm = DLL('rt', 'System' if OSX else 'kernel' if WIN else 'gcc_s'), DLL('m', 'ucrtbase' if WIN else 'm') + rt_lib, libm = DLL('rt', 'System' if OSX else 'gcc_s'), DLL('m', 'm') - def _load(self, lib, base=0): return lib if lib[:4] != libc.ELFMAG.encode() else jit_loader(lib, base=base, link_libs=list(DLL._loaded_.values())) + def _load(self, lib, base=0): return lib if lib[:4] != libc.ELFMAG.encode() else jit_loader(lib, base=base, link_libs=list(DLL._loaded_.values()), + link_syms={n:p.buf.buf for n,p in self.links.items()}) - def __init__(self, dev:CPUDevice, obj:TinyELF): - self.dev, self.name, self.signature, self.profile_key = dev, obj.name, obj.signature, obj.profile_key + def __init__(self, dev:CPUDevice, obj:TinyELF, links:dict[str,CPUProgram]): + self.dev, self.name, self.signature, self.profile_key, self.links = dev, obj.name, obj.signature, obj.profile_key, links self.lvp = obj.target.renderer == "LVP" - if dev.remote is not None: self.fxn:Any = dev.remote.rpc(RemoteCmd.LOAD_PROG, len(obj.lib), payload=obj.lib)[0] - elif sys.platform == "win32": # mypy doesn't understand when WIN is used here - PAGE_EXECUTE_READWRITE, MEM_COMMIT, MEM_RESERVE = 0x40, 0x1000, 0x2000 - ctypes.windll.kernel32.VirtualAlloc.restype = ctypes.c_void_p - self.addr = ctypes.windll.kernel32.VirtualAlloc(ctypes.c_void_p(0), ctypes.c_size_t(len(obj.lib)), MEM_COMMIT | MEM_RESERVE, - PAGE_EXECUTE_READWRITE) - ctypes.memmove(self.addr, (loaded:=self._load(obj.lib, self.addr)), len(loaded)) - ctypes.windll.kernel32.GetCurrentProcess.restype = ctypes.c_void_p - proc = ctypes.windll.kernel32.GetCurrentProcess() - ctypes.windll.kernel32.FlushInstructionCache(ctypes.c_void_p(proc), ctypes.c_void_p(self.addr), ctypes.c_size_t(len(loaded))) - self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) + self.packer = struct.Struct(''.join(unwrap(dt.fmt) for _, dt in TinyELF.iter_sig(self.signature))) + if dev.remote is not None: + link_ids = json.dumps({n:p.fxn for n,p in links.items()}).encode() + self.fxn:Any = dev.remote.rpc(RemoteCmd.LOAD_PROG, len(obj.lib), len(link_ids), payload=obj.lib + link_ids)[0] else: - # On apple silicon with SPRR enabled (it always is in macos) RWX pages are unrepresentable: https://blog.svenpeter.dev/posts/m1_sprr_gxf/ - # MAP_JIT allows us to easily flip pages from RW- to R-X and vice versa. It is a noop on intel cpus. (man pthread_jit_write_protect_np) - self.mem = mmap.mmap(-1, len(obj.lib), mmap.MAP_ANON|mmap.MAP_PRIVATE|(MAP_JIT if OSX else 0), mmap.PROT_READ|mmap.PROT_WRITE|mmap.PROT_EXEC) - self.addr = mv_address(self.mem) + self.buf = dev.allocator.alloc(len(obj.lib), BufferSpec(nolru=True)) + loaded = self._load(obj.lib, self.buf.buf) + unwrap(self.buf.host)[:len(loaded)] = loaded - if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(False) - self.mem.write(loaded:=self._load(obj.lib, mv_address(self.mem))) - if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(True) + # RWX pages are unrepresentable on apple silicon (https://blog.svenpeter.dev/posts/m1_sprr_gxf/), so flip the written pages to R-X + libc.mprotect(ctypes.c_void_p(self.buf.buf), len(obj.lib), libc.PROT_READ | libc.PROT_EXEC) # __clear_cache isn't a normal libc function, but a compiler support routine found in libgcc_s for gcc and compiler-rt for clang. # libgcc_s comes as shared library but compiler-rt is only a bunch of static library archives which we can't directly load, but fortunately # it somehow found its way into libSystem on macos (likely because it used __builtin_clear_cache) and libgcc_s is ~always present on linux # Using ["name"] instead of .name because otherwise name is getting mangled: https://docs.python.org/3.12/reference/expressions.html#index-5 - if 'rt' in DLL._loaded_: CPUProgram.rt_lib["__clear_cache"](ctypes.c_void_p(self.addr), ctypes.c_void_p(self.addr + len(loaded))) + if 'rt' in DLL._loaded_: CPUProgram.rt_lib["__clear_cache"](ctypes.c_void_p(self.buf.buf), ctypes.c_void_p(self.buf.buf + len(loaded))) else: # msync should be a universal POSIX way to do this - libc.msync(ctypes.c_void_p(self.addr), len(loaded), libc.MS_SYNC | libc.MS_INVALIDATE) + libc.msync(ctypes.c_void_p(self.buf.buf), len(loaded), libc.MS_SYNC | libc.MS_INVALIDATE) - self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) + self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.buf.buf) - def __call__(self, *bufs:int, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), - vals:tuple[int|None, ...]=(), wait:bool=False, timeout:int|None=None) -> float|None: - args = [*bufs, *cast(tuple[int, ...], vals)] + def __call__(self, *args:int, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), + wait:bool=False, timeout:int|None=None) -> float|None: + hdr = 12 if self.lvp else 0 + kernargs = ctypes.create_string_buffer(hdr + len(args) * 8) + if self.lvp: struct.pack_into('<3I', kernargs, 0, *data64_le(ctypes.addressof(kernargs) + hdr), len(args) * 2) + self.packer.pack_into(kernargs, hdr, *args) if (remote:=self.dev.remote) is not None: - data = struct.pack(f'<{len(args)}Q', *(a & 0xffffffffffffffff for a in args)) - ret = (remote._rpc if wait else remote._post)(remote.sock, RemoteCmd.EXEC_PROG, self.fxn, len(args), int(wait), payload=data) + ret = (remote._rpc if wait else remote._post)(remote.sock, RemoteCmd.EXEC_PROG, self.fxn, len(kernargs), int(wait), payload=bytes(kernargs)) return ret[0] / 1e9 if ret is not None else None - with cpu_profile(self.name, self.dev.device, profile_key=self.profile_key) as prof: - if self.lvp: - lvp_args = bytearray(12 + (len(bufs) + len(vals)) * 8) - addr = mv_address(lvp_args) - struct.pack_into(f'<3I{len(bufs)}Q', lvp_args, 0, *data64_le(addr+12), (len(bufs)+len(vals))*2, *bufs) - for v,(off,dt) in zip(vals, TinyELF.iter_sig(self.signature[-len(vals):], len(bufs)*8)): struct.pack_into(f'<{dt.fmt}', lvp_args, 12+off, v) - self.fxn(addr) - else: self.fxn(*[ctypes.c_uint64(x) for x in args]) + with cpu_profile(self.name, self.dev.device, profile_key=self.profile_key) as prof: self.fxn(kernargs) return float(unwrap(prof.en) - prof.st) * 1e-6 if wait else None - @suppress_finalizing - def __del__(self): - if self.dev.remote is None and sys.platform == 'win32': - ctypes.windll.kernel32.VirtualFree(ctypes.c_void_p(self.addr), ctypes.c_size_t(0), 0x8000) #0x8000 - MEM_RELEASE - class CPUDevice(Compiled): - wait_timeout_ms = 30000 - @property def has_copy_queue(self) -> bool: return False diff --git a/tinygrad_repo/tinygrad/runtime/ops_cuda.py b/tinygrad_repo/tinygrad/runtime/ops_cuda.py index 8d74851e20..80e5428f52 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_cuda.py +++ b/tinygrad_repo/tinygrad/runtime/ops_cuda.py @@ -1,64 +1,62 @@ from __future__ import annotations import ctypes, functools, mmap, struct, time -from typing import Any -from tinygrad.helpers import DEBUG, DEV, getenv, unwrap -from tinygrad.device import Buffer, BufferStorage, BufferSpec, Allocator, Compiled, MMIOInterface, HCQ_RUNTIME_DEV +from typing import Any, Callable +from tinygrad.helpers import DEBUG, DEV, getenv, unwrap, to_name +from tinygrad.device import Device, Buffer, BufferStorage, BufferSpec, Allocator, Compiled, MMIOInterface, HCQ_RUNTIME_DEV from tinygrad.dtype import dtypes -from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, uopfunc -from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops +from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo +from tinygrad.engine.realize import get_call_kernel_args from tinygrad.renderer.cstyle import CUDARenderer, NVCCRenderer from tinygrad.renderer.ptx import PTXRenderer from tinygrad.runtime.autogen import cuda from tinygrad.runtime.support.compiler_cuda import pretty_ptx from tinygrad.runtime.support.c import init_c_var -from tinygrad.runtime.support.hcq2 import HWQueue, ccall, layout_args, pack_args, encode_cmdbuf +from tinygrad.runtime.support.hcq2 import ccall, layout_args, patch, make_program if getenv("IOCTL"): import extra.nv_gpu_driver.nv_ioctl # noqa: F401 # pylint: disable=unused-import if DEV.target("CUDA").interface == "MOCK": import test.mockgpu.cuda.cuda # noqa: F401 # pylint: disable=unused-import def check(status:int): if status != 0: raise RuntimeError(f"CUDA Error {status}, {cuda.enum_cudaError_enum.get(status, 'unknown')}") -def as_int(handle) -> int: return unwrap(ctypes.cast(handle, ctypes.c_void_p).value) - -def host_stamp(slot:int): ctypes.c_uint64.from_address(slot).value = time.perf_counter_ns() -def extern(ptr:int, meta=None) -> Buffer: return Buffer(HCQ_RUNTIME_DEV.device, 8, opaque=BufferStorage(ptr, meta)) - # ***************** -# queue +# runtime -@uopfunc -def cuda_run(cmdbuf:UOp, rt_vars:UOp, cmds:tuple, copy:bool) -> UOp: # rt_vars: [ctx, compute stream, copy stream, status] - words, h = cmdbuf.bitcast(dtypes.uint64), ccall(cuda.cuCtxSetCurrent, rt_vars.index(0).load()) - for fn, *args in cmds: - s = rt_vars.after(h).index(2 if copy else 1).load() # the stream after the last call - h = ccall(fn, *[s if a is None else cmdbuf.index(a[0]) if isinstance(a, tuple) else words.index(a).load() for a in args]) - return rt_vars.after(h).index(3).store(h.cast(dtypes.uint64)).sink() +def cu_launch_kernel(ctx:CUDADevice, call:UOp, prg:UOp) -> tuple: + rows = layout_args(get_call_kernel_args(call, prg, (ctx.device,)), 48) + buf = UOp.alloc((max([o + w.dtype.itemsize for o, w in rows], default=48),), dtypes.uint8, device=HCQ_RUNTIME_DEV.device, name="kernargs") + addr = buf.getaddr(HCQ_RUNTIME_DEV.device) # [extra: buffer pointer, &args, buffer size, &size, end][size][args] + hdr = layout_args([UOp.const(v, dtypes.uint64) if isinstance(v, int) else v for v in (1, addr+48, 2, addr+40, ctx.launch_end, buf.max_numel()-48)]) + func = (p:=make_program(prg, 8, ctx.device)).after(p.store(prg)).bitcast(dtypes.u64).index(0).load() + return (cuda.cuLaunchKernel, func, *prg.arg.global_size, *prg.arg.local_size, 0, None, 0, patch(buf, hdr + rows).index(0)) -class CUDAQueue(HWQueue): - dev:CUDADevice - def call(self, fn, *args): # the values go in the cmdbuf as words - vals = [UOp.const(a, dtypes.uint64) if isinstance(a, int) else a for a in args] - self.cmds.append((fn, *[a if a is None or isinstance(a, tuple) else self.q(a.cast(dtypes.uint64)) // 8 - 1 for a in vals])) - def extern(self, tag) -> UOp: return UOp.alloc((1,), dtypes.uint64, 0, device=self.devs[0]).rtag(tag).getaddr(self.dev.host) +pm_cuda = PatternMatcher([ # None is the stream + (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cu_launch_kernel), + (UPat(Ops.CALL, src=(UPat(Ops.STORE), UPat(name="dst"), UPat(name="src")), allow_any_len=True), + lambda ctx, dst, src: (cuda.cuMemcpyAsync, dst.getaddr(ctx.device), src.getaddr(ctx.device), src.max_numel() * src.dtype.itemsize, None)), + (UPat(Ops.CALL, src=(UPat.custom_function("copy"), UPat(name="dst"), UPat(name="src"), UPat(name="n"))), + lambda ctx, dst, src, n: (cuda.cuMemcpyAsync, dst.getaddr(ctx.device), src.getaddr(ctx.device), n.val, None)), + (UPat(Ops.CALL, src=(UPat.custom_function("wait"), UPat(name="sig"), UPat(name="val"))), + lambda ctx, sig, val: (cuda.cuStreamWaitValue64_v2, None, sig.getaddr(ctx.device), val, cuda.CU_STREAM_WAIT_VALUE_GEQ)), + (UPat(Ops.CALL, src=(UPat.custom_function("store"), UPat(name="sig"), UPat(name="val"))), + lambda ctx, sig, val: (cuda.cuStreamWriteValue64_v2, None, sig.getaddr(ctx.device), val, cuda.CU_STREAM_WRITE_VALUE_DEFAULT)), + (UPat(Ops.CALL, src=(UPat.custom_function("timestamp"), UPat(name="sig"))), # [sig][stamp] + lambda ctx, sig: (cuda.cuLaunchHostFunc, None, UOp.alloc((1,), dtypes.uint64, 0, device=ctx.device, name="stamp").getaddr(ctx.host), + sig[1:2].getaddr(ctx.device))), +]) - def exec(self, call:UOp, prg:UOp): - obj, bufs, vals = prg.to_elf(), get_call_arg_uops(call), get_call_var_uops(call, prg) - rows = layout_args([bufs[i].getaddr(self.devs) for i in prg.arg.globals] + [v.ccast(var.dtype) for v, var in zip(vals, prg.arg.vars)], 8) - size = max([o + w.dtype.itemsize for o, w in rows], default=8) - 8 - addr = UOp(Ops.LINEAR, src=tuple(pack_args([(0, UOp.const(size, dtypes.uint64))] + rows, 8 + size)), arg="kernargs").getaddr(self.devs) - # extra: [buffer pointer, &args, buffer size, &size, end] - self.call(cuda.cuLaunchKernel, self.extern((self.dev.tag("function"), obj.lib, obj.name)), *prg.arg.global_size, *prg.arg.local_size, 0, None, 0, - (self.q(*[UOp.const(v, dtypes.uint64) if isinstance(v, int) else v for v in (1, addr + 8, 2, addr, self.dev.launch_end)]) - 40,)) +def cuda_submit(s:UOp, ccallfn:Callable[[Any, *tuple[UOp, ...]], UOp]=ccall) -> UOp: + (call, *deps), q = s.src, s.src[0].src[1] + rt_vars = UOp.param(0, dtypes.uint64, 3, q.device, name="rt_vars") # [ctx, compute stream, copy stream] + h = ccallfn(cuda.cuCtxSetCurrent, rt_vars.index(0).load()) + for fn, *args in [x for c in call.body.src if (x:=pm_cuda.rewrite(c, ctx=Device[q.device])) is not None]: + stream = rt_vars.after(h).index(2 if q.arg.name.startswith("COPY") else 1).load() # the stream after the last call + h = ccallfn(fn, *[stream if a is None else UOp.const(a, dtypes.uint64) if isinstance(a, int) else a for a in args]) + return h.sink().call(UOp.alloc((3,), dtypes.uint64, 0, device=q.device, name="handles").after(*deps), name=call.body.arg.name) - def copy(self, dst:UOp, src:UOp, sz:int): self.call(cuda.cuMemcpyAsync, dst.getaddr(self.devs), src.getaddr(self.devs), sz, None) - def wait(self, sig:UOp, val:UOp): self.call(cuda.cuStreamWaitValue64_v2, None, sig.getaddr(self.devs), val, cuda.CU_STREAM_WAIT_VALUE_GEQ) - def signal(self, sig:UOp, val:UOp): self.call(cuda.cuStreamWriteValue64_v2, None, sig.getaddr(self.devs), val, cuda.CU_STREAM_WRITE_VALUE_DEFAULT) - def timestamp(self, sig:UOp): self.call(cuda.cuLaunchHostFunc, None, self.extern(self.dev.tag("stamp")), sig[1:2].getaddr(self.devs)) # [sig][stamp] - - def encode(self) -> UOp: - self.cmds:list[tuple] = [] - rt_vars = UOp.alloc((4,), dtypes.uint64, 0, device=self.devs[0]).rtag(self.dev.tag("cuda")) - return cuda_run(encode_cmdbuf(self, self.lin), rt_vars, tuple(self.cmds), self.queue.startswith("COPY")) +Compiled.pm_encode += PatternMatcher([ + (UPat.sink(arg=KernelInfo("cuda_compute_submit")).call(UPat()).after(allow_any_len=True, name="s"), cuda_submit), + (UPat.sink(arg=KernelInfo("cuda_copy_submit")).call(UPat()).after(allow_any_len=True, name="s"), cuda_submit), +]) # ***************** # device @@ -82,19 +80,16 @@ class CUDAAllocator(Allocator['CUDADevice']): check((api:=self.dev.api).cuCtxSetCurrent(self.dev.context)) if (status:=api.cuMemHostRegister_v2(host.addr, buf.nbytes, 0)) != cuda.CUDA_ERROR_HOST_MEMORY_ALREADY_REGISTERED: check(status) # another device pinned it already - return BufferStorage(host.addr, status == cuda.CUDA_SUCCESS) + addr = init_c_var(api.CUdeviceptr, lambda x: check(api.cuMemHostGetDevicePointer_v2(ctypes.byref(x), host.addr, 0))).value + return BufferStorage(addr, status == cuda.CUDA_SUCCESS) def _unmap(self, mapping:BufferStorage): - if mapping.meta: check(self.dev.api.cuMemHostUnregister(mapping.buf)) + if mapping.meta: check(self.dev.api.cuMemHostUnregister(unwrap(mapping.host).addr)) def _offset(self, buf:int, size:int, offset:int) -> int: return buf + offset class CUDADevice(Compiled): api:Any = cuda launch_end = 0 # CU_LAUNCH_PARAM_END renderers = [CUDARenderer, PTXRenderer, NVCCRenderer] - pm_encode = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat.custom_function("submit_cuda_compute"), UPat()), name="s"), lambda s: CUDAQueue(s).encode()), - (UPat(Ops.CALL, src=(UPat.custom_function("submit_cuda_copy"), UPat()), name="s"), lambda s: CUDAQueue(s).encode()), - ]) def __init__(self, device:str=""): device_id = int(device.split(":")[1]) if ":" in device else 0 @@ -103,29 +98,26 @@ class CUDADevice(Compiled): self.context = init_c_var(api.CUcontext, lambda x: check(api.cuCtxCreate_v2(ctypes.byref(x), 0, self.cu_device))) self.streams = [init_c_var(api.CUstream, lambda x: check(api.cuStreamCreate(ctypes.byref(x), cuda.CU_STREAM_NON_BLOCKING))) for _ in range(2)] super().__init__(device, CUDAAllocator(self), self.renderers, None, arch=self.get_arch(device_id)) - Compiled.pm_bufferize += PatternMatcher([ - (UPat(Ops.ALLOC, tag=self.tag("cuda")), lambda d=self: d.handles), - (UPat(Ops.ALLOC, tag=self.tag("stamp")), lambda d=self: d.stamp), - (UPat(Ops.ALLOC, name="b"), lambda b, d=self: d.function(*b.tag[1:]) if isinstance(b.tag, tuple) and b.tag[0] == d.tag("function") else None)]) + + # [ctx, compute stream, copy stream], and the host function that writes a timestamp + handles = Buffer(HCQ_RUNTIME_DEV.device, 24, initial_value=b"".join(map(bytes, (self.context, *self.streams)))) + fn = cuda.CUhostFn(lambda slot: ctypes.memmove(slot, struct.pack("Q", time.perf_counter_ns()), 8)) + stamp = Buffer(HCQ_RUNTIME_DEV.device, 8, opaque=BufferStorage(int.from_bytes(bytes(fn), "little"), fn)) + Compiled.link_bufs |= {to_name(self.device, "handles"): handles, to_name(self.device, "stamp"): stamp} + Compiled.pm_link += PatternMatcher([(UPat(Ops.ALLOC, name="p").store(UPat(Ops.PROGRAM, name="prg")), lambda p, prg, d=self: + p.store(UOp(Ops.BINARY, arg=d.function(prg.src[3].arg, prg.src[0].arg.function_name))) if p.device == d.device else None)]) def get_arch(self, device_id:int) -> str: check(cuda.cuDeviceComputeCapability(ctypes.byref(major:=ctypes.c_int()), ctypes.byref(minor:=ctypes.c_int()), device_id)) return f"sm_{major.value}{minor.value}" - @functools.cached_property - def handles(self) -> Buffer: - return Buffer(HCQ_RUNTIME_DEV.device, 32, initial_value=struct.pack("4Q", *[as_int(h) for h in (self.context, *self.streams)], 0)) - - @functools.cached_property - def stamp(self) -> Buffer: return extern(as_int(fn:=cuda.CUhostFn(host_stamp)), fn) - @functools.cache - def function(self, lib:bytes, name:str) -> Buffer: + def function(self, lib:bytes, name:str) -> bytes: if DEBUG >= 5 and self.api is cuda: print("\n".join([f"{i+1:>3} {line}" for i, line in enumerate(pretty_ptx(lib.decode()).split("\n"))])) check(self.api.cuCtxSetCurrent(self.context)) check(self.api.cuModuleLoadData(ctypes.byref(module:=self.api.CUmodule()), lib)) check(self.api.cuModuleGetFunction(ctypes.byref(func:=self.api.CUfunction()), module, name.encode())) - return extern(as_int(func)) + return bytes(func) def count(self) -> int: return init_c_var(ctypes.c_int, lambda x: check(self.api.cuDeviceGetCount(ctypes.byref(x)))).value diff --git a/tinygrad_repo/tinygrad/runtime/ops_disk.py b/tinygrad_repo/tinygrad/runtime/ops_disk.py index fa2fe1c36b..54fc646027 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_disk.py +++ b/tinygrad_repo/tinygrad/runtime/ops_disk.py @@ -1,4 +1,4 @@ -import os, sys, mmap, io, ctypes, contextlib, pathlib, functools, collections, struct, itertools +import os, sys, mmap, io, ctypes, contextlib, pathlib, functools, collections, struct, itertools, errno from dataclasses import replace from typing import cast from tinygrad.helpers import OSX, mv_address, flatten, to_tuple, unwrap, ceildiv @@ -25,15 +25,15 @@ class DiskDevice(Compiled): return filename = self.device[len("disk:"):] - if sys.platform != "win32" and filename.startswith("shm:"): + if filename.startswith("shm:"): fd = _posixshmem.shm_open("/"+filename[4:].lstrip("/"), os.O_RDWR, 0o600) self.mem = mmap.mmap(fd, size, mmap.MAP_SHARED | MAP_POPULATE | MAP_LOCKED) os.close(fd) else: - try: self.fd = os.open(filename, os.O_RDWR|os.O_CREAT|getattr(os, "O_DIRECT", 0)) - except OSError: self.fd = os.open(filename, os.O_RDWR|os.O_CREAT) + try: self.fd, rdonly = os.open(filename, os.O_RDWR|os.O_CREAT|getattr(os, "O_DIRECT", 0)), False + except OSError as e: self.fd = os.open(filename, os.O_RDONLY if (rdonly:=e.errno in (errno.EROFS, errno.EACCES)) else os.O_RDWR|os.O_CREAT) if not pathlib.Path(filename).is_block_device() and os.fstat(self.fd).st_size < size: os.ftruncate(self.fd, size) - self.mem = mmap.mmap(self.fd, size) + self.mem = mmap.mmap(self.fd, size, access=mmap.ACCESS_COPY if rdonly else mmap.ACCESS_DEFAULT) self.info.host.view()[:16] = struct.pack("2Q", self.fd, mv_address(memoryview(self.mem))) self.size = size if hasattr(self.mem, 'madvise') and (hp := getattr(mmap, "MADV_HUGEPAGE", None)) is not None: @@ -115,39 +115,40 @@ def disk_read(file:UOp, srcs:UOp, counts:UOp, rings:UOp, sqes:UOp) -> UOp: # src def is_disk_read(c:UOp) -> bool: dev, src = [to_tuple(b.device)[0] for b in c.src[1:3]] if c.op is Ops.CALL and c.body.op is Ops.STORE else ("", "") - return src.startswith("DISK:") and not src.startswith("DISK:shm:") and Device[dev].has_copy_queue and not dev.startswith("NULL") and bool(uring()) + return src.startswith("DISK:") and not src.startswith("DISK:shm:") and Device[dev].has_copy_queue and not dev.startswith("NULL") and bool(uring()) \ + and not getattr(Device[dev], "is_usb", False) def disk_copy_rewriter(s:UOp) -> UOp|None: - lins = [submit.without_after.src[1].without_after for submit in s.src] - if not (copies:=[c for lin in lins for c in lin.src if is_disk_read(c)]): return None - rings = UOp.alloc((unwrap(uring())[2].nbytes // 4,), dtypes.uint32, 0, device=HCQ_RUNTIME_DEV.device).rtag("uring_rings") - sqes = UOp.alloc((8 * RING_ENTRIES,), dtypes.uint64, 0, device=HCQ_RUNTIME_DEV.device).rtag("uring_sqes") + lin = s.without_after.body + if not (copies:=[c for c in lin.src if is_disk_read(c)]): return None + rings = UOp.alloc((unwrap(uring())[2].nbytes // 4,), dtypes.uint32, 0, device=HCQ_RUNTIME_DEV.device, name="uring_rings") + sqes = UOp.alloc((8 * RING_ENTRIES,), dtypes.uint64, 0, device=HCQ_RUNTIME_DEV.device, name="uring_sqes") # [mmap address, bytes] per copy in a table: an arg each would hit ctypes' 1024 limit words = flatten((c.src[2].getaddr(HCQ_RUNTIME_DEV.device), UOp.const(c.src[2].nbytes(), dtypes.uint64)) for c in copies) table = UOp.alloc((8 * len(words),), dtypes.uint8, device=HCQ_RUNTIME_DEV.device) srcs = patch(table, [(8 * i, w) for i, w in enumerate(words)]).bitcast(dtypes.uint64) - # per gpu: [chunks read, chunks copied] at the start of its stage - devs = [to_tuple(c.src[1].device)[0] for c in copies] - counts = {d: UOp.alloc((STAGE_SZ,), dtypes.uint8, 0, device=d).rtag("disk_stage")[:16].bitcast(dtypes.uint64) for d in devs} + # [chunks read, chunks copied] at the start of the gpu's stage + dev = s.without_after.src[1].device + counts = UOp.alloc((STAGE_SZ,), dtypes.uint8, 0, device=dev, name="disk_stage")[:16].bitcast(dtypes.uint64) # cpu part - runs = [(key, list(ks)) for key, ks in itertools.groupby(range(len(copies)), lambda k: (devs[k], to_tuple(copies[k].src[2].device)[0]))] - done, offs, gpu_ops = s.src[-1], collections.Counter[str](), collections.defaultdict[UOp, list[UOp]](list) - for (dev, disk), ks in runs: - file = UOp.alloc((2,), dtypes.uint64, 0, device=disk).rtag("disk_info") - done = disk_read(file, srcs[2 * ks[0]:2 * ks[-1] + 2], counts[dev].after(done), rings, sqes) + done, off, gpu_ops = s, 0, collections.defaultdict[UOp, list[UOp]](list) + for disk, grp in itertools.groupby(range(len(copies)), lambda k: to_tuple(copies[k].src[2].device)[0]): + file, ks = UOp.alloc((2,), dtypes.uint64, 0, device=disk, name="disk_info"), list(grp) + done = disk_read(file, srcs[2 * ks[0]:2 * ks[-1] + 2], counts.after(done), rings, sqes) # gpu part - for k, (c, dev) in enumerate(zip(copies, devs)): - first, slots = counts[dev].index(0).load() + offs[dev], counts[dev].getaddr(dev) + 4096 + srcs.index(2 * k).load() % 4096 - offs[dev] += ceildiv(c.src[2].nbytes(), CHUNK_SZ) + for k, c in enumerate(copies): + first, slots = counts.index(0).load() + off, counts.getaddr(dev) + 4096 + srcs.index(2 * k).load() % 4096 + off += ceildiv(c.src[2].nbytes(), CHUNK_SZ) for chunk in range(ceildiv(c.src[2].nbytes(), CHUNK_SZ)): n, dst, nb = first + chunk, c.src[1].getaddr() + chunk * CHUNK_SZ, min(CHUNK_SZ, c.src[2].nbytes() - chunk * CHUNK_SZ) - gpu_ops[c] += [ins("wait", counts[dev], n + 1), ins("copy", dst, slots + n % SLOTS * SLOT_SZ, nb), ins("store", counts[dev][1:], n + 1)] - return s.replace(src=(*s.src, done)).substitute({lin: lin.replace(src=tuple(flatten(gpu_ops.get(c, [c]) for c in lin.src))) for lin in lins}) -Compiled.pm_batch = Compiled.pm_batch + PatternMatcher([(UPat(Ops.SINK, name="s"), disk_copy_rewriter)]) + gpu_ops[c] += [ins("wait", counts, n + 1), ins("copy", dst, slots + n % SLOTS * SLOT_SZ, nb), ins("store", counts[1:], n + 1)] + return done.substitute({lin: lin.replace(src=tuple(flatten(gpu_ops.get(c, [c]) for c in lin.src)))}, enter_calls=True) +Compiled.pm_encode = PatternMatcher([(UPat.sink().call(UPat(Ops.PARAM)).after(allow_any_len=True, name="s"), + disk_copy_rewriter)]) + Compiled.pm_encode # ***************** # 3. bufferize @@ -165,6 +166,5 @@ def uring() -> tuple[int, io_uring.struct_io_uring_params, Buffer, Buffer]|None: @functools.cache def _stage(dev) -> Buffer: return Buffer(Device[dev].host, STAGE_SZ, options=BufferSpec(cpu_access=True), initial_value=bytes(STAGE_SZ)) -Compiled.pm_bufferize += PatternMatcher([(UPat(Ops.ALLOC, tag="disk_info", name="b"), lambda b: cast(DiskDevice, Device[b.device]).info), - (UPat(Ops.ALLOC, tag="disk_stage", name="b"), lambda b: _stage(b.device)), - (UPat(Ops.ALLOC, tag="uring_rings"), lambda: unwrap(uring())[2]), (UPat(Ops.ALLOC, tag="uring_sqes"), lambda: unwrap(uring())[3])]) +Compiled.link_bufs |= {"disk_info": lambda b: cast(DiskDevice, Device[b.device]).info, "disk_stage": lambda b: _stage(b.device), + "uring_rings": lambda b: unwrap(uring())[2], "uring_sqes": lambda b: unwrap(uring())[3]} diff --git a/tinygrad_repo/tinygrad/runtime/ops_dsp.py b/tinygrad_repo/tinygrad/runtime/ops_dsp.py index f4f24d6b3c..70c4a7fcb9 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_dsp.py +++ b/tinygrad_repo/tinygrad/runtime/ops_dsp.py @@ -1,6 +1,5 @@ from __future__ import annotations -import ctypes, os, mmap, tempfile, pathlib, array, threading, contextlib, sys, subprocess, struct -assert sys.platform != 'win32' +import ctypes, os, mmap, tempfile, pathlib, array, threading, contextlib, subprocess, struct from tinygrad.device import BufferStorage, MMIOInterface, BufferSpec, Compiled, Allocator, Compiler, Program, TinyELF from tinygrad.dtype import dtypes, AddrSpace from tinygrad.uop.ops import Ops, UOp @@ -32,12 +31,12 @@ class DSPRenderer(ClangRenderer): msrc += [f'{self._render_dtype(b[1][0].dtype) if b[1][0].addrspace == AddrSpace.ALU else "int"} sz_or_val_{i} = ' f'*({self._render_dtype(b[1][0].dtype) if b[1][0].addrspace == AddrSpace.ALU else "int"}*)((char*)pra[0].buf.pv+{i*8});' for i,b in enumerate(bufs)] - msrc += [f'int off{i} = ((int*)pra[1].buf.pv)[{i}];' for i,b in enumerate(bufs) if b[1][0].addrspace == AddrSpace.GLOBAL] - msrc += [f'void *buf_{i} = HAP_mmap(0,sz_or_val_{i},3,0,pra[{i+3}].dma.fd,0)+off{i};' - for i,b in enumerate(bufs) if b[1][0].addrspace == AddrSpace.GLOBAL] + gbufs = [i for i,b in enumerate(bufs) if b[1][0].addrspace == AddrSpace.GLOBAL] + msrc += [f'int off{i} = ((int*)pra[1].buf.pv)[{j}];' for j,i in enumerate(gbufs)] + msrc += [f'void *buf_{i} = HAP_mmap(0,sz_or_val_{i},3,0,pra[{j+3}].dma.fd,0)+off{i};' for j,i in enumerate(gbufs)] msrc += ["unsigned long long start = HAP_perf_get_time_us();"] fbufs = [(f'buf_{i}' if b[1][0].addrspace == AddrSpace.GLOBAL else f'sz_or_val_{i}') for i,b in enumerate(bufs)] - msrc += [f"{function_name}({', '.join(fbufs)});"] + msrc += [f"{function_name}(&(struct {function_name}_args){{{', '.join(fbufs)}}});"] msrc += ["*(unsigned long long *)(pra[2].buf.pv) = HAP_perf_get_time_us() - start;"] msrc += [f'HAP_munmap(buf_{i}, sz_or_val_{i});' for i,b in enumerate(bufs) if b[1][0].addrspace == AddrSpace.GLOBAL] msrc += ["return 0; }"] @@ -57,15 +56,16 @@ def rpc_prep_args(ins=None, outs=None, in_fds=None): return pra, fds, attrs, (ins, outs) class DSPProgram(Program['DSPDevice']): - def __init__(self, dev:DSPDevice, obj:TinyELF): self.dev, self.lib, self.signature = dev, obj.lib, obj.signature + def __init__(self, dev:DSPDevice, obj:TinyELF, links=None): self.dev, self.lib, self.signature = dev, obj.lib, obj.signature - def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw): + def __call__(self, *args, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), wait=False, **kw): + bufs = [b for b,(_,a,_,_) in zip(args, self.signature) if a is not AddrSpace.ALU] if len(bufs) >= 16: raise RuntimeError(f"Too many buffers to execute: {len(bufs)}") - pra, fds, attrs, _ = rpc_prep_args(ins=[var_vals_mv:=memoryview(bytearray((len(bufs)+len(vals))*8)), off_mv:=memoryview(bytearray(len(bufs)*4))], + pra, fds, attrs, _ = rpc_prep_args(ins=[var_vals_mv:=memoryview(bytearray(len(args)*8)), off_mv:=memoryview(bytearray(len(bufs)*4))], outs=[timer:=memoryview(bytearray(8)).cast('Q')], in_fds=[b.share_info.fd for b in bufs]) - for i,b in enumerate(bufs): struct.pack_into('i', var_vals_mv, i*8, b.size) - for i,(v,(_,_,dt,_)) in enumerate(zip(vals, self.signature[len(bufs):]), start=len(bufs)): struct.pack_into(unwrap(dt.fmt), var_vals_mv, i*8, v) + for i,(b,(_,a,dt,_)) in enumerate(zip(args, self.signature)): + struct.pack_into(unwrap(dt.fmt) if a is AddrSpace.ALU else 'i', var_vals_mv, i*8, b if a is AddrSpace.ALU else b.size) off_mv.cast('I')[:] = array.array('I', tuple(b.offset for b in bufs)) self.dev.exec_lib(self.lib, rpc_sc(method=2, ins=2, outs=1, fds=len(bufs)), pra, fds, attrs) return timer[0] / 1e6 @@ -266,7 +266,7 @@ class MockDSPRenderer(DSPRenderer): msrc.append(f"{self._render_dtype(b[1][0].dtype)} val{i}; read(0, &val{i}, {b[1][0].dtype.itemsize});") msrc.append("unsigned int st = inscount();") params = [(f'(void*)buf{i}' if b[1][0].addrspace == AddrSpace.GLOBAL else f'val{i}') for i,b in enumerate(bufs)] - msrc.append(f"{function_name}({', '.join(params)});") + msrc.append(f"{function_name}(&(struct {function_name}_args){{{', '.join(params)}}});") msrc.append("unsigned int et = inscount() - st; write(1, &et, sizeof(et));") for i,b in enumerate(bufs): if b[1][0].addrspace == AddrSpace.GLOBAL: msrc.append(f"write(1, buf{i}, {b[1][0].max_numel()*b[1][0].dtype.itemsize});") @@ -274,18 +274,18 @@ class MockDSPRenderer(DSPRenderer): return '\n'.join(msrc) class MockDSPProgram(Program[DSPDevice]): - def __init__(self, dev:DSPDevice, obj:TinyELF): self.lib, self.signature = obj.lib, obj.signature - def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw): + def __init__(self, dev:DSPDevice, obj:TinyELF, links=None): self.lib, self.signature = obj.lib, obj.signature + def __call__(self, *args, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), wait=False, **kw): with tempfile.NamedTemporaryFile(suffix=".out") as dsp_lib: dsp_lib.write(self.lib) dsp_lib.flush() os.chmod(dsp_lib.name, 0o0777) proc = subprocess.run(["qemu-hexagon-static", *(['-strace'] if DEBUG >= 5 else []), dsp_lib.name], - input=b''.join([bytes(to_mv(x.va_addr, x.size)) for x in bufs] + - [struct.pack(unwrap(dt.fmt), x) for x,(_,_,dt,_) in zip(vals, self.signature[len(bufs):])]), + input=b''.join(struct.pack(unwrap(dt.fmt), x) if a is AddrSpace.ALU else bytes(to_mv(x.va_addr, x.size)) + for x,(_,a,dt,_) in zip(args, self.signature)), stdout=subprocess.PIPE, check=True) offset = 4 - for x in bufs: + for x in (x for x,(_,a,_,_) in zip(args, self.signature) if a is not AddrSpace.ALU): to_mv(x.va_addr, x.size)[:] = proc.stdout[offset:offset+x.size] offset += x.size assert offset == len(proc.stdout) diff --git a/tinygrad_repo/tinygrad/runtime/ops_hip.py b/tinygrad_repo/tinygrad/runtime/ops_hip.py index 495c68367e..4ecb9c37a4 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_hip.py +++ b/tinygrad_repo/tinygrad/runtime/ops_hip.py @@ -1,35 +1,42 @@ +from typing import Any from types import SimpleNamespace -from tinygrad.helpers import getenv +from tinygrad.helpers import DEV, getenv from tinygrad.dtype import dtypes -from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite +from tinygrad.uop.ops import UOp, UPat, PatternMatcher, KernelInfo +from tinygrad.device import Compiled from tinygrad.renderer.cstyle import HIPRenderer from tinygrad.runtime.autogen import hip from tinygrad.runtime.support.hcq2 import ccall from tinygrad.runtime.support.c import init_c_var -from tinygrad.runtime.ops_cuda import CUDAQueue, CUDADevice, check +from tinygrad.runtime.ops_cuda import cuda_submit, CUDADevice, check if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl # noqa: F401 # pylint: disable=unused-import HIPIFY = dict(CUdevice=hip.hipDevice_t, CUcontext=hip.hipCtx_t, CUstream=hip.hipStream_t, CUdeviceptr=hip.hipDeviceptr_t, CUmodule=hip.hipModule_t, CUfunction=hip.hipFunction_t, cuInit=hip.hipInit, cuDeviceGet=hip.hipDeviceGet, cuDeviceGetCount=hip.hipGetDeviceCount, - cuCtxCreate_v2=hip.hipCtxCreate, cuCtxSetCurrent=hip.hipCtxSetCurrent, cuCtxSynchronize=hip.hipDeviceSynchronize, - cuStreamCreate=hip.hipStreamCreateWithFlags, cuMemAlloc_v2=hip.hipMalloc, cuMemFree_v2=hip.hipFree, + cuCtxCreate_v2=hip.hipCtxCreate, cuCtxSetCurrent=hip.hipCtxSetCurrent, cuCtxSynchronize=hip.hipDeviceSynchronize, cuMemAlloc_v2=hip.hipMalloc, + cuStreamCreate=hip.hipStreamCreateWithFlags, cuMemHostGetDevicePointer_v2=hip.hipHostGetDevicePointer, cuMemFree_v2=hip.hipFree, cuMemHostAlloc=lambda p, sz, _: hip.hipHostMalloc(p, sz, hip.hipHostMallocCoherent), cuMemFreeHost=hip.hipHostFree, cuMemHostRegister_v2=hip.hipHostRegister, cuMemHostUnregister=hip.hipHostUnregister, cuModuleLoadData=hip.hipModuleLoadData, cuModuleGetFunction=hip.hipModuleGetFunction, cuLaunchKernel=hip.hipModuleLaunchKernel, cuMemcpyAsync=hip.hipMemcpyAsync, cuStreamWaitValue64_v2=hip.hipStreamWaitValue64, cuStreamWriteValue64_v2=hip.hipStreamWriteValue64, cuLaunchHostFunc=hip.hipLaunchHostFunc) +if DEV.target("HIP").interface == "MOCK": import test.mockgpu.hip.hip # noqa: F401 # pylint: disable=unused-import -def hipify(c:UOp, f:UOp) -> UOp|None: - if (fn:=HIPIFY.get(f.arg.name)) is None: return None - args = list(c.src[1:]) +def hipify(cufn:Any, *cuargs:UOp): + if (fn:=HIPIFY.get(cufn.__name__)) is None: return None + args = list(cuargs) if fn is hip.hipMemcpyAsync: args.insert(3, UOp.const(hip.hipMemcpyDefault, dtypes.int)) if fn is hip.hipStreamWaitValue64: args.append(UOp.const((1<<64)-1, dtypes.uint64)) return ccall(fn, *args) -pm_hipify = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, name="f"),), allow_any_len=True, name="c"), hipify)]) + +def hip_submit(s:UOp)->UOp: return cuda_submit(s, hipify) + +Compiled.pm_encode += PatternMatcher([ + (UPat.sink(arg=KernelInfo("hip_compute_submit")).call(UPat()).after(allow_any_len=True, name="s"), hip_submit), + (UPat.sink(arg=KernelInfo("hip_copy_submit")).call(UPat()).after(allow_any_len=True, name="s"), hip_submit), +]) class HIPDevice(CUDADevice): api, launch_end, renderers = SimpleNamespace(**HIPIFY), 3, [HIPRenderer] - pm_encode = PatternMatcher([(UPat(Ops.CALL, src=(UPat.custom_function(f"submit_hip_{q}"), UPat()), name="s"), - lambda s: graph_rewrite(CUDAQueue(s).encode(), pm_hipify, enter_calls=True, name="hipify")) for q in ("compute", "copy")]) def get_arch(self, device_id:int) -> str: return init_c_var(hip.hipDeviceProp_t, lambda x: check(hip.hipGetDeviceProperties(x, device_id))).gcnArchName.decode().split(":")[0] diff --git a/tinygrad_repo/tinygrad/runtime/ops_metal.py b/tinygrad_repo/tinygrad/runtime/ops_metal.py index b76d0c684f..deb6b56094 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_metal.py +++ b/tinygrad_repo/tinygrad/runtime/ops_metal.py @@ -7,9 +7,9 @@ from tinygrad.dtype import dtypes, AddrSpace from tinygrad.renderer.cstyle import MetalRenderer from tinygrad.runtime.autogen import metal from tinygrad.runtime.support.c import DLL -from tinygrad.runtime.support.hcq2 import HWQueue, ccall, patch, layout_args, to_name, bufferize_buf -from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, uopfunc -from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops +from tinygrad.runtime.support.hcq2 import HWQueue, ccall, patch, layout_args, to_name +from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo, uopfunc +from tinygrad.engine.realize import get_call_kernel_args # 13 is requestType that metal uses to compile source code into MTLB, there aren't any docs or symbols. REQUEST_TYPE_COMPILE = 13 @@ -87,7 +87,7 @@ MSGSEND, SELNAME = [metal.dll.bind(ctypes.c_void_p)(f) for f in (metal.dll.objc_ # the slot of a handle of the device def mtl_handle(d, name:str) -> UOp: - return UOp.alloc((len(HANDLES),), dtypes.uint64, 0, device=to_tuple(d)[0]).rtag(to_name(to_tuple(d)[0], "handles"))[(i:=HANDLES.index(name)):i+1] + return UOp.alloc((len(HANDLES),), dtypes.uint64, 0, device=to_tuple(d)[0], name="handles")[(i:=HANDLES.index(name)):i+1] @uopfunc def mtl_send(obj:UOp, sel:UOp, a:UOp, b:UOp, c:UOp, out:UOp|None=None) -> UOp: # objc_msgSend by the selector's name, the result to out @@ -101,7 +101,7 @@ def mtl_msg(obj:UOp, sel:str, *args:UOp|int, out:UOp|None=None) -> UOp: @uopfunc def mtl_run(icb:UOp, value:UOp, first:UOp|int, count:int, last:bool, q:MetalQueue, stamp:UOp|None=None) -> UOp: cmds, hdr, devs, dev = q.cmds, round_up(q.nbytes, 8) + 24, q.devs, q.dev # a command buffer for the commands [first, first + count) - cb, enc = [UOp.alloc((1,), dtypes.uint64, 0, device=devs[0]).rtag(t) for t in ("mtl_cb", "mtl_enc")] + cb, enc = [UOp.alloc((1,), dtypes.uint64, 0, device=devs[0], name=t) for t in ("mtl_cb", "mtl_enc")] fence, event = [mtl_handle(devs, h).index(0).load() for h in ("fence", "event")] c = mtl_msg(mtl_handle(devs, "queue"), "commandBuffer", out=cb) c = mtl_msg(cb.after(c), "computeCommandEncoder", out=enc) @@ -111,7 +111,7 @@ def mtl_run(icb:UOp, value:UOp, first:UOp|int, count:int, last:bool, q:MetalQueu # before apple9 the encoder must use the pipelines if not dev.arch.startswith("Apple") or int(dev.arch[5:]) < 9: - r = UOp.range(len(dedup(c[:2] for c in cmds)), next(UOp.unique_num), dtype=dtypes.uint64) + r = UOp.range(len(dedup(c.src[:2] for c in cmds)), next(UOp.unique_num), dtype=dtypes.uint64) c = mtl_msg(enc.after(c), "setComputePipelineState:", icb.bitcast(dtypes.uint64).index(hdr // 8 + 1 + len(cmds) + r).load()) c = mtl_msg(enc.after(c), "dispatchThreadgroups:threadsPerThreadgroup:", icb.index(hdr - 24), icb.index(hdr - 24)).end(r) @@ -129,11 +129,10 @@ class MetalQueue(HWQueue): dev:MetalDevice def __init__(self, submit:UOp): super().__init__(submit) - self.rows, self.cmds, self.sizes, self.stamps, self.nbytes = list[tuple[int, UOp]](), list[tuple](), list[tuple[int, int]](), list[UOp](), 0 + self.rows, self.cmds, self.sizes, self.stamps, self.nbytes = list[tuple[int, UOp]](), list[UOp](), list[tuple[int, int]](), list[UOp](), 0 def exec(self, call:UOp, prg:UOp): - bufs, vals, obj = get_call_arg_uops(call), get_call_var_uops(call, prg), prg.to_elf() - args = [bufs[i].getaddr(self.devs) for i in prg.arg.globals] + [v.ccast(var.dtype) for v, var in zip(vals, prg.arg.vars)] + args, obj = get_call_kernel_args(call, prg, self.devs), prg.to_elf() self.rows += (rows:=layout_args(args, off:=round_up(self.nbytes, 256))) self.nbytes = max([o + w.dtype.itemsize for o, w in rows], default=off + 8) @@ -143,17 +142,17 @@ class MetalQueue(HWQueue): self.sizes.append((len(self.cmds), at:=round_up(self.nbytes, 8))) self.rows += layout_args([UOp.const(d, dtypes.uint64) for d in dims], at) self.nbytes = at + 48 - self.cmds.append((obj.lib, obj.name, tuple(1 if isinstance(d, UOp) else int(d) for d in dims), off)) + consts = [UOp.const(1 if isinstance(d, UOp) else d, dtypes.uint64) for d in (*dims, off)] + self.cmds.append(UOp(Ops.LINEAR, src=(UOp(Ops.BINARY, arg=obj.lib), UOp(Ops.BINARY, arg=obj.name.encode()), *consts))) # [lib, name, dims, off] def wait(self, dst:UOp, val:UOp, eq=False): pass # the fence orders the queue def timestamp(self, dst:UOp): self.stamps.append(dst) def signal(self, dst:UOp, val:UOp): self.value = val def submit(self, cmdbuf:UOp) -> UOp: - n, zero, pipes = len(self.cmds), round_up(self.nbytes, 8), dedup(c[:2] for c in self.cmds) - tag = (self.dev.tag("mtl_icb"), tuple(self.cmds), zero + 24) - buf = UOp.alloc((zero + 24 + 8 * (1 + n + len(pipes)),), dtypes.uint8, device=self.devs[0]).rtag(tag).after(*self.deps) - icb = patch(buf, self.rows + [(zero + 8 * i, UOp.const(0, dtypes.uint64)) for i in range(3)]) + n, zero, pipes, cmds = len(self.cmds), round_up(self.nbytes, 8), dedup(c.src[:2] for c in self.cmds), UOp(Ops.LINEAR, src=tuple(self.cmds)) + buf = UOp.alloc((zero + 24 + 8 * (1 + n + len(pipes)),), dtypes.uint8, device=self.devs[0], name="icb") + icb = patch(buf.after(buf.store(cmds), *self.deps), self.rows + [(zero + 8 * i, UOp.const(0, dtypes.uint64)) for i in range(3)]) # symbolic sizes for ci, off in self.sizes: @@ -183,15 +182,20 @@ class MetalAllocator(Allocator['MetalDevice']): storage.meta.release() def _offset(self, buf:int, size:int, offset:int) -> int: return buf + offset +Compiled.pm_encode += PatternMatcher([ + (UPat.sink(arg=KernelInfo("metal_compute_submit")).call(UPat()).after(allow_any_len=True, name="s"), lambda s: MetalQueue(s).encode()), +]) + +def poll_event(p:UOp, t:UOp) -> UOp|None: + if p.arg.name != "tl" or not str(p.device).startswith("METAL"): return None + event = mtl_handle(p.device, "event").after(t) + done = (r:=UOp.placeholder((1,), dtypes.uint64, None, AddrSpace.REG)).after(mtl_msg(event, "signaledValue", out=r))[0] + # then mirror it into the timeline + return r.after(t.index(0).store(done))[0] +Compiled.pm_lower += PatternMatcher([(UPat(Ops.PARAM, name="p").f(Ops.AFTER, allow_any_len=True, name="t").index(UPat.const(0)).load(), poll_event)]) + class MetalDevice(Compiled): has_copy_queue = False - pm_encode = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat.custom_function("submit_metal_compute"), UPat()), name="s"), lambda s: MetalQueue(s).encode()), - ]) - pm_lower = PatternMatcher([ - (UPat(Ops.PARAM, name="p").f(Ops.AFTER, allow_any_len=True, name="t").index(UPat.const(0)).load(), lambda p, t: None if p.arg.name != "tl" else \ - (r:=UOp.placeholder((1,), dtypes.uint64, None, AddrSpace.REG)).after(mtl_msg(mtl_handle(p.device, "event").after(t), "signaledValue", out=r))[0]) - ]) def __init__(self, device:str=""): self.sysdevice = metal.MTLCreateSystemDefaultDevice() @@ -207,18 +211,15 @@ class MetalDevice(Compiled): self.resources, self.table = list[int](), (ctypes.c_uint64 * 1)() self.event, self.fence = self.sysdevice.newSharedEvent(), self.sysdevice.newFence() - self.icbs, self.profile_slots = weakref.WeakKeyDictionary[Buffer, tuple](), weakref.WeakSet[Buffer]() + self.icbs = weakref.WeakKeyDictionary[Buffer, tuple]() # https://developer.apple.com/documentation/metal/mtlgpufamily def check_family(f): return next(filter(self.sysdevice.supportsFamily, reversed([v for v, nm in metal.enum_MTLGPUFamily.items() if f in nm])), 0) super().__init__(device, MetalAllocator(self), [MetalRenderer], None, arch=metal.enum_MTLGPUFamily[check_family("Apple") or check_family("Mac")][12:]) - Compiled.pm_bufferize += PatternMatcher([ - (UPat(Ops.ALLOC, tag=self.tag("handles")), lambda d=self: d.handles), - (UPat(Ops.ALLOC, tag="slots", name="b"), # with stamps - lambda b, d=self: d.new_slots(b.max_numel()) if b.device == d.device and b.max_numel() > 4 else None), - (UPat(Ops.ALLOC, name="b"), lambda ctx, b, d=self: - d.new_icb(bufferize_buf(ctx, b.rtag(None)).buffer, *b.tag[1:]) if isinstance(b.tag, tuple) and b.tag[0] == d.tag("mtl_icb") else None)]) + Compiled.link_bufs |= {to_name(self.device, "handles"): lambda b: self.handles} + Compiled.pm_link += PatternMatcher([ + (UPat(Ops.BUFFER, name="b").store(UPat(Ops.LINEAR, name="cmds")), lambda b, cmds, d=self: d.new_icb(b.buffer, cmds.src))]) @functools.cached_property def handles(self) -> Buffer: @@ -236,10 +237,6 @@ class MetalDevice(Compiled): # update sels table if "handles" in self.__dict__: self.handles.host.view(fmt='Q')[3:5] = array.array('Q', [ctypes.addressof(self.table), len(self.resources)]) - def new_slots(self, n:int) -> Buffer: - self.profile_slots.add(buf:=Buffer(self.host, n * 8, initial_value=bytes(8 * n))) - return buf - @functools.cache def pipeline(self, lib:bytes, name:str) -> metal.MTLComputePipelineState: library = checked(self.sysdevice.newLibraryWithData_error, objc.dispatch_data_create(lib, len(lib), None, None)) @@ -248,23 +245,24 @@ class MetalDevice(Compiled): descriptor.setSupportIndirectCommandBuffers(True) return checked(self.sysdevice.newComputePipelineStateWithDescriptor_options_reflection_error, descriptor, metal.MTLPipelineOptionNone, None) - def new_icb(self, buf:Buffer, cmds:tuple[tuple[bytes, str, tuple[int, ...], int], ...], header:int) -> Buffer: - pipes = dedup(c[:2] for c in cmds) + def new_icb(self, buf:Buffer, cmds:tuple[UOp, ...]) -> UOp|None: # the icb handles are the tail of the buffer: [icb, commands, pipelines] + if buf.device != self.device: return None + pipes = [self.pipeline(lib.arg, name.arg.decode()) for lib, name in dedup(c.src[:2] for c in cmds)] desc = metal.MTLIndirectCommandBufferDescriptor.new() desc.setCommandTypes(metal.MTLIndirectCommandTypeConcurrentDispatch) desc.setMaxKernelBufferBindCount(1) icb = self.sysdevice.newIndirectCommandBufferWithDescriptor_maxCommandCount_options(desc, max(len(cmds), 1), 0) if icb.value is None: raise RuntimeError("create indirect command buffer failed, does your system support this?") commands = [icb.indirectComputeCommandAtIndex(i).own() for i in range(len(cmds))] - for cmd, (lib, name, dims, off) in zip(commands, cmds): + for cmd, (lib, name, *dims, off) in zip(commands, [(c.src[0].arg, c.src[1].arg.decode(), *[v.val for v in c.src[2:]]) for c in cmds]): cmd.setComputePipelineState(state:=self.pipeline(lib, name)) if prod(dims[3:]) > (mx:=state.maxTotalThreadsPerThreadgroup()): raise RuntimeError(f"local size {dims[3:]} bigger than {mx}") cmd.setKernelBuffer_offset_atIndex(buf.meta, buf.offset + off, 0) cmd.concurrentDispatchThreadgroups_threadsPerThreadgroup(metal.MTLSize(*dims[:3]), metal.MTLSize(*dims[3:])) cmd.setBarrier() - buf.host.view(fmt='Q')[header // 8:] = array.array('Q', [icb.value, *[c.value for c in commands], *[self.pipeline(*p).value for p in pipes]]) + buf.host.view(fmt='Q')[-1 - len(cmds) - len(pipes):] = array.array('Q', [icb.value, *[c.value for c in commands], *[p.value for p in pipes]]) self.icbs[buf] = (icb, commands) - return buf + return UOp(Ops.NOOP) def _wait_signal(self, sig:MMIOInterface|memoryview, value:int, timeout:int|None=None): if sys.is_finalizing(): return # the event doesn't wake at exit @@ -272,12 +270,10 @@ class MetalDevice(Compiled): if not wait(self.event, value, int(self.wait_timeout_ms)): raise RuntimeError(f"{self.device} signal wait timed out") def synchronize(self, timeout:int|None=None): - for buf in list(self.profile_slots): # pending: [command buffer, 0] - slots = buf.host.view(fmt='Q') - for start in range(5, buf.nbytes // 8, 4): - if slots[start] and not slots[start + 2]: - (cb:=metal.MTLCommandBuffer(slots[start])).waitUntilCompleted() - slots[start], slots[start + 2] = int(cb.GPUStartTime() * 1e9), int(cb.GPUEndTime() * 1e9) + for slots, start in [(buf.host.view(fmt='Q'), start) for buf, start in self.prof_ents]: + if slots[start] and not slots[start + 2]: # [command buffer, 0] + (cb:=metal.MTLCommandBuffer(slots[start])).waitUntilCompleted() + slots[start], slots[start + 2] = int(cb.GPUStartTime() * 1e9), int(cb.GPUEndTime() * 1e9) super().synchronize(timeout) if (pool:=getattr(pools, "pool", None)) is not None: objc.lib.objc_autoreleasePoolPop(pool) diff --git a/tinygrad_repo/tinygrad/runtime/ops_null.py b/tinygrad_repo/tinygrad/runtime/ops_null.py index a6c863b9c8..c89f29660f 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_null.py +++ b/tinygrad_repo/tinygrad/runtime/ops_null.py @@ -2,10 +2,10 @@ import inspect, functools, itertools from tinygrad.device import BufferStorage, BufferSpec, Buffer, Compiled, HostAllocator, MMIOInterface, Program, ProfileGraphEntry, ProfileGraphEvent from tinygrad.renderer import Renderer, cstyle, nir, ptx, llvmir, wgsl from tinygrad.renderer.cstyle import CStyleLanguage -from tinygrad.uop.ops import UOp, Ops, UPat, PatternMatcher, uopfunc +from tinygrad.uop.ops import UOp, Ops, UPat, PatternMatcher, KernelInfo, uopfunc from tinygrad.dtype import dtypes from tinygrad.helpers import getenv, dedup, prod, panic, cpu_events, perf_counter_us, NULL_ALLOW_COPYOUT, PROFILE -from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops +from tinygrad.engine.realize import get_call_kernel_args from tinygrad.runtime.support.hcq2 import HWQueue, layout_args, pack_args class NullRenderer(CStyleLanguage): @@ -29,38 +29,42 @@ class NullQueue(HWQueue): def cmd(self, op, *args): self.q(*[a.getaddr(self.devs) if isinstance(a, UOp) else UOp.const(a, dtypes.uint64) for a in (op, *args, 0, 0, 0)][:4]) def event(self, device:str, name:str, key:bytes|None=None) -> int: return null_events.setdefault((device, name, key), len(null_events)) def exec(self, call:UOp, prg:UOp): - args = [a.getaddr(self.devs) for a in get_call_arg_uops(call)] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] + args = get_call_kernel_args(call, prg, self.devs) kernargs = UOp(Ops.LINEAR, src=tuple(pack_args(layout_args(args), 8 * max(len(args), 1))), arg="kernargs") self.cmd(EXEC, kernargs, len(args), self.event(self.devs[0], prg.src[0].arg.function_name, prg.key)) def copy(self, dst:UOp, src:UOp, sz:int): self.cmd(COPY, dst, src, self.event(f"{src.device}:SDMA:0", f"{src.device} -> {dst.device}")) def wait(self, signal:UOp, value:UOp, eq:bool=False): self.cmd(WAIT, signal, value, int(eq)) def signal(self, signal:UOp, value:UOp): self.cmd(STORE, signal, value) def timestamp(self, signal:UOp): self.cmd(TIMESTAMP, signal.getaddr(self.devs) + UOp.const(8, dtypes.uint64)) - def submit(self, cmdbuf): return null_submit(cmdbuf, UOp.alloc((1,), dtypes.uint8, device=self.devs[0]).rtag("doorbell")) + def submit(self, cmdbuf): return null_submit(cmdbuf, UOp.alloc((1,), dtypes.uint8, device=self.devs[0], name="doorbell")) class NullProgram(Program['NullDevice']): - def __init__(self, dev, obj): self.streams = [(i, prod(s)) for i, (n, _, _, s) in enumerate(obj.signature) if (n or "").startswith("cmdbuf")] - def __call__(self, *bufs, **kwargs): - st, words = perf_counter_us(), [w for i, n in self.streams for w in MMIOInterface(bufs[i], n, fmt='Q')[:]] + def __init__(self, dev, obj, links=None): self.streams = [(i,prod(s)) for i,(n,_,_,s) in enumerate(obj.signature) if (n or "").startswith("cmdbuf")] + def __call__(self, *args, **kwargs): + st, words = perf_counter_us(), [w for i, n in self.streams for w in MMIOInterface(args[i], n, fmt='Q')[:]] # timestamps are emulated: every exec and copy takes 1us for op, addr, done in zip(words[0::4], words[1::4], itertools.accumulate(op in (EXEC, COPY) for op in words[0::4])): if op == TIMESTAMP: MMIOInterface(addr, 8, fmt='Q')[0] = int((st + done) * 1000) descs = [list(null_events)[event] for op, event in zip(words[0::4], words[3::4]) if op in (EXEC, COPY)] if PROFILE else [] sigs = [st + sum(x[0] == d[0] for x in descs[:i]) + k for i, d in enumerate(descs) for k in (0, 1)] - if descs: cpu_events.append(ProfileGraphEvent([ProfileGraphEntry(d, n, 2*i, 2*i+1, k) for i, (d, n, k) in enumerate(descs)], [], sigs)) + if descs: cpu_events.append(ProfileGraphEvent([ProfileGraphEntry(d, n, 2*i, 2*i+1, k) for i, (d, n, k) in enumerate(descs)], sigs)) class NullAllocator(HostAllocator): - def _alloc(self, size:int, options, va=itertools.count(1 << 40, 1 << 32)) -> BufferStorage: return BufferStorage(next(va)) + def _alloc(self, size:int, options, va=itertools.count(1 << 40, 1 << 32)) -> BufferStorage: + return super()._alloc(size, options) if options.host or options.cpu_access else BufferStorage(next(va)) def _copyin(self, dest, src:memoryview): pass def _copyout(self, dest:memoryview, src): NULL_ALLOW_COPYOUT or panic(RuntimeError, "no copyout on NULL") def _map(self, buf:Buffer) -> BufferStorage: return BufferStorage(buf._buf if buf.device.startswith("NULL") else buf.host.addr) +Compiled.pm_encode += PatternMatcher([ + (UPat.sink(arg=KernelInfo("null_compute_submit")).call(UPat()).after(allow_any_len=True, name="s"), lambda s: NullQueue(s).encode()), + (UPat.sink(arg=KernelInfo("null_copy_submit")).call(UPat()).after(allow_any_len=True, name="s"), lambda s: NullQueue(s).encode()), +]) + class NullDevice(Compiled): - pm_encode = PatternMatcher([(UPat(Ops.CALL, src=(UPat.custom_function(f"submit_null_{q}"), UPat()), name="submit"), - lambda submit: NullQueue(submit).encode()) for q in ("compute", "copy")]) host = property(lambda self: self.device) - timeline = functools.cached_property(lambda self: self.link_buffer(2, dtypes.uint64)) - error_state = functools.cached_property(lambda self: self.link_buffer(1, dtypes.int64)) + timeline = functools.cached_property(lambda self: self.host_buffer(16)) + error_state = functools.cached_property(lambda self: self.host_buffer(8)) def __init__(self, device:str): assert (emu:=getenv("EMULATE", "")) == "", \ @@ -68,8 +72,6 @@ class NullDevice(Compiled): renderers = [NullRenderer] + [r for m in [cstyle, nir, ptx, llvmir, wgsl] for r in m.__dict__.values() if inspect.isclass(r) and issubclass(r, Renderer)] super().__init__(device, NullAllocator(self), dedup(renderers), NullProgram) - Compiled.pm_bufferize += PatternMatcher([ # its memory is fake: every placeholder on it is a link buffer - (UPat(Ops.ALLOC, name="b"), lambda b, d=self: d.link_buffer(b.max_numel(), b.dtype) if b.device == d.device else None)]) - def link_buffer(self, n, dt): - return Buffer(self.device, n * dt.itemsize, opaque=memoryview(bytearray(n * dt.itemsize)), options=BufferSpec(external_ptr=1)) + # its memory is fake: every placeholder on it gets host memory + def host_buffer(self, n:int) -> Buffer: return Buffer(self.device, n, opaque=memoryview(bytearray(n)), options=BufferSpec(external_ptr=1)) diff --git a/tinygrad_repo/tinygrad/runtime/ops_nv.py b/tinygrad_repo/tinygrad/runtime/ops_nv.py index bfc3a8d31f..0a1bfa7756 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_nv.py +++ b/tinygrad_repo/tinygrad/runtime/ops_nv.py @@ -1,24 +1,21 @@ from __future__ import annotations -import os, ctypes, contextlib, re, functools, mmap, struct, array, sys, itertools -assert sys.platform != 'win32' +import os, ctypes, contextlib, re, functools, mmap, struct, array, itertools from typing import Any from dataclasses import dataclass, replace from tinygrad.runtime.support.hcq2 import HWQueue, patch, to_name, unwrap_view, make_submit, timeline, HCQInfo, lower_call, hcq_link -from tinygrad.runtime.support.hcq2 import layout_args, make_program +from tinygrad.runtime.support.hcq2 import layout_args, make_program, ins from tinygrad.runtime.support.memory import MMIOInterface, BumpAllocator -from tinygrad.runtime.support.system import FileIOInterface, filter_visible_devices +from tinygrad.runtime.support.system import FileIOInterface, filter_visible_devices, PCIIfaceBase, MAP_FIXED from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo, uopfunc -from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops, lower_and_compile, run_linear +from tinygrad.engine.realize import get_call_arg_uops, get_call_kernel_args, lower_and_compile, run_linear from tinygrad.device import BufferStorage, Buffer, BufferSpec, Allocator, Compiled, Device, TinyELF -from tinygrad.dtype import dtypes, DType -from tinygrad.helpers import getenv, mv_address, round_up, data64, data64_le, prod, OSX, PROFILE, ContextVar, VIZ -from tinygrad.helpers import ProfileEvent, unwrap +from tinygrad.dtype import dtypes, DType, AddrSpace +from tinygrad.helpers import getenv, mv_address, round_up, data64, data64_le, prod, OSX, PROFILE, ContextVar, VIZ, ProfileEvent, unwrap from tinygrad.renderer.ptx import PTXRenderer from tinygrad.renderer.cstyle import CUDARenderer, NVCCRenderer -from tinygrad.runtime.autogen import nv_570, nv_580, nv_610, mesa +from tinygrad.runtime.autogen import nv_570, nv_580, nv_610, nv_615, mesa from tinygrad.runtime.support.elf import elf_loader from tinygrad.runtime.support.nv.nvdev import NVDev, NVMemoryManager -from tinygrad.runtime.support.system import PCIIfaceBase, MAP_FIXED from tinygrad.renderer.nir import NAKRenderer if getenv("IOCTL"): import extra.nv_gpu_driver.nv_ioctl # noqa: F401 # pylint: disable=unused-import @@ -106,7 +103,7 @@ def nv_submit(cmdbuf:UOp, ring:UOp, gpput:UOp, doorbell:UOp, put:UOp, token:int) class NVQueue(HWQueue): dev:NVDevice q_rewrite = HWQueue.q_rewrite + PatternMatcher([ - (UPat(Ops.INS, arg=("nv", dtypes.void), name="u"), lambda ctx, u: ctx.q(*u.src)), + (UPat(Ops.CALL, src=(UPat.custom_function("nv"),), allow_any_len=True, name="u"), lambda ctx, u: ctx.q(*u.src[1:])), ]) def nvm(self, subc:int, mthd:int|UOp, *vals, typ=2): self.q(*nvm(subc, mthd, *vals, typ=typ)) @@ -125,17 +122,17 @@ class NVQueue(HWQueue): fifo, dev = self.dev.fifos[self.queue], self.devs[0] bufs = (("ring", dtypes.uint64, fifo.entries, dev), ("gpput", dtypes.uint32, 1, dev), ("doorbell", dtypes.uint32, 1, dev), ("put_value", dtypes.uint64, 1, self.dev.host)) - ring, gpput, doorbell, put = [UOp.alloc((sz,), dt, device=d).rtag(self.dev.tag(nm, self.queue)) for nm, dt, sz, d in bufs] + ring, gpput, doorbell, put = [UOp.alloc((sz,), dt, device=d, name=to_name(self.dev.device, nm, self.queue)) for nm, dt, sz, d in bufs] return nv_submit(cmdbuf, ring, gpput, doorbell, put, fifo.token) class NVComputeQueue(NVQueue): def __init__(self, submit): super().__init__(submit) - progs = [nv_build_program(self.dev, u.body, self.devs)[0] for u in self.lin.src if u.op is Ops.CALL] + progs = [nv_build_program(self.dev, u.body, self.devs)[0] for u in self.lin.src if u.op is Ops.CALL and u.body.op is Ops.PROGRAM] self.qmd_sz = round_up(QMD(self.dev).sz * 4, 256) self.stride = self.qmd_sz + max([p.kernargs_size for p in progs], default=0) - self.qmd_buf = UOp.alloc((len(progs) * self.stride,), dtypes.uint8, device=self.devs[0]).rtag(to_name("qmd", self.queue)) + self.qmd_buf = UOp.alloc((len(progs) * self.stride,), dtypes.uint8, device=self.devs[0], name=to_name("qmd", self.queue)) self.qmds:list[QMD] = [] self.prev_qmd:QMD|None = None # the launch the next one chains onto @@ -174,9 +171,9 @@ class NVComputeQueue(NVQueue): qmd.set_program_addr(lib.getaddr(self.devs) + data.prog_off) for j, (off, _) in data.constbufs.items(): qmd.set_constant_buf_addr(j, qmd_addr + UOp.const(self.qmd_sz, dtypes.uint64) if j == 0 else lib.getaddr(self.devs) + off) - bufs, vals = [get_call_arg_uops(call)[j] for j in prg.arg.globals], get_call_var_uops(call, prg) + args = get_call_kernel_args(call, prg, self.devs) qmd.mv[self.qmd_sz:(at:=self.qmd_sz + len(data.cbuf_0) * 4)] = array.array('I', data.cbuf_0).tobytes() # constant buffer 0: the driver params - qmd.patches |= dict(layout_args([b.getaddr(self.devs) for b in bufs] + [v.ccast(dt) for v, dt in zip(vals, data.vars)], at)) + qmd.patches |= dict(layout_args([a.ccast(dtypes.uint64) for a in args] if data.mock else args, at)) # mockgpu wants every arg 64 bit if self.prev_qmd is None: if self.dev.pma_enabled: self.nvm(1, nv_gpu.NVC6C0_PM_TRIGGER, 0) @@ -269,10 +266,9 @@ class NVProgramData: min_cbuf0_entries = 224 if dev.iface.compute_class >= nv_gpu.BLACKWELL_COMPUTE_A else 12 self.cbuf_0 = [0] * max(cbuf0_size // 4, min_cbuf0_entries) - # the arguments follow the driver params in constant buffer 0: the buffers as 64 bit addresses, then the vars packed by their width - nbufs = sum(name is None for name, *_ in signature) - self.vars = [dtypes.uint64 if mock else dt for _,_,dt,_ in signature[nbufs:]] # mockgpu wants every var 64 bit - if mock: self.cbuf_0[80:82] = [nbufs, len(self.vars)] # mockgpu reads the arg counts out of cbuf0 + # the arguments follow the driver params in constant buffer 0 in the kernel's order: buffers as 64 bit addresses, vars packed by their width + self.mock, nvars = mock, sum(a is AddrSpace.ALU for _,a,_,_ in signature) + if mock: self.cbuf_0[80:82] = [len(signature) - nvars, nvars] # mockgpu reads the arg counts out of cbuf0 # NOTE: Ensure at least 4KB of space after the program to mitigate prefetch memory faults. self.image = image.ljust(round_up(len(image), 0x1000) + 0x1000, b'\x00') @@ -358,8 +354,9 @@ class NVKIface: NVKIface.root = self.rm_alloc(0, nv_gpu.NV01_ROOT_CLIENT, None, root=0) drvver = self.rm_control(self.root, nv_gpu.NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION_V2, nv_gpu.NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_V2_PARAMS()) - if int(drvver.driverVersionBuffer.decode().split('.')[0], 10) >= 610: nv_gpu = nv_610 - elif int(drvver.driverVersionBuffer.decode().split('.')[0], 10) >= 580: nv_gpu = nv_580 + if (major:=int(drvver.driverVersionBuffer.decode().split('.')[0], 10)) >= 615: nv_gpu = nv_615 + elif major >= 610: nv_gpu = nv_610 + elif major >= 580: nv_gpu = nv_580 self.uvm(nv_gpu.UVM_INITIALIZE, nv_gpu.UVM_INITIALIZE_PARAMS()) @@ -554,15 +551,15 @@ class PCIIface(PCIIfaceBase): class MOCKIface(NVKIface): count = 1 +Compiled.pm_encode += PatternMatcher([ + (UPat.sink(arg=KernelInfo("nv_compute_submit")).call(UPat()).after(allow_any_len=True, name="s"), lambda s: NVComputeQueue(s).encode()), + (UPat.sink(arg=KernelInfo("nv_copy_submit")).call(UPat()).after(allow_any_len=True, name="s"), lambda s: NVCopyQueue(s).encode()), + (UPat.sink(arg=KernelInfo("nv_encdec_submit")).call(UPat()).after(allow_any_len=True, name="s"), lambda s: NVEncDecQueue(s).encode()), +]) + class NVDevice(Compiled): ifaces = [NVKIface, PCIIface, MOCKIface] sleep_timeout_ms = 200 - pm_encode = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat.custom_function("submit_nv_compute"), UPat()), name="s"), lambda s: NVComputeQueue(s).encode()), - (UPat(Ops.CALL, src=(UPat.custom_function("submit_nv_copy"), UPat()), name="s"), lambda s: NVCopyQueue(s).encode()), - (UPat(Ops.CALL, src=(UPat.custom_function("submit_nv_encdec"), UPat()), name="s"), lambda s: NVEncDecQueue(s).encode()), - (UPat(Ops.CALL, src=(UPat.custom_function("submit_nv_raw"), UPat()), name="s"), lambda s: NVQueue(s).encode()), - ]) def is_nvd(self) -> bool: return isinstance(self.iface, PCIIface) @@ -655,8 +652,7 @@ class NVDevice(Compiled): gpput=self.gpfifo_buf.view(4, gpput_off).ensure_allocated(), doorbell=Buffer("CPU", 4, options=BufferSpec(external_ptr=self.gpu_mmio.addr + 0x90), preallocate=True), put_value=Buffer("CPU", 8, initial_value=bytes(8)), notifier=notifier, entries=entries, token=ws_token_params.workSubmitToken) - Compiled.pm_bufferize += PatternMatcher([(UPat(Ops.ALLOC, tag=self.tag(n, name)), lambda b=getattr(fifo, n): b) - for n in ("ring", "gpput", "doorbell", "put_value")]) + Compiled.link_bufs |= {to_name(self.device, n, name): getattr(fifo, n) for n in ("ring", "gpput", "doorbell", "put_value")} return fifo def _query_gpu_info(self, *reqs): @@ -673,15 +669,13 @@ class NVDevice(Compiled): return [x.data for x in infos] def _submit_cmds(self, queue:str, *cmds:int): # channel setup uses the same runtime submit as kernels - tl = timeline(devs:=(self.device,)) + tl = timeline(self.device) value = tl.index(1).load() - submit = make_submit( - UOp(Ops.INS, arg=("wait", dtypes.void), src=(tl, value)), - UOp(Ops.INS, arg=("nv", dtypes.void), src=(UOp(Ops.BINARY, arg=array.array('I', cmds).tobytes()),)), - UOp(Ops.INS, arg=("store", dtypes.void), src=(tl, value + 1)), devs=devs, queue=queue, fn="submit_nv_raw") - call = UOp.sink(tl.after(submit).index(1).store(value + 1), arg=KernelInfo("nv_setup")).call(aux=HCQInfo(devs)) + submit = make_submit(ins("wait", tl, value), ins("nv", UOp(Ops.BINARY, arg=array.array('I', cmds).tobytes())), ins("store", tl, value + 1), + dev=self.device, queue=queue, deps=(tl,)) + call = UOp.sink(tl.after(submit).index(1).store(value + 1), arg=KernelInfo("nv_setup")).call(aux=HCQInfo((self.device,))) linear = lower_and_compile(UOp(Ops.LINEAR, src=(unwrap(lower_call(call)),))) - run_linear(hcq_link(linear, allow_cache=True), jit=True, update_stats=False, wait=True) + run_linear(hcq_link(linear), jit=True, update_stats=False, wait=True) def _ensure_has_local_memory(self, required): if self.slm_per_thread >= required: return @@ -722,8 +716,7 @@ class NVDevice(Compiled): if self.is_nvd(): self.iface.rm_control(self.subdevice, nv_gpu.NV2080_CTRL_CMD_INTERNAL_BUS_FLUSH_WITH_SYSMEMBAR, None) else: self.iface.rm_control(self.subdevice, nv_gpu.NV2080_CTRL_CMD_FB_FLUSH_GPU_CACHE, nv_gpu.NV2080_CTRL_FB_FLUSH_GPU_CACHE_PARAMS( - flags=((nv_gpu.NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_WRITE_BACK_YES << 2) | (nv_gpu.NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_INVALIDATE_YES << 3) | - (nv_gpu.NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FLUSH_MODE_FULL_CACHE << 4)))) + flags=nv_flags("NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS", write_back="yes", invalidate="yes", flush_mode="full_cache"))) def on_device_hang(self): # Prepare fault report. diff --git a/tinygrad_repo/tinygrad/runtime/ops_python.py b/tinygrad_repo/tinygrad/runtime/ops_python.py index 4782334a28..7dfb8087ee 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_python.py +++ b/tinygrad_repo/tinygrad/runtime/ops_python.py @@ -8,9 +8,8 @@ from dataclasses import replace from tinygrad.dtype import bitcast, DType, dtypes, AddrSpace, truncate, storage_fmt_for_dtype, to_storage_scalar, from_storage_scalar from tinygrad.helpers import all_same, getenv, Target, IMAGE, is_image_shape, to_mv, mv_address from tinygrad.device import HostAllocator, Compiled, Compiler, Program, TinyELF -from tinygrad.renderer import tc +from tinygrad.renderer import tc, Renderer from tinygrad.uop.ops import exec_alu, python_alu, Ops, UOp, GroupOp -from tinygrad.renderer import Renderer from tinygrad.runtime.support.elf import link_sym from tinygrad.runtime.support.c import DLL @@ -46,20 +45,18 @@ def wmma(tensor_cores:list[tc.TensorCore], arg, inp, warp_size:int): return out class PythonProgram(Program['PythonDevice']): - def __init__(self, dev:'PythonDevice', obj:TinyELF, uops:list[UOp]|None=None): - self.uops: list[UOp] = uops or pickle.loads(obj.lib) - self.fxns = {u: PythonProgram(dev, obj, list(u.body.src)) for u in self.uops if u.op is Ops.CALL and u.body.op is Ops.LINEAR} - self.tensor_cores = PythonRenderer(obj.target).tensor_cores + def __init__(self, dev:'PythonDevice', obj:TinyELF, links:dict[str,'PythonProgram']): + self.uops: list[UOp] = pickle.loads(obj.lib) + self.tensor_cores, self.links = PythonRenderer(obj.target).tensor_cores, links self.uop_to_index: dict[UOp, int] = {u:i for i,u in enumerate(self.uops)} self.loop_ends: dict[UOp, int] = {u.src[1]:i for i, u in enumerate(self.uops) if u.op in {Ops.END, Ops.BACKEDGE}} - def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw): + def __call__(self, *args, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), wait=False, **kw): st = time.perf_counter() warp = list(itertools.product(*[range(x) for x in local_size[::-1]])) warp_size = len(warp) for idxs in itertools.product(*[range(x) for x in global_size[::-1]]): values: dict[UOp, Any] = dict(env:=kw.get("env", ())) # a function runs with its params bound to the caller's args - pbufs: list[int] = list(bufs) - pvals: list[int] = list(vals) + pargs: list[int] = list(args) # the PARAMs are first in the uops, in the args' order exec_masks = [[True] * warp_size] i = 0 while i < len(self.uops): @@ -95,7 +92,7 @@ class PythonProgram(Program['PythonDevice']): i += 1 continue if u.op is Ops.AFTER or (u.op is Ops.BITCAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)): values[u] = src_values[0] - elif u.op is Ops.PARAM and u.addrspace is AddrSpace.ALU: values[u] = [pvals.pop(0)] * warp_size + elif u.op is Ops.PARAM and u.addrspace is AddrSpace.ALU: values[u] = [pargs.pop(0)] * warp_size elif u.op in {Ops.PARAM, Ops.BUFFER}: storage_fmt = storage_fmt_for_dtype(u.dtype) if storage_fmt is None: raise RuntimeError(f"dtype={u.dtype} is not supported") @@ -105,7 +102,7 @@ class PythonProgram(Program['PythonDevice']): values[u] = [memoryview(bytearray(u.max_numel()*u.dtype.itemsize)).cast(storage_fmt) for _ in range(warp_size)] else: size = u.max_numel() * u.dtype.itemsize - buf = memoryview(bytearray(size)) if u.op is not Ops.PARAM else to_mv(pbufs.pop(0), size) + buf = memoryview(bytearray(size)) if u.op is not Ops.PARAM else to_mv(pargs.pop(0), size) values[u] = [buf.cast(storage_fmt)] * warp_size elif u.op is Ops.BINARY: values[u] = [memoryview(u.arg)] * warp_size elif u.op is Ops.SPECIAL: @@ -144,7 +141,8 @@ class PythonProgram(Program['PythonDevice']): for k in range(len(src_values))], j, u.dtype) for j in range(load_sz)] else: values[u] = load(src_values, 0, u.dtype) - elif u in self.fxns: values[u] = [self.fxns[u](env={p: values[u.src[p.arg.slot+1]] for p in u.body.src if p.op is Ops.PARAM})] + elif u.op is Ops.CALL and u.body.op is Ops.PROGRAM: + values[u] = [self.links[u.body.src[0].arg.function_name](env={p: values[u.src[p.arg.slot+1]] for p in u.body.arg.params})] elif u.op is Ops.CALL: # a C function by symbol, linked against the loaded libraries restype = None if u.dtype is dtypes.void else getattr(ctypes, f"c_{'u' if u.dtype in dtypes.uints else ''}int{u.dtype.bitsize}") cfunc = ctypes.CFUNCTYPE(restype, *[ctypes.c_uint64] * len(src_values))(link_sym(u.src[0].arg.name, list(DLL._loaded_.values()))) diff --git a/tinygrad_repo/tinygrad/runtime/ops_qcom.py b/tinygrad_repo/tinygrad/runtime/ops_qcom.py index e778c1f8bc..53804bdb43 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_qcom.py +++ b/tinygrad_repo/tinygrad/runtime/ops_qcom.py @@ -1,20 +1,17 @@ from __future__ import annotations -import os, ctypes, functools, mmap, struct, array, math, sys, contextlib -assert sys.platform != 'win32' +import os, ctypes, functools, mmap, struct, array, math, contextlib from typing import Any from tinygrad.device import Compiled, BufferStorage, BufferSpec, Buffer, Device, Allocator, TinyELF from tinygrad.runtime.support.hcq2 import HWQueue, HCQ_RUNTIME_DEV, ccall, cstruct, patch, unwrap_view, layout_args, pack_args, make_program from tinygrad.runtime.support.memory import MMIOInterface -from tinygrad.runtime.support.system import FileIOInterface +from tinygrad.runtime.support.system import FileIOInterface, System from tinygrad.runtime.autogen import kgsl, mesa, libc from tinygrad.renderer.cstyle import QCOMCLRenderer from tinygrad.renderer.nir import IR3Renderer -from tinygrad.helpers import getenv, mv_address, round_up, ceildiv, prod, is_image_shape -from tinygrad.helpers import next_power2, flatten, PROFILE, IMAGE +from tinygrad.helpers import getenv, mv_address, round_up, ceildiv, prod, is_image_shape, next_power2, flatten, to_name, PROFILE, IMAGE from tinygrad.dtype import dtypes, AddrSpace -from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, uopfunc -from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops -from tinygrad.runtime.support.system import System +from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo, uopfunc +from tinygrad.engine.realize import get_call_kernel_args if getenv("IOCTL"): import extra.qcom_gpu_driver.opencl_ioctl # noqa: F401 # pylint: disable=unused-import BUFTYPE_BUF, BUFTYPE_TEX, BUFTYPE_IBO = 0, 1, 2 @@ -27,7 +24,7 @@ def dcache_flush(): i = UOp.range(n, 0, dtype=dtypes.int) flush = UOp(Ops.CUSTOM, src=(buf.index(i * 64),), arg=('__asm__ volatile("dc cvac, %0" :: "r"({0}) : "memory");', dtypes.void)) sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg=('__asm__ volatile("dsb sy" ::: "memory");', dtypes.void)), - arg=KernelInfo(name="dcache_flush"), tag=1) + arg=KernelInfo(name="dcache_flush", opts_to_apply=())) prg = to_program(sink, Device["CPU"].renderer) return Device["CPU"].runtime(prg.to_elf()) @@ -67,7 +64,7 @@ class QCOMComputeQueue(HWQueue): def _cache_flush(self, write_back=True, invalidate=False, sync=True, memsync=False): # TODO: 7xx support. if write_back: # dirty cache write-back, into the device's dummy buffer - dummy = UOp.alloc((0x1000,), dtypes.uint8, 0, device=self.devs[0]).rtag(self.dev.tag("dummy")) + dummy = UOp.alloc((0x1000,), dtypes.uint8, 0, device=self.devs[0], name="dummy") self.cmd(mesa.CP_EVENT_WRITE, mesa.CACHE_FLUSH_TS, dummy.getaddr(self.devs), 0) if invalidate: self.cmd(mesa.CP_EVENT_WRITE, mesa.CACHE_INVALIDATE) # invalidate cache lines (following reads from RAM). if memsync: self.cmd(mesa.CP_WAIT_MEM_WRITES) @@ -93,27 +90,26 @@ class QCOMComputeQueue(HWQueue): value.cast(dtypes.uint32), qreg.cp_wait_reg_mem_4(mask=0xFFFFFFFF), qreg.cp_wait_reg_mem_5(delay_loop_cycles=32)) def kernargs(self, call:UOp, prg:UOp, data:QCOMProgramData) -> UOp: - bufs, vals = [get_call_arg_uops(call)[g] for g in prg.arg.globals], get_call_var_uops(call, prg) - ubos = [bufs[slot] for _,slot,_,shape in data.signature if slot < len(bufs) and not is_image_shape(shape)] - uavs = [(dt,shape,bufs[slot]) for _,slot,dt,shape in data.signature if slot < len(bufs) and is_image_shape(shape)] + kargs = list(zip(get_call_kernel_args(call, prg, self.devs), data.signature)) + uavs = [(dt,shape,a) for a,(_,_,dt,shape) in kargs if is_image_shape(shape)] # NIR can reorder images to different texture slots ibos, texs = uavs[:data.ibo_cnt], [uavs[data.ibo_cnt + (data.tex_to_image[i] if data.NIR else i)] for i in range(data.tex_cnt)] args = [(off, UOp.const(val, dtypes.uint32 if sz == 4 else dtypes.uint16)) for val,off,sz in data.consts_info] args += layout_args(data.samplers, data.samp_off) - vals = [v.ccast(dt) for v,(_,_,dt,_) in zip(vals, data.signature[len(bufs):])] + ubos = [a for a,(_,_,_,shape) in kargs if not is_image_shape(shape)] if data.NIR: - args += layout_args([b.getaddr(self.devs) for b in ubos] + vals, data.buf_off) + args += layout_args(ubos, data.buf_off) if data.wgsz != 0xfc: args += layout_args(list(prg.arg.local_size), data.wgsz * 4) - else: args += list(zip(data.buf_offs, [b.getaddr(self.devs) for b in ubos] + vals)) + else: args += list(zip(data.buf_offs, ubos)) def _tex(b, ibo=False): - imgdt, shape, buf = b + imgdt, shape, addr = b pitch = shape[1] * 4 * imgdt.itemsize fmt = mesa.FMT6_32_32_32_32_FLOAT if imgdt.itemsize == 4 else mesa.FMT6_16_16_16_16_FLOAT return [qreg.a6xx_tex_const_0(fmt=fmt) if ibo else qreg.a6xx_tex_const_0(0x8, swiz_x=0, swiz_y=1, swiz_z=2, swiz_w=3, fmt=fmt), qreg.a6xx_tex_const_1(width=shape[1], height=shape[0]), - qreg.a6xx_tex_const_2(type=mesa.A6XX_TEX_2D, pitch=pitch, pitchalign=ctz(pitch)-6), 0, buf.getaddr(self.devs), + qreg.a6xx_tex_const_2(type=mesa.A6XX_TEX_2D, pitch=pitch, pitchalign=ctz(pitch)-6), 0, addr, qreg.a6xx_tex_const_6(plane_pitch=0x400000), qreg.a6xx_tex_const_7(13), 0, 0, 0, 0, 0, 0, 0, 0] args += layout_args(flatten(map(_tex, texs)), data.tex_off) + layout_args(flatten(map(functools.partial(_tex, ibo=True), ibos)), data.ibo_off) return UOp(Ops.LINEAR, src=tuple(pack_args(args, data.kernargs_alloc_size)), arg="kernargs") @@ -129,7 +125,7 @@ class QCOMComputeQueue(HWQueue): global_size_mp = [cast_int(g*l) for g,l in zip(global_size, local_size)] args_addr, lib_addr = self.kernargs(call, prg, data).getaddr(self.devs), lib.getaddr(self.devs) - stack_addr = UOp.alloc((data.hw_stack_offset * 4,), dtypes.uint8, 0, device=self.devs[0]).rtag(self.dev.tag("stack")).getaddr(self.devs) + stack_addr = UOp.alloc((data.hw_stack_offset * 4,), dtypes.uint8, 0, device=self.devs[0], name="stack").getaddr(self.devs) self.cmd(mesa.CP_SET_MARKER, qreg.a6xx_cp_set_marker_0(mode=mesa.RM6_COMPUTE)) self.reg(mesa.REG_A6XX_SP_UPDATE_CNTL, qreg.a6xx_sp_update_cntl(cs_state=True, cs_uav=True)) @@ -173,7 +169,7 @@ class QCOMComputeQueue(HWQueue): state_block=mesa.SB6_CS_TEX, num_unit=data.samp_cnt), args_addr + data.samp_off) self.reg(mesa.REG_A6XX_SP_CS_SAMPLER_BASE, args_addr + data.samp_off) self.reg(mesa.REG_A6XX_TPL1_CS_BORDER_COLOR_BASE, - UOp.alloc((0x1000,), dtypes.uint8, 0, device=self.devs[0]).rtag(self.dev.tag("border_color")).getaddr(self.devs)) + UOp.alloc((0x1000,), dtypes.uint8, 0, device=self.devs[0], name="border_color").getaddr(self.devs)) if data.tex_cnt > 0: self.cmd(mesa.CP_LOAD_STATE6_FRAG, qreg.cp_load_state6_0(state_type=mesa.ST_CONSTANTS, state_src=mesa.SS6_INDIRECT, @@ -203,7 +199,7 @@ class QCOMComputeQueue(HWQueue): obj = cstruct(kgsl.struct_kgsl_command_object, gpuaddr=ib.getaddr(self.devs) + ib_off, size=cmdbuf.max_numel(), flags=kgsl.KGSL_CMDLIST_IB) req = cstruct(kgsl.struct_kgsl_gpu_command, cmdlist=obj.getaddr(HCQ_RUNTIME_DEV.device), cmdsize=ctypes.sizeof(kgsl.struct_kgsl_command_object), numcmds=1, context_id=ctxid) - return qcom_submit(req.after(cmdbuf), UOp.alloc((1,), dtypes.int32, device=self.devs[0]).rtag("submit_ret"), fd) + return qcom_submit(req.after(cmdbuf), UOp.alloc((1,), dtypes.int32, device=self.devs[0], name="submit_ret"), fd) class QCOMProgramData: def __init__(self, dev:QCOMDevice, obj:TinyELF): @@ -299,11 +295,12 @@ class QCOMAllocator(Allocator['QCOMDevice']): def flag(nm, val): return (val << getattr(kgsl, f"{nm}_SHIFT")) & getattr(kgsl, f"{nm}_MASK") +Compiled.pm_encode += PatternMatcher([ + (UPat.sink(arg=KernelInfo("qcom_compute_submit")).call(UPat()).after(allow_any_len=True, name="s"), lambda s: QCOMComputeQueue(s).encode()), +]) + class QCOMDevice(Compiled): timestamp_divider = 19.2 - pm_encode = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat.custom_function("submit_qcom_compute"), UPat()), name="s"), lambda s: QCOMComputeQueue(s).encode()), - ]) @property def has_copy_queue(self) -> bool: return False @@ -335,17 +332,12 @@ class QCOMDevice(Compiled): arch=("a%d%d%d" + (",IMAGE_PITCH_ALIGNMENT=64" if IMAGE else "")) % self.gpu_id) self.var_vals = {"kgsl_fd": self.fd.fd, "kgsl_ctx": self.ctx} - Compiled.pm_bufferize += PatternMatcher([ - (UPat(Ops.ALLOC, tag=self.tag("stack"), name="b"), lambda b, d=self: d._ensure_stack_size(b.max_numel())), - (UPat(Ops.ALLOC, tag=self.tag("dummy")), lambda d=self: d.dummy), - (UPat(Ops.ALLOC, tag=self.tag("border_color")), lambda d=self: d.border_color)]) - @functools.cached_property - def dummy(self) -> Buffer: return Buffer(self.device, 0x1000, options=BufferSpec(nolru=True), preallocate=True) # cache flush target - - @functools.cached_property - def border_color(self) -> Buffer: # zeros: the samplers clamp to a black border - return Buffer(self.device, 0x1000, options=BufferSpec(nolru=True), initial_value=bytes(0x1000)) + # the cache flush target, and zeros: the samplers clamp to a black border + dummy = Buffer(self.device, 0x1000, options=BufferSpec(nolru=True), preallocate=True, allocator=self.allocator) + border_color = Buffer(self.device, 0x1000, options=BufferSpec(nolru=True), initial_value=bytes(0x1000), allocator=self.allocator) + stack = {to_name(self.device, "stack"): lambda b: self._ensure_stack_size(b.max_numel())} + Compiled.link_bufs |= stack | {to_name(self.device, "dummy"): dummy, to_name(self.device, "border_color"): border_color} def _gpu_alloc(self, size:int, flags:int=0, uncached=False, fill_zeroes=False) -> BufferStorage: flags |= flag("KGSL_MEMALIGN", alignment_hint:=12) | kgsl.KGSL_MEMFLAGS_USE_CPU_MAP @@ -359,7 +351,7 @@ class QCOMDevice(Compiled): def _gpu_map(self, ptr:int, size:int) -> BufferStorage: ptr_aligned, size_aligned = (ptr & ~0xfff), round_up(size + (ptr & 0xfff), 0x1000) - dcache_flush().fxn(ctypes.c_uint64(ptr_line_aligned:=ptr & ~63), ceildiv(ptr + size - ptr_line_aligned, 64)) + dcache_flush()(ptr_line_aligned:=ptr & ~63, ceildiv(ptr + size - ptr_line_aligned, 64)) try: mi = kgsl.IOCTL_KGSL_MAP_USER_MEM(self.fd, hostptr=ptr_aligned, len=size_aligned, memtype=kgsl.KGSL_USER_MEM_TYPE_ADDR) return BufferStorage(mi.gpuaddr + (ptr - ptr_aligned), (mi, False), MMIOInterface(ptr, size, fmt='B')) diff --git a/tinygrad_repo/tinygrad/runtime/ops_rdma.py b/tinygrad_repo/tinygrad/runtime/ops_rdma.py index 2ab57f877e..459571bb88 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_rdma.py +++ b/tinygrad_repo/tinygrad/runtime/ops_rdma.py @@ -9,8 +9,7 @@ from tinygrad.runtime.autogen import bnxt from tinygrad.runtime.support.rdma.bnxtdev import BNXTDev, BNXTQP, db_value, send_wqe, recv_wqe, WQE_SIZE, RING_ENTRIES, CQ_ENTRIES, MTU from tinygrad.runtime.support.hcq2 import unwrap_view, to_name, ins from tinygrad.runtime.support.memory import AddrSpace, MMIOInterface, VirtMapping, MemoryManager -from tinygrad.runtime.support.system import PCIIfaceBase, PCIAllocationMeta, System -from tinygrad.runtime.support.system import filter_visible_devices +from tinygrad.runtime.support.system import PCIIfaceBase, PCIAllocationMeta, System, filter_visible_devices from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat RDMA_CHUNK = 1 << 30 # a wqe length is 32 bits @@ -81,12 +80,12 @@ def rdma_qp(pair:tuple[str, str]) -> dict[str, BNXTQP]: for nic, q in zip(nics, qps.values()): bufs = {name: nic.iface.buffer(getattr(q, name).ring, getattr(q, name).paddrs) for name in ("sq", "rq", "scq", "rcq")} bufs |= {name: Buffer(nic.device, 8, initial_value=bytes(8)) for name in ("sq_seq", "rq_seq", "psn")} | {"db": nic.iface.doorbell} - Compiled.pm_bufferize += PatternMatcher([(UPat(Ops.ALLOC, tag=to_name(nic.device, *pair, n)), lambda b=b: b) for n, b in bufs.items()]) + Compiled.link_bufs |= {to_name(nic.device, *pair, n): b for n, b in bufs.items()} for a, b in (nics, nics[::-1]): qps[a.device].connect(qps[b.device].qpn, b.iface.dev_impl.local_gid, b.iface.dev_impl.mac) return qps def rdma_mem(nic:str, pair:tuple[str, str], name:str, size:int, dtype:DType=dtypes.uint8) -> UOp: - return UOp.alloc((size,), dtype, 0, device=nic).rtag(to_name(nic, *pair, name)) + return UOp.alloc((size,), dtype, 0, device=nic, name=to_name(*pair, name)) def rdma_ring(nic:str, pair:tuple[str, str], is_recv:bool) -> UOp: return rdma_mem(nic, pair, "rq" if is_recv else "sq", RING_ENTRIES * (WQE_SIZE if is_recv else WQE_SIZE + 8)) def rdma_cq(nic:str, pair:tuple[str, str], is_recv:bool) -> UOp: return rdma_mem(nic, pair, "rcq" if is_recv else "scq", CQ_ENTRIES * 32) @@ -101,8 +100,8 @@ def is_rdma(call:UOp) -> bool: return rdma_wire(call) is not None def queue_of(call:UOp) -> tuple[tuple[str, str], bool]: (dst, src), wire = get_call_arg_uops(call), unwrap(rdma_wire(call)) - gpu = to_tuple((dst if wire is src else src).device)[0] - return (min(gpu, wire.tag), max(gpu, wire.tag)), wire is src + gpu, peer = to_tuple((dst if wire is src else src).device)[0], wire.base.arg.name + return (min(gpu, peer), max(gpu, peer)), wire is src def rdma_copies(devs:tuple[str, ...], calls:list[UOp]) -> list[list[UOp]]: # the ops of each copy of a submit on one queue (pair, is_recv), nic = queue_of(calls[0]), cast(RDMADevice, Device[unwrap(rdma_wire(calls[0])).device]) @@ -148,19 +147,19 @@ def rdma_copies(devs:tuple[str, ...], calls:list[UOp]) -> list[list[UOp]]: # the ins("write", db, ((n + 1) % CQ_ENTRIES | ((n + 1) // CQ_ENTRIES & 1) << bnxt.BNXT_QPLIB_DBR_EPOCH_SHIFT) | cq_db)] # and invalidate the gpu caches on recv - copies.append(ops + ([ins("barrier")] if is_recv else [])) + copies.append(ops + ([ins("memory_barrier")] if is_recv else [])) return copies # ***************** # encode rewrite -def rdma_submit(ctx, submit:UOp, arg:UOp) -> UOp|None: - lin = arg.without_after +def rdma_submit(s:UOp) -> UOp|None: + lin, queue = s.without_after.body, s.without_after.src[1] if not (queues:={i: queue_of(u) for i, u in enumerate(lin.src) if is_rdma(u)}): return None ops = [[u] for u in lin.src] for q in dict.fromkeys(queues.values()): positions = [i for i in queues if queues[i] == q] - for i, copy_ops in zip(positions, rdma_copies(lin.arg[0], [lin.src[i] for i in positions])): ops[i] = copy_ops - return submit.replace(src=(submit.body, lin.replace(src=tuple(flatten(ops))).after(*arg.src[1:] if arg.op is Ops.AFTER else ()))) -pm_rdma_encode = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION), UPat(Ops.LINEAR).or_after("arg")), name="submit"), rdma_submit)]) + for i, copy_ops in zip(positions, rdma_copies((queue.device,), [lin.src[i] for i in positions])): ops[i] = copy_ops + return s.without_after.replace(src=(lin.replace(src=tuple(flatten(ops))), queue)).after(*s.src[1:]) +Compiled.pm_encode = PatternMatcher([(UPat.sink().call(UPat(Ops.PARAM)).after(allow_any_len=True, name="s"), rdma_submit)]) + Compiled.pm_encode diff --git a/tinygrad_repo/tinygrad/runtime/ops_webgpu.py b/tinygrad_repo/tinygrad/runtime/ops_webgpu.py index d3b03a21a6..96fd33f06f 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_webgpu.py +++ b/tinygrad_repo/tinygrad/runtime/ops_webgpu.py @@ -1,5 +1,6 @@ import functools, struct from tinygrad.device import BufferStorage, Compiled, Allocator, BufferSpec, Program, TinyELF +from tinygrad.dtype import AddrSpace from tinygrad.renderer.wgsl import WGSLRenderer from tinygrad.helpers import round_up, suppress_finalizing, getenv, to_mv from tinygrad.runtime.autogen import webgpu @@ -50,8 +51,8 @@ AdapterRequestDevice = synchronous(webgpu.enum_WGPURequestDeviceStatus, True)(we QueueOnSubmittedWorkDone = synchronous(webgpu.enum_WGPUQueueWorkDoneStatus)(webgpu.wgpuQueueOnSubmittedWorkDone2) class WebGPUProgram(Program['WebGpuDevice']): - def __init__(self, dev:'WebGpuDevice', obj:TinyELF): - self.dev, self.name = dev, to_wgpu_str(obj.name) + def __init__(self, dev:'WebGpuDevice', obj:TinyELF, links=None): + self.dev, self.name, self.signature = dev, to_wgpu_str(obj.name), obj.signature # Creating shader module shader = webgpu.WGPUShaderModuleWGSLDescriptor(code=to_wgpu_str(obj.lib.decode()), @@ -66,16 +67,15 @@ class WebGPUProgram(Program['WebGpuDevice']): @suppress_finalizing def __del__(self): webgpu.wgpuShaderModuleRelease(self.prg) - def __call__(self, *bufs:webgpu.WGPUBuffer, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), - vals:tuple[int, ...]=(), wait=False, **kw) -> float|None: + def __call__(self, *args, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), wait=False, **kw) -> float|None: wait = wait and webgpu.WGPUFeatureName_TimestampQuery in self.dev.features # Creating bind group layout def bgl_entry(n:int, ty:str): return webgpu.WGPUBindGroupLayoutEntry(binding=n, visibility=webgpu.WGPUShaderStage_Compute, buffer=webgpu.WGPUBufferBindingLayout(type=getattr(webgpu, f'WGPUBufferBindingType_{ty}'))) - bind_entries = (webgpu.WGPUBindGroupLayoutEntry * (1+len(bufs)+len(vals)))( - bgl_entry(0, 'Uniform'), *(bgl_entry(i+1, 'Uniform' if i >= len(bufs) else 'Storage') for i in range(len(bufs)+len(vals)))) + bind_entries = (webgpu.WGPUBindGroupLayoutEntry * (1+len(args)))( + bgl_entry(0, 'Uniform'), *(bgl_entry(i+1, 'Uniform' if a is AddrSpace.ALU else 'Storage') for i,(_,a,_,_) in enumerate(self.signature))) webgpu.wgpuDevicePushErrorScope(self.dev.device_res, webgpu.WGPUErrorFilter_Validation) bind_layout = webgpu.wgpuDeviceCreateBindGroupLayout(self.dev.device_res, @@ -94,12 +94,12 @@ class WebGPUProgram(Program['WebGpuDevice']): def bg_entry(n:int, x:webgpu.WGPUBuffer|int|float): buf = x if isinstance(x, webgpu.WGPUBuffer) else self.dev.create_uniform(x) return webgpu.WGPUBindGroupEntry(binding=n, buffer=buf, offset=0, size=webgpu.wgpuBufferGetSize(buf)) - bindings = (webgpu.WGPUBindGroupEntry * (1+len(bufs)+len(vals)))(bg_entry(0, float('inf')), *(bg_entry(i+1, x) for i,x in enumerate(bufs+vals))) + bindings = (webgpu.WGPUBindGroupEntry * (1+len(args)))(bg_entry(0, float('inf')), *(bg_entry(i+1, x) for i,x in enumerate(args))) bind_group_desc = webgpu.WGPUBindGroupDescriptor(layout=bind_layout, entryCount=len(bindings), entries=bindings) webgpu.wgpuDevicePushErrorScope(self.dev.device_res, webgpu.WGPUErrorFilter_Validation) bind_group = webgpu.wgpuDeviceCreateBindGroup(self.dev.device_res, bind_group_desc) - for binding in (bindings[0], *bindings[1+len(bufs):]): webgpu.wgpuBufferRelease(binding.buffer) + for b in bindings[:1] + [b for b,(_,a,_,_) in zip(bindings[1:], self.signature) if a is AddrSpace.ALU]: webgpu.wgpuBufferRelease(b.buffer) if err := self.dev.pop_error(): raise RuntimeError(f"Error creating bind group: {err}") # Creating compute pipeline diff --git a/tinygrad_repo/tinygrad/runtime/support/am/amdev.py b/tinygrad_repo/tinygrad/runtime/support/am/amdev.py index 0f3a0b4be0..a07baf266e 100644 --- a/tinygrad_repo/tinygrad/runtime/support/am/amdev.py +++ b/tinygrad_repo/tinygrad/runtime/support/am/amdev.py @@ -201,9 +201,8 @@ class AMDev: if self.is_hive(): if reset_mode: return # in reset mode, do not raise raise RuntimeError("Malformed state. Use extra/amdpci/hive_reset.py to reset the hive") - # Quiesce first: mode1 reset over live engines at max clocks can wedge the GPU until power cycled. + # Quiesce first: mode1 reset over live engines can wedge the GPU until power cycled. self.gfx.fini_hw() - self.smu.set_clocks(level=0) self.gfx.halt_engines() self.sdma.halt_engines() time.sleep(0.1) diff --git a/tinygrad_repo/tinygrad/runtime/support/c.py b/tinygrad_repo/tinygrad/runtime/support/c.py index 059704d8cc..04f8014c9d 100644 --- a/tinygrad_repo/tinygrad/runtime/support/c.py +++ b/tinygrad_repo/tinygrad/runtime/support/c.py @@ -1,10 +1,9 @@ from __future__ import annotations import ctypes, functools, os, pathlib, re, sys, sysconfig -from tinygrad.helpers import ceildiv, getenv, DEBUG, OSX, WIN +from tinygrad.helpers import ceildiv, getenv, DEBUG, OSX from typing import TYPE_CHECKING, get_args, Generic, ParamSpec, TypeVar def _do_ioctl(__idir, __base, __nr, __struct, __fd, *args, __payload=None, **kwargs): - assert not WIN, "ioctl not supported" import tinygrad.runtime.support.system as system, fcntl ioctl = __fd.ioctl if isinstance(__fd, system.FileIOInterface) else functools.partial(fcntl.ioctl, __fd) if __struct is None: return ioctl((__base<<8)|__nr, __payload or (args[0] if args else 0)) @@ -39,6 +38,7 @@ class Array(Generic[T, U], ctypes.Array): class Struct(ctypes.Structure): SIZE = 0 + _real_fields_:list[tuple[str, type, int]|tuple[str, type, int, int, int]] # (name, type, offset[, bit width, bit offset]) def __init__(self, *args, **kwargs): ctypes.Structure.__init__(self) @@ -103,8 +103,8 @@ class DLL(ctypes.CDLL): else: continue for pre in (pathlib.Path(pre) for pre in ([path] if path else []) + libpaths.get(os.name, []) + libpaths.get(sys.platform, []) + extra_paths): if not pre.is_dir(): continue - if WIN or OSX: - for base in ([f"lib{p}.dylib", f"{p}.dylib", str(p)] if OSX else [f"{p}.dll"]): + if OSX: + for base in [f"lib{p}.dylib", f"{p}.dylib", str(p)]: if (l:=pre / base).is_file() or (OSX and 'framework' in str(l) and l.is_symlink()): return str(l) else: for l in (l for l in pre.iterdir() if l.is_file() and re.fullmatch(f"lib{p}\\.so[.0-9]*", l.name)): diff --git a/tinygrad_repo/tinygrad/runtime/support/compiler_amd.py b/tinygrad_repo/tinygrad/runtime/support/compiler_amd.py index eef5e07f88..174c569845 100644 --- a/tinygrad_repo/tinygrad/runtime/support/compiler_amd.py +++ b/tinygrad_repo/tinygrad/runtime/support/compiler_amd.py @@ -1,5 +1,5 @@ import ctypes, hashlib, tempfile, subprocess, pathlib -from tinygrad.helpers import amdgpu_disassemble, getenv +from tinygrad.helpers import amdgpu_disassemble, getenv, to_char_p_p from tinygrad.runtime.autogen import comgr try: comgr.amd_comgr_get_version(ctypes.byref(major:=ctypes.c_uint64()), ctypes.byref(minor:=ctypes.c_uint64())) @@ -10,7 +10,6 @@ try: except AttributeError: pass # ignore if ROCm isn't installed from tinygrad.device import Compiler, CompileError from tinygrad.runtime.support import c -from tinygrad.helpers import to_char_p_p def check(status): if status != 0: diff --git a/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py b/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py index d2a7446c38..2eb41c9d3d 100644 --- a/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py +++ b/tinygrad_repo/tinygrad/runtime/support/compiler_llvm.py @@ -51,6 +51,9 @@ class LLVMCompiler(Compiler): self.diag_msgs.clear() src_buf = llvm.LLVMCreateMemoryBufferWithMemoryRangeCopy(ctypes.create_string_buffer(src_bytes:=src.encode()), len(src_bytes), b'src') mod = expect(llvm.LLVMParseIRInContext(self.context, src_buf, ctypes.pointer(m:=llvm.LLVMModuleRef()), err:=cerr()), err, m) + layout = llvm.LLVMCreateTargetDataLayout(self.target_machine) + llvm.LLVMSetModuleDataLayout(mod, layout) + llvm.LLVMDisposeTargetData(layout) expect(llvm.LLVMVerifyModule(mod, llvm.LLVMReturnStatusAction, err:=cerr()), err) expect(llvm.LLVMRunPasses(mod, self.passes, self.target_machine, self.pbo), 'failed to run passes') if DEBUG >= 7: print(ctypes.string_at(llvm.LLVMPrintModuleToString(mod)).decode()) diff --git a/tinygrad_repo/tinygrad/runtime/support/elf.py b/tinygrad_repo/tinygrad/runtime/support/elf.py index 30cabd7af1..071701948c 100644 --- a/tinygrad_repo/tinygrad/runtime/support/elf.py +++ b/tinygrad_repo/tinygrad/runtime/support/elf.py @@ -6,13 +6,15 @@ from tinygrad.runtime.autogen import libc @dataclass(frozen=True) class ElfSection: name:str; header:libc.Elf64_Shdr|libc.Elf32_Shdr; content:bytes # noqa: E702 -def link_sym(sym:str, libs:list[ctypes.CDLL]) -> int: +def link_sym(sym:str, libs:list[ctypes.CDLL], link_syms:dict[str,int]|None=None) -> int: + if link_syms is not None and (addr:=link_syms.get(sym)): return addr for lib in libs: try: return unwrap(ctypes.cast(getattr(lib, sym), ctypes.c_void_p).value) except (OSError, AttributeError, TypeError): pass raise RuntimeError(f'Attempting to relocate against an undefined symbol {sym}') -def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[ctypes.CDLL]|None=None) -> tuple[memoryview, list[ElfSection], list[tuple]]: +def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[ctypes.CDLL]|None=None, + link_syms:dict[str,int]|None=None) -> tuple[memoryview, list[ElfSection], list[tuple]]: assert blob[:4] == libc.ELFMAG.encode(), "blob is not an ELF, missing magic bytes" ecls = {libc.ELFCLASS32: "Elf32", libc.ELFCLASS64: "Elf64"}[blob[libc.EI_CLASS]] @@ -44,13 +46,13 @@ def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[ctypes.CDLL target_image_off = next(tsh for tsh in sections if tsh.name == trgt_sh_name).header.sh_addr rels = [(r.r_offset, unwrap(symtab)[getattr(libc, f"{ecls.upper()}_R_SYM")(r.r_info)], getattr(libc, f"{ecls.upper()}_R_TYPE")(r.r_info), getattr(r, "r_addend", 0)) for r in c_rels] - relocs += [(target_image_off + roff, link_sym(_strtab(sh_strtab, sym.st_name), link_libs or []) if sym.st_shndx == 0 else + relocs += [(target_image_off + roff, link_sym(_strtab(sh_strtab, sym.st_name), link_libs or [], link_syms) if sym.st_shndx == 0 else sections[sym.st_shndx].header.sh_addr + sym.st_value, rtype, raddend) for roff, sym, rtype, raddend in rels] return memoryview(image), sections, relocs -def jit_loader(obj: bytes, base:int=0, link_libs:list[ctypes.CDLL]|None=None) -> bytes: - image_, _, relocs = elf_loader(obj, link_libs=link_libs) +def jit_loader(obj: bytes, base:int=0, link_libs:list[ctypes.CDLL]|None=None, link_syms:dict[str,int]|None=None) -> bytes: + image_, _, relocs = elf_loader(obj, link_libs=link_libs, link_syms=link_syms) image = bytearray(image_) def relocate(instr: int, base: int, ploc: int, tgt: int, r_type: int, addend: int): diff --git a/tinygrad_repo/tinygrad/runtime/support/hcq2.py b/tinygrad_repo/tinygrad/runtime/support/hcq2.py index f2b28d326e..26afb00881 100644 --- a/tinygrad_repo/tinygrad/runtime/support/hcq2.py +++ b/tinygrad_repo/tinygrad/runtime/support/hcq2.py @@ -1,31 +1,24 @@ from __future__ import annotations from typing import cast, Any, Sequence -import itertools, weakref, ctypes, struct, time +import itertools, ctypes, struct, time from dataclasses import replace, dataclass, field from collections import defaultdict -from tinygrad.helpers import dedup, pluralize, unwrap, to_tuple, ContextVar, Context, panic, partition, getenv, to_name -from tinygrad.helpers import DEBUG, VIZ, ALL2ALL, PROFILE -from tinygrad.device import Device, Buffer, BufferSpec, Compiled, TinyELF, HCQ_RUNTIME_DEV, ProfileProgramEvent -from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo, GroupOp, graph_rewrite, rewrite_group, exec_alu, uopfunc, sym_infer -from tinygrad.uop.ops import pm_renumber_slots -from tinygrad.dtype import dtypes, DType, AddrSpace, truncate, to_dtype +from tinygrad.helpers import dedup, pluralize, unwrap, to_tuple, Context, panic, partition, to_name, DEBUG, VIZ, ALL2ALL, PROFILE +from tinygrad.device import Device, Buffer, BufferSpec, Compiled, TinyELF, HCQ_RUNTIME_DEV, HCQ_DEVS, HOST_DEVS, UVM_DEVS, ProfileProgramEvent +from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, KernelInfo, GroupOp, graph_rewrite, rewrite_group, exec_alu, uopfunc +from tinygrad.uop.ops import pm_renumber_slots, CountCtx, sint_to_uop +from tinygrad.dtype import dtypes, AddrSpace, truncate from tinygrad.renderer import Estimates -from tinygrad.engine.realize import get_call_arg_uops, get_call_name, get_call_outs_ins -from tinygrad.engine.realize import estimate_uop, pm_flatten_linear, lower_and_compile +from tinygrad.runtime.support.c import Struct +from tinygrad.engine.realize import get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop, pm_flatten_linear, lower_and_compile -# ***************** -# 0. helpers - -HCQ_CACHE_THRESH = ContextVar("HCQ_CACHE_THRESH", 64) -HCQ_DEVS = frozenset(("AMD", "NV", "QCOM", "CUDA", "HIP", "NULL", "METAL")) -HOST_DEVS = frozenset(("CPU", "PYTHON")) -UVM_DEVS = frozenset(("QCOM", "METAL")) # the host maps their memory, so copies are host memcpys +# **** calls @dataclass(frozen=True) class HCQInfo: device:tuple[str, ...] - kernels:tuple[tuple, ...] = () # (devices, name, estimates, timestamp slots, profile key, input slots of the buffers, (outs, ins)) + kernels:tuple[tuple, ...] = () # (device, name, estimates, timestamp slots, profile key, input slots of the buffers, (outs, ins)) estimates:Estimates = Estimates() slots:tuple[tuple[str, int], ...] = () # per device, the position of its batch slots in the args @@ -37,7 +30,7 @@ def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for def get_enqueue_devs(call:UOp) -> tuple[str, ...]: if call.op is not Ops.CALL: return () # entries can be AFTER-wrapped calls if call.body.op not in (Ops.PROGRAM, Ops.STORE) and not (call.body.op is Ops.CUSTOM_FUNCTION and call.body.arg.name == "encdec"): return () - runs_on = call.device if call.body.op is Ops.STORE and all_devices_in(call.device, HCQ_DEVS) else call.src[1].device + runs_on = call.device if call.body.op is Ops.STORE and all_devices_in(call.device, HCQ_DEVS - UVM_DEVS) else get_call_arg_uops(call)[0].device devs = tuple(Device.canonicalize(d) for d in to_tuple(runs_on)) if not all_devices_in(devs, HCQ_DEVS) or (call.body.op is Ops.STORE and all_devices_in(devs, UVM_DEVS)): return () return devs @@ -53,32 +46,73 @@ def unwrap_lane(v:UOp) -> tuple[UOp, int|None, int]: # look through views and a if sel.op is not Ops.MSELECT: return sel, None, off return (inner:=unwrap_view(sel.src[0]))[0], sel.arg, off + inner[1] -def select_lane(u:UOp, lane:int) -> UOp: return u.src[lane] if u.op is Ops.MSTACK else u.mselect(lane) if len(to_tuple(u.device)) > 1 else u +# **** commands - -def timeline(devs:tuple[str, ...]) -> UOp: return UOp.alloc((2,), dtypes.u64, 0, device=devs[0]).rtag("timeline") -def timeline_value(devs:tuple[str, ...]) -> UOp: return timeline(devs).index(1).load() +def timeline(dev:str) -> UOp: return UOp.alloc((2,), dtypes.u64, 0, device=dev, name="timeline") +def timeline_value(dev:str) -> UOp: return timeline(dev).index(1).load() def make_program(prg:UOp, size:int, device:str) -> UOp: slot = int.from_bytes(prg.key[:8], "little") if PROFILE: Compiled.profile_events.append(ProfileProgramEvent(device, prg.src[0].arg.function_name, prg.src[3].arg, None, slot, prg.key)) - return UOp.alloc((size,), dtypes.u8, slot, device=device).rtag("program") + return UOp.alloc((size,), dtypes.u8, slot, device=device, name="program") -def make_submit(*cmds, devs:str|tuple[str, ...], queue:str, fn:str|None=None, deps:tuple[UOp, ...]=()) -> UOp: # the order is on the arg - lin = UOp(Ops.LINEAR, src=tuple(cmds), arg=(to_tuple(devs), queue)).after(*deps) - return UOp.custom_function(fn or to_name("submit", to_tuple(devs)[0].split(":")[0], queue.split(":")[0])).call(lin) +def make_submit(*cmds:UOp, dev:str, queue:str, deps:tuple[UOp, ...]) -> UOp: # a submit calls its commands on a queue, after deps + kind = to_name(dev.split(":")[0], queue.split(":")[0], "submit") # nv_compute_submit: the backend matches the kind + q = UOp.param(0, dtypes.u8, 1, dev, name=queue) # the queue instance: its device and name + return UOp.sink(*cmds, arg=KernelInfo(kind)).call(q).after(*deps) -def ins(name, *src) -> UOp: return UOp(Ops.INS, tuple(UOp.const(s, dtypes.u32) if isinstance(s, int) else s for s in src), (name, dtypes.void)) +@uopfunc +def do_get_time_ms(ms:UOp) -> UOp: + ts = UOp.placeholder((2,), dtypes.u64, addrspace=AddrSpace.REG) + ts = ts.after(UOp.custom_function("clock_gettime", dtype=dtypes.i32).call(UOp.const(time.CLOCK_MONOTONIC, dtypes.i32), ts.index(0))) + return ms.index(0).store(ts[0] * 1000 + ts[1] // 1000000).sink() +def get_time_ms(dep:UOp) -> UOp: return (ms:=UOp.placeholder((1,), dtypes.u64, addrspace=AddrSpace.REG)).after(do_get_time_ms(ms.after(dep)))[0] -def chunks(nbytes:int, chunk_sz:int, dtype:DType=dtypes.i32) -> list[tuple[UOp|int, int]]: # (index, bytes): full chunks as one range, then the tail - full, tail = divmod(nbytes, chunk_sz) # no one-trip loops - return ([(UOp.range(full, next(UOp.unique_num), dtype=dtype) if full > 1 else 0, chunk_sz)] if full else []) + ([(full, tail)] if tail else []) +@uopfunc +def hcq_fence(slots:UOp, tl:UOp, last:int) -> UOp: # wait for the previous run of this schedule + tl = tl.replace(arg=replace(tl.arg, volatile=True)) # make it volatile, since it's polled -# C FFI + loop = UOp.range(UOp(Ops.NOOP).after(start:=get_time_ms(target:=slots.index(last).load())), next(UOp.unique_num), dtype=dtypes.void) + done = tl.after(target, loop).index(0).load() + return done.backedge(loop, (done < target) & (get_time_ms(done) - start < int(Compiled.wait_timeout_ms))).sink() + +def make_fence(slots:dict[str, UOp], num_queues:dict[str, int]) -> UOp: + fences = [hcq_fence(patch(b, [], bytes(b.nbytes())), timeline(dev), 2 * num_queues[dev], name=to_name("fence", dev)) for dev, b in slots.items()] + rearmed = [slots[dev].after(*fences).index(2 * i).store(0) for dev, n in num_queues.items() for i in range(n)] + return UOp.group(*rearmed).barrier() + +def announce(slots:dict[str, UOp], num_queues:dict[str, int], dep:UOp) -> list[UOp]: # bump the timeline and record it for the next run's fence + bumped = [timeline(dev).after(dep).index(1).store(timeline_value(dev) + 1) for dev in slots] + return bumped + [buf.after(dep).index(2 * num_queues[dev]).store(timeline_value(dev) + 1) for dev, buf in slots.items()] + +def ins(name:str, *src:UOp|int) -> UOp: return UOp.custom_function(name).call(*[UOp.const(s, dtypes.u32) if isinstance(s, int) else s for s in src]) + +# **** patchers + +def _is_link_patch(w:UOp) -> bool: + if w.op is Ops.GETADDR: return not ((base:=unwrap_lane(w.src[0])[0]).op is Ops.PARAM and base.arg.name != "lt_input") # an input's is runtime + if w.op is Ops.BUFFER: return w.addrspace is AddrSpace.GLOBAL # a register is written at runtime + if w.op in {Ops.LOAD, Ops.CALL, Ops.PARAM}: return False + if w.op is Ops.PROGRAM: return True + return all(_is_link_patch(s) for s in w.src) + +def patch(buf:UOp, rows:Sequence[tuple[int|UOp, UOp]], blob:bytes|None=None) -> UOp: + # group into stacks based on dtype, alignment, is_link (rt/lt can't share a store) and ranges (a ranged offset is a loop) + keys = [(w.dtype, (u:=sint_to_uop(o)).vmin % w.dtype.itemsize, _is_link_patch(w), tuple(u.ranges)) for o, w in rows] + groups = [(key, [row for row, k in zip(rows, keys) if k == key]) for key in dedup(keys)] + + # a link patch writes the bare buffer after the blob, a runtime patch writes it after its deps + dep = [buf.without_after.store(UOp(Ops.BINARY, arg=blob).bitcast(buf.dtype))] if blob is not None else [] + base, stores = buf.after(*dep), [] + for (dt, phase, link, rngs), grp in groups: + view = (buf.without_after.after(*dep) if link else base)[phase:phase + (buf.max_numel() - phase) // dt.itemsize * dt.itemsize].bitcast(dt) + offs = [UOp.const(i) if isinstance(i:=(o - phase) // dt.itemsize, int) else i for o, _ in grp] + stores.append(view.index(UOp.stack(*offs)).store(UOp.stack(*[w for _, w in grp])).end(*rngs)) + return buf.after(*dep, *stores) def layout_args(args:Sequence[UOp|int], offset:int=0) -> list[tuple[int, UOp]]: words = [a if isinstance(a, UOp) else UOp.const(a, dtypes.u32) for a in args] - return [(offset + o, w) for (o, _), w in zip(TinyELF.iter_sig(tuple((None, i, w.dtype, ()) for i, w in enumerate(words))), words)] + return [(offset + o, w) for (o, _), w in zip(TinyELF.iter_sig(tuple((None, AddrSpace.ALU, w.dtype, ()) for w in words)), words)] def pack_args(args:list[tuple[int, UOp]], size:int) -> list[UOp]: words, end = [], 0 @@ -88,45 +122,30 @@ def pack_args(args:list[tuple[int, UOp]], size:int) -> list[UOp]: return words + [UOp(Ops.BINARY, arg=bytes(size - end))] def ccall(fn:Any, *args:UOp|int) -> UOp: - ret = {None: dtypes.void, ctypes.c_void_p: dtypes.u64}.get(fn.restype) or to_dtype(fn.restype.__name__[2:]) # void, void*, c_int -> int + ret = dtypes.void if fn.restype is None else getattr(dtypes, fn.restype.__name__[2:], dtypes.u64) # void, c_int -> int, pointers -> u64 return UOp.custom_function(fn.__name__, dtype=ret).call(*[UOp.const(a, dtypes.i32) if isinstance(a, int) else a for a in args]) -@uopfunc -def do_get_time_ms(ms:UOp) -> UOp: - ts = UOp.placeholder((2,), dtypes.u64, addrspace=AddrSpace.REG) - ts = ts.after(UOp.custom_function("clock_gettime", dtype=dtypes.i32).call(UOp.const(time.CLOCK_MONOTONIC, dtypes.i32), ts.index(0))) - return ms.index(0).store(ts[0] * 1000 + ts[1] // 1000000).sink() -def get_time_ms(dep:UOp) -> UOp: return (ms:=UOp.placeholder((1,), dtypes.u64, addrspace=AddrSpace.REG)).after(do_get_time_ms(ms.after(dep)))[0] - -def cstruct(struct_t, **fields:UOp|int) -> UOp: +def cstruct(struct_t:type[Struct], **fields:UOp|int) -> UOp: flds = {n: (o, {d.itemsize: d for d in dtypes.uints}[ctypes.sizeof(t)]) for n, t, o, *_ in struct_t._real_fields_ if ctypes.sizeof(t)} rows = [(flds[n][0], UOp.const(v, flds[n][1])) for n, v in fields.items()] - buf = UOp.alloc((ctypes.sizeof(struct_t),), dtypes.u8, device=HCQ_RUNTIME_DEV.device).rtag(struct_t.__name__) + buf = UOp.alloc((ctypes.sizeof(struct_t),), dtypes.u8, device=HCQ_RUNTIME_DEV.device, name=struct_t.__name__) return patch(buf, rows, bytes(ctypes.sizeof(struct_t))) -# ***************** -# 0.1. prep: eager buffers become tagged params +# **** prep: params, unwrap multi, staging copies -def replace_buffer(ctx:tuple[bool, list[UOp], dict[UOp, int]], b:UOp) -> UOp: - use_rt, bufs, slots = ctx +def replace_buffer(ctx:tuple[list[UOp], dict[UOp, int]], b:UOp) -> UOp: + bufs, slots = ctx if slots.setdefault(b, len(bufs)) == len(bufs): bufs.append(b) - param = UOp.param(slots[b], b.dtype, b.max_numel(), b.device) - return param if use_rt else param.replace(tag="lt_input") + return UOp.param(slots[b], b.dtype, b.max_numel(), b.device, name="lt_input") pm_replace_buffers = PatternMatcher([(UPat(Ops.BUFFER, name="b"), lambda ctx, b: replace_buffer(ctx, b))]) -# ***************** -# 1.1. prep: unwrap multi - def unwrap_call(call:UOp) -> UOp|None: if not get_enqueue_devs(call) or (n:=max(len(to_tuple(a.device)) for a in get_call_arg_uops(call))) == 1: return None dnum = UOp.variable("_device_num", 0, n - 1, dtypes.i32) - return UOp(Ops.LINEAR, src=tuple(call.replace(src=(call.body, *[a if a.is_bound_var else select_lane(a, i) for a in call.src[1:]], dnum.bind(i))) - for i in range(n))) + return UOp(Ops.LINEAR, src=tuple(call.replace(src=(call.body, *[a.src[i] if a.op is Ops.MSTACK else a.mselect(i) if len(to_tuple(a.device)) > 1 + else a for a in call.src[1:]], dnum.bind(i))) for i in range(n))) pm_unwrap_multi = PatternMatcher([(UPat(Ops.CALL, name="call"), unwrap_call)]) -# ***************** -# 1.2. prep: staging copies - def split_rdma(call:UOp, dst:UOp, src:UOp) -> UOp|None: devs = [to_tuple(b.device)[0] for b in (dst, src)] if not all(hasattr(Device[d], "iface") for d in devs) or Device[devs[0]].peer_group == Device[devs[1]].peer_group: return None # not 2 nodes @@ -134,8 +153,8 @@ def split_rdma(call:UOp, dst:UOp, src:UOp) -> UOp|None: from tinygrad.runtime.ops_rdma import rdma_nic_for if None in (nics:=[rdma_nic_for(Device[d], Device[min(devs)]) for d in devs]): return None - # wires: a placeholder per nic in place of the far gpu, tagged by it - wires = [UOp.alloc(src.max_shape, src.dtype, 0, device=unwrap(nic).device).rtag(peer) for nic, peer in zip(nics, devs[::-1])] + # wires: a placeholder per nic in place of the far gpu, named by it + wires = [UOp.alloc(src.max_shape, src.dtype, 0, device=unwrap(nic).device, name=peer) for nic, peer in zip(nics, devs[::-1])] send = call.replace(src=wires[1].store_call(src).src) return UOp(Ops.LINEAR, src=(send, call.replace(src=dst.store_call(wires[0]).src))) @@ -162,155 +181,163 @@ pm_insert_copy_staging = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.STORE), UPat(name="dst"), UPat(name="src")), name="call", allow_any_len=True), stage_copy), ]) -# ***************** -# 2. deps +# **** batch: calls become queue programs class DepsTracker: def __init__(self): - self.writes:dict[Any, list[tuple[int, int, Any]]] = defaultdict(list) - self.reads:dict[Any, list[tuple[int, int, Any]]] = defaultdict(list) + self.ranges:dict[tuple[UOp, int|None], list[tuple[int, int, bool, int]]] = defaultdict(list) # per (base, lane): (start, end, is_write, tag) - def access_resources(self, bufs:Sequence[UOp], write:list[int], new_dependency:Any) -> list: - accesses = [(i in write, (base, lane), off, off + b.max_numel() * b.dtype.itemsize) - for i, b in enumerate(bufs) for base, lane, off in [unwrap_lane(b)]] + def access_resources(self, accesses:Sequence[tuple[UOp, bool]], tag:int) -> list[int]: # (buffer, is_write) accesses -> earlier tags to wait on + lanes = [(*unwrap_lane(b), b, is_write) for b, is_write in accesses] + spans = [((base, lane), off, off + b.max_numel() * b.dtype.itemsize, is_write) for base, lane, off, b, is_write in lanes] - waits = [dep for is_write, key, start, end in accesses for lo, hi, dep in (self.writes[key] + self.reads[key] if is_write else self.writes[key]) - if lo < end and start < hi] + # a write waits for the overlapping reads and writes, a read for the overlapping writes + waits = [t for key, start, end, is_write in spans for lo, hi, was_write, t in self.ranges[key] + if lo < end and start < hi and (is_write or was_write)] - for is_write, key, start, end in accesses: + # a write owns its span: the earlier accesses keep only the parts outside it + for key, start, end, is_write in spans: if is_write: - for ranges in (self.writes, self.reads): - before_and_after = [((lo, min(hi, start), dep), (max(lo, end), hi, dep)) for lo, hi, dep in ranges[key]] - ranges[key] = [(lo, hi, dep) for parts in before_and_after for lo, hi, dep in parts if lo < hi] - (self.writes if is_write else self.reads)[key].append((start, end, new_dependency)) + self.ranges[key] = [(lo, hi, w, t) for a, b, w, t in self.ranges[key] for lo, hi in ((a, min(b, start)), (max(a, end), b)) if lo < hi] + self.ranges[key].append((start, end, is_write, tag)) return dedup(waits) @dataclass class BatchCtx: - batch:list[tuple[UOp, tuple[str, ...], str]] # (call, devices, queue) per enqueued call + batch:list[tuple[UOp, str, str]] # (call, device, queue) per enqueued call profile:bool - tracker:DepsTracker = field(default_factory=DepsTracker) - queues:dict[str, list[str]] = field(init=False) - last:dict[tuple[str, str], int] = field(init=False) - prev:list[int|None] = field(init=False) - signal_tags:set[int] = field(init=False) - slots:dict[str, UOp] = field(init=False) + queues:dict[str, list[str]] = field(init=False) # per device, its queues in first-use order: none for a peer without calls of its own + calls:dict[tuple[str, str], list[int]] = field(init=False) # per queue, the tags of its calls in order peers:dict[tuple[str, str], set[str]] = field(init=False) # the other devices whose memory a queue touches + slots:dict[str, UOp] = field(init=False) def __post_init__(self): - self.queues, self.last, self.prev, self.peers = {}, {}, [], {} - for tag, (c, devs, q) in enumerate(self.batch): - if q not in self.queues.setdefault(devs[0], []): self.queues[devs[0]].append(q) - self.prev.append(self.last.get((devs[0], q))) - self.last[(devs[0], q)] = tag - peers = HCQ_DEVS if getattr(Device[devs[0]], "is_usb", False) else HCQ_DEVS | HOST_DEVS # usb gpu can't reach host memory - for d in {Device.canonicalize(x) for b in get_call_arg_uops(c) for x in to_tuple(b.device) if all_devices_in(x, peers)} - {devs[0]}: - self.peers.setdefault((devs[0], q), set()).add(d) - self.queues.setdefault(d, []) - self.signal_tags = {tag for (dev, q), tag in self.last.items() if q != self.epilogue_queue(dev) or (dev, q) in self.peers} + # the other devices whose memory a call touches: a usb gpu can't reach host memory + reach = [HCQ_DEVS if getattr(Device[d], "is_usb", False) else HCQ_DEVS | HOST_DEVS for _, d, _ in self.batch] + arg_devs = [{Device.canonicalize(x) for b in get_call_arg_uops(c) for x in to_tuple(b.device)} for c, _, _ in self.batch] + touched = [{x for x in xs if all_devices_in(x, r)} - {d} for xs, r, (_, d, _) in zip(arg_devs, reach, self.batch)] + + # queues and devices in first-use order, a call's own device then the peers it touches + used = dedup(keys:=[(d, q) for _, d, q in self.batch]) + devs = dedup([x for (d, _), t in zip(keys, touched) for x in [d, *sorted(t)]]) + self.queues = {dev: [q for d, q in used if d == dev] for dev in devs} + + # the tags of a queue in order, and the devices its calls touch + self.calls = {k: [tag for _, tag in grp] for k, grp in itertools.groupby(sorted((key, tag) for tag, key in enumerate(keys)), key=lambda e: e[0])} + self.peers = {k: {d for tag in tags for d in touched[tag]} for k, tags in self.calls.items()} + # a slot is [signal][timestamp], 16 bytes: the queue signals, the timeline, then two per call if profiling self.slots = {dev: UOp.alloc((2 * (len(qs) + 1 + (2 * len(self.batch) if self.profile else 0)),), dtypes.u64, device=dev, - spec=BufferSpec(host=True, uncached=True, cpu_access=True)).rtag("slots") for dev, qs in self.queues.items()} + spec=BufferSpec(host=True, uncached=True, cpu_access=True), name="slots") for dev, qs in self.queues.items()} - def epilogue_queue(self, dev:str) -> str: return "COMPUTE:0" if len(self.queues[dev]) != 1 else self.queues[dev][0] # closes the device + def slot(self, dev:str, i:int) -> UOp: return self.slots[dev].shrink(((2 * i, 2 * i + 2),)) # not a slice: 10x the cost + def queue_signal(self, dev:str, queue:str) -> UOp: return self.slot(dev, self.queues[dev].index(queue)) + def stamps(self, dev:str, tag:int) -> tuple[int, ...]: return (st:=len(self.queues[dev])+1+2*tag, st + 1) if self.profile else () - def slot(self, devs:tuple[str, ...], i:int) -> UOp: return self.slots[devs[0]].shrink(((2 * i, 2 * i + 2),)) # not a slice: 10x the cost - def queue_signal(self, devs:tuple[str, ...], queue:str) -> UOp: return self.slot(devs, self.queues[devs[0]].index(queue)) - def sched_timeline(self, devs:tuple[str, ...]) -> UOp: return self.slot(devs, len(self.queues[devs[0]])) - def stamps(self, devs:tuple[str, ...], tag:int) -> tuple[int, ...]: return (st:=len(self.queues[devs[0]])+1+2*tag, st + 1) if self.profile else () +def _wait_ins(ctx:BatchCtx, waits:dict[tuple[str, str], int]) -> list[UOp]: + return [ins("wait", ctx.queue_signal(d, q), UOp.const(t + 1, dtypes.u64)) for (d, q), t in waits.items()] -def _wait_ins(ctx:BatchCtx, call:UOp, device:str, queue:str, tag:int) -> list[UOp]: - bufs, write = list(get_call_arg_uops(call)), get_call_outs_ins(call)[0] - latest:dict[tuple[str, str], int] = {} # (producer device, queue) -> the latest submit tag to wait on, same-queue submits are fifo - for d, q, t in ctx.tracker.access_resources(bufs, list(range(len(bufs)) if write is None else write), (device, queue, tag)): - if t < tag and (d, q) != (device, queue): latest[(d, q)] = max(latest.get((d, q), 0), t) +def _start_ins(ctx:BatchCtx, dev:str, queue:str) -> list[UOp]: + return [ins("wait", timeline(d), timeline_value(d)) for d in [dev, *sorted(ctx.peers.get((dev, queue), ()))]] + [ins("memory_barrier")] - # NV waits break QMD chaining, so also wait for the previous launch - if latest and device.split(":")[0] == "NV" and queue.startswith("COMPUTE") and (p:=ctx.prev[tag]) is not None: latest[(device, queue)] = p +def _build_queues(ctx:BatchCtx) -> dict[tuple[str, str], list[UOp]]: + queue_of = [(dev, queue) for _, dev, queue in ctx.batch] # the (device, queue) of each call by tag - ctx.signal_tags |= set(latest.values()) - return [ins("wait", ctx.queue_signal((d,), q), UOp.const(t + 1, dtypes.u64)) for (d, q), t in latest.items()] + # calc deps for each tag + tracker = DepsTracker() + accs = [[(s, k in get_call_outs_ins(c)[0]) for k, s in enumerate(c.src[1:]) if not s.is_variable] for c, _, _ in ctx.batch] + dep_tags:list[list[int]] = [tracker.access_resources(acc, tag) for tag, acc in enumerate(accs)] -def _start_ins(ctx:BatchCtx, dev:str, queue:str) -> list[UOp]: # a queue first waits for prior work of its device and of the peers it touches - return [ins("barrier")] + [ins("wait", timeline((d,)), timeline_value((d,))) for d in [dev, *sorted(ctx.peers.get((dev, queue), ()))]] + # add barriers when a call deps on another call of its queue + barriers, last_barrier = set[int](), dict[tuple[str, str], int]() + for tag, (deps, queue) in enumerate(zip(dep_tags, queue_of)): + if any(queue_of[t] == queue and t >= last_barrier.get(queue, 0) for t in deps): + barriers.add(tag) + last_barrier[queue] = tag -def _build_queues(ctx:BatchCtx) -> dict[tuple[tuple[str, ...], str], list[UOp]]: - # find all waits first to mark calls that must signal - call_waits = [_wait_ins(ctx, c, d[0], q, tag) for tag, (c, d, q) in enumerate(ctx.batch)] - queues:dict[tuple[tuple[str, ...], str], list[UOp]] = {} - for tag, ((call, devices, queue), waits) in enumerate(zip(ctx.batch, call_waits)): - if not (q:=queues.setdefault((devices, queue), [])): q += _start_ins(ctx, devices[0], queue) # first use of a queue + # keep only latest tags to wait for each queue + call_waits = [{queue_of[t]: t for t in sorted(deps) if queue_of[t] != queue_of[tag]} for tag, deps in enumerate(dep_tags)] - # dependency waits, then the call between its timestamps - ts_ins = [ins("timestamp", ctx.slot(devices, i)) for i in ctx.stamps(devices, tag)] - q += waits + ts_ins[:1] + [call] + ts_ins[1:] + # NV waits break QMD chaining, so a call on an NV compute queue that waits also waits for the previous launch there + # TODO: can we handle that better for nv? + nv_compute = [queue for queue in ctx.calls if queue[0].split(":")[0] == "NV" and queue[1].startswith("COMPUTE")] + prev = {tag: p for queue in nv_compute for p, tag in itertools.pairwise(ctx.calls[queue])} + call_waits = [waits | {queue_of[tag]: prev[tag]} if waits and tag in prev else waits for tag, waits in enumerate(call_waits)] - # signal the queue if someone waits for us - if tag in ctx.signal_tags: q += [ins("store", ctx.queue_signal(devices, queue), UOp.const(tag + 1, dtypes.u64))] + # for each device collect what queues it depends on (i.e. which queues touch its memory) + dev_users = {dev: sorted(queue for queue, peers in ctx.peers.items() if dev == queue[0] or dev in peers) for dev in ctx.queues} - # one queue advances the device timeline after all other queues finish, and after the queues of the peers that touched the device - for dev in ctx.queues: - touched = sorted(k for k, ds in ctx.peers.items() if dev in ds) - qdev, queue = touched[0] if dev.split(":")[0] in HOST_DEVS else (dev, ctx.epilogue_queue(dev)) # the host has no queues, a toucher closes it - waits = [ins("wait", ctx.queue_signal((d,), q), UOp.const(ctx.last[(d, q)] + 1, dtypes.u64)) - for d, q in [(dev, q) for q in ctx.queues[dev]] + touched if (d, q) != (qdev, queue)] - bump = ins("store", timeline((dev,)), timeline_value((dev,)) + 1) + # to finalize device, closer waits for other users to complete + own_closers = {dev: (dev, queues[0] if len(queues) == 1 else "COMPUTE:0") for dev, queues in ctx.queues.items()} + closers = {dev: dev_users[dev][0] if dev.split(":")[0] in HOST_DEVS else own_closers[dev] for dev in ctx.queues} + close_waits = {dev: {queue: ctx.calls[queue][-1] for queue in dev_users[dev] if queue != closers[dev]} for dev in ctx.queues} - # multiple copy queues may need a new compute stream. a peer without calls of its own starts like any queue - if not (q:=queues.setdefault(((qdev,), queue), [])) and qdev not in {d for d, _ in ctx.last}: q += _start_ins(ctx, qdev, queue) - q.extend([*waits, bump]) - return queues + # all tags that should signal + signal_tags = {tag for waits in [*call_waits, *close_waits.values()] for tag in waits.values()} + + def call_cmds(tag:int, call:UOp, dev:str, queue:str) -> list[UOp]: + barrier = [ins("exec_barrier")] if tag in barriers else [] + stamps = [ins("timestamp", ctx.slot(dev, i)) for i in ctx.stamps(dev, tag)] + signal = [ins("store", ctx.queue_signal(dev, queue), UOp.const(tag + 1, dtypes.u64))] if tag in signal_tags else [] + return barrier + _wait_ins(ctx, call_waits[tag]) + [ins("memory_barrier")] * bool(call_waits[tag]) + stamps[:1] + [call] + stamps[1:] + signal + + # closers advance device timelines + close_cmds = {dev: _wait_ins(ctx, close_waits[dev]) + [ins("store", timeline(dev), timeline_value(dev) + 1)] for dev in ctx.queues} + + # wait for timeline accessing devices for the first time + order = dedup(queue_of + list(closers.values())) + open_cmds = {queue: _start_ins(ctx, *queue) if queue in ctx.calls or not ctx.queues[queue[0]] else [] for queue in order} + + programs = {queue: [cmd for tag in ctx.calls.get(queue, []) for cmd in call_cmds(tag, *ctx.batch[tag])] for queue in order} + closings = {queue: [cmd for dev in ctx.queues if closers[dev] == queue for cmd in close_cmds[dev]] for queue in order} + return {queue: open_cmds[queue] + programs[queue] + closings[queue] for queue in order} def _finalize_batch(ctx:BatchCtx, skip_wait:bool=False) -> UOp: queues = _build_queues(ctx) - # re-arm the batch signals before submitting queues in first-use order + # wait for the previous (this) schedule run + fence = make_fence(ctx.slots, num_queues:={dev: len(qs) for dev, qs in ctx.queues.items()}) submits:list[UOp] = [] - timelines = [ctx.sched_timeline((dev,)) for dev in ctx.queues] - signals = [ctx.queue_signal((dev,), q) for dev, qs in ctx.queues.items() for q in qs] - fence = UOp.custom_function("hcq_fence").call(*timelines, *signals) - for (devs, queue), cmds in queues.items(): submits.append(make_submit(*cmds, devs=devs, queue=queue, deps=(fence, *submits[-1:]))) - sink = UOp.sink(*submits, arg=KernelInfo("hcq_submit", estimates=Estimates()), tag=1) - for pm in [Device[d].pm_batch for d in ctx.queues]: # a device adds its own work to the batch - if (r:=pm.rewrite(sink)) is not None: sink = r + for (dev, queue), cmds in queues.items(): submits.append(make_submit(*cmds, dev=dev, queue=queue, deps=(fence, *submits[-1:]))) + sink = UOp.sink(submits[-1], *announce(ctx.slots, num_queues, submits[-1]), arg=KernelInfo("hcq_submit", estimates=Estimates(), optimized=True)) # per call metadata names = [get_call_name(c, get_call_arg_uops(c)) for c, _, _ in ctx.batch] estimates = [estimate_uop(c) for c, _, _ in ctx.batch] stamps = [tuple(2 * s + 1 for s in ctx.stamps(d, tag)) for tag, (_, d, _) in enumerate(ctx.batch)] profile_keys = [c.body.key if c.body.op is Ops.PROGRAM else None for c, _, _ in ctx.batch] - args = [[unwrap_lane(bs[g])[:2] for g in getattr(c.body.arg, "globals", range(len(bs)))] for c, _, _ in ctx.batch for bs in [get_call_arg_uops(c)]] + args = [[unwrap_lane(c.src[1 + g])[:2] for g in getattr(c.body.arg, "globals", range(len(c.src) - 1))] for c, _, _ in ctx.batch] bufs = [tuple(b.arg.slot for b, _ in a) if all(b.op is Ops.PARAM and lane is None for b, lane in a) else () for a in args] kerns = tuple(zip([d for _, d, _ in ctx.batch], names, estimates, stamps, profile_keys, bufs, [get_call_outs_ins(c) for c, _, _ in ctx.batch])) slots = ctx.slots if ctx.profile else {} # profiling reads them - info = HCQInfo(tuple(ctx.queues), skip_wait=skip_wait, kernels=kerns, estimates=sum(estimates, start=Estimates()).simplify(), - slots=tuple((d, i) for i, d in enumerate(slots))) + info = HCQInfo(tuple(ctx.queues), skip_wait=skip_wait, kernels=kerns, slots=tuple((d, i) for i, d in enumerate(slots)), + estimates=Estimates(*[sum(x*t.count(x) for x in dict.fromkeys(t)) for t in zip(*[vars(e).values() for e in estimates])]).simplify()) return sink.call(*slots.values(), aux=info) @rewrite_group(new_ctx=False) def sched_batches(l:UOp, profile:bool) -> UOp: + copies = [c for c in l.src if c.op is Ops.CALL and c.body.op is Ops.STORE] + ring = sorted({Device.canonicalize(d) for c in copies for b in get_call_arg_uops(c) for d in to_tuple(b.device) if all_devices_in(d, HCQ_DEVS)}) + # TODO: handle max queue amt better + num_copy = max(1, min(len(ring), 8 if {d.split(":")[0] for d in ring} == {"AMD"} else 1) if ALL2ALL >= 1 else 1) + copy_of = {(dst, src): (i - j - 1) % len(ring) % num_copy for i, dst in enumerate(ring) for j, src in enumerate(ring)} + + def queue_for(c:UOp) -> str: + if c.body.op is Ops.PROGRAM: return "COMPUTE:0" + if c.body.op is Ops.CUSTOM_FUNCTION: return "ENCDEC:0" + return f"COPY:{copy_of.get((c.src[1].device, c.src[2].device), 0)}" + devs = [get_enqueue_devs(c) for c in l.src] - - # assign to queues - peers = sorted({Device.canonicalize(d) for c in l.src if c.op is Ops.CALL and c.body.op is Ops.STORE - for b in get_call_arg_uops(c) for d in to_tuple(b.device) if d.split(":")[0] == "AMD"}) - num_queues = max(1, getenv("HCQ_NUM_SDMA", min(len(peers), 8) if ALL2ALL >= 1 else 1)) - queues = ["COMPUTE:0" if c.op is Ops.CALL and c.body.op is Ops.PROGRAM else "COPY:0" for c in l.src] - for i, c in enumerate(l.src): - if c.op is Ops.CALL and c.body.op is Ops.CUSTOM_FUNCTION: queues[i] = "ENCDEC:0" - if c.op is Ops.CALL and c.body.op is Ops.STORE and all(b.device in peers for b in get_call_arg_uops(c)): - queues[i] = f"COPY:{(peers.index(c.src[1].device) - peers.index(c.src[2].device) - 1) % len(peers) % num_queues}" - srcs:list[UOp] = [] - for hcq, grp in itertools.groupby(zip(l.src, devs, queues), key=lambda e: bool(e[1])): - nodes:dict[str, list] = {} - for e in grp: nodes.setdefault(Device[e[1][0]].peer_group if hcq else "", []).append(e) - batches = list(nodes.values()) - for batch in batches: srcs += [_finalize_batch(BatchCtx(batch, profile), batch is not batches[-1])] if hcq else [c for c, _, _ in batch] + for hcq, grp in itertools.groupby(zip(l.src, devs), key=lambda e: bool(e[1])): + run = list(grp) + peer_groups = dedup([Device[d[0]].peer_group for _, d in run if d]) + batches = [[(c, d[0], queue_for(c)) for c, d in run if Device[d[0]].peer_group == pg] for pg in peer_groups] + if hcq: srcs += [_finalize_batch(BatchCtx(batch, profile), batch is not batches[-1]) for batch in batches] + else: srcs += [c for c, _ in run] return l.replace(src=tuple(srcs)) -# ***************** -# 3. encode +# **** encode: queue programs become cmdbufs class HWQueue: q_rewrite = PatternMatcher([ @@ -321,36 +348,38 @@ class HWQueue: (UPat(Ops.CALL, src=(UPat.custom_function("encdec", name="s"),), name="c", allow_any_len=True), lambda ctx, c, s: ctx.encdec(c, s)), # ins - (UPat(Ops.INS, arg=("copy", dtypes.void), src=(UPat(name="dst"), UPat(name="src"), UPat(name="n"))), + (UPat(Ops.CALL, src=(UPat.custom_function("copy"), UPat(name="dst"), UPat(name="src"), UPat(name="n"))), lambda ctx, dst, src, n: ctx.copy(dst, src, n.val)), - (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda ctx: ctx.memory_barrier()), - (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: ctx.wait(dst, val)), - (UPat(Ops.INS, arg=("wait_eq", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: ctx.wait(dst, val, eq=True)), - (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), lambda ctx, dst: ctx.timestamp(dst)), - (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: ctx.signal(dst, val)), - (UPat(Ops.INS, arg=("write", dtypes.void), name="u"), lambda ctx, u: ctx.write(*u.src)), + (UPat(Ops.CALL, src=(UPat.custom_function("memory_barrier"),)), lambda ctx: ctx.memory_barrier()), + (UPat(Ops.CALL, src=(UPat.custom_function("exec_barrier"),)), lambda ctx: ctx.exec_barrier()), + (UPat(Ops.CALL, src=(UPat.custom_function("wait"), UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: ctx.wait(dst, val)), + (UPat(Ops.CALL, src=(UPat.custom_function("wait_eq"), UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: ctx.wait(dst, val, eq=True)), + (UPat(Ops.CALL, src=(UPat.custom_function("timestamp"), UPat(name="dst"))), lambda ctx, dst: ctx.timestamp(dst)), + (UPat(Ops.CALL, src=(UPat.custom_function("store"), UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: ctx.signal(dst, val)), + (UPat(Ops.CALL, src=(UPat.custom_function("write"),), allow_any_len=True, name="u"), lambda ctx, u: ctx.write(*u.src[1:])), # loop in cmdbufs (UPat(Ops.END, src=(UPat(Ops.LINEAR, name="body"), UPat(Ops.RANGE, name="r"))), lambda ctx, body, r: ctx.loop(body, r)), ]) - def __init__(self, submit:UOp): - self.lin, self.deps = (lin:=submit.src[1]).without_after, lin.src[1:] if lin.op is Ops.AFTER else () - self.devs, self.queue = self.lin.arg + def __init__(self, submit:UOp): # submit: sink(commands).call(queue).after(deps) + call, self.deps = submit.src[0], submit.src[1:] + self.lin, q = call.body, call.src[1] + self.devs, self.queue = (q.device,), q.arg.name self.dev = Device[self.devs[0]] self.blob, self.patches = bytearray(), list[tuple[int|UOp, UOp]]() - def q(self, *words) -> int: + def q(self, *words:UOp|int) -> int: for w in words: - c = w - while isinstance(c, UOp) and c.op is Ops.CAST: c = c.src[0] - if isinstance(c, UOp) and c.op is Ops.BINARY: self.blob += c.arg - elif isinstance(c, UOp) and c.op is not Ops.CONST: - self.patches.append((len(self.blob), w)) - self.blob += bytes(w.dtype.itemsize) + if isinstance(w, int): self.blob += struct.pack("I", truncate[dtypes.u32](w)) else: - v, dt = (c.val, w.dtype) if isinstance(w, UOp) else (c, dtypes.u32) - self.blob += struct.pack(unwrap(dt.fmt), truncate[dt](v)) + c = w + while c.op is Ops.CAST: c = c.src[0] + if c.op is Ops.BINARY: self.blob += c.arg + elif c.op is not Ops.CONST: + self.patches.append((len(self.blob), w)) + self.blob += bytes(w.dtype.itemsize) + else: self.blob += struct.pack(unwrap(w.dtype.fmt), truncate[w.dtype](c.val)) return len(self.blob) def loop(self, body:UOp, r:UOp): @@ -362,72 +391,22 @@ class HWQueue: self.blob += self.blob[start:] * int(r.vmax) def memory_barrier(self): pass # a copy queue has nothing to flush + def exec_barrier(self): pass def submit(self, cmdbuf:UOp) -> UOp: raise NotImplementedError("queues need a submit") def encode(self) -> UOp: return self.submit(encode_cmdbuf(self, self.lin)) # submit(linear) becomes submit(cmdbuf) -# ***************** -# 3.1. hcq special functions - -def _is_link_patch(w:UOp) -> bool: - if w.op is Ops.GETADDR: return not ((base:=unwrap_lane(w.src[0])[0]).op is Ops.PARAM and base.tag is None) # an input's is runtime - if w.op is Ops.PARAM: return w.tag is not None - if w.op is Ops.BUFFER: return w.addrspace is AddrSpace.GLOBAL # a register is written at runtime - if w.op in {Ops.LOAD, Ops.AFTER}: return False - return all(_is_link_patch(s) for s in w.src) - -def patch(buf:UOp, rows:Sequence[tuple[int|UOp, UOp]], blob:bytes|None=None) -> UOp: - # group into stacks based on dtype, alignment, is_link (rt/lt can't share a store) and ranges (a ranged offset is a loop) - keys = [(w.dtype, getattr(o, "vmin", o) % w.dtype.itemsize, _is_link_patch(w), tuple(getattr(o, "ranges", ()))) for o, w in rows] - groups = [(key, [row for row, k in zip(rows, keys) if k == key]) for key in dedup(keys)] - - # a link patch writes the bare buffer after the blob, a runtime patch writes it after its deps - dep = [buf.without_after.store(UOp(Ops.BINARY, arg=blob).bitcast(buf.dtype))] if blob is not None else [] - base, stores = buf.after(*dep), [] - for (dt, phase, link, rngs), grp in groups: - view = (buf.without_after.after(*dep) if link else base)[phase:phase + (buf.max_numel() - phase) // dt.itemsize * dt.itemsize].bitcast(dt) - offs = [UOp.const(i) if isinstance(i:=(o - phase) // dt.itemsize, int) else i for o, _ in grp] - stores.append(view.index(UOp.stack(*offs)).store(UOp.stack(*[w for _, w in grp])).end(*rngs)) - return buf.after(*dep, *stores) - -@uopfunc -def hcq_fence(slots:UOp, tl:UOp, tv:UOp, last:int) -> UOp: # wait for the previous run of this schedule, then announce and record this one - tl = tl.replace(arg=replace(tl.arg, volatile=True)) # make it volatile, since it's polled - - loop = UOp.range(UOp(Ops.NOOP).after(start:=get_time_ms(target:=slots.index(last).load())), next(UOp.unique_num), dtype=dtypes.void) - done = tl.after(target, loop).index(0).load() - bumped = tl.after(done.backedge(loop, (done < target) & (get_time_ms(done) - start < 30000))).index(1).store(nxt:=tv + 1) - return slots.after(bumped).index(last).store(nxt).sink() - -def encode_fence(f:UOp) -> UOp: - devs = dedup(to_tuple(s.device)[0] for s in f.src[1:]) - lasts, sigs = f.src[1:1 + len(devs)], f.src[1 + len(devs):] - last:tuple[UOp, ...] = () - - # wait for prev schedule to not collide, the slots are zeroed at link - for dev, (slots, off) in zip(devs, map(unwrap_view, lasts)): - slots = patch(slots, [], bytes(slots.nbytes())).after(*last) - last = (hcq_fence(slots, timeline((dev,)), timeline_value((dev,)), off // slots.dtype.itemsize),) - - # re-arm the signals - for slots, off in map(unwrap_view, sigs): last = (slots.after(*last).index(off // slots.dtype.itemsize).store(0),) - return last[0].barrier() -pm_hcq_encode = PatternMatcher([(UPat(Ops.CALL, src=(UPat.custom_function("hcq_fence"),), allow_any_len=True, name="f"), encode_fence)]) - -# ***************** -# 3.2. encode - def encode_cmdbuf(hq:HWQueue, lin:UOp|None=None, name:str="cmdbuf", device:str|tuple[str, ...]|None=None) -> UOp: for u in lin.src if lin is not None else (): hq.q_rewrite.rewrite(u, ctx=hq) # the commands of the linear go to the stream - stream, patches = bytes(hq.blob), hq.patches + stream = bytes(hq.blob) # loop over pathes with the same value - rt = [(o, w) for o, w in patches if isinstance(o, int) and o % 4 == 0 and w.dtype.itemsize in (4, 8) and not _is_link_patch(w)] + rt = [(o, w) for o, w in hq.patches if isinstance(o, int) and o % 4 == 0 and w.dtype.itemsize in (4, 8) and not _is_link_patch(w)] uses = {w: [o for o, _ in grp] for w, grp in itertools.groupby(sorted(rt, key=lambda p: p[1].key), key=lambda p: p[1])} - looped = {w: (at, [*w.ranges][0] if w.ranges else UOp.range(len(at), next(UOp.unique_num))) for w, at in uses.items() if len(at) > 1 or w.ranges} + looped = {w: (at, UOp.range(len(at), next(UOp.unique_num))) for w, at in uses.items() if len(at) > 1} dwords = [(UOp(Ops.BINARY, arg=struct.pack(f"<{len(at)}I", *at)).bitcast(dtypes.u32).index(r).load() + 4 * k, (w >> 32 * k).cast(dtypes.u32)) for w, (at, r) in looped.items() for k in range(w.dtype.itemsize // 4)] - patches = [(o, w) for o, w in patches if w not in looped] + dwords - nested = dedup([g.src[0] for _, w in patches for g in w.toposort() if g.op is Ops.GETADDR and g.src[0].op is Ops.LINEAR]) + patches = [(o, w) for o, w in hq.patches if w not in looped] + dwords + nested = dedup([g.src[0] for g in UOp.sink(*[w for _, w in patches]).toposort() if g.op is Ops.GETADDR and g.src[0].op is Ops.LINEAR]) # nested linears (like kernargs) merge into a buffer per name, patched before the stream bufs = [] @@ -437,15 +416,14 @@ def encode_cmdbuf(hq:HWQueue, lin:UOp|None=None, name:str="cmdbuf", device:str|t bufs.append((offs, encode_cmdbuf(hq, name=lname))) views = {l: buf.without_after[o:e] for offs, buf in bufs for l, (o, e) in offs.items()} - buf = UOp.alloc((len(stream),), dtypes.u8, device=device or hq.devs[0]).rtag(to_name(name, hq.queue)).after(*hq.deps) + buf = UOp.alloc((len(stream),), dtypes.u8, device=device or hq.devs[0], name=to_name(name, hq.queue)).after(*hq.deps) words = UOp.sink(*[w for _, w in patches]).substitute(views).src return patch(buf, list(zip([o for o, _ in patches], words)), stream).after(*[b for _, b in bufs]) -# ***************** -# 3.3. lift +# **** lower: cmdbufs become a host program def hoist_links(ctx:list[UOp], a:UOp) -> UOp|None: - links, rest = partition(a.src[1:], lambda s: (s.src[0] if s.op is Ops.END else s).op is Ops.STORE and _is_link_patch(s)) # a store or its loop + links, rest = partition(a.src[1:], lambda s: s.op is Ops.STORE and _is_link_patch(s)) if not links: return None ctx.extend(links) return a.src[0].after(*rest) @@ -461,91 +439,70 @@ pm_lift_deps = PatternMatcher([ ]) def _needs_arg(u:UOp, root:bool) -> bool: - return u.addrspace is AddrSpace.GLOBAL if u.op in (Ops.BUFFER, Ops.ALLOC) else u.op is Ops.PARAM and u.is_variable != root + if u.op in (Ops.BUFFER, Ops.ALLOC): return u.addrspace is AddrSpace.GLOBAL + return u.op is Ops.PARAM and (u.is_variable != root or u.arg.name == "lt_input") def _param_for(u:UOp, slot:int) -> UOp: if u.op is Ops.GETADDR or u.is_variable: return UOp.param(slot, u.commit_dtype(dtypes.i32), name=u.arg.name if u.is_variable else None, addrspace=AddrSpace.ALU).cast(u.dtype) - return UOp.param(slot, u.dtype, u.max_numel(), HCQ_RUNTIME_DEV.device, name=f"{u.tag}_{slot}" if isinstance(u.tag, str) else None) + return UOp.param(slot, u.dtype, u.max_numel(), HCQ_RUNTIME_DEV.device, name=f"{u.arg.name}_{slot}" if u.op is Ops.ALLOC and u.arg.name else None) def lift(call:UOp, root:bool=False) -> UOp: # callees are lifted already - body, args = graph_rewrite(call.body, pm_lift_deps + pm_renumber_slots, ctx=itertools.count(), walk=True, name="lift deps"), list(call.src[1:]) + body, args = graph_rewrite(call.body, pm_lift_deps + pm_renumber_slots, init_ctx=CountCtx, walk=True, name="lift deps"), list(call.src[1:]) nodes = body.toposort(gate=lambda u: u.op is not Ops.GETADDR) leaves = dedup([u for u in nodes if _needs_arg(u, root)] + [g for u in nodes for g in u.src if g.op is Ops.GETADDR]) slots = args + (new:=[u for u in leaves if u not in args]) body = body.substitute({u: _param_for(u, slots.index(u)) for u in leaves}, walk=True) # new args in the caller - own = {p: args[p.arg.slot].without_after for u in new for p in u.toposort() if p.op is Ops.PARAM and not _needs_arg(p, root)} + own = {p: args[p.arg.slot].without_after for u in new for p in u.toposort() if p.op is Ops.PARAM and p.arg.name and not _needs_arg(p, root)} return call.replace(src=(body, *args, *UOp.sink(*new).substitute(own, walk=True).src)) pm_lift = PatternMatcher([ - (UPat(Ops.PARAM, name="u"), lambda u: u.replace(arg=replace(u.arg, device=None)) if u.arg.name else None), # of a lowered function + # a param of a lowered function, an "lt_input" keeps its device + (UPat(Ops.PARAM, name="u"), lambda u: u.replace(arg=replace(u.arg, device=None)) if u.arg.name not in (None, "lt_input") else None), (UPat(Ops.CALL, src=(UPat(Ops.SINK),), allow_any_len=True, name="call"), lift), # lift allocs and getaddrs ]) +Compiled.pm_encode = PatternMatcher([]) +Compiled.pm_lower = PatternMatcher([]) + def lower_call(call:UOp) -> UOp|None: if not isinstance(call.arg.aux, HCQInfo): return None # encode bodies - from tinygrad.runtime.ops_rdma import pm_rdma_encode - devs = [Device[d] for d in dedup([d.split(":")[0] for d in call.arg.aux.device])] - body = graph_rewrite(call.body, pm_rdma_encode + sum([d.pm_encode for d in devs], PatternMatcher([])) + pm_hcq_encode, - ctx=(lt_patches:=list[UOp]()), bpm=pm_hoist_links, name="encode") - body = graph_rewrite(body, sum([d.pm_lower for d in devs], PatternMatcher([])), - ctx=lt_patches, bpm=pm_hoist_links, enter_calls=True, name="lower") + body = graph_rewrite(call.body, Compiled.pm_encode, ctx=(lt_patches:=list[UOp]()), bpm=pm_hoist_links, name="encode") + body = graph_rewrite(body, Compiled.pm_lower, ctx=lt_patches, bpm=pm_hoist_links, enter_calls=True, name="lower") body = graph_rewrite(body, pm_lift, walk=True, enter_calls=True, name="lift") if VIZ: graph_rewrite(UOp.sink(*dedup(lt_patches)), PatternMatcher([]), name="View Link-Time Patches") # batch args, link patches wait after the call return lift(call.replace(src=(body, *call.src[1:])), root=True).after(*dedup(lt_patches)) +pm_lower_call = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.SINK),), name="call", allow_any_len=True), lower_call)]) -pm_encode = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.SINK),), name="call", allow_any_len=True), lower_call)]) +# **** link: allocs to buffers, addresses to consts -# ***************** -# 4. compile - -hcq_compile_cache:dict[tuple[UOp, bool, bool], UOp] = {} # eager templates: a buffer-free linear (uops are hash-consed) to its compiled form - -@rewrite_group(lambda linear,input_uops,profile,cache=False,ret=None: f"HCQ Compile {pluralize('Kernel', len(ret.src))}") -def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool, cache=False) -> UOp: - if any(isinstance(getattr(c.without_after.arg, "aux", None), HCQInfo) for c in linear.src): return linear # compiled already - - if cache and input_uops is not None: - use_rt = len(linear.src) < HCQ_CACHE_THRESH # small schedules use runtime address patches so linked schedules can be cached without input buffers - slots = {u:i for i,u in reversed(tuple(enumerate(input_uops)))} - linear = graph_rewrite(linear, pm_replace_buffers, ctx=(use_rt, input_uops, slots), walk=True, name="replace buffers") - linear = graph_rewrite(linear, pm_unwrap_multi+pm_insert_copy_staging+pm_flatten_linear, name="prep calls") - if cache and input_uops is not None and (cached:=hcq_compile_cache.get(key:=(linear, profile, ALL2ALL >= 1))) is not None: return cached - lin = graph_rewrite(sched_batches(linear, profile), pm_encode, walk=True, name="encode") - with Context(EMULATED_DTYPES=""): final_linear = lower_and_compile(lin, verbose=DEBUG>=3) - if cache and input_uops is not None and final_linear is not linear: hcq_compile_cache[key] = final_linear - return final_linear - -# ***************** -# 5. link - -Compiled.pm_batch = Compiled.pm_encode = Compiled.pm_lower = PatternMatcher([]) # a device adds its own rules -Compiled.pm_bufferize = PatternMatcher([(UPat(Ops.ALLOC, tag="timeline", name="b"), lambda b: Device[b.device].timeline)]) +Compiled.link_bufs |= {"timeline": lambda b: Device[b.device].timeline} +Compiled.pm_link = PatternMatcher([]) @dataclass -class LinkCtx: inputs:dict[UOp, UOp]; use_rt:bool; refs:list[UOp] = field(default_factory=list) # noqa: E702 +class LinkCtx: inputs:list[UOp]; refs:list[UOp] = field(default_factory=list) # noqa: E702 -def bufferize_buf(ctx:LinkCtx, b:UOp) -> UOp: # ctx: a kept link (the jit's) owns the linear's buffers, a one-shot borrows ring slots - dev, spec = Device[b.device], b.arg.spec or BufferSpec(cpu_access=True) # data the device reads, unless the alloc says otherwise +def bufferize_buf(ctx:LinkCtx, b:UOp) -> UOp: + if b.arg.name is None: return UOp.new_buffer(b.device, b.max_numel(), b.dtype) + dev, spec = Device[b.device], b.arg.spec or BufferSpec(cpu_access=True) - # a device owns the placeholders it names, the rest are allocated where they live - if (r:=cast(Buffer|None, Compiled.pm_bufferize.rewrite(b, ctx))) is not None: pass - elif not ctx.use_rt or b.tag is None: r = Buffer(dev.device, max(b.max_numel(), 1) * b.dtype.itemsize, options=spec, preallocate=True) - else: r = dev.rt_buffer(spec).view(b.nbytes(), dev.rt_allocator(spec).alloc(max(b.nbytes(), 1), alignment=256)).ensure_allocated() + owned = Compiled.link_bufs.get(b.arg.name, Compiled.link_bufs.get(to_name(b.device, b.arg.name))) + if owned is not None: r = owned if isinstance(owned, Buffer) else owned(b) + else: r = Buffer(dev.device, max(b.max_numel(), 1) * b.dtype.itemsize, options=spec, preallocate=True) return UOp.from_buffer(r, b.dtype, HCQ_RUNTIME_DEV.device) def resolve_getaddr(ctx:LinkCtx, g:UOp) -> UOp|None: - if unwrap_lane(buf:=unwrap_view(g.src[0])[0])[0].op is not Ops.BUFFER: return None # input address, resolved per run + if unwrap_lane(buf:=(v:=unwrap_view(g.src[0]))[0])[0].op is not Ops.BUFFER: return None # input address, resolved per run ctx.refs.append(buf) # add to refs - return UOp.const(g.val, dtypes.u64) + return UOp.const(cast(Buffer, buf.buffer).view(g.src[0].nbytes(), v[1]).get_buf(to_tuple(g.arg)[0]), dtypes.u64) def _write_host(buf:UOp, writes:list[tuple[int, bytes]]) -> UOp: base, off = unwrap_view(buf) @@ -556,45 +513,48 @@ def _write_host(buf:UOp, writes:list[tuple[int, bytes]]) -> UOp: def fold_words(buf:UOp, offs:UOp, ws:UOp) -> UOp: return _write_host(buf, [(o.val * w.dtype.itemsize, struct.pack(w.dtype.fmt, truncate[w.dtype](w.val))) for o, w in zip(offs.src, ws.src)]) -def fold_ranged(buf:UOp, offs:UOp, ws:UOp, e:UOp) -> UOp: # the words of every trip - vs = {r: UOp.variable(f"r{i}", 0, r.vmax) for i, r in enumerate(e.src[1:])} - ows = list(zip(offs.substitute(vs).src, ws.substitute(vs).src)) - trips = [dict(zip([v.expr for v in vs.values()], t)) for t in itertools.product(*[range(int(r.vmax) + 1) for r in vs])] - data = [(sym_infer(o, t) * w.dtype.itemsize, struct.pack(w.dtype.fmt, truncate[w.dtype](sym_infer(w, t)))) for t in trips for o, w in ows] - return _write_host(buf, data) - -pm_link = PatternMatcher([ +pm_resolve_link = PatternMatcher([ # collapse committed const conversions (UPat(Ops.CAST, src=(UPat(Ops.CAST, src=(UPat.cvar(),), name="inner"),), name="c"), lambda c, inner: inner.src[0].cast(c.dtype)), # an alloc and a link time input become buffers (UPat(Ops.ALLOC, name="b"), bufferize_buf), - (UPat(Ops.PARAM, name="b"), lambda ctx, b: ctx.inputs.get(b)), + (UPat(Ops.PARAM, name="b"), lambda ctx, b: ctx.inputs[b.arg.slot] if b.arg.name == "lt_input" else None), # the address of a buffer is a const (UPat(Ops.GETADDR, name="g"), resolve_getaddr), # math on consts is a const - (UPat(GroupOp.ALU, src=UPat.cvar().or_casted(), name="a"), lambda a: UOp.const(exec_alu(a.op, a.dtype, [s.val for s in a.src], False), a.dtype)), + (UPat(GroupOp.ALU, src=UPat.cvar().or_casted(), name="a"), lambda a: UOp.const(exec_alu(a.op, a.dtype, [s.val for s in a.src]), a.dtype)), # fold rules (UPat(name="buf").store(UPat(Ops.BINARY, name="blob").or_bitcasted()), lambda buf, blob: _write_host(buf, [(0, blob.arg)])), (UPat(name="buf").index(UPat(Ops.STACK, src=UPat.cvar(), name="offs")).store(UPat(Ops.STACK, src=UPat.cvar().or_casted(), name="ws")), fold_words), (UPat(name="buf").index(UPat.cvar("off")).store(UPat.cvar().or_casted("w")), lambda buf, off, w: fold_words(buf, UOp.stack(off), UOp.stack(w))), - (UPat(name="buf").index(UPat(Ops.STACK, name="offs")).store(UPat(Ops.STACK, name="ws")).end(allow_any_len=True, name="e"), fold_ranged), # a call keeps the deps that are not written yet (UPat(Ops.AFTER, src=(UPat(Ops.CALL),), allow_any_len=True, name="a"), lambda a: a.src[0].after(*(s for s in a.src[1:] if s.op is not Ops.NOOP))), (UPat(Ops.AFTER, name="a"), lambda a: None if a.without_after.op is Ops.CALL else a.src[0] if all(s.op is Ops.NOOP for s in a.src[1:]) else panic(RuntimeError, f"unresolved link words on {a.src[0].op}")), ]) -link_linear_cache:weakref.WeakKeyDictionary[UOp, UOp] = weakref.WeakKeyDictionary() # a baked link lives as long as its bound linear +# **** compile and link -@rewrite_group(lambda _,input_uops=None,allow_cache=True,ret=None: f"HCQ Link {pluralize('Kernel', len(ret.src))}") -def hcq_link(linear:UOp, input_uops:list[UOp]|None=None, allow_cache=True) -> UOp: - if allow_cache and (linked:=link_linear_cache.get(linear)) is not None: return linked +hcq_compile_cache:dict[tuple[UOp, bool, bool], UOp] = {} # eager templates: a buffer-free linear (uops are hash-consed) to its compiled form - # if we have any link time buffers, do not cache this linear - cache = allow_cache and not any(u.tag == "lt_input" for u in linear.toposort() if u.op is Ops.PARAM) +@rewrite_group(lambda linear,input_uops,profile,cache=False,ret=None: f"HCQ Compile {pluralize('Kernel', len(ret.src))}") +def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool, cache:bool=False) -> UOp: + if any(isinstance(getattr(c.without_after.arg, "aux", None), HCQInfo) for c in linear.src): return linear # compiled already - inputs = {UOp.param(i, b.dtype, b.max_numel(), b.device).replace(tag="lt_input"): b for i, b in enumerate(input_uops or ())} - linked = graph_rewrite(linear, pm_link, ctx=(ctx:=LinkCtx(inputs, use_rt=allow_cache and not cache)), walk=True, name="link") + if cache and input_uops is not None: + slots = {u:i for i,u in reversed(tuple(enumerate(input_uops)))} + linear = graph_rewrite(linear, pm_replace_buffers, ctx=(input_uops, slots), walk=True, name="replace buffers") + linear = graph_rewrite(linear, pm_unwrap_multi+pm_insert_copy_staging+pm_flatten_linear, name="prep calls") + if cache and input_uops is not None and (cached:=hcq_compile_cache.get(key:=(linear, profile, ALL2ALL >= 1))) is not None: return cached + lin = graph_rewrite(sched_batches(linear, profile), pm_lower_call, walk=True, name="lower call") + with Context(EMULATED_DTYPES=""): final_linear = lower_and_compile(lin, verbose=DEBUG>=3) + if cache and input_uops is not None and final_linear is not linear: hcq_compile_cache[key] = final_linear + return final_linear + +@rewrite_group(lambda _,input_uops=None,ret=None: f"HCQ Link {pluralize('Kernel', len(ret.src))}") +def hcq_link(linear:UOp, input_uops:list[UOp]|None=None) -> UOp: + linked = graph_rewrite(linear, Compiled.pm_link, walk=True, name="link programs") if Compiled.pm_link.patterns else linear + ctx = LinkCtx(input_uops or []) + linked = graph_rewrite(linked, pm_resolve_link+Compiled.pm_link, ctx=ctx, walk=True, name="link") if ctx.refs: linked = linked.replace(src=(linked.src[0].after(*dedup(ctx.refs)), *linked.src[1:])) # attach refs to linear - if cache and linked is not linear: link_linear_cache[linear] = linked return linked diff --git a/tinygrad_repo/tinygrad/runtime/support/memory.py b/tinygrad_repo/tinygrad/runtime/support/memory.py index d831c99e22..6ae812eb03 100644 --- a/tinygrad_repo/tinygrad/runtime/support/memory.py +++ b/tinygrad_repo/tinygrad/runtime/support/memory.py @@ -37,52 +37,28 @@ class TLSFAllocator: # self.blocks is more like a linked list, where each entry is a contiguous block. self.blocks:dict[int, tuple[int, int|None, int|None, bool]] = {0: (size, None, None, True)} # size, next, prev, is_free - if size > 0: self._insert_block(0, size) + if size > 0: self._set_free(0, size, True) - @functools.cache # pylint: disable=method-cache-max-size-none - def lv1(self, size): return size.bit_length() + def lv(self, size:int) -> tuple[int, int]: return (l1:=size.bit_length()), (size - (1 << (l1 - 1))) >> max(0, l1 - self.l2_cnt) - @functools.cache # pylint: disable=method-cache-max-size-none - def lv2(self, size): return (size - (1 << (size.bit_length() - 1))) // (1 << max(0, size.bit_length() - self.l2_cnt)) - - def _insert_block(self, start:int, size:int, prev:int|None=None): - if prev is None: prev = self.blocks[start][2] - self.storage[self.lv1(size)][self.lv2(size)].append(start) - self.lv1_entries[self.lv1(size)] += 1 - self.blocks[start] = (size, start + size, prev, True) - return self - - def _remove_block(self, start:int, size:int, prev:int|None=None): - if prev is None: prev = self.blocks[start][2] - self.storage[self.lv1(size)][self.lv2(size)].remove(start) - self.lv1_entries[self.lv1(size)] -= 1 - self.blocks[start] = (size, start + size, prev, False) + def _set_free(self, start:int, size:int, free:bool): + # Free blocks are kept in the bucket of their size, allocated ones are not. + l1, l2 = self.lv(size) + if free: self.storage[l1][l2].append(start) + else: self.storage[l1][l2].remove(start) + self.lv1_entries[l1] += 1 if free else -1 + self.blocks[start] = (size, start + size, self.blocks[start][2], free) return self def _split_block(self, start:int, size:int, new_size:int): - nxt = self.blocks[start][1] - assert self.blocks[start][3], "block must be free" - self._remove_block(start, size)._insert_block(start, new_size)._insert_block(start + new_size, size - new_size, prev=start) + # Splits an allocated block into two allocated blocks, the caller gives the unused one back with _set_free. + _, nxt, prev, is_free = self.blocks[start] + assert not is_free, "block must be allocated" + self.blocks[start] = (new_size, start + new_size, prev, False) + self.blocks[start + new_size] = (size - new_size, nxt, start, False) if nxt in self.blocks: self.blocks[nxt] = (self.blocks[nxt][0], self.blocks[nxt][1], start + new_size, self.blocks[nxt][3]) return self - def _merge_right(self, start:int): - size, nxt, _, is_free = self.blocks[start] - assert is_free, "block must be free" - - while is_free and nxt in self.blocks: - if (blk:=self.blocks[nxt])[3] is False: break - self._remove_block(start, size)._remove_block(nxt, blk[0])._insert_block(start, size:=size + blk[0]) - assert self.blocks[start][1] == blk[1] - _, nxt, _, _ = self.blocks.pop(nxt) - - if nxt in self.blocks: self.blocks[nxt] = (self.blocks[nxt][0], self.blocks[nxt][1], start, self.blocks[nxt][3]) - - def _merge_block(self, start:int): - # Go left while blocks are free. Then merge all them right. - while (x:=self.blocks[start][2]) is not None and self.blocks[x][3] is True: start = x - self._merge_right(start) - def alloc(self, req_size:int, align:int=1) -> int: req_size = max(self.block_size, req_size) # at least block size. size = max(self.block_size, req_size + align - 1) @@ -91,28 +67,44 @@ class TLSFAllocator: size = round_up(size, (1 << size.bit_length() - self.l2_cnt)) # Search for the smallest block that can fit the requested size. Start with its bucket and go up until any block is found. - for l1 in range(self.lv1(size), len(self.storage)): + size_l1, size_l2 = self.lv(size) + for l1 in range(size_l1, len(self.storage)): if self.lv1_entries[l1] == 0: continue - for l2 in range(self.lv2(size) if l1 == size.bit_length() else 0, (1 << self.l2_cnt)): + for l2 in range(size_l2 if l1 == size_l1 else 0, (1 << self.l2_cnt)): if len(self.storage[l1][l2]) > 0: - # Block start address. + # Block start address. Take the whole block, the unused parts are given back below. start = self.storage[l1][l2][0] nsize = self.blocks[start][0] assert nsize >= size, "block must be larger" + self._set_free(start, nsize, False) - # If request contains alignment, split the block into two parts. + # If request contains alignment, give back the unaligned head. if (new_start:=round_up(start, align)) != start: - self._split_block(start, nsize, new_start - start) - start, nsize = new_start, self.blocks[new_start][0] + head_size = new_start - start + self._split_block(start, nsize, head_size)._set_free(start, head_size, True) + start, nsize = new_start, nsize - head_size - # If the block is larger than the requested size, split it into two parts. - if nsize > req_size: self._split_block(start, nsize, req_size) - self._remove_block(start, req_size) # Mark the block as allocated. + # If the block is larger than the requested size, give back the tail. + if nsize > req_size: self._split_block(start, nsize, req_size)._set_free(start + req_size, nsize - req_size, True) return start + self.base raise MemoryError(f"Can't allocate {req_size} bytes") def free(self, start:int): - self._insert_block(start - self.base, self.blocks[start - self.base][0])._merge_block(start - self.base) + start -= self.base + size, nxt, prev, _ = self.blocks[start] + + # Merge with the free neighbors. There is at most one on each side, since free blocks are always merged. + if nxt in self.blocks and self.blocks[nxt][3]: + self._set_free(nxt, nxt_size:=self.blocks[nxt][0], False) + del self.blocks[nxt] + size, nxt = size + nxt_size, nxt + nxt_size + if prev is not None and self.blocks[prev][3]: + self._set_free(prev, prev_size:=self.blocks[prev][0], False) + del self.blocks[start] + start, size = prev, size + prev_size + + if nxt in self.blocks: self.blocks[nxt] = (self.blocks[nxt][0], self.blocks[nxt][1], start, self.blocks[nxt][3]) + self._set_free(start, size, True) # Memory Management diff --git a/tinygrad_repo/tinygrad/runtime/support/nv/ip.py b/tinygrad_repo/tinygrad/runtime/support/nv/ip.py index 0fa8614a1e..59405fcb8a 100644 --- a/tinygrad_repo/tinygrad/runtime/support/nv/ip.py +++ b/tinygrad_repo/tinygrad/runtime/support/nv/ip.py @@ -2,7 +2,7 @@ from __future__ import annotations import ctypes, time, array, struct, itertools, dataclasses from typing import cast, Any from tinygrad.runtime.autogen import nv, nv_570 as nv_gpu, pci -from tinygrad.helpers import lo32, hi32, DEBUG, round_up, round_down, fetch_fw, wait_cond, ceildiv +from tinygrad.helpers import lo32, hi32, DEBUG, round_up, round_down, fetch_fw, getenv, wait_cond, ceildiv from tinygrad.runtime.support.system import System from tinygrad.runtime.support.memory import MMIOInterface from tinygrad.runtime.support.elf import elf_loader @@ -183,8 +183,9 @@ class NV_FLCN(NV_IP): _, self.booter_image_paddr, _ = self.nvdev._alloc_boot_mem(len(patched_image), data=patched_image, sysmem=False) self.booter_data_off, self.booter_data_sz, self.booter_code_off, self.booter_code_sz = lh.os_data_offset, lh.os_data_size, app.offset, app.size - def init_hw(self): + def init_hw(self) -> bool: self.falcon, self.sec2 = 0x00110000, 0x00840000 + speed_override = getenv("NV_OVERRIDE", 1) and self.nvdev.pci_dev.read_config(pci.PCI_DEVICE_ID, 2) in {0x2203,0x2204,0x2206,0x220a,0x2216,0x2684} self.reset(self.falcon) self.execute_hs(self.falcon, self.frts_image_paddr, code_off=0x0, data_off=self.desc_v3.IMEMLoadSize, @@ -193,6 +194,20 @@ class NV_FLCN(NV_IP): pkc_off=self.desc_v3.PKCDataOffset, engid=self.desc_v3.EngineIdMask, ucodeid=self.desc_v3.UcodeId) assert self.nvdev.NV_PFB_PRI_MMU_WPR2_ADDR_HI.read() != 0, "WPR2 is not initialized" + if needs_mid_reset:=(speed_override and self.nvdev.rreg(0x823804) != 0xffffffff): + # unlock FEAT registers + code = ([0xffffffff, 0x19c, 0xd44, 0x19c, 0x19c, 0x19c, 0x823804, 0x19c, 0x1fce, 0x19c, 0x19c, 0x0, 0x19c, 0x1101, 0x84c8, 0x8e18, 0x19c, + 0x84c8, 0x0, 0x19c, 0x1fce, 0x19c, 0x19c, 0xffbc, 0x19c, 0x5789, 0x19c, 0x19c, 0x19c, 0xd44, 0x19c, 0x19c, 0x19c, 0x3, 0x19c, 0x1fce, + 0x19c, 0x19c, 0x19c, 0x19c, 0xd52, 0x81ee, 0x19c, 0x19c, 0x19c, 0x19c] if self.nvdev.chip_name.startswith("GA1") else + [0xfffff0ff, 0x19c, 0x21b2, 0x19c, 0x19c, 0x19c, 0x1fa7cc, 0x19c, 0x33fc, 0x19c, 0x19c, 0xffffffff, 0x19c, 0x256f, 0x7a02, 0x9a0148, + 0x19c, 0x33fc, 0x19c, 0x19c, 0xffffffff, 0x19c, 0x256f, 0x7a02, 0x1fa7c4, 0x19c, 0x33fc, 0x19c, 0x19c, 0xffffffff, 0x19c, 0x256f, + 0x7a02, 0x823804, 0x19c, 0x33fc, 0x19c, 0x19c, 0x67b00001, 0x19c, 0x256f, 0x7a02, 0x19c, 0x19c, 0x31b5, 0x771e]) + desc = bytes(array.array('I', [7 if i == 0x1100 else 0x19c for i in range(0, 0xf948, 4)] + code)) + _, _, desc_addrs = self.nvdev._alloc_boot_mem(len(desc), data=desc, contiguous=True) + meta = nv.GspFwWprMeta.from_buffer_copy(bytes(self.nvdev.gsp.wpr_meta[:ctypes.sizeof(nv.GspFwWprMeta)])) + meta.sysmemAddrOfSignature, meta.sizeOfSignature = desc_addrs[0], len(desc) + self.nvdev.gsp.wpr_meta[:ctypes.sizeof(meta)] = bytes(meta) + self.reset(self.falcon, riscv=True) # set up the mailbox @@ -203,11 +218,19 @@ class NV_FLCN(NV_IP): self.reset(self.sec2) mbx = self.execute_hs(self.sec2, self.booter_image_paddr, code_off=self.booter_code_off, data_off=self.booter_data_off, imemPa=0x0, imemVa=self.booter_code_off, imemSz=self.booter_code_sz, dmemPa=0x0, dmemVa=0x0, dmemSz=self.booter_data_sz, - pkc_off=0x10, engid=1, ucodeid=3, mailbox=self.nvdev.gsp.wpr_meta_sysmem) - assert mbx[0] == 0x0, f"Booter failed to execute, mailbox is {mbx[0]:08x}, {mbx[1]:08x}" + pkc_off=0x10, engid=1, ucodeid=3, mailbox=self.nvdev.gsp.wpr_meta_sysmem, + wait_for=(lambda: self.nvdev.rreg(0x823804) == 0xffffffff) if needs_mid_reset and self.nvdev.chip_name.startswith("GA1") else None) + assert needs_mid_reset or mbx[0] == 0x0, f"Booter failed to execute, mailbox is {mbx[0]:08x}, {mbx[1]:08x}" + if needs_mid_reset: return True + + if speed_override: + for off, val in (regs:={0x82381c:0x88888888, 0x823820:0x8}).items(): self.nvdev.wreg(off, val) + assert all(self.nvdev.rreg(off) == v for off, v in regs.items()) and \ + (self.nvdev.chip_name.startswith("GA1") or not self.nvdev.rreg(0x823818) & 0x07ffffff), "FEAT not latched" self.nvdev.NV_PFALCON_FALCON_OS.with_base(self.falcon).write(0x0) assert self.nvdev.NV_PRISCV_RISCV_CPUCTL.with_base(self.falcon).read_bitfields()['active_stat'] == 1, "GSP Core is not active" + return False def execute_dma(self, base:int, cmd:int, dest:int, mem_off:int, src:int, size:int): wait_cond(lambda: self.nvdev.NV_PFALCON_FALCON_DMATRFCMD.with_base(base).read_bitfields()['full'], value=0, msg="DMA does not progress") @@ -233,7 +256,8 @@ class NV_FLCN(NV_IP): def wait_cpu_halted(self, base): wait_cond(lambda: self.nvdev.NV_PFALCON_FALCON_CPUCTL.with_base(base).read_bitfields()['halted'], msg="not halted") - def execute_hs(self, base, img_paddr, code_off, data_off, imemPa, imemVa, imemSz, dmemPa, dmemVa, dmemSz, pkc_off, engid, ucodeid, mailbox=None): + def execute_hs(self, base, img_paddr, code_off, data_off, imemPa, imemVa, imemSz, dmemPa, dmemVa, dmemSz, pkc_off, engid, ucodeid, + mailbox=None, wait_for=None): self.disable_ctx_req(base) # target=0 is FB (not in published headers) @@ -259,7 +283,8 @@ class NV_FLCN(NV_IP): self.nvdev.NV_PFALCON_FALCON_MAILBOX1.with_base(base).write(hi32(mailbox)) self.start_cpu(base) - self.wait_cpu_halted(base) + if wait_for is None: self.wait_cpu_halted(base) + else: wait_cond(wait_for, msg="Firmware did not complete") if mailbox is not None: return self.nvdev.NV_PFALCON_FALCON_MAILBOX0.with_base(base).read(), self.nvdev.NV_PFALCON_FALCON_MAILBOX1.with_base(base).read() diff --git a/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py b/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py index e27af2fa1f..89bec02b1f 100644 --- a/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py +++ b/tinygrad_repo/tinygrad/runtime/support/nv/nvdev.py @@ -2,10 +2,9 @@ from __future__ import annotations import time, functools, tinygrad.runtime.autogen.nv_regs from tinygrad.helpers import getenv, DEBUG, getbits, round_up from tinygrad.runtime.autogen import pci -from tinygrad.runtime.support.memory import TLSFAllocator, MemoryManager, AddrSpace +from tinygrad.runtime.support.memory import TLSFAllocator, MemoryManager, AddrSpace, MMIOInterface from tinygrad.runtime.support.nv.ip import NV_FLCN, NV_FLCN_COT, NV_GSP from tinygrad.runtime.support.system import PCIDevice -from tinygrad.runtime.support.memory import MMIOInterface NV_DEBUG = getenv("NV_DEBUG", 0) @@ -83,7 +82,8 @@ class NVDev: self.is_booting = False for ip in [self.flcn, self.gsp]: ip.init_sw() - for ip in [self.flcn, self.gsp]: ip.init_hw() + if self.flcn.init_hw(): return NVDev.__init__(self, pci_dev) + self.gsp.init_hw() def fini(self): for ip in [self.gsp, self.flcn]: ip.fini_hw() diff --git a/tinygrad_repo/tinygrad/runtime/support/system.py b/tinygrad_repo/tinygrad/runtime/support/system.py index dc63d73b31..93242f09be 100644 --- a/tinygrad_repo/tinygrad/runtime/support/system.py +++ b/tinygrad_repo/tinygrad/runtime/support/system.py @@ -1,11 +1,10 @@ from __future__ import annotations import os, mmap, array, functools, ctypes, ctypes.util, select, contextlib, dataclasses, sys, struct, socket, enum, itertools -try: import fcntl # windows misses that -except ImportError: fcntl = None #type:ignore[assignment] +import fcntl from tinygrad.device import BufferStorage, Buffer, Device from tinygrad.helpers import round_up, getenv, OSX, temp, DEBUG, pluralize, DEV from tinygrad.runtime.autogen import libc, pci, vfio -from tinygrad.runtime.support.memory import VirtMapping, AddrSpace, BumpAllocator, MMIOInterface +from tinygrad.runtime.support.memory import VirtMapping, AddrSpace, BumpAllocator, MMIOInterface, TLSFAllocator from tinygrad.runtime.support.usb import USB3, CustomASM24Controller, USBMMIOInterface def filter_visible_devices(devs, device): @@ -53,6 +52,8 @@ MAP_LOCKED, MAP_POPULATE, MAP_NORESERVE = 0 if OSX else 0x2000, getattr(mmap, "M def ipv4_to_gid(ip:str) -> bytes: return bytes(10) + b'\xff\xff' + socket.inet_aton(ip) class _System: + vfio:FileIOInterface|None = None + iova = TLSFAllocator(12 << 40, base=4 << 40, block_size=0x1000) def write_sysfs(self, path:str, value:str, msg:str, expected:str|None=None): if FileIOInterface(path, os.O_RDONLY).read().splitlines()[0] != (expected or value): os.system(cmd:=f"sudo sh -c 'echo {value} > {path}'") @@ -69,18 +70,6 @@ class _System: self.write_sysfs("/proc/sys/vm/compact_unevictable_allowed", "0", "Failed to disable migration of locked pages") return FileIOInterface("/proc/self/pagemap", os.O_RDONLY) - @functools.cached_property - def vfio(self) -> FileIOInterface|None: - try: - if not FileIOInterface.exists("/sys/module/vfio"): os.system("sudo modprobe vfio-pci disable_idle_d3=1") - - FileIOInterface("/sys/module/vfio/parameters/enable_unsafe_noiommu_mode", os.O_RDWR).write("1") - vfio_fd = FileIOInterface("/dev/vfio/vfio", os.O_RDWR) - vfio.VFIO_CHECK_EXTENSION(vfio_fd, vfio.VFIO_NOIOMMU_IOMMU) - - return vfio_fd - except OSError: return None - @functools.cache def reserve_va(self, va_start, va_size): # cached, runs only once per range. used to not collide with other mappings. @@ -91,7 +80,18 @@ class _System: def lock_memory(self, addr:int, size:int): if libc.mlock(ctypes.c_void_p(addr), size): raise RuntimeError(f"Failed to lock memory at {addr:#x} with size {size:#x}") + def vfio_map(self, vaddr:int, size:int, align:int=1) -> int: + iova = self.iova.alloc(size, align) + flags = vfio.VFIO_DMA_MAP_FLAG_READ | vfio.VFIO_DMA_MAP_FLAG_WRITE + vfio.VFIO_IOMMU_MAP_DMA(self.vfio, vfio.struct_vfio_iommu_type1_dma_map(argsz=32, flags=flags, vaddr=vaddr, iova=iova, size=size)) + return iova + + def vfio_unmap(self, iova:int, size:int): + vfio.VFIO_IOMMU_UNMAP_DMA(self.vfio, vfio.struct_vfio_iommu_type1_dma_unmap(argsz=24, iova=iova, size=size)) + self.iova.free(iova) + def system_paddrs(self, vaddr:int, size:int) -> list[int]: + if self.vfio is not None: return list(range(iova:=self.vfio_map(vaddr, size), iova + size, mmap.PAGESIZE)) self.pagemap.seek(vaddr // mmap.PAGESIZE * 8) return [(x & ((1<<55) - 1)) * mmap.PAGESIZE for x in array.array('Q', self.pagemap.read(size//mmap.PAGESIZE*8, binary=True))] @@ -186,8 +186,6 @@ class _System: return bars def flock_acquire(self, name:str) -> int: - import fcntl # to support windows - os.umask(0) # Set umask to 0 to allow creating files with 0666 permissions # Avoid O_CREAT because we don’t want to re-create/replace an existing file (triggers extra perms checks) when opening as non-owner. @@ -219,15 +217,15 @@ class PCIDevice: for fn in range(1, 8): if FileIOInterface.exists(sib:=f"/sys/bus/pci/devices/{self.pcibus[:-1]}{fn}"): FileIOInterface(f"{sib}/remove", os.O_WRONLY).write("1") - if getenv("VFIO", 0) and (vfio_fd:=System.vfio) is not None: + if FileIOInterface.exists(grp:=f"/sys/bus/pci/devices/{self.pcibus}/iommu_group") and FileIOInterface(f"{grp}/type").read().strip() != "identity": + if not FileIOInterface.exists("/sys/module/vfio_pci"): raise RuntimeError("IOMMU is on: run `sudo modprobe vfio-pci disable_idle_d3=1`") FileIOInterface(f"/sys/bus/pci/devices/{self.pcibus}/driver_override", os.O_WRONLY).write("vfio-pci") FileIOInterface("/sys/bus/pci/drivers_probe", os.O_WRONLY).write(self.pcibus) - iommu_group = FileIOInterface.readlink(f"/sys/bus/pci/devices/{self.pcibus}/iommu_group").split('/')[-1] - - self.vfio_group = FileIOInterface(f"/dev/vfio/noiommu-{iommu_group}", os.O_RDWR) + vfio_fd = System.vfio = System.vfio or FileIOInterface("/dev/vfio/vfio", os.O_RDWR) + self.vfio_group = FileIOInterface(f"/dev/vfio/{FileIOInterface.readlink(grp).split('/')[-1]}", os.O_RDWR) vfio.VFIO_GROUP_SET_CONTAINER(self.vfio_group, ctypes.c_int(vfio_fd.fd)) - with contextlib.suppress(OSError): vfio.VFIO_SET_IOMMU(vfio_fd, vfio.VFIO_NOIOMMU_IOMMU) # set iommu works only once for the fd. + with contextlib.suppress(OSError): vfio.VFIO_SET_IOMMU(vfio_fd, vfio.VFIO_TYPE1v2_IOMMU) # set iommu works only once for the fd. self.vfio_dev = FileIOInterface(fd=vfio.VFIO_GROUP_GET_DEVICE_FD(self.vfio_group, ctypes.create_string_buffer(self.pcibus.encode()))) self.irq_fd = FileIOInterface.eventfd(0, 0) @@ -257,7 +255,9 @@ class PCIDevice: @functools.cache def bar_info(self, bar_idx:int) -> tuple[int, int]: s, e, _ = FileIOInterface(f"/sys/bus/pci/devices/{self.pcibus}/resource", os.O_RDONLY).read().splitlines()[bar_idx].split() - return (int(s, 16), int(e, 16) - int(s, 16) + 1) + addr, sz = int(s, 16), int(e, 16) - int(s, 16) + 1 + if hasattr(self, "vfio_dev"): addr = System.vfio_map(self.map_bar(bar_idx, size=sz).addr, sz, sz) + return addr, sz def map_bar(self, bar:int, off:int=0, addr:int=0, size:int|None=None, fmt='B') -> MMIOInterface: fd, sz = self.bar_fd(bar), size or (self.bar_info(bar)[1] - off) libc.madvise(loc:=fd.mmap(addr, sz, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED | (MAP_FIXED if addr else 0), off), sz, libc.MADV_DONTFORK) @@ -326,15 +326,18 @@ class PCIIfaceBase: def free(self, storage:BufferStorage): if storage.meta.mapping.aspace is AddrSpace.PHYS: self.dev_impl.mm.vfree(storage.meta.mapping) - if storage.meta.has_cpu_mapping and self.remote is None: FileIOInterface.munmap(storage.buf, storage.meta.mapping.size) + elif hasattr(self.pci_dev, "vfio_dev"): System.vfio_unmap(storage.meta.hMemory, storage.meta.mapping.size) + if storage.meta.has_cpu_mapping and type(storage.host) is MMIOInterface: FileIOInterface.munmap(storage.host.addr, storage.host.nbytes) - def unmap(self, mapping:BufferStorage): self.dev_impl.mm.unmap_range(*mapping.meta) + def unmap(self, mapping:BufferStorage): + self.dev_impl.mm.unmap_range(*mapping.meta[:2]) + if mapping.meta[2] is not None: System.vfio_unmap(mapping.meta[2], mapping.meta[1]) def p2p_paddrs(self, paddrs:list[tuple[int,int]]) -> tuple[list[tuple[int,int]], AddrSpace]: return [(p + self.pci_dev.bar_info(self.vram_bar)[0], sz) for p, sz in paddrs], AddrSpace.SYS def map(self, b:Buffer) -> BufferStorage: - if b.device.split(":")[0] in {"CPU", "PYTHON", "NPY"}: + if (host:=b.device.split(":")[0] in {"CPU", "PYTHON", "NPY"}): if self.dev.host != Device[b.device].host: raise RuntimeError(f"host memory is not on the node of {self.dev.device}") if b._buf % 0x1000: raise RuntimeError("Host mapping requires a page-aligned address") lo, size = b._buf, round_up(b.nbytes, 0x1000) @@ -352,7 +355,7 @@ class PCIIfaceBase: else: raise RuntimeError(f"map failed: {b.device} -> {self.dev.device}") self.dev_impl.mm.map_range(lo, size, paddrs, aspace=aspace, snooped=snooped, uncached=uncached) - return BufferStorage(lo, (lo, size)) + return BufferStorage(lo, (lo, size, paddrs[0][0] if host and hasattr(self.pci_dev, "vfio_dev") else None)) # *** Remote PCI: memory and program execution on extra/remote/serve.py diff --git a/tinygrad_repo/tinygrad/runtime/support/usb.py b/tinygrad_repo/tinygrad/runtime/support/usb.py index b3677577c3..0ec56089e5 100644 --- a/tinygrad_repo/tinygrad/runtime/support/usb.py +++ b/tinygrad_repo/tinygrad/runtime/support/usb.py @@ -1,10 +1,11 @@ import ctypes, struct, time, functools, itertools +from typing import cast from tinygrad.runtime.autogen import libusb, libc -from tinygrad.helpers import DEBUG, DEV, HCQ_RUNTIME_DEV, to_mv, round_up, ceildiv, flatten +from tinygrad.helpers import DEBUG, DEV, HCQ_RUNTIME_DEV, to_mv, round_up, ceildiv, flatten, to_tuple from tinygrad.dtype import dtypes, DType, AddrSpace from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, uopfunc -from tinygrad.device import Buffer, BufferSpec, Compiled -from tinygrad.runtime.support.hcq2 import HCQ_DEVS, ccall, patch, unwrap_view, all_devices_in, to_name, get_time_ms, ins, chunks +from tinygrad.device import Buffer, BufferSpec, Compiled, Device +from tinygrad.runtime.support.hcq2 import HCQ_DEVS, ccall, patch, unwrap_view, all_devices_in, to_name, get_time_ms, ins from tinygrad.runtime.support.memory import MMIOInterface from tinygrad.runtime.support import c from tinygrad.runtime.support.am.startup_trace import short_read @@ -209,14 +210,14 @@ class USBMMIOInterface(MMIOInterface): HALF, CHUNK, SLOT, STREAM = 0x40000, 0x40000 - 512, 0x4000, 1 << 20 # sram half, payload, slot, stream HOST_SIZE = 64 + 2 * HALF + STREAM # link, staging, zeros -def usb_host(dev:str) -> UOp: return UOp.alloc((HOST_SIZE,), dtypes.uint8, 0, device=HCQ_RUNTIME_DEV.device).rtag(to_name(dev, "usb_host")) +def usb_host(dev:str) -> UOp: return UOp.alloc((HOST_SIZE,), dtypes.uint8, 0, device=HCQ_RUNTIME_DEV.device, name=to_name(dev, "usb_host")) def usb_link(dev:str) -> UOp: return usb_host(dev)[:48].bitcast(dtypes.uint64) # handle, context, prev chunks, two transfers, error state def usb_stage(dev:str) -> UOp: return usb_host(dev)[64:64 + 2 * HALF] def usb_zeros(dev:str) -> UOp: return usb_host(dev)[64 + 2 * HALF:] -def usb_go(dev:str) -> UOp: return UOp.alloc((1,), dtypes.uint32, 0, device=dev).rtag(to_name(dev, "usb_go")) # chunk + 1: read armed +def usb_go(dev:str) -> UOp: return UOp.alloc((1,), dtypes.uint32, 0, device=dev, name="usb_go") # chunk + 1: read armed -def usb_asm24(dev:str) -> UOp: return UOp.alloc((0x85000,), dtypes.uint8, 0, device=dev).rtag(to_name(dev, "usb_asm24")) # sys, cq, sram +def usb_asm24(dev:str) -> UOp: return UOp.alloc((0x85000,), dtypes.uint8, 0, device=dev, name="usb_asm24") # sys, cq, sram def usb_fence(dev:str) -> UOp: return usb_asm24(dev)[0x800:0x804].bitcast(dtypes.uint32) # gpu chunks done def usb_cq(dev:str) -> UOp: return usb_asm24(dev)[0x100c:0x1010].bitcast(dtypes.uint32) # completion queue def usb_sram(dev:str) -> UOp: return usb_asm24(dev)[0x5000:0x5000 + 2 * HALF] @@ -240,12 +241,14 @@ def usb_poke(link:UOp, addr:UOp, val:UOp) -> UOp: # 0xF0 mode 0: a dword @uopfunc def usb_stream(link:UOp, addr:UOp, data:UOp, n:UOp, write:bool, fill:bool=False) -> UOp: # 0xF0 mode 1/2: header, then bulk per STREAM - i = UOp.range(((n + (STREAM - 1)) // STREAM).after(link), next(UOp.unique_num), dtype=dtypes.int) - off, cnt = i * STREAM, (n - i * STREAM).minimum(STREAM) - payload = usb_stack(dtypes.uint64, addr + off.cast(dtypes.uint64), cnt // 4) - live = UOp.range(link.after(i).index(5).load().eq(0).cast(dtypes.int).after(link), next(UOp.unique_num), dtype=dtypes.int) # no trips once failed - header = usb_ctrl(link.after(live), 0x40, 0xF0, (0x60 if write else 0x20) | 0x0F00, 1 if write else 2, payload.index(0), 12, 5000) - return usb_bulk(link.after(header), 0x02 if write else 0x81, data.index(0 if fill else off // data.dtype.itemsize), cnt).end(live, i).sink() + live = UOp.range((link.index(5).load().eq(0) & (n > 0)).cast(dtypes.int), next(UOp.unique_num), dtype=dtypes.int) # no trips once failed + loop, at = UOp.range(UOp(Ops.NOOP).after(link, live), next(UOp.unique_num), dtype=dtypes.void), usb_stack(dtypes.int, 0) + off = at.after(loop).index(0).load() + cnt = (n - off).minimum(STREAM) + header = usb_ctrl(link.after(loop), 0x40, 0xF0, (0x60 if write else 0x20) | 0x0F00, 1 if write else 2, + usb_stack(dtypes.uint64, addr + off.cast(dtypes.uint64), cnt // 4).index(0), 12, 5000) + sent = usb_bulk(link.after(header), 0x02 if write else 0x81, data.index(0 if fill else off // data.dtype.itemsize), cnt) + return at.after(sent).index(0).store(off + STREAM).backedge(loop, (off + STREAM < n) & link.after(sent).index(5).load().eq(0)).end(live).sink() def usb_poke_word(link:UOp, addr:UOp, val:UOp) -> UOp: low = usb_poke(link, addr, val.cast(dtypes.uint32)) @@ -259,25 +262,16 @@ def usb_patch(link:UOp, addr:UOp, val:UOp, slot:UOp) -> UOp: # poke only on chan # ***************** # 3. lower -def is_host(b:UOp) -> bool: return b.device is None or not all_devices_in(b.device, HCQ_DEVS - {"CPU"}) # register or host memory -def is_remote(b:UOp) -> bool: return (p:=unwrap_view(b)[0]).op in (Ops.PARAM, Ops.ALLOC) and not is_host(p) +def is_host(b:UOp) -> bool: return not all_devices_in(b.device, HCQ_DEVS - {"CPU"}) +def is_usb(dev) -> bool: return getattr(Device[to_tuple(dev)[0]], "is_usb", False) +def is_remote(b:UOp) -> bool: return (p:=unwrap_view(b)[0]).op in (Ops.PARAM, Ops.ALLOC) and p.device is not None and is_usb(p.device) def usb_addr(b:UOp, idx:UOp, dt:DType) -> UOp: return b.getaddr("CPU") + (idx * dt.itemsize).cast(dtypes.uint64) # byte address of b[idx] def usb_deps(b:UOp) -> tuple[UOp, ...]: # deps through views return (b.src[1:] if b.op is Ops.AFTER else ()) + (usb_deps(b.src[0]) if b.op in (Ops.BITCAST, Ops.SHRINK, Ops.AFTER) else ()) -def usb_affine(idx:UOp, r:UOp) -> UOp|None: # base of base + r - if idx is r: return UOp.const(0, r.dtype) - if idx.op is not Ops.ADD or r not in idx.src: return None - base = idx.src[1] if idx.src[0] is r else idx.src[0] - return base if r not in base.ranges else None - -def usb_copy(dst:UOp, di:UOp, v:UOp, r:UOp) -> UOp|None: # ranged store: one stream - if not is_remote(dst): return None - if v.op is Ops.LOAD and not is_remote(sb:=v.src[0].src[0]): s0, deps, fill = usb_affine(v.src[0].src[1], r), usb_deps(sb), False - elif v.vmin == v.vmax == 0: sb, s0, deps, fill = usb_zeros(dst.device), UOp.const(0, dtypes.int), (), True - else: return usb_store(dst, di, v).end(r) - if s0 is None or (d0:=usb_affine(di, r)) is None: return usb_store(dst, di, v).end(r) - link = usb_link(dst.device).after(*usb_deps(dst), *deps, *r.src[1:]) - return usb_stream(link, usb_addr(dst, d0, v.dtype), sb.index(s0), r.src[0] * v.dtype.itemsize, True, fill) +def usb_memcpy(dst:UOp, src:UOp, n:UOp, fill:bool=False) -> UOp|None: + if not is_remote(b:=dst.src[0]): return None + link = usb_link(b.device).after(*usb_deps(b), *usb_deps(src.src[0])) + return usb_stream(link, usb_addr(b, dst.src[1], b.dtype), src, n.cast(dtypes.int), True, fill) def usb_store(b:UOp, idx:UOp, v:UOp) -> UOp: # kernargs: poke on change if idx.op is Ops.STACK: # word by word @@ -286,8 +280,8 @@ def usb_store(b:UOp, idx:UOp, v:UOp) -> UOp: # kernargs: poke on change return stored link, addr = usb_link(b.device).after(*usb_deps(b)), usb_addr(b, idx, v.dtype) v = v.bitcast({d.itemsize: d for d in dtypes.uints}[v.dtype.itemsize]) # bitcast to uint raw bits - if str(unwrap_view(b)[0].tag).startswith("kernargs"): - cache = UOp.alloc((int(idx.vmax - idx.vmin) + 1,), v.dtype, device=HCQ_RUNTIME_DEV.device).rtag("usb_arg_cache") + if str(unwrap_view(b)[0].arg.name).startswith("kernargs"): + cache = UOp.alloc((int(idx.vmax - idx.vmin) + 1,), v.dtype, device=HCQ_RUNTIME_DEV.device, name="usb_arg_cache") return usb_patch(link, addr, v, patch(cache, [], bytes(v.dtype.itemsize * cache.max_numel())).index(idx - idx.vmin)) return usb_poke_word(link, addr, v) @@ -296,12 +290,6 @@ def usb_load(b:UOp, idx:UOp, ld:UOp) -> UOp: # all ones once the link failed, li read = usb_stream(link, usb_addr(b, idx, ld.dtype), slot.index(0), n, False) return link.after(read).index(5).load().ne(0).where(UOp.const(ld.dtype.max, ld.dtype), slot.after(read).index(0).load()) -pm_usb_lower = PatternMatcher([ - (UPat.var("dst").index(UPat.var("di")).store(UPat.var("v")).end(UPat(Ops.RANGE, name="r")), usb_copy), - (UPat.var("b").index(UPat.var("idx")).store(UPat.var("v")), lambda b, idx, v: None if idx.ranges or not is_remote(b) else usb_store(b, idx, v)), - (UPat.var("b").index(UPat.var("idx")).load(name="ld"), lambda b, idx, ld: usb_load(b, idx, ld) if is_remote(b) else None), -]) - # ***************** # 4. copies @@ -371,10 +359,14 @@ def usb_hostaddr(host:UOp, dev:str) -> UOp: base, boff = unwrap_view(host) return base.bitcast(dtypes.uint8)[boff:boff + host.nbytes()].getaddr(dev) +def chunks(nbytes:int, chunk_sz:int) -> list[tuple[UOp|int, int]]: # (index, bytes): full chunks as one range, then the tail + full, tail = divmod(nbytes, chunk_sz) # no one-trip loops + return ([(UOp.range(full, next(UOp.unique_num), dtype=dtypes.int) if full > 1 else 0, chunk_sz)] if full else []) + ([(full, tail)] if tail else []) + def usb_table(hosts:list[tuple[UOp, int]], n:int, win:int, dev:str) -> UOp: # [address, bytes] per chunk rows = [(16 * (k + r) + o, w) for host, k in hosts for r, nb in chunks(host.nbytes(), win) for o, w in ((0, usb_hostaddr(host, dev) + UOp.const(r, dtypes.uint64) * win), (8, UOp.const(nb, dtypes.uint64)))] - return patch(UOp.alloc((2 * n,), dtypes.uint64, device=HCQ_RUNTIME_DEV.device).rtag("usb_table"), rows) + return patch(UOp.alloc((2 * n,), dtypes.uint64, device=HCQ_RUNTIME_DEV.device, name="usb_table"), rows) def usb_chunks(call:UOp, first:int, run:int) -> list[UOp]: # gpu side. first: chunk id of the copy and of its run dst, src = call.src[1:] @@ -395,9 +387,8 @@ def usb_chunks(call:UOp, first:int, run:int) -> list[UOp]: # gpu side. first: ch return ops def usb_copy_rewriter(s:UOp) -> UOp|None: - lins = [submit.without_after.src[1].without_after for submit in s.src] - if not (copies:=[call for lin in lins for call in lin.src if is_staged(call)]): return None - dev = lins[0].arg[0][0] + call, lin = s.without_after, s.without_after.body + if not is_usb(dev:=call.src[1].device) or not (copies:=[c for c in lin.src if is_staged(c)]): return None # a run: one direction, one table runs, nums, n = list[tuple[bool, int, UOp, int]](), list[tuple[int, int]](), 0 @@ -408,21 +399,30 @@ def usb_copy_rewriter(s:UOp) -> UOp|None: # gpu side chunks = (usb_chunks(call, *num) for call, num in zip(copies, nums)) - s = s.substitute({lin: lin.replace(src=tuple(flatten(next(chunks) if is_staged(c) else [c] for c in lin.src))) for lin in lins}) + s = call.replace(src=(lin.replace(src=tuple(flatten(next(chunks) if is_staged(c) else [c] for c in lin.src))), call.src[1])).after(*s.src[1:]) - # host side, after the submits - link = usb_link(dev).after(s.src[-1]) + # host side, after the submit + link = usb_link(dev).after(s) done = UOp.custom_function("usb_begin").call(link, usb_fence(dev), link.index(2).load()) for cin, run, table, k in runs: if run: done = UOp.custom_function("usb_drain").call(link.after(done), usb_fence(dev), UOp.const(run + 1, dtypes.uint64)) # a run uses both halves - i = UOp.range(k, next(UOp.unique_num), dtype=dtypes.int) - word = usb_fence(dev) if cin else usb_go(dev) - live = UOp.range(link.after(done, i).index(5).load().eq(0).cast(dtypes.int), next(UOp.unique_num), dtype=dtypes.int) # no trips once failed - done = UOp.custom_function("usb_send" if cin else "usb_recv").call(link.after(live), table, i, UOp.const(run, dtypes.int), word).end(live, i) - return s.replace(src=(*s.src, link.after(done).index(2).store(UOp.const(n, dtypes.uint64)))) -pm_usb_batch = PatternMatcher([(UPat(Ops.SINK, name="s"), usb_copy_rewriter)]) + live = UOp.range(link.after(done).index(5).load().eq(0).cast(dtypes.int), next(UOp.unique_num), dtype=dtypes.int) # no trips once failed + loop, at = UOp.range(UOp(Ops.NOOP).after(link, live), next(UOp.unique_num), dtype=dtypes.void), usb_stack(dtypes.int, 0) + i, word = at.after(loop).index(0).load(), usb_fence(dev) if cin else usb_go(dev) + sent = UOp.custom_function("usb_send" if cin else "usb_recv").call(link.after(loop), table, i, UOp.const(run, dtypes.int), word) + done = at.after(sent).index(0).store(i + 1).backedge(loop, (i + 1 < k) & link.after(sent).index(5).load().eq(0)).end(live) + return link.after(done).index(2).store(UOp.const(n, dtypes.uint64)) -pm_usb_encode = PatternMatcher([ +Compiled.pm_encode = PatternMatcher([(UPat.sink().call(UPat(Ops.PARAM)).after(allow_any_len=True, name="s"), + usb_copy_rewriter)]) + Compiled.pm_encode # the copies of a submit go through the sram window +Compiled.pm_lower += PatternMatcher([ + # the gpu's memory is reached over usb + (UPat.custom_function("memcpy", dtypes.uint64).call(UPat.var("dst"), UPat.var("src"), UPat.var("n")), usb_memcpy), + (UPat.custom_function("memset", dtypes.uint64).call(UPat.var("dst"), UPat.const(0), UPat.var("n")), + lambda dst, n: usb_memcpy(dst, usb_zeros(dst.src[0].device).index(0), n, fill=True)), + (UPat.var("b").index(UPat.var("idx")).store(UPat.var("v")), lambda b, idx, v: usb_store(b, idx, v) if is_remote(b) else None), + (UPat.var("b").index(UPat.var("idx")).load(name="ld"), lambda b, idx, ld: usb_load(b, idx, ld) if is_remote(b) else None), + # the usb host calls get their bodies (UPat(Ops.CALL, src=(UPat.custom_function("usb_begin"), UPat.var("link"), UPat.var("fence"), UPat.var("prev"))), lambda link, fence, prev: usb_begin(link, fence.getaddr("CPU"), prev)), (UPat(Ops.CALL, src=(UPat.custom_function("usb_drain"), UPat.var("link"), UPat.var("fence"), UPat.var("need"))), @@ -436,26 +436,18 @@ pm_usb_encode = PatternMatcher([ # ***************** # 6. bufferize -@functools.cache -def _host_block(dev) -> Buffer: +def usb_reset(dev): + host = cast(Buffer, Compiled.link_bufs[to_name(dev.device, "usb_host")]) + for buf, off, n in ((dev.iface.ctrl, 0x800, 4), (dev.iface.ctrl, 0x5000, 0x80000), (host, 16, 8)): buf.host.view(off, n)[:] = bytes(n) + for t in map(libusb.struct_libusb_transfer.from_address, host.host.view(fmt='Q')[3:5]): t.status = 0xff * (t.status == 0xff) + +def setup_usb_rules(dev): # placeholders the gpu owns xfers = [libusb.libusb_alloc_transfer(0).contents for _ in range(2)] for t in xfers: t.dev_handle, t.endpoint, t.type, t.timeout = dev.iface.pci_dev.usb.usb.handle, 0x02, libusb.LIBUSB_TRANSFER_TYPE_BULK, 10000 words = [ctypes.addressof(x.contents) for x in (dev.iface.pci_dev.usb.usb.handle, USB3.ctx())] + [0] + [ctypes.addressof(t) for t in xfers] - return Buffer("CPU", HOST_SIZE, options=BufferSpec(nolru=True), initial_value=struct.pack('5Q', *words).ljust(HOST_SIZE, b'\0')) - -@functools.cache -def _go(dev) -> Buffer: - return Buffer(dev.device, 4, options=BufferSpec(uncached=True, cpu_access=True, nolru=True), initial_value=bytes(4)) - -def usb_reset(dev): - for buf, off, n in ((dev.iface.ctrl, 0x800, 4), (dev.iface.ctrl, 0x5000, 0x80000), (_host_block(dev), 16, 8)): buf.host.view(off, n)[:] = bytes(n) - for t in map(libusb.struct_libusb_transfer.from_address, _host_block(dev).host.view(fmt='Q')[3:5]): t.status = 0xff * (t.status == 0xff) - -def setup_usb_rules(dev): - dev.pm_batch, dev.pm_lower, dev.pm_encode = dev.pm_batch + pm_usb_batch, dev.pm_lower + pm_usb_lower, dev.pm_encode + pm_usb_encode - Compiled.pm_bufferize += PatternMatcher([(UPat(Ops.ALLOC, tag=dev.tag("usb_host")), lambda d=dev: _host_block(d)), # placeholders the gpu owns - (UPat(Ops.ALLOC, tag=dev.tag("usb_go")), lambda d=dev: _go(d)), - (UPat(Ops.ALLOC, tag=dev.tag("usb_asm24")), lambda d=dev: d.iface.ctrl)]) - dev.error_state = _host_block(dev).view(8, 40) # the link's error word + host = Buffer("CPU", HOST_SIZE, options=BufferSpec(nolru=True), initial_value=struct.pack('5Q', *words).ljust(HOST_SIZE, b'\0')) + go = Buffer(dev.device, 4, options=BufferSpec(uncached=True, cpu_access=True, nolru=True), initial_value=bytes(4), allocator=dev.allocator) + Compiled.link_bufs |= {to_name(dev.device, n): b for n, b in (("usb_host", host), ("usb_go", go), ("usb_asm24", dev.iface.ctrl))} + dev.error_state = host.view(8, 40) # the link's error word if DEV.interface.startswith("MOCK"): from test.mockgpu.usb import MockUSB3 as USB3 # type: ignore # noqa: F811 diff --git a/tinygrad_repo/tinygrad/schedule/__init__.py b/tinygrad_repo/tinygrad/schedule/__init__.py index d8227b8221..b6b36094a3 100644 --- a/tinygrad_repo/tinygrad/schedule/__init__.py +++ b/tinygrad_repo/tinygrad/schedule/__init__.py @@ -2,7 +2,7 @@ import time, inspect from collections import deque from dataclasses import dataclass, field, replace from tinygrad.dtype import AddrSpace -from tinygrad.uop.ops import GroupOp, remove_all_tags, UOp, Ops, UOpMetaClass, graph_rewrite, gate_kernel_sink, KernelInfo +from tinygrad.uop.ops import GroupOp, UOp, Ops, UOpMetaClass, graph_rewrite, MapCtx, gate_kernel_sink, KernelInfo from tinygrad.uop.spec import type_verify, spec_tensor from tinygrad.helpers import DEBUG, cpu_profile, TracingKey, SPEC, SCACHE, BASEDIR, partition, dedup, all_int, VIZ from tinygrad.helpers import diskcache_get, diskcache_put, colored @@ -11,14 +11,14 @@ from tinygrad.helpers import diskcache_get, diskcache_put, colored # unwrap VIEW/CAST/etc to find the actual data source (kernel output, buffer, or multi-device op) def _unwrap_src(s: UOp) -> UOp: - while len(s.src) and s.op not in {Ops.AFTER, Ops.BUFFER, Ops.ALLOC, Ops.PARAM, Ops.MSELECT, Ops.MSTACK}: s = s.src[0] + while len(s.src) and s.op not in GroupOp.Defines|{Ops.AFTER, Ops.MSELECT, Ops.MSTACK}: s = s.src[0] return s -# a buffer state is AFTER | BUFFER | ALLOC | PARAM. MSELECT/MSTACK join per-device states +# a buffer state is a define or an AFTER. MSELECT/MSTACK join per-device states def _states(s: UOp) -> list[UOp]: s = _unwrap_src(s) if s.op in {Ops.MSELECT, Ops.MSTACK}: return [st for ss in s.src for st in _states(ss)] - assert s.op in {Ops.AFTER, Ops.BUFFER, Ops.ALLOC, Ops.PARAM}, f"input to kernel must resolve to a buffer state, not {s.op}" + assert s.op in GroupOp.Defines|{Ops.AFTER}, f"input to kernel must resolve to a buffer state, not {s.op}" return [s] def _split_after(after: UOp) -> tuple[tuple[UOp, ...], tuple[UOp, ...]]: @@ -59,7 +59,7 @@ def create_schedule(sched_sink:UOp) -> UOp: for a, write_kernels in writes.get(s, []): if a is u: continue for t in write_kernels: - if t is not k and t not in k.backward_slice: + if t is not k: children.setdefault(k, []).append(t) in_degree[t] += 1 @@ -107,7 +107,7 @@ def resolve_linear_call(linear_call:UOp, outer_binds:dict[int, UOp]|None=None): def apply_binds(si:UOp) -> UOp: if si.op is Ops.CALL and si.body.op is Ops.LINEAR: return resolve_linear_call(si, binds) if si.op is Ops.CALL and si.body.op is Ops.PROGRAM: return si # compiled parameters already have ABI slots - subs = {v:binds[v.arg.slot] for s in si.src for v in s.variables() if v.arg.slot in binds} + subs = {v:binds[v.arg.slot] for s in si.src for v in s.variables() if v.arg.name is None and v.arg.slot in binds} return si.replace(src=tuple(s.substitute(subs, name="resolve scalar params") for s in si.src)) return linear.replace(src=tuple(apply_binds(si) for si in linear.src)) @@ -164,7 +164,7 @@ def simplify_copy_kernel(call:UOp, ast:UOp, dst:UOp, src:UOp): from tinygrad.codegen.simplify import pm_flatten_range, pm_simplify_ranges from tinygrad.schedule.prepare import pm_mops from tinygrad.uop.symbolic import sym - sink = graph_rewrite(ast, sym+pm_mops+pm_flatten_range+pm_simplify_ranges, ctx={}, name="simplify ranges in copy") + sink = graph_rewrite(ast, sym+pm_mops+pm_flatten_range+pm_simplify_ranges, init_ctx=MapCtx, name="simplify ranges in copy") return call.replace(src=(sink,) + call.src[1:]) pm_copy_from_store = PatternMatcher([ @@ -233,7 +233,7 @@ def canonicalize_alloc(ctx:CallifyCtx, b:UOp): return ctx.allocs.get(b) def canonicalize_call_body(c:UOp): - return c.replace(src=(graph_rewrite(c.body, pm_canonicalize_alloc, ctx=CallifyCtx(), bottom_up=True),)+c.src[1:]) + return c.replace(src=(graph_rewrite(c.body, pm_canonicalize_alloc, init_ctx=CallifyCtx, bottom_up=True),)+c.src[1:]) pm_canonicalize_alloc = PatternMatcher([ # NOTE: lambda for late binding, canonicalize_call_body references pm_canonicalize_alloc @@ -260,7 +260,7 @@ def transform_to_call(big_sink:UOp) -> UOp: # The tensor replacement map is collected before these rewrites change node identities. graph_rewrite(big_sink, pm_callify_ctx_collect, ctx=(ctx:=CallifyCtx()), name="early transform tensor graph") - ret = graph_rewrite(UOp.sink(*ctx.stores), pm_canonicalize_alloc+pm_replace_buf+remove_all_tags, ctx=ctx, bottom_up=True, name="replace bufs") + ret = graph_rewrite(UOp.sink(*ctx.stores), pm_canonicalize_alloc+pm_replace_buf, ctx=ctx, bottom_up=True, name="replace bufs") ret = ret.call(*ctx.replacements, precompile=True) if VIZ: graph_rewrite(ret, PatternMatcher([]), name="View Call") return ret diff --git a/tinygrad_repo/tinygrad/schedule/allreduce.py b/tinygrad_repo/tinygrad/schedule/allreduce.py index b92aa5d929..13fab4bdcc 100644 --- a/tinygrad_repo/tinygrad/schedule/allreduce.py +++ b/tinygrad_repo/tinygrad/schedule/allreduce.py @@ -47,8 +47,8 @@ def handle_allreduce(buf:UOp, red:UOp) -> UOp|None: chunk, reduced = buf.reshape((numel,)).shrink(((s,e),)), buf.reshape((numel,)).shrink(((s,e),)) for step in range(ndev-1): src, dest = (i+step)%ndev, (i+step+1)%ndev - cp = reduced.copy_to_device(buf.device[dest], src if isinstance(reduced.device, tuple) else None) - reduced = cp.alu(op, chunk.copy_to_device(buf.device[dest], dest)) + cp = (reduced.mselect(src) if isinstance(reduced.device, tuple) else reduced).copy_to_device(buf.device[dest]) + reduced = cp.alu(op, chunk.mselect(dest).copy_to_device(buf.device[dest])) reduced_chunks.append(reduced) # allgather diff --git a/tinygrad_repo/tinygrad/schedule/indexing.py b/tinygrad_repo/tinygrad/schedule/indexing.py index 2d50c5f056..7c13ba6509 100644 --- a/tinygrad_repo/tinygrad/schedule/indexing.py +++ b/tinygrad_repo/tinygrad/schedule/indexing.py @@ -21,9 +21,7 @@ class IndexingContext: return UOp.range(s, next(self.range_idx)) if resolve(s!=1) else UOp.const(0) -ALWAYS_CONTIGUOUS: set[Ops] = {Ops.AFTER, Ops.BUFFER, Ops.ALLOC, - Ops.CONST, Ops.MSELECT, Ops.MSTACK, Ops.PARAM, - Ops.LOAD, Ops.CALL} +ALWAYS_CONTIGUOUS: set[Ops] = GroupOp.Defines|{Ops.AFTER, Ops.CONST, Ops.MSELECT, Ops.MSTACK, Ops.LOAD, Ops.CALL} def realize(ctx:IndexingContext, tr:UOp) -> None: ctx.realize_map[tr] = None @@ -66,7 +64,7 @@ def broadcast_rngs(x:UOp, src:UOp, rngs:tuple[UOp, ...]) -> tuple[UOp, ...]: # TODO: srcs contain (real data srcs, something else, ranges) and the boundary is confusing. see range_start def data_srcs(op:Ops, src:tuple[UOp, ...]) -> tuple[UOp, ...]: - if op in {Ops.PARAM, Ops.BUFFER, Ops.ALLOC, Ops.RANGE, Ops.SPECIAL}: return () + if op in GroupOp.Defines|{Ops.RANGE, Ops.SPECIAL}: return () if op in GroupOp.Movement|{Ops.INDEX, Ops.STAGE, Ops.REDUCE, Ops.AFTER, Ops.END, Ops.BACKEDGE, Ops.COPY}: return src[:1] return src @@ -77,7 +75,7 @@ def create_bufferize_and_index_srcs(ctx:IndexingContext, x:UOp) -> list[UOp]: for i, s in enumerate(x.src): new_src = s src_rngs = broadcast_rngs(x, s, ctx.range_map[x][0]) if x in ctx.range_map else () - if s.op in {Ops.PARAM, Ops.BUFFER, Ops.ALLOC, Ops.MSTACK, Ops.MSELECT, Ops.AFTER}: + if s.op in GroupOp.Defines|{Ops.MSTACK, Ops.MSELECT, Ops.AFTER}: if x in ctx.range_map and i < data_src_count: new_src = new_src.index(*src_rngs) elif s in ctx.realize_map: assert isinstance(ctx.realize_map[s], list), "realize map must contain range list" @@ -99,7 +97,8 @@ def create_bufferize_and_index_based_on_ranges(ctx:IndexingContext, x:UOp): def convert_pad_to_where_to_keep_behavior_local(ctx:IndexingContext, x:UOp): if x not in ctx.range_map: return None - bx = create_bufferize_and_index_based_on_ranges(ctx, x) + if x.op is Ops.INDEX and len(ctx.range_map[x][0]) == len(ctx.range_map[x][1]): return None + if (bx:=create_bufferize_and_index_based_on_ranges(ctx, x)).src[0] is x: return None valid: UOp = UOp.const(True).uprod([r.get_valid() for r in ctx.range_map[x][0]]) return valid.where(bx.src[0], UOp.const(x.dtype.const(0))) @@ -111,23 +110,18 @@ def convert_reduce_to_reduce_with_ranges(ctx:IndexingContext, x:UOp): new_ranges = list(ctx.range_map[x][0][:x.arg[1]]) return UOp(Ops.REDUCE, src=(bx.src[0],)+tuple(new_ranges), arg=(x.arg[0], 0)) -def _stack_select(r0:UOp, srcs:list[UOp], lo:int, hi:int) -> UOp: - # Bound lookup depth for large constant tables, rather than building a linear chain of thousands of WHEREs. - if hi-lo <= 8: - ret = srcs[hi-1] - for k in range(hi-2, lo-1, -1): ret = r0.eq(k).where(srcs[k], ret) - return ret - mid = (lo+hi)//2 - return (r0 < mid).where(_stack_select(r0, srcs, lo, mid), _stack_select(r0, srcs, mid, hi)) +def _stack_select(r0:UOp, srcs:list[UOp]) -> UOp: + # MSB-first bit tree + if len(srcs) == 1: return srcs[0] + half = 1 << ((len(srcs)-1).bit_length()-1) + return (r0 % (2*half) < half).where(_stack_select(r0, srcs[:half]), _stack_select(r0, srcs[half:])) def convert_stack_to_where(ctx:IndexingContext, x:UOp): # only data STACKs: shape tuple STACKs aren't in range_map if x not in ctx.range_map: return None # use the src list directly, a transient STACK of mid-rangeify srcs violates the spec shape rule srcs = create_bufferize_and_index_srcs(ctx, x) - r0 = ctx.range_map[x][1][0] - ret = _stack_select(r0, srcs, 0, len(srcs)) - return (r0 < 0).where(srcs[-1], ret) if len(srcs) > 8 else ret + return _stack_select(ctx.range_map[x][1][0], srcs) def remove_movement_op_after_rangeify(ctx:IndexingContext, x:UOp): if x in ctx.range_map or x.src[0].op is Ops.INDEX: return x.src[0] @@ -136,7 +130,7 @@ pm_apply_rangeify = PatternMatcher([ # REDUCE(op, axis) -> REDUCE(op) with ranges (UPat(Ops.REDUCE, name="x"), convert_reduce_to_reduce_with_ranges), # PAD -> WHERE - (UPat(Ops.PAD, name="x"), convert_pad_to_where_to_keep_behavior_local), + (UPat((Ops.PAD, Ops.INDEX), name="x"), convert_pad_to_where_to_keep_behavior_local), # STACK -> WHERE select on the leading range (UPat(Ops.STACK, name="x"), convert_stack_to_where), # finally, apply_rangeify @@ -289,6 +283,8 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> UOp: if x.op in GroupOp.Movement: rngs = apply_movement_op(x.op, x.src[0].shape, x.marg, rngs) # STACK: the leading range selects the src, srcs get the trailing ranges if x.op is Ops.STACK: rngs = out_rngs[1:] + # when the index srcs and output ranges of INDEX address every axis of its src, the index srcs select the leading axes + if x.op is Ops.INDEX and len(x.src)-1+len(out_rngs) == x.src[0].ndim: rngs = x.src[1:]+out_rngs # if the EXPAND is used to inject a range, we don't mark it as ending_ranges. otherwise we do. # NOTE: this doesn't actually always end a range, but this is why convs are realized, so for now we need it if x.op is Ops.EXPAND and all(isinstance(y, int) or y.op is not Ops.RANGE for y in x.shape): diff --git a/tinygrad_repo/tinygrad/schedule/multi.py b/tinygrad_repo/tinygrad/schedule/multi.py index 758753c7d4..7284c1419a 100644 --- a/tinygrad_repo/tinygrad/schedule/multi.py +++ b/tinygrad_repo/tinygrad/schedule/multi.py @@ -175,7 +175,7 @@ def shrink_multi(root:UOp, multi:UOp): if len(multi.sharding) != 1 or not isinstance(multi.device, tuple) or (s, l) not in part_bounds: raise RuntimeError(f"shrinking not supported for {root.marg=}") non_shard_shrink = tuple((0, shard_sz) if i == ax else t for i, t in enumerate(root.marg)) - return multi.src[0].copy_to_device(multi.device, arg=part_bounds.index((s, l)))._mop(Ops.SHRINK, non_shard_shrink) + return multi.src[0].mselect(part_bounds.index((s, l))).copy_to_device(multi.device)._mop(Ops.SHRINK, non_shard_shrink) val = multi.src[0]._mop(Ops.SHRINK, tuple(local_marg)) return val if not remaining else val.unshard(tuple(a for a,_ in remaining), tuple(r for _,r in remaining)) @@ -278,7 +278,7 @@ multi_pm = PatternMatcher([ (UPat(Ops.EXPAND, src=(UPat(Ops.UNSHARD, name="multi"), UPat()), name="root"), expand_multi), (UPat(Ops.PAD, src=(UPat(Ops.UNSHARD, name="multi"), UPat(), UPat()), name="root"), pad_multi), (UPat(Ops.SHRINK, src=(UPat(Ops.UNSHARD, name="multi"), UPat(), UPat()), name="root"), shrink_multi), - (UPat(Ops.PERMUTE, src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), permute_multi), + (UPat(Ops.PERMUTE, src=(UPat(Ops.UNSHARD, name="multi"), UPat()), name="root"), permute_multi), (UPat(Ops.STACK, name="root", custom_early_reject=set([Ops.UNSHARD])), stack_multi), (UPat(Ops.INDEX, src=(UPat(Ops.UNSHARD, name="multi"),), name="root", allow_any_len=True), index_multi), (UPat(Ops.AFTER, src=(UPat(Ops.UNSHARD), UPat(Ops.STORE, src=(UPat(Ops.UNSHARD, name="dest"), UPat(Ops.UNSHARD, name="src"))))), store_after_multi), diff --git a/tinygrad_repo/tinygrad/schedule/prepare.py b/tinygrad_repo/tinygrad/schedule/prepare.py index 82c5146813..76b792b5a2 100644 --- a/tinygrad_repo/tinygrad/schedule/prepare.py +++ b/tinygrad_repo/tinygrad/schedule/prepare.py @@ -1,7 +1,7 @@ from dataclasses import replace from tinygrad.dtype import dtypes, to_dtype from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, GroupOp -from tinygrad.uop.ops import graph_rewrite, rewrite_group, identity_element, resolve_returned_after +from tinygrad.uop.ops import graph_rewrite, MapCtx, rewrite_group, identity_element, resolve_returned_after from tinygrad.uop.movement import mop_cleanup from tinygrad.helpers import prod, getenv, all_int, DEBUG, SPLIT_REDUCEOP, OPENPILOT_HACKS, FLOAT16, argsort from tinygrad.schedule.indexing import apply_movement_op @@ -90,21 +90,6 @@ pm_mops = PatternMatcher([ # ***************** # 0. do some cleanup rewrites, mostly copied from the old stuff -# stop at materialization boundaries, including COPYs/STAGEs already lowered to AFTER+STORE -def store_hazard_boundary(s:UOp): - if s.op in {Ops.COPY, Ops.STAGE}: return False - if s.op is Ops.AFTER: return not any(d.op is Ops.STORE and d.src[0].base is s.src[0].base for d in s.src[1:]) - return True - -def fix_store_hazard(target:UOp, src:UOp): - if (base:=target.base) not in src.toposort(): return None - # PERMUTE reorders indices, SHRINK can have overlapping regions when dest is also shrunk - unsafe = {Ops.PERMUTE} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) - reaches_base: dict[UOp, bool] = {} - for s in src.toposort(gate=store_hazard_boundary): - reaches_base[s] = s is base or any(reaches_base.get(c) for c in s.src) - if reaches_base[s] and s.op in unsafe and not (s is target and s.op is Ops.SHRINK): return target.store(src.contiguous()) - def split_reduceop(reduce:UOp, x:UOp): if prod(reduce.shape) == 0: return None if not SPLIT_REDUCEOP or not all_int(x.shape) or prod(x.shape[:reduce.arg[1]]) UOp: # prepare for rangeify tsink = graph_rewrite(forward_call_outputs(sink), multi_pm, name="multi_pm") tsink = graph_rewrite(tsink, pm_mops+pm_inline_calls+pm_disk_copy, name="inline calls") - if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, ctx={}, name="fold moved afters") + if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, init_ctx=MapCtx, name="fold moved afters") tsink = graph_rewrite(tsink, pm_mops+earliest_rewrites, bottom_up=True, name="earliest rewrites") return tsink diff --git a/tinygrad_repo/tinygrad/schedule/rangeify.py b/tinygrad_repo/tinygrad/schedule/rangeify.py index bb4d8308ae..8238ec207d 100644 --- a/tinygrad_repo/tinygrad/schedule/rangeify.py +++ b/tinygrad_repo/tinygrad/schedule/rangeify.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import cast import itertools from tinygrad.dtype import dtypes, AddrSpace, Invalid @@ -159,7 +159,7 @@ pm_no_indexing_calls = PatternMatcher([ # the kernel graph is what gets executed: no shape views left in it, the storage of a value is just the storage pm_no_views = PatternMatcher([ (UPat((Ops.RESHAPE, Ops.SHRINK), name="v", - src=(UPat((Ops.AFTER, Ops.PARAM, Ops.UNSHARD, Ops.MSTACK, Ops.BUFFER, Ops.ALLOC)),), allow_any_len=True), lambda v: v.src[0]), + src=(UPat(GroupOp.Defines|{Ops.AFTER, Ops.UNSHARD, Ops.MSTACK}),), allow_any_len=True), lambda v: v.src[0]), ]) DEVICE_MAX_BUFS = {"WEBGPU": 8} # TODO: get from device? @@ -168,6 +168,11 @@ class LimitBufsContext: buf_cache: dict[UOp, frozenset[UOp]] = field(default_factory=dict) range_idx: itertools.count = field(default_factory=itertools.count) +def stage(ctx:LimitBufsContext, s:UOp) -> UOp: + # Insert bufferize: use fresh WEAK ranges, while the DEVICE range stays a launched axis + end_ranges = [x.replace(arg=(AxisType.WEAK, next(ctx.range_idx))) if x.axis_type is not AxisType.DEVICE else x for x in s.ranges] + return s.substitute(dict(zip(s.ranges, end_ranges))).bufferize(*end_ranges, arg=BufferizeOpts(device=s.device)).index(*s.ranges) + def _limit_bufs(ctx:LimitBufsContext, root:UOp): if (device:=root.device) is None: return None # no device, index related calculations device = device if isinstance(device, str) else device[0].split(":")[0] @@ -180,17 +185,14 @@ def _limit_bufs(ctx:LimitBufsContext, root:UOp): bufs = root.topovisit(visitor, ctx.buf_cache) if len(bufs) > MAX_BUFS - 1: # NOTE: this -1 is for the output buffer - srcs = [] - for s in root.src: - if s.op in GroupOp.Elementwise and s.device is not None: - # Insert bufferize: use fresh WEAK ranges, while the DEVICE range stays a launched axis - orig_ranges = s.ranges - end_ranges = [x.replace(arg=(AxisType.WEAK, next(ctx.range_idx))) if x.axis_type is not AxisType.DEVICE else x - for x in s.ranges] - s = s.substitute(dict(zip(orig_ranges, end_ranges))).bufferize(*end_ranges, arg=BufferizeOpts(device=s.device)).index(*orig_ranges) - srcs.append(s) - return root.replace(src=tuple(srcs)) -pm_limit_bufs = PatternMatcher([(UPat(set.union(GroupOp.Binary, GroupOp.Ternary), name="root"), _limit_bufs)]) + return root.replace(src=tuple(stage(ctx, s) if s.op in GroupOp.Elementwise and s.device is not None else s for s in root.src)) + +def fix_store_hazard(ctx:LimitBufsContext, dst:UOp, src:UOp): + loads = [u for u in src.toposort(gate=lambda u: u.op not in {Ops.STAGE, Ops.AFTER}) if u.op is Ops.INDEX and u.buf_uop is dst.buf_uop] + if any(u.src[1].get_idx() is not dst.src[1].get_idx() for u in loads): return dst.store(stage(ctx, src)) + +pm_limit_bufs = PatternMatcher([(UPat(set.union(GroupOp.Binary, GroupOp.Ternary), name="root"), _limit_bufs), + (UPat(Ops.INDEX, name="dst").store(UPat.var("src")), fix_store_hazard)]) # ***************** # 4. put in buffers for bufferize @@ -271,17 +273,18 @@ pm_add_buffers = pm_mops+pm_flatten_bufferize+PatternMatcher([ @dataclass class LocalAddBufferContext: - dg:int = 0 map:dict = field(default_factory=dict) - range:int = 0 + ranges:dict = field(default_factory=dict) -def debuf(ctx:LocalAddBufferContext, buf:UOp): - param = UOp.param(ctx.dg, buf.dtype, buf.max_numel(), addrspace=buf.addrspace, device=buf.device) +def add_buf(ctx:LocalAddBufferContext, buf:UOp): + if buf not in ctx.map: ctx.map[buf] = buf + raise BottomUpGate + +def debuf(slot:int, buf:UOp): + param = UOp.param(slot, buf.dtype, buf.max_numel(), addrspace=buf.addrspace, device=buf.device) ret = param.reshape(buf.max_shape) # if the buffer has symbolic shape, shrink the max-sized view to the actual shape if buf.max_shape != buf.shape: ret = ret.shrink(tuple((0, s) for s in buf.shape)) - if buf not in ctx.map: ctx.map[buf] = buf - ctx.dg += 1 return ret def handle_after(ctx:LocalAddBufferContext, after:UOp): @@ -291,12 +294,6 @@ def handle_after(ctx:LocalAddBufferContext, after:UOp): if buf not in ctx.map: ctx.map[buf] = after return buf -def renumber_range(ctx:LocalAddBufferContext, r:UOp): - if r.tag != (): return None - ret = r.replace(arg=(r.axis_type, ctx.range)+r.axis_id[1:], tag=None) - ctx.range += 1 - return ret - def check_buf_states(x:UOp): idxs = [s for s in x.toposort(gate=lambda x: x.op is not Ops.AFTER) if s.op is Ops.INDEX] read_from: dict[UOp, UOp] = {} @@ -305,27 +302,21 @@ def check_buf_states(x:UOp): to_define_global = PatternMatcher([ (UPat(Ops.STORE, name="x"), check_buf_states), - (UPat((Ops.BUFFER, Ops.ALLOC, Ops.MSTACK, Ops.MSELECT), name="buf"), debuf), + (UPat((Ops.BUFFER, Ops.ALLOC, Ops.MSTACK, Ops.MSELECT), name="buf"), add_buf), # Only storage parameters get kernel-local slots; scalar parameters retain their enclosing call's slots. (UPat(Ops.PARAM, name="buf"), lambda ctx, buf: - None if buf.tag != () or buf.addrspace is AddrSpace.ALU else debuf(ctx, buf)), + None if buf.addrspace is AddrSpace.ALU else add_buf(ctx, buf)), # ALU params are scalar symbolic values, not buffers. (UPat(Ops.INDEX, src=(UPat(Ops.PARAM, name="v"),)), lambda v: v if v.addrspace == AddrSpace.ALU else None), (UPat(Ops.AFTER, name="after"), handle_after), - # remove device from local BUFFERIZE - (UPat(Ops.STAGE, name="b"), lambda b: b.replace(arg=replace(b.arg, device=None))), - - # renumber the ranges starting with 0 so that kernel deduping works - (UPat(Ops.RANGE, name="r"), renumber_range), + (UPat(Ops.RANGE, name="r"), lambda ctx, r: ctx.ranges.setdefault(r.arg)), ]) -pm_add_param_range_tags = PatternMatcher([ - # Scalar parameters keep their identity across the call boundary. - (UPat((Ops.PARAM, Ops.RANGE), name="x"), lambda x: None if x.op is Ops.PARAM and x.addrspace is AddrSpace.ALU else x.rtag(())), -]) +# renumber the ranges starting with 0 so that kernel deduping works +pm_renumber_range = PatternMatcher([(UPat(Ops.RANGE, name="r"), lambda ctx, r: r.replace(arg=(r.axis_type, ctx[r.arg])+r.axis_id[1:]))]) def split_store(x:UOp) -> UOp|None: # if we have any open ranges here, we don't split. open DEVICE ranges are fine, they are bound per device at launch @@ -334,6 +325,8 @@ def split_store(x:UOp) -> UOp|None: # local kernel rewrite lctx = LocalAddBufferContext() ret = graph_rewrite(x, to_define_global+pm_flatten_range, ctx=lctx, name="kernel split", bottom_up=True) + ret = ret.substitute({b:debuf(i, b) for i,b in enumerate(lctx.map)}, walk=True) + ret = graph_rewrite(ret, pm_renumber_range, ctx={a:i for i,a in enumerate(lctx.ranges)}, walk=True) # create the Kernel. NOTE: buffers can be on different devices here now, they are compiled to SDMA copies later by schedule return ret.sink(arg=KernelInfo()).call(*lctx.map.values()) @@ -358,9 +351,9 @@ def get_kernel_graph(tsink:UOp) -> UOp: # bufferize -> store slots = [x.arg.slot for x in tsink.toposort() if x.op is Ops.ALLOC] paramarg_start: int = max([-1]+slots) + 1 - tsink = graph_rewrite(tsink, pm_add_buffers+pm_add_param_range_tags, ctx=itertools.count(paramarg_start), bottom_up=True, name="stage to store") - tsink = graph_rewrite(tsink, split_kernels, bottom_up=True, name="split kernels") + tsink = graph_rewrite(tsink, pm_add_buffers, ctx=itertools.count(paramarg_start), bottom_up=True, name="stage to store") tsink = graph_rewrite(tsink, pm_no_indexing_calls, name="remove indexing from call args") + tsink = graph_rewrite(tsink, split_kernels, bottom_up=True, name="split kernels") tsink = graph_rewrite(tsink, pm_no_views, name="remove views from the kernel graph") if VIZ: graph_rewrite(tsink, PatternMatcher([]), name="View Kernel Graph") diff --git a/tinygrad_repo/tinygrad/tensor.py b/tinygrad_repo/tinygrad/tensor.py index cdad2de291..506d6488e8 100644 --- a/tinygrad_repo/tinygrad/tensor.py +++ b/tinygrad_repo/tinygrad/tensor.py @@ -1,12 +1,12 @@ # inspired by https://github.com/karpathy/micrograd/blob/master/micrograd/engine.py from __future__ import annotations import time, functools, sys, inspect, pathlib, hashlib, weakref -from typing import Any, Callable, cast, get_args, ParamSpec, TypeVar, Generic, TYPE_CHECKING +from typing import Callable, cast, get_args, ParamSpec, TypeVar, TYPE_CHECKING if TYPE_CHECKING: import numpy from tinygrad.dtype import DType, DTypeLike, dtypes, ConstType, least_upper_dtype, to_dtype, _from_np_dtype, _to_np_dtype, PyConst -from tinygrad.helpers import all_int, getenv, fetch, Metadata, TRACEMETA, TracingKey, is_numpy_ndarray, prod +from tinygrad.helpers import all_int, getenv, fetch, PROFILE, TracingKey, is_numpy_ndarray, prod from tinygrad.helpers import cpu_profile, suppress_finalizing, disable_gc, VIZ -from tinygrad.uop.ops import UOp, Ops, sint, all_metadata, Variable, ConstLike, PatternMatcher, GroupOp, graph_rewrite, rewrite_group, pm_drop_after +from tinygrad.uop.ops import UOp, Ops, sint, Variable, ConstLike, PatternMatcher, GroupOp, graph_rewrite, rewrite_group, pm_drop_after from tinygrad.mixin.rand import RandMixin from tinygrad.schedule import create_linear_with_vars, contiguous_mops_to_view, is_store_after from tinygrad.device import Buffer, canonicalize_device, is_disk_device @@ -27,8 +27,7 @@ def _apply_map_to_tensors(applied_map:dict[UOp, UOp], name:str) -> None: new_sink = sink.substitute(applied_map, name=f"substitute {name}", walk=True) # set the relevant uop to the realized UOps - for t,s,ns in zip(scope_tensors, sink.src, new_sink.src): - if s is ns: continue + for t,ns in zip(scope_tensors, new_sink.src): t.uop = ns def _tensor_holds(u:UOp) -> bool: return any((t:=tref()) is not None and t.uop is u for tref in list(all_tensors)) @@ -104,7 +103,6 @@ class Tensor(RandMixin): def _apply_uop(self, fxn:Callable[..., UOp], *x:Tensor, **kwargs) -> Tensor: srcs = (self,)+x new_uop: UOp = fxn(*[t.uop for t in srcs], **kwargs) - if TRACEMETA >= 1 and (metadata:=_METADATA.get()) is not None: all_metadata[new_uop] = (metadata,) # directly create the Tensor ret = Tensor.__new__(Tensor) ret.uop, ret.grad, ret.is_param = new_uop, None, True @@ -201,8 +199,8 @@ class Tensor(RandMixin): sink = tensor_map.get(sink, sink) # Realized outputs become the storage their AFTER sequenced a store into. Compose with tensor_map before updating # Tensors so map values reference final storage. - becomes_map = {u:graph_rewrite(u.src[0], pm_drop_after, bottom_up=True, name="drop after").shrink_to(u.shape) - for u in sink.toposort() if is_store_after(u)} + storage = graph_rewrite(stores:=UOp.sink(*(u for u in sink.toposort() if is_store_after(u))), pm_drop_after, bottom_up=True, name="drop after") + becomes_map = {u:s.shrink_to(u.shape) for u,s in zip(stores.src, storage.src)} assert not any(x in becomes_map for x in becomes_map.values()) tensor_map = dict(zip(tensor_map, UOp.sink(*tensor_map.values()).substitute(becomes_map, walk=True).src)) _apply_map_to_tensors(becomes_map | tensor_map, name="bufferize") @@ -234,7 +232,7 @@ class Tensor(RandMixin): return self def assign(self, x:Tensor|PyConst|list|tuple) -> Tensor: - if self.dtype in dtypes.weaks: self.uop = self.uop.clone() + if self.dtype in dtypes.weaks: raise RuntimeError("cannot assign into a weak tensor; it has no storage") is_disk = self.uop.on_disk() if not isinstance(x, Tensor): x = Tensor(x, device="CPU" if is_disk else self.device, dtype=self.dtype) if self.uop is x.uop: return self # a self assign is a NOOP @@ -250,28 +248,26 @@ class Tensor(RandMixin): (b:=self._buffer()).copy_from(Buffer("PYTHON", b.nbytes, opaque=x._data())) return self assigned_to = self.uop.storage_base - # assigning to a value is initialization, not a write: the whole tensor is overwritten, so the pending value is dead. - # a pending CONTIGUOUS counts only if it's the whole target: writes through views of it store into its storage - if not assigned_to.has_buffer_identity() and (assigned_to.op is not Ops.STAGE or self.uop is assigned_to): + views = [self.uop] + while views[-1].op in GroupOp.Movement|{Ops.BITCAST, Ops.DETACH}: views.append(views[-1].src[0]) + # assigning to a pending value that views no other live tensor is initialization, not a write: the whole tensor is + # overwritten, so the pending value is dead. (a broadcast const like `const_like` is EXPAND(CONST) but views nothing) + if not assigned_to.has_buffer_identity() and not any(_tensor_holds(u) for u in views[1:]): self.uop = (x.uop.src[0] if x.uop.op is Ops.STAGE else x.uop).clone() return self # STORE+AFTER: STORE is the write effect (void), AFTER wraps the view for correct shape/ranging - assign = self.uop.after(store := self.uop.store(x.uop)) - views = [self.uop] - while views[-1].op in GroupOp.Movement|{Ops.BITCAST, Ops.DETACH}: views.append(views[-1].src[0]) ib = next((u for u in reversed(views) if u.has_buffer_identity() and (u is self.uop or _tensor_holds(u))), views[-1]) if ib is not self.uop: # a partial write needs storage to land in: a pending value gets explicit storage (a clone) target = ib if ib.has_buffer_identity(after_ok=True) else ib.clone() - if target is not ib: - assign = assign.substitute({ib: target}, walk=True) - store = assign.src[1] + store = self.uop.substitute({ib: target}, walk=True).store(x.uop) # view assign: the base reads "after the store into the view" (one AFTER level). replace the node under the # views (e.g. RESHAPE(BUFFER)) so @function's substitution catches it - _apply_map_to_tensors({ib: target.after(store)}, name="Embed View Assign") + for t in [t for tref in list(all_tensors) if (t:=tref()) is not None and t.uop.storage_base is assigned_to]: + t.uop = t.uop.substitute({ib: target.after(store)}, walk=True) else: # simple assign - self.uop = assign + self.uop = self.uop.after(self.uop.store(x.uop)) return self def _buffer(self) -> Buffer: @@ -303,29 +299,6 @@ class Tensor(RandMixin): assert fmt != "e" or sys.version_info >= (3, 12) return buf.as_memoryview().cast(fmt, self.shape) # type: ignore[arg-type,return-value] - # NOTE: list[Any] because return type is recursive (list[list[...]] for higher dimensions) - def tolist(self) -> PyConst|list[Any]: - """ - Returns the value of this tensor as a nested list. - Returns single value for const tensor. - - ```python exec="true" source="above" session="tensor" result="python" - t = Tensor([1, 2, 3, 4]) - print(t.tolist()) - ``` - ```python exec="true" source="above" session="tensor" result="python" - t = Tensor(5) - print(t.tolist()) - ``` - """ - # TODO: remove half once minimum python supports it - if self.dtype in (dtypes.half, dtypes.bfloat16, *dtypes.fp8s): return self.cast(dtypes.float32).tolist() - if 0 in self.shape: - assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" - def _tolist(shape:tuple[int, ...]): return [_tolist(shape[1:]) for _ in range(shape[0])] - return _tolist(self.shape) - return self.data().tolist() - def numpy(self) -> 'numpy.ndarray': """ Returns the value of this tensor as a `numpy.ndarray`. @@ -464,14 +437,10 @@ class Tensor(RandMixin): if device not in Tensor._device_seeds: seed = [int.from_bytes(hashlib.sha256(len(Tensor._device_seeds).to_bytes(4, "big")).digest(), "big"), Tensor._seed] Tensor._device_seeds[device] = Tensor(seed, device=device, dtype=dtypes.uint32) - Tensor._device_rng_counters[device] = Tensor([0, 0], device=device, dtype=dtypes.uint32) + Tensor._device_rng_counters[device] = Tensor([0], device=device, dtype=dtypes.uint64) counter = Tensor._device_rng_counters[device] - new_low = counter[0:1] + (num & 0xffffffff) - new_high = counter[1:2] + (num >> 32) + (new_low < counter[0]) - counter.assign(new_low.cat(new_high)) - low = counter[0:1] - (num & 0xffffffff) - high = counter[1:2] - (num >> 32) - (counter[0] < (num & 0xffffffff)) - return Tensor._device_seeds[device], low.cat(high) + counter.assign(counter + num) + return Tensor._device_seeds[device], (counter - num).bitcast(dtypes.uint32) # ***** toposort and backward pass ***** @@ -517,7 +486,7 @@ class Tensor(RandMixin): idx = [indices] if (isinstance(indices, list) and all_int(indices)) or not isinstance(indices, (tuple, list)) else list(indices) is_disk = self.uop.on_disk() advanced = any(isinstance(i, (Tensor, list, tuple)) for i in idx) - realized = is_disk or self.uop.base.op in {Ops.BUFFER, Ops.ALLOC} or self.uop._base_buffer_is_realized() + realized = is_disk or self.uop.base.op in {Ops.BUFFER, Ops.ALLOC} or self.uop.base.without_after.is_realized if (not self.uop.base.is_realized and self.is_floating_point()) or not (advanced or realized): if not isinstance(v, Tensor): v = Tensor(v, device=self.device, dtype=self.dtype) self.replace(self._getitem(indices, v)) @@ -526,9 +495,7 @@ class Tensor(RandMixin): if not isinstance(v, Tensor): v = Tensor(v, device=self.device, dtype=self.dtype) self.assign(self._getitem(indices, v)) else: # basic setitem - view = self[indices] - if isinstance(v, Tensor) and v.uop.op is Ops.AFTER and v.uop in view.uop.base.src: return - view.assign(v) + self[indices].assign(v) def __delitem__(self, indices) -> None: raise TypeError("Tensor does not support deleting items") @@ -569,26 +536,14 @@ class Tensor(RandMixin): P = ParamSpec("P") T = TypeVar("T") -# this tracks the tensor.py METADATA, contextvars.ContextVar was switched to this due to thread safety issues -class _ContextVar(Generic[T]): - def __init__(self, default:T): self.state:T = default - def get(self) -> T: return self.state - def set(self, x:T) -> T: - ret, self.state = self.state, x - return ret -_METADATA: _ContextVar[Metadata|None] = _ContextVar(default=None) - -def _metadata_wrapper(fn: Callable[P, T]) -> Callable[P, T]: +def _profile_wrapper(fn: Callable[P, T]) -> Callable[P, T]: + @functools.wraps(fn) def _wrapper(*args: P.args, **kwargs: P.kwargs) -> T: - if TRACEMETA < 1 or _METADATA.get() is not None: return fn(*args, **kwargs) - token = _METADATA.set(Metadata(name=fn.__name__)) + if not PROFILE: return fn(*args, **kwargs) with cpu_profile(TracingKey(fn.__name__), "USER"): - ret = fn(*args, **kwargs) - _METADATA.set(token) - return ret + return fn(*args, **kwargs) return _wrapper -if TRACEMETA >= 1: - for name, fn in inspect.getmembers(Tensor, inspect.isfunction): - if name in ["__class__", "__del__", "__init__", "__new__", "__repr__", "backward", "sequential", "gradient"]: continue - setattr(Tensor, name, functools.wraps(fn)(_metadata_wrapper(fn))) +for name, fn in inspect.getmembers(Tensor, inspect.isfunction): + if name in ["__class__", "__del__", "__init__", "__new__", "__repr__", "backward", "sequential", "gradient"]: continue + setattr(Tensor, name, _profile_wrapper(fn)) diff --git a/tinygrad_repo/tinygrad/uop/__init__.py b/tinygrad_repo/tinygrad/uop/__init__.py index 7991eae746..60d6f9204d 100644 --- a/tinygrad_repo/tinygrad/uop/__init__.py +++ b/tinygrad_repo/tinygrad/uop/__init__.py @@ -54,14 +54,14 @@ class Ops(FastEnum): # UnaryOps CAST = auto(); BITCAST = auto(); EXP2 = auto(); LOG2 = auto(); SIN = auto() - SQRT = auto(); RECIPROCAL = auto(); NEG = auto(); TRUNC = auto() + SQRT = auto(); RECIP = auto(); NEG = auto(); TRUNC = auto() # BinaryOps - ADD = auto(); MUL = auto(); SHL = auto(); SHR = auto(); CDIV = auto(); MAX = auto(); CMOD = auto() + ADD = auto(); MUL = auto(); SHL = auto(); SHR = auto(); TRUNCDIV = auto(); MAX = auto(); TRUNCMOD = auto() CMPLT = auto(); CMPNE = auto(); CMPEQ = auto() XOR = auto(); OR = auto(); AND = auto() THREEFRY = auto(); SUB = auto(); FDIV = auto(); POW = auto() - FLOORDIV = auto(); FLOORMOD = auto() + DIV = auto(); MOD = auto() # TernaryOps WHERE = auto(); MULACC = auto() @@ -101,9 +101,9 @@ class Ops(FastEnum): PYLITERAL = auto() class GroupOp: - Unary = {Ops.EXP2, Ops.LOG2, Ops.SIN, Ops.SQRT, Ops.RECIPROCAL, Ops.NEG, Ops.TRUNC} - Binary = {Ops.ADD, Ops.MUL, Ops.CDIV, Ops.MAX, Ops.CMOD, Ops.CMPLT, Ops.CMPNE, Ops.CMPEQ, - Ops.XOR, Ops.SHL, Ops.SHR, Ops.OR, Ops.AND, Ops.THREEFRY, Ops.SUB, Ops.FDIV, Ops.POW, Ops.FLOORDIV, Ops.FLOORMOD} + Unary = {Ops.EXP2, Ops.LOG2, Ops.SIN, Ops.SQRT, Ops.RECIP, Ops.NEG, Ops.TRUNC} + Binary = {Ops.ADD, Ops.MUL, Ops.TRUNCDIV, Ops.MAX, Ops.TRUNCMOD, Ops.CMPLT, Ops.CMPNE, Ops.CMPEQ, + Ops.XOR, Ops.SHL, Ops.SHR, Ops.OR, Ops.AND, Ops.THREEFRY, Ops.SUB, Ops.FDIV, Ops.POW, Ops.DIV, Ops.MOD} Ternary = {Ops.WHERE, Ops.MULACC} ALU = set.union(Unary, Binary, Ternary) Broadcastable = set.union(Binary, Ternary) @@ -120,7 +120,7 @@ class GroupOp: Commutative = {Ops.ADD, Ops.MUL, Ops.MAX, Ops.CMPNE, Ops.CMPEQ, Ops.XOR, Ops.AND, Ops.OR} # BinaryOps where f(f(a,b),c) = f(a,f(b,c)) - Associative = {Ops.ADD, Ops.MUL, Ops.AND, Ops.OR, Ops.MAX} + Associative = {Ops.ADD, Ops.MUL, Ops.AND, Ops.OR, Ops.MAX, Ops.XOR} # BinaryOps that satisfy f(x,x)=x see https://en.wikipedia.org/wiki/Idempotence Idempotent = {Ops.OR, Ops.AND, Ops.MAX} diff --git a/tinygrad_repo/tinygrad/uop/divandmod.py b/tinygrad_repo/tinygrad/uop/divandmod.py index 5a9905eff1..1157bb0e13 100644 --- a/tinygrad_repo/tinygrad/uop/divandmod.py +++ b/tinygrad_repo/tinygrad/uop/divandmod.py @@ -8,11 +8,9 @@ from tinygrad.helpers import unwrap def fold_divmod_general(d: UOp) -> UOp|None: x, y = d.src - if y.vmin==y.vmax==0: raise ZeroDivisionError(f"{'Division' if d.op is Ops.FLOORDIV else 'Mod'} by zero trying to rewrite {x.alu(d.op, y)}") + if y.vmin==y.vmax==0: raise ZeroDivisionError(f"{'Division' if d.op is Ops.DIV else 'Mod'} by zero trying to rewrite {x.alu(d.op, y)}") # x//y is constant - if (xdiv:=x//y).vmin == xdiv.vmax: return x - xdiv.vmin*y if d.op is Ops.FLOORMOD else xdiv.const_like(xdiv.vmin) - # PARAM // c is irreducible - if x.op is Ops.PARAM and y.op is Ops.CONST and x.arg.multiple_of % y.val == 0: return d.const_like(0) if d.op is Ops.FLOORMOD else None + if d.op is Ops.MOD and (xdiv:=x//y).vmin == xdiv.vmax: return x - xdiv.vmin*y # split uops for the rest of the processing x_peeled, const = x.pop_const() @@ -22,18 +20,21 @@ def fold_divmod_general(d: UOp) -> UOp|None: # these rules strictly require y to be a scalar constant > 0 if y.op is Ops.CONST and (c := y.val) > 0: # nested_div: (x%(k*c))//c -> (x//c)%k (requires k>0); the mod case is handled by remove_nested_mod below - if d.op is Ops.FLOORDIV and x.op is Ops.FLOORMOD and (k := x.src[1].divides(c)) is not None and k.vmin > 0: return x.src[0] // y % k + if d.op is Ops.DIV and x.op is Ops.MOD and (k := x.src[1].divides(c)) is not None and k.vmin > 0: return x.src[0] // y % k # remove_nested_mod in sum: (a%4 + b)%2 -> (a+b)%2 - if d.op is Ops.FLOORMOD: + if d.op is Ops.MOD: new_xs, changed = [], False for u in uops_no_const: - if u.op is Ops.FLOORMOD and u.src[1].divides(c) is not None: + if u.op is Ops.MOD and u.src[1].divides(c) is not None: u = u.src[0] changed = True new_xs.append(u) if changed: return (UOp.usum(*new_xs) + const) % y + # exact division keeps the structure of x: (q*c+k)//c -> q+k//c + if d.op is Ops.DIV and (q:=x_peeled.divides(c)) is not None: return q + const//c + # Shared decomposition for folding rules decomp = [(u.divides(f:=u.const_factor()),f) for u in uops_no_const] terms, factors = zip(*decomp) @@ -44,22 +45,16 @@ def fold_divmod_general(d: UOp) -> UOp|None: rem_choices = [(r, r-c) if (r:=f%c)*2 == c or len(terms)==1 else (min(r, r-c, key=abs),) for f in factors] for rems in itertools.product(*rem_choices): if (rem:=sum(r*v for r,v in zip(rems,terms))+const%c).vmin//c==rem.vmax//c: - if d.op is Ops.FLOORMOD: return rem - rem.vmin//c*c + if d.op is Ops.MOD: return rem - rem.vmin//c*c return sum((f-r)//c * v for f,r,v in zip(factors,rems,terms)) + const//c + rem.vmin//c - # gcd_with_remainder: factor out common gcd from numerator - if (g:=math.gcd(*factors, c)) > 1: - new_x = unwrap(x_peeled.divides(g)).simplify() + (const//g)%(c//g) - if new_x.vmin >= 0: - if d.op is Ops.FLOORMOD: return new_x % (c//g) * g + const%g - return new_x // (c//g) + const//c - # nest_by_factor: x//c -> (x//f)//(c//f), x%c -> (x//f%(c//f))*f + b where b=x%f - # FLOORDIV identity holds for any sign of x; FLOORMOD reconstruction needs x.vmin>=0 + # f is a term factor dividing c or the gcd of all factors and c + # DIV identity holds for any sign of x; MOD reconstruction needs x.vmin>=0 results = [] - for div in {abs(f) for u, f in zip(uops_no_const, factors) if u.op is not Ops.CONST and 1 < abs(f) < c and (c%f)==0}: + for div in {abs(f) for f in (*factors, math.gcd(*factors, c)) if 1 < abs(f) < c and c%f==0}: if (newxs := fold_divmod_general(x//div)) is not None: - if d.op is Ops.FLOORDIV: + if d.op is Ops.DIV: results.append((len(newxs.backward_slice), newxs // (c // div))) elif x.vmin >= 0 and newxs.vmin >= 0: b_parts = [f%div*t for f, t in zip(factors, terms) if f%div] @@ -78,7 +73,7 @@ def fold_divmod_general(d: UOp) -> UOp|None: gcd = UOp.gcd(*all_uops, y).simplify() if not (gcd.op is Ops.CONST and gcd.val==1): ret = unwrap(x.divide_exact(gcd)).alu(d.op, unwrap(y.divide_exact(gcd))) - return ret*gcd if d.op is Ops.FLOORMOD else ret + return ret*gcd if d.op is Ops.MOD else ret # factor_remainder: (d*x+y)//d -> x+y//d if y.vmin<0 or x.vmin<0: return None @@ -87,23 +82,23 @@ def fold_divmod_general(d: UOp) -> UOp|None: if (q:=u.divide_exact(y)) is not None: quo.append(q) elif y.op is Ops.CONST and (c:=u.const_factor())%y.val!=c: rem.append(u.divides(c)*(c%y.val)) - quo.append(u.divides(c)*(c//y.val) if d.op is Ops.FLOORDIV else u.const_like(0)) + quo.append(u.divides(c)*(c//y.val) if d.op is Ops.DIV else u.const_like(0)) else: rem.append(u) if not quo: return None new_x = sum(rem)+x.const_like(0) if new_x.vmin<0: return None - return new_x%y if d.op is Ops.FLOORMOD else new_x//y+sum(quo) + return new_x%y if d.op is Ops.MOD else new_x//y+sum(quo) div_and_mod_symbolic = PatternMatcher([ # ** 1. Fast Inline Rules ** # (x//c+a)//d -> (x+a*c)//(c*d) for c>0, d>0 - ((UPat.var("x")//UPat.cvar("c") + UPat.cvar("a"))//UPat.cvar("d"), lambda x,c,a,d: (x+a*c)//(c*d) if d.vmin>0 else None), + ((UPat.var("x", dtypes.weakint)//UPat.cvar("c") + UPat.cvar("a"))//UPat.cvar("d"), lambda x,c,a,d: (x+a*c)//(c*d) if d.vmin>0 else None), # (x+c)//d -> (x+c%d)//d + c//d ; (x+c)%d -> (x+c%d)%d (split the multiple of d out of the const, holds for any d!=0) - (UPat((Ops.FLOORDIV, Ops.FLOORMOD), src=(UPat.var("x", dtypes.weakint)+UPat.cvar("c"), UPat.cvar("d")), name="n"), + (UPat((Ops.DIV, Ops.MOD), src=(UPat.var("x", dtypes.weakint)+UPat.cvar("c"), UPat.cvar("d")), name="n"), lambda n,x,c,d: None if d.val==0 or c.val%d.val==c.val else - (x+c.val%d.val)//d + c.val//d.val if n.op is Ops.FLOORDIV else (x+c.val%d.val)%d), + (x+c.val%d.val)//d + c.val//d.val if n.op is Ops.DIV else (x+c.val%d.val)%d), # ** 2. Slow Rules ** - (UPat((Ops.FLOORDIV, Ops.FLOORMOD), dtypes.weakint, name="d"), lambda d: fold_divmod_general(d)), + (UPat((Ops.DIV, Ops.MOD), dtypes.weakint, name="d"), lambda d: fold_divmod_general(d)), ]) diff --git a/tinygrad_repo/tinygrad/uop/movement.py b/tinygrad_repo/tinygrad/uop/movement.py index 790ef23661..8237ba8eaf 100644 --- a/tinygrad_repo/tinygrad/uop/movement.py +++ b/tinygrad_repo/tinygrad/uop/movement.py @@ -11,9 +11,8 @@ mop_cleanup = PatternMatcher([ # remove noop RESHAPEs (UPat(Ops.RESHAPE, src=(UPat(name="x2"), UPat()), name="x"), lambda x,x2: x2 if x2._shape is not None and x2.shape == x.shape else None), # merge PERMUTEs - (UPat(Ops.PERMUTE, src=(UPat(Ops.PERMUTE, name="x2"),), name="x"), lambda x,x2: x2.replace(arg=tuple(x2.arg[i] for i in x.arg))), - # remove noop PERMUTEs - (UPat(Ops.PERMUTE, name="x"), lambda x: x.src[0] if list(x.arg) == list(range(len(x.arg))) else None), + (UPat(Ops.PERMUTE, src=(UPat(Ops.PERMUTE, name="x2"), UPat()), name="x"), + lambda x,x2: x2.src[0].permute(tuple(x2.marg[i] for i in x.marg))), # STACK on INDEX CONST (UPat(Ops.STACK, src=UPat(Ops.INDEX, src=(UPat.var("src"), UPat(Ops.CONST))), name="stk"), lambda src,stk: src if stk.shape == src.shape and list(range(len(stk.src))) == [x.src[1].val for x in stk.src] else None), diff --git a/tinygrad_repo/tinygrad/uop/ops.py b/tinygrad_repo/tinygrad/uop/ops.py index f07de64a52..9286a4eb76 100644 --- a/tinygrad_repo/tinygrad/uop/ops.py +++ b/tinygrad_repo/tinygrad/uop/ops.py @@ -7,7 +7,7 @@ from tinygrad.uop import Ops, GroupOp from tinygrad.dtype import ConstType, dtypes, DType, DTypeLike, truncate, least_upper_dtype, least_upper_float, Invalid, AddrSpace, strong_dtype from tinygrad.dtype import PyConst, InvalidType, bitcast from tinygrad.device import Buffer, BufferSpec, MultiBuffer, canonicalize_device, is_disk_device, TinyELF -from tinygrad.helpers import ContextVar, all_int, prod, getenv, all_same, Context, partition, temp, unwrap, T, argfix, Metadata, flatten, TRACEMETA +from tinygrad.helpers import ContextVar, all_int, prod, getenv, all_same, Context, partition, temp, unwrap, T, argfix, flatten from tinygrad.helpers import PROFILE, dedup, cdiv, cmod, floordiv, floormod, diskcache_put, to_function_name, cpu_profile, TracingKey from tinygrad.helpers import VIZ, SPEC, CAPTURE_PROCESS_REPLAY, DISALLOW_BROADCAST, get_shape, fully_flatten, to_tuple from tinygrad.helpers import colored, ansilen, printable, Target, is_image_shape, strides_for_shape @@ -26,9 +26,8 @@ class ParamArg: slot: int dtype: DType vmin_vmax: tuple[PyConst, PyConst]|None = None - multiple_of: int|None = None name: str|None = None - addrspace: AddrSpace|None = AddrSpace.GLOBAL + addrspace: AddrSpace = AddrSpace.GLOBAL device: str|tuple[str, ...]|None = None volatile: bool = False # (h, w) if this is an image2d buffer, then the size CONST is h*w*4 @@ -41,7 +40,7 @@ class ParamArg: # the memory the linker gives an ALLOC spec: BufferSpec|None = None def __repr__(self): - fields = (("vmin_vmax", None), ("multiple_of", None), ("name", None), ("addrspace", AddrSpace.GLOBAL), ("device", None), + fields = (("vmin_vmax", None), ("name", None), ("addrspace", AddrSpace.GLOBAL), ("device", None), ("volatile", False), ("image", None), ("bind_on_realize", False), ("val", None), ("spec", None)) args = [repr(self.slot), repr(self.dtype)] + [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] if self.buffer is not None: @@ -52,10 +51,13 @@ axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.LOCAL: "l", axis_colors = {AxisType.DEVICE: "green", AxisType.GLOBAL: "blue", AxisType.LOCAL: "cyan", AxisType.WARP: "CYAN", AxisType.WEAK: "red", AxisType.LOOP: "red", AxisType.UPCAST: "yellow"} -range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.END: 1, Ops.CALL: 1, Ops.LINEAR: 0} +range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.END: 1, Ops.LINEAR: 0} # https://en.wikipedia.org/wiki/Identity_element def identity_element(op:Ops, dt:DType) -> PyConst: return dt.const({Ops.ADD:0, Ops.MUL:1, Ops.MAX:dt.min}[op]) +# unsigned arithmetic wraps mod 2^n, so its interval holds only if it fits the dtype +def wrapped(dt:DType, lo:PyConst, hi:PyConst) -> tuple[PyConst, PyConst]: + return (lo, hi) if not dtypes.is_unsigned(dt) or 0 <= lo and hi <= dt.max else (dt.min, dt.max) # With True as the default, this matches the old symbolic behavior def resolve(x:UOp|bool, default:bool=True): @@ -128,8 +130,8 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: assert isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[1], DType), f"CUSTOM/CUSTOMI arg must be (str, DType), got {arg}" return arg[1] case Ops.INS: - # arg is (instruction, dtype), a queue command or an asm line is void - assert isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[1], DType), f"INS arg must be (instruction, DType), got {arg}" + # arg is (instruction, dtype, *registers), a queue command or an asm line is void + assert isinstance(arg, tuple) and len(arg) >= 2 and isinstance(arg[1], DType), f"INS arg must be (instruction, DType, *registers), got {arg}" return arg[1] case Ops.INDEX: # an image access is always float, no matter the storage dtype @@ -143,7 +145,7 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: return src[0].dtype case Ops.CMPLT | Ops.CMPNE | Ops.CMPEQ: return dtypes.bool - case Ops.SIN | Ops.LOG2 | Ops.EXP2 | Ops.SQRT | Ops.RECIPROCAL: + case Ops.SIN | Ops.LOG2 | Ops.EXP2 | Ops.SQRT | Ops.RECIP: return dtypes.bool if src[0].base.is_invalid else least_upper_float(src[0].dtype) case Ops.WHERE: if src[0].dtype != dtypes.bool: raise RuntimeError(f"where cond must be bool, got {src[0].dtype}") @@ -187,11 +189,10 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: class UOpMetaClass(type): ucache:dict[tuple, weakref.ReferenceType[UOp]] = {} - def __call__(cls, op:Ops, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, metadata:tuple[Metadata,...]|None=None): + def __call__(cls, op:Ops, src:tuple[UOp,...]=tuple(), arg:Any=None): # NOTE: the key must separate nodes of different dtype: a CONST's dtype is the type of its arg, and True == 1 as dict keys - if (wret:=UOpMetaClass.ucache.get(key:=(op, src, arg, tag, type(arg)), None)) is not None and (ret:=wret()) is not None: return ret - UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(op, src, arg, tag)) - if metadata is not None: all_metadata[created] = metadata + if (wret:=UOpMetaClass.ucache.get(key:=(op, src, arg, type(arg)), None)) is not None and (ret:=wret()) is not None: return ret + UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(op, src, arg)) if SPEC > 1: from tinygrad.uop.spec import spec_full if SPEC > 2: @@ -201,9 +202,6 @@ class UOpMetaClass(type): if fret is not True: raise RuntimeError(f"SPEC ISSUE {fret}: {created}") return created -# some uops map to other stuff -all_metadata:weakref.WeakKeyDictionary[UOp, tuple[Metadata, ...]] = weakref.WeakKeyDictionary() # TODO: should this be here? - # recursive_property replaces functools.cached_property in recursive UOp functions to prevent RecursionError class recursive_property: def __init__(self, fxn): @@ -231,20 +229,18 @@ class UOp(RandMixin, metaclass=UOpMetaClass): op:Ops src:tuple[UOp, ...] = tuple() arg:Any = None - tag:Any = None @recursive_property def dtype(self) -> DType: return dtype_from_uop(self.op, self.src, self.arg) def __del__(self): # NOTE: getattr because this object may be partially constructed (e.g. if __init__ raised, like the BEAM timeout SIGALRM) - try: del UOpMetaClass.ucache[(self.op, self.src, self.arg, self.tag, type(self.arg))] + try: del UOpMetaClass.ucache[(self.op, self.src, self.arg, type(self.arg))] except (AttributeError, KeyError): pass - def __reduce__(self): return UOp, (self.op, self.src, self.arg, self.tag, self.metadata) + def __reduce__(self): return UOp, (self.op, self.src, self.arg) def replace(self, **kwargs) -> UOp: - new_args = (kwargs.pop("op", self.op), kwargs.pop("src", self.src), kwargs.pop("arg", self.arg), kwargs.pop("tag", self.tag)) + new_args = (kwargs.pop("op", self.op), kwargs.pop("src", self.src), kwargs.pop("arg", self.arg)) assert len(kwargs) == 0, f"unused kwargs in replace {list(kwargs)}" - if (self.op, self.src, self.arg, self.tag) == new_args: return self + if (self.op, self.src, self.arg) == new_args: return self return UOp(*new_args) - def rtag(self, tag=True): return self.replace(tag=tag) @property def val(self): if self.op is Ops.CONST: return self.arg @@ -263,7 +259,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): def argstr(self): if self.op is Ops.REDUCE: return f'({", ".join(map(str, self.arg))})' return repr(self.arg) - def tagstr(self): return f", tag={self.tag}" if self.tag is not None else "" @functools.cached_property def backward_slice(self:UOp) -> dict[UOp, None]: @@ -448,7 +443,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @functools.cached_property def ended_ranges(self) -> tuple[UOp, ...]: - if self.op is Ops.CALL and self.body.op is Ops.CUSTOM_FUNCTION: return () if self.op is Ops.END: return tuple(r for r in self.src[1:] if r.op is Ops.RANGE) if self.op is Ops.BACKEDGE: return self.src[1:2] # the condition's other ranges remain live if self.op in range_start: return self.src[range_start[self.op]:] @@ -490,12 +484,12 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # *** uop evaluation *** - def simplify(self, tracked=False): + def simplify(self): if self.op is Ops.CONST: return self - if self.op is Ops.SINK and all(s.op is Ops.CONST or (s.op is Ops.STACK and len(s.src) == 0) for s in self.src): return self + if self.op is Ops.SINK and all(s.op is Ops.CONST or (s.op is Ops.STACK and all(x.op is Ops.CONST for x in s.src)) for s in self.src): return self # late import! from tinygrad.uop.symbolic import symbolic - with Context(TRACK_MATCH_STATS=0 if not tracked else TRACK_MATCH_STATS.value): + with Context(TRACK_MATCH_STATS=0): return graph_rewrite(self, symbolic, name="simplify") def ssimplify(self) -> UOp|ConstType: if (ret := self.simplify()).op is Ops.CAST and ret.src[0].op is Ops.CONST: return ret.dtype.const(ret.src[0].val) @@ -523,20 +517,18 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @recursive_property def trace_num(self): num = next(ucount) - # tags can contain UOps (callify tags nodes with their originals): store them as trace_nums, same as srcs - tag = tuple(t.trace_num if isinstance(t, UOp) else t for t in self.tag) if isinstance(self.tag, tuple) else self.tag # the trace must not retain the device Buffer: store a placeholder instead (the real one would pin memory and fail pickling) arg = replace(self.arg, buffer=cast("Buffer", object())) if isinstance(self.arg, ParamArg) and self.arg.buffer is not None else self.arg - uop_fields[num] = (self.op, tuple(s.trace_num for s in self.src), arg, tag)+((self.metadata,) if TRACEMETA>=2 else ()) + uop_fields[num] = (self.op, tuple(s.trace_num for s in self.src), arg) return num # *** uop syntactic sugar *** def sink(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument return UOp(Ops.SINK, src=tuple([x for x in srcs if x is not None]), **kwargs) - def group(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument + def group(*srcs:UOp|None): # pylint: disable=no-self-argument if len(srcs) == 1 and isinstance(srcs[0], UOp): return srcs[0] - return UOp(Ops.STACK).after(*[x for x in srcs if x is not None], **kwargs) + return UOp(Ops.STACK).after(*[x for x in srcs if x is not None]) @property def body(self) -> UOp: """the body of a CALL: the program, copy or function reference being called (its first src)""" @@ -557,7 +549,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): def index(self, *srcs:UOp|int|None, **kwargs): new_srcs: list[UOp] = [UOp.const(x) if isinstance(x, int) else x for x in srcs if x is not None] if len(new_srcs) == 1 and new_srcs[0].op is Ops.CONST and self.op is Ops.STACK: return self.src[new_srcs[0].val] - return UOp(Ops.INDEX, src=(self,)+tuple(new_srcs), **kwargs) + ret = UOp(Ops.INDEX, src=(self,)+tuple(new_srcs), **kwargs) + return UOp.invalid().expand(ret.shape) if self.base.is_invalid else ret def __getitem__(self, idx): # buffers index into INDEX UOps (scalar lookup); everything else uses the shared mixin view path if self.addrspace in (None, AddrSpace.ALU) or self.device is not None: return super(UOp, self).__getitem__(idx) @@ -589,16 +582,16 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if count == 1: return self return UOp(Ops.STACK, src=(self,)*count) def load(self, *src:UOp, **kwargs): return UOp(Ops.LOAD, src=(self,)+src, **kwargs) - def store(self, src:UOp|ConstType, gate:UOp|None=None, **kwargs): + def store(self, src:UOp|ConstType, gate:UOp|None=None): srcs = (self, self.const_like(src) if not isinstance(src, UOp) else src) + ((gate,) if gate is not None else ()) - return UOp(Ops.STORE, src=srcs, **kwargs) + return UOp(Ops.STORE, src=srcs) def end(self, *src:UOp): return UOp(Ops.END, src=(self,)+src) if len(src) else self def backedge(self, loop:UOp, cond:UOp): return UOp(Ops.BACKEDGE, src=(self, loop, cond)) - def after(self, *src:UOp, **kwargs): return UOp(Ops.AFTER, src=(self,)+src, **kwargs) if len(src) else self + def after(self, *src:UOp): return UOp(Ops.AFTER, src=(self,)+src) if len(src) else self @property def without_after(self) -> UOp: return self.src[0].without_after if self.op is Ops.AFTER else self def barrier(self, *src:UOp): return UOp(Ops.BARRIER, src=(self,)+src) - def ins(self, arg, **kwargs): return UOp(Ops.INS, kwargs.pop("src", self.src), (arg, kwargs.pop("dtype", self.dtype)), kwargs.pop("tag", self.tag)) + def ins(self, arg, *regs, **kwargs): return UOp(Ops.INS, kwargs.pop("src", self.src), (arg, kwargs.pop("dtype", self.dtype), *regs)) def contract(self, *rngs:UOp): assert all(x.axis_type == AxisType.UPCAST for x in rngs), "all contract ranges must be upcast" return UOp.stack(*[self.substitute(dict(zip(rngs, [r.const_like(i) for r,i in zip(rngs, idx)]))) @@ -618,8 +611,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @staticmethod def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, src=(UOp.const(b),), arg=dtype) @staticmethod - def range(end:sint, axis_id, axis_type=AxisType.WEAK, *, dtype=dtypes.weakint, **kwargs): - return UOp(Ops.RANGE, src=(sint_to_uop(end, dtype),), arg=(axis_type, axis_id), **kwargs) + def range(end:sint, axis_id, axis_type=AxisType.WEAK, *, dtype=dtypes.weakint): + return UOp(Ops.RANGE, src=(sint_to_uop(end, dtype),), arg=(axis_type, axis_id)) @staticmethod def loop(axis_id:int): return UOp(Ops.RANGE, src=(UOp(Ops.NOOP),), arg=(AxisType.WEAK, axis_id)) @staticmethod @@ -684,12 +677,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): """(axis, RANGE) pairs this value is sharded over (the source of truth for shard bounds/counts).""" return tuple(zip(self.arg, self.src[1:])) if self.op is Ops.UNSHARD else () - @property - def bounds(self): - if self.axis is None: raise RuntimeError("bounds is not defined when axis is None") - dcount = int(self.src[1].vmax)+1 if self.op is Ops.UNSHARD else len(self.device) - return tuple(itertools.pairwise(itertools.accumulate([self.src[0].shape[self.axis] for _ in range(dcount)], initial=0))) - @functools.cached_property def axis(self) -> int|None: # COPY removes axis. TODO: add more tests for this, and consider MSELECT/MSTACK @@ -733,21 +720,17 @@ class UOp(RandMixin, metaclass=UOpMetaClass): copied = self.copy_to_device(devices) return copied if axis is None else copied._shard(axis, UOp.range(len(devices), 0, AxisType.DEVICE)).unshard(axis) - def copy_to_device(self, device:str|tuple[str, ...], arg=None): + def copy_to_device(self, device:str|tuple[str, ...]): if is_disk_device(device): raise RuntimeError("COPY to DISK is not allowed; use STORE to an explicit disk buffer") - assert arg is None or isinstance(self.device, tuple) - inp = self if arg is None else UOp(Ops.MSELECT, src=(self,), arg=arg) - if inp.dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {inp.dtype}") + if self.dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {self.dtype}") # multi-device COPYs carry the DEVICE range as src[1] (like UNSHARD's sharding ranges) - return UOp(Ops.COPY, src=(inp, *UOp.device_range_src(device)), arg=device) + return UOp(Ops.COPY, src=(self, *UOp.device_range_src(device)), arg=device) def store_call(self, src:UOp) -> UOp: """Executable bulk transfer into this buffer.""" return self.param_like(0).store(src.param_like(1)).call(self, src) def mselect(self, arg:int) -> UOp: return UOp(Ops.MSELECT, src=(self,), arg=arg) def mstack(self, *srcs: UOp) -> UOp: return UOp(Ops.MSTACK, src=(self,)+srcs) if len(srcs) else self - @property - def metadata(self) -> tuple[Metadata, ...]|None: return all_metadata.get(self, None) # little helpers def on_disk(self:UOp): return isinstance(self.device, str) and self.device.startswith("DISK") @@ -764,19 +747,15 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # base with UNSHARD @property - def unsharded_base(self) -> UOp: - if self.op in GroupOp.Movement: return self.src[0].base - if self.op is Ops.DETACH: return self.src[0].base # DETACH can't change base - # TODO: why can't this be in normal base? - if self.op is Ops.UNSHARD: return self.src[0].base - return self + def unsharded_base(self) -> UOp: return self.src[0].base if self.op is Ops.UNSHARD else self.base # the storage this uop ultimately targets: base with UNSHARD, BITCAST and AFTER stripped + @recursive_property + def _storage_base(self) -> UOp|None: + if self.op in GroupOp.Movement|{Ops.DETACH, Ops.BITCAST, Ops.AFTER, Ops.UNSHARD}: return self.src[0].storage_base + return None @property - def storage_base(self) -> UOp: - b = self.unsharded_base - while b.op in {Ops.BITCAST, Ops.AFTER, Ops.UNSHARD}: b = b.src[0].unsharded_base - return b + def storage_base(self) -> UOp: return self if (base:=self._storage_base) is None else base # cached property here makes external_uop_gc fail, why? @property @@ -790,7 +769,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): match self.op: case Ops.RESHAPE | Ops.EXPAND: return self.src[1].as_shape case Ops.PAD | Ops.SHRINK: return tuple(zip(self.src[1].as_shape, self.src[2].as_shape)) - case Ops.PERMUTE: return self.arg + case Ops.PERMUTE: return self.src[1].as_shape case _: raise RuntimeError(f"{self.op} is not a MovementOp") def _mop(self, op:Ops, arg) -> UOp: @@ -802,14 +781,13 @@ class UOp(RandMixin, metaclass=UOpMetaClass): match op: case Ops.RESHAPE | Ops.EXPAND: src_args = [arg] case Ops.PAD | Ops.SHRINK: src_args = list(zip(*arg)) - case Ops.PERMUTE: src_args = [] + case Ops.PERMUTE: src_args = [arg] case Ops.STACK: srcs = (self,)+tuple(arg) dtype = dtype_from_uop(Ops.STACK, srcs, None) return UOp(Ops.STACK, src=tuple(u if u.base.is_invalid else u.ccast(dtype) for u in srcs)) case _: raise RuntimeError(f"{op} is not a MovementOp") usrcs = [shape_to_shape_arg(arg) for arg in src_args] - if len(usrcs) == 0: return UOp(op, src=(self,), arg=arg) return UOp(op, src=(self,)+UOp.sink(*usrcs).simplify().src) # *** uop Buffer stuff *** @@ -817,8 +795,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): unique_num = itertools.count(0) def getaddr(self, device=None) -> UOp: - if self.without_after.op not in {Ops.BUFFER, Ops.ALLOC, Ops.SHRINK, Ops.BITCAST, Ops.BINARY, - Ops.MSTACK, Ops.MSELECT, Ops.PARAM, Ops.LINEAR}: return self + if self.without_after.op not in GroupOp.Defines|{Ops.SHRINK, Ops.BITCAST, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.LINEAR}: return self return UOp(Ops.GETADDR, src=(self,), arg=device or to_tuple(self.device)[0]) @staticmethod def device_range_src(device:str|tuple[str, ...]|None) -> tuple[UOp, ...]: @@ -891,6 +868,8 @@ class UOp(RandMixin, metaclass=UOpMetaClass): return self.device is None or self.dtype in dtypes.weaks @recursive_property def addrspace(self) -> AddrSpace|None: + # anything that has dtype void has no memory backing, it lives in the ALU + if self.dtype is dtypes.void: return AddrSpace.ALU if self.op is Ops.PARAM: return self.arg.addrspace if self.op in {Ops.BUFFER, Ops.ALLOC}: return self.arg.addrspace if self.op in {Ops.SPECIAL, Ops.RANGE, Ops.CONST}: return AddrSpace.ALU @@ -911,7 +890,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if self.op is Ops.MSTACK: return UOp(Ops.MSTACK, src=tuple(x.buf_uop for x in self.src)) if self.base.op is Ops.AFTER: return self.base.src[0].buf_uop.base s = self - while len(s.src) and s.op not in {Ops.BUFFER, Ops.ALLOC, Ops.PARAM, Ops.STAGE, Ops.MSTACK}: s = s.src[0] + while len(s.src) and s.op not in GroupOp.Defines|{Ops.STAGE, Ops.MSTACK}: s = s.src[0] return s def contiguous_view(self) -> tuple[UOp, int]|None: @@ -944,12 +923,6 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if after_ok and self.op == Ops.AFTER: return self.src[0].has_buffer_identity(after_ok) return self.op in GroupOp.Defines - def _base_buffer_is_realized(self) -> bool: - """Walk through AFTER chain to find if the underlying buffer is realized (has allocated memory).""" - u = self.base - while u.op is Ops.AFTER: u = u.src[0] - return u.is_realized - @functools.cached_property def _buffer_view(self) -> tuple[UOp, int]: # Cache only the UOp and byte offset, never an allocated Buffer view. @@ -998,10 +971,9 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # *** uop Variable stuff *** @staticmethod - def variable(name:str, min_val:PyConst, max_val:PyConst, dtype:DType=dtypes.weakint, multiple_of:int=1) -> UOp: + def variable(name:str, min_val:PyConst, max_val:PyConst, dtype:DType=dtypes.weakint) -> UOp: # a Variable is a scalar ALU PARAM with a name and a value range; binding it sets the val payload on the arg - return UOp(Ops.PARAM, src=(UOp(Ops.STACK),), arg=ParamArg(-1, dtype, name=name, vmin_vmax=(min_val, max_val), - multiple_of=multiple_of, addrspace=AddrSpace.ALU)) + return UOp(Ops.PARAM, src=(UOp(Ops.STACK),), arg=ParamArg(-1, dtype, name=name, vmin_vmax=(min_val, max_val), addrspace=AddrSpace.ALU)) @property def is_variable(self) -> bool: # a Variable is a scalar ALU PARAM that carries a value range @@ -1019,12 +991,10 @@ class UOp(RandMixin, metaclass=UOpMetaClass): uval = UOp.const(val) if isinstance(val, int) else val assert uval.op is Ops.CONST, f"bind value must be a CONST, not {uval.op}" assert self.vmin <= uval.vmin and uval.vmax <= self.vmax, f"bind {val} not in range [{self.vmin}, {self.vmax}]" - assert uval.divides(self.arg.multiple_of) is not None, f"bind {val} not divisible by {self.arg.multiple_of}" return self.replace(arg=replace(self.arg, val=uval.val)) def unbound(self) -> Variable: assert self.is_variable, f"op is {self.op}, need Variable" - # strip the tag too: tags are kernel-graph processing state, the unbound Variable is the canonical node - return self.replace(arg=replace(self.arg, val=None), tag=None) + return self.replace(arg=replace(self.arg, val=None)) def unbind(self) -> tuple[Variable, int]: assert self.is_bound_var, f"can't unbind {self}" return self.unbound(), self.arg.val @@ -1046,29 +1016,22 @@ class UOp(RandMixin, metaclass=UOpMetaClass): """largest known int that divides self""" # TODO: for negatives it's not the largest if self.op is Ops.CONST: return self.val - if self.op is Ops.STACK: return math.gcd(*[x.const_factor() for x in self.src]) if self.op is Ops.ADD: return math.gcd(self.src[0].const_factor(), self.src[1].const_factor()) if self.op is Ops.MUL: return self.src[0].val if self.src[0].op is Ops.CONST else self.src[1].val if self.src[1].op is Ops.CONST else 1 - if self.op in GroupOp.Defines and self.arg.multiple_of is not None: return self.arg.multiple_of return 1 def divides(self, v:int) -> UOp|None: if v==1: return self if self.op is Ops.CONST: return self.const_like(self.val//v) if self.val%v == 0 else None - if self.op is Ops.STACK: - srcs = tuple(s.divides(v) for s in self.src) - return None if any(s is None for s in srcs) else UOp(Ops.STACK, src=cast(tuple[UOp, ...], srcs)) if self.op is Ops.ADD: return d0+d1 if (d0:=self.src[0].divides(v)) is not None and (d1:=self.src[1].divides(v)) is not None else None if self.op is Ops.MUL: if (d0:=self.src[0].divides(v)) is not None: return d0 * self.src[1] if (d1:=self.src[1].divides(v)) is not None: return self.src[0] * d1 - if self.op in GroupOp.Defines and self.arg.multiple_of is not None: - return self // v if self.arg.multiple_of%v == 0 else None return None # generic None if we aren't sure def pop_const(self, op=Ops.ADD) -> tuple[UOp, PyConst]: # NOTE: assume Invalid ALU is resolved return (self.src[0], self.src[1].val) if self.op is op and self.src[1].op is Ops.CONST else (self, identity_element(op, self.dtype)) @staticmethod def gcd(*uops: UOp) -> UOp: - # Strip explicit coefficients, not multiple_of, so symbolic factors stay recognizable by divide_exact. + # Strip explicit coefficients so symbolic factors stay recognizable by divide_exact. terms, factors = zip(*[u.pop_const(Ops.MUL) for u in uops]) count = functools.reduce(operator.and_, [collections.Counter(term.split_uop(Ops.MUL)) for term in terms]) if not count: factors = tuple(u.const_factor() for u in uops) @@ -1091,28 +1054,28 @@ class UOp(RandMixin, metaclass=UOpMetaClass): def _min_max(self) -> tuple[PyConst, PyConst]: if self.op in GroupOp.Binary and not dtypes.is_float(self.dtype): (s0_vmin, s0_vmax), (s1_vmin, s1_vmax) = self.src[0]._min_max, self.src[1]._min_max - if self.op is Ops.ADD: return s0_vmin+s1_vmin, s0_vmax+s1_vmax - if self.op is Ops.SUB: return s0_vmin-s1_vmax, s0_vmax-s1_vmin - if self.op is Ops.AND and dtypes.is_int(self.dtype) and s1_vmin == s1_vmax >= 0: + if self.op is Ops.ADD: return wrapped(self.dtype, s0_vmin+s1_vmin, s0_vmax+s1_vmax) + if self.op is Ops.SUB: return wrapped(self.dtype, s0_vmin-s1_vmax, s0_vmax-s1_vmin) + if self.op is Ops.AND and dtypes.is_int(self.dtype) and s1_vmin == s1_vmax and (s0_vmin >= 0 or s1_vmax >= 0): return 0, s1_vmax if s0_vmin < 0 else min(s0_vmax, int(s1_vmax) & ((1 << int(s0_vmax).bit_length()) - 1)) - if self.op is Ops.MUL: return min(vals:=(s0_vmin*s1_vmin, s0_vmin*s1_vmax, s0_vmax*s1_vmin, s0_vmax*s1_vmax)), max(vals) + if self.op is Ops.MUL: return wrapped(self.dtype, min(vals:=(s0_vmin*s1_vmin, s0_vmin*s1_vmax, s0_vmax*s1_vmin, s0_vmax*s1_vmax)), max(vals)) # SHL/SHR on consts only - if self.op is Ops.SHL and s1_vmin == s1_vmax and all_int(t:=(s0_vmin, s0_vmax, s1_vmin)): return t[0] << t[2], t[1] << t[2] + if self.op is Ops.SHL and s1_vmin == s1_vmax and all_int(t:=(s0_vmin, s0_vmax, s1_vmin)): return wrapped(self.dtype, t[0] << t[2], t[1] << t[2]) if self.op is Ops.SHR and s1_vmin == s1_vmax and all_int(t:=(s0_vmin, s0_vmax, s1_vmin)): return t[0] >> t[2], t[1] >> t[2] - if self.op is Ops.CMOD: + if self.op is Ops.TRUNCMOD: if (c:=s1_vmin) == s1_vmax > 0: return (0 if s0_vmin > 0 else s0_vmin if 0 >= s0_vmin > -c else -(s1_vmax-1), 0 if s0_vmax < 0 else s0_vmax if 0 <= s0_vmax < c else c-1) if s1_vmin > 0: return (0, s1_vmax-1) if s0_vmin >= 0 else (-(s1_vmax-1), 0) if s0_vmax <= 0 else (-(s1_vmax-1), s1_vmax-1) if s1_vmax < 0: return (0, -s1_vmin-1) if s0_vmin >= 0 else (-(-s1_vmin-1), 0) if s0_vmax <= 0 else (-(-s1_vmin-1), -s1_vmin-1) - if self.op is Ops.CDIV: + if self.op is Ops.TRUNCDIV: assert isinstance(s0_vmin, int) and isinstance(s0_vmax, int) and isinstance(s1_vmin, int) and isinstance(s1_vmax, int) if s1_vmin*s1_vmax>0: return min(vals:=(cdiv(s0_vmin, s1_vmin), cdiv(s0_vmin, s1_vmax), cdiv(s0_vmax, s1_vmin), cdiv(s0_vmax, s1_vmax))), max(vals) - if self.op is Ops.FLOORDIV: + if self.op is Ops.DIV: assert isinstance(s0_vmin, int) and isinstance(s0_vmax, int) and isinstance(s1_vmin, int) and isinstance(s1_vmax, int) if s0_vmin > s0_vmax: return 0, 0 # numerator range is empty (e.g. RANGE with end=0) if s1_vmin*s1_vmax>0: return min(vals:=(s0_vmin//s1_vmin, s0_vmin//s1_vmax, s0_vmax//s1_vmin, s0_vmax//s1_vmax)), max(vals) - if self.op is Ops.FLOORMOD: + if self.op is Ops.MOD: assert isinstance(s0_vmin, int) and isinstance(s0_vmax, int) and isinstance(s1_vmin, int) and isinstance(s1_vmax, int) if s0_vmin > s0_vmax: return 0, 0 # numerator range is empty (e.g. RANGE with end=0) if (c:=s1_vmin) == s1_vmax > 0: return (s0_vmin%c, s0_vmax%c) if s0_vmin//c == s0_vmax//c else (0, c-1) @@ -1120,15 +1083,19 @@ class UOp(RandMixin, metaclass=UOpMetaClass): if s1_vmin > 0: return (0, s1_vmax-1) if s1_vmax < 0: return (s1_vmin+1, 0) if self.op is Ops.XOR and s1_vmin == s1_vmax == -1 and isinstance(s0_vmin, int) and isinstance(s0_vmax, int): - return ~int(s0_vmax), ~int(s0_vmin) + return wrapped(self.dtype, ~int(s0_vmax), ~int(s0_vmin)) if self.op is Ops.XOR and dtypes.is_int(self.dtype) and s0_vmin >= 0 and s1_vmin >= 0: return 0, (1 << int(max(s0_vmax, s1_vmax)).bit_length()) - 1 if self.op is Ops.MAX: return max(s0_vmin, s1_vmin), max(s0_vmax, s1_vmax) if self.op is Ops.CMPLT: return (s0_vmax 1: ret = ret.reshape(shape) return ret - def placeholder_like(self, slot:int, addrspace=AddrSpace.GLOBAL): + def placeholder_like(self, slot:int): assert all_int(self.shape), "no placeholder-like on symbolic shape" - return UOp.placeholder(self.max_shard_shape, self.dtype, slot, addrspace) + return UOp.placeholder(self.max_shard_shape, self.dtype, slot) # set is store+end+after def set(self:UOp, val:UOp|ConstType, end:UOp|tuple[UOp, ...]|list[UOp]=()) -> UOp: @@ -1213,14 +1179,14 @@ class UOp(RandMixin, metaclass=UOpMetaClass): # TODO: this should replace placeholder @staticmethod def param(slot:int, dtype:DType, shape:tuple[sint, ...]|sint|None=None, device=None, vmin_vmax:tuple[PyConst, PyConst]|None=None, - multiple_of:int|None=None, name=None, addrspace=AddrSpace.GLOBAL, volatile:bool=False): + name=None, addrspace=AddrSpace.GLOBAL, volatile:bool=False): """create a PARAM: a single sint or 1-d shape gives a flat param of that size, a None shape gives a scalar param. src[0] stores the concrete max size (never symbolic): a multi-dim shape is a RESHAPE on top of the flat param, a symbolic shape is a max-size param shrunk to the real shape""" if dtype in dtypes.weaks: raise RuntimeError(f"cannot create param for weak dtype {dtype}") shape = (shape,) if isinstance(shape, (int, UOp)) else shape or () ret = UOp(Ops.PARAM, src=(shape_to_shape_arg((prod(to_max_shape(shape)),) if shape else ()),), - arg=ParamArg(slot, dtype, vmin_vmax, multiple_of, name, addrspace, device, volatile)) + arg=ParamArg(slot, dtype, vmin_vmax, name, addrspace, device, volatile)) return ret.view_as(shape) def param_like(self, slot:int, name:str|None=None): # Scalar arguments bind by slot; names and values stay at the call site, not in schedule cache keys. @@ -1255,7 +1221,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @staticmethod def call_with_outputs(values:tuple[UOp, ...], *srcs:UOp, grad_fxn:Callable|None=None, - name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None, + name:str|None=None, precompile:bool=False, precompile_backward:bool=False, output_pos:tuple[int, ...]|None=None) -> tuple[UOp, ...]: """call a body producing the given values, returning the outputs. the body stores into output PARAMs, and the outputs are ALLOCs passed as extra inputs to the call (you AFTER on them like normal buffers). @@ -1288,24 +1254,26 @@ class UOp(RandMixin, metaclass=UOpMetaClass): for p, r in zip(pos, rets): args[p] = r it = iter(x.contiguous() if precompile else x for x in srcs) call = body.call(*[r if r is not None else next(it) for r in args], grad_fxn=grad_fxn, name=name, precompile=precompile, - precompile_backward=precompile_backward, aux=aux) + precompile_backward=precompile_backward) return tuple(r.after(call) for r in rets) # one-line convenience for the single-output case: self is the value def call_with_output(self, *srcs:UOp, **kwargs) -> UOp: return UOp.call_with_outputs((self,), *srcs, **kwargs)[0] def custom_kernel(*srcs:UOp, fxn:Callable, grad_fxn:Callable|None=None) -> list[UOp]: - placeholders = [UOp.placeholder_like(s, slot=i) for i,s in enumerate(srcs)] + placeholders = [s.replace(arg=replace(s.arg, slot=i, val=None)) if s.is_variable else UOp.placeholder_like(s, slot=i) + for i,s in enumerate(srcs)] kernel = fxn(*placeholders).call(*srcs, grad_fxn=grad_fxn) return [s.after(kernel) for s in srcs] + @functools.cached_property + def kernel_params(self) -> tuple[UOp, ...]: + # the kernel's params in its arg order, which is slot order. the LINEAR keeps two IMAGE params of one slot in order, x86 isel replaces every PARAM + params = [u for u in self.src[1].src if u.op is Ops.PARAM] + params += [p for p in self.arg.params if p.arg.slot not in {u.arg.slot for u in params}] + return tuple(sorted(params, key=lambda p: p.arg.slot)) + def to_elf(self) -> TinyELF: - assert self.op is Ops.PROGRAM and isinstance(self.arg, ProgramInfo), "to_elf should only be called on a PROGRAM ast" - params = tuple(u for u in self.src[1].src if u.op is Ops.PARAM and u.addrspace != AddrSpace.ALU) - # sig slots are compact: buffers in globals order (runtimes launch buffers in that order), then vars. raw call-arg - # positions skip buffers for kernels using a sparse subset of the call's buffers (CL binds bufs[slot]) - gmap = {s:j for j, s in enumerate(self.arg.globals)} - sig = tuple((u.arg.name, gmap[u.arg.slot], u.dtype, u._shape) for u in params) + \ - tuple((v.arg.name, len(self.arg.globals)+j, v.dtype, v._shape) for j, v in enumerate(self.arg.vars)) + sig = tuple((p.arg.name, p.arg.addrspace, p.dtype, p._shape) for p in self.kernel_params) return TinyELF(self.src[3].arg, self.src[0].arg.function_name, self.arg.target, sig, self.key) @property @@ -1315,11 +1283,12 @@ def uopfunc(fn:Callable[..., UOp]) -> Callable[..., UOp]: # sugar for body.call( def param(i:int, n:str, a:UOp) -> UOp: if a.addrspace in (None, AddrSpace.ALU): return UOp.param(i, a.commit_dtype(dtypes.int), name=n, addrspace=AddrSpace.ALU) return UOp.param(i, a.dtype, 1 if a.op is Ops.INDEX else a.max_numel(), a.device, name=n, addrspace=a.addrspace) - def outlined(*args, **kwargs) -> UOp: + def outlined(*args, name:str|None=None, **kwargs) -> UOp: bound = inspect.signature(fn).bind(*args, **kwargs).arguments ins = {n: a for n, a in bound.items() if isinstance(a, UOp)} - f = graph_rewrite(fn(**(bound|{n: param(i, n, a) for i, (n, a) in enumerate(ins.items())})), pm_renumber_slots, ctx=itertools.count(), walk=True) - with Context(TRACK_MATCH_STATS=0): return f.call(*ins.values(), name=fn.__name__) + f = graph_rewrite(fn(**(bound|{n: param(i, n, a) for i, (n, a) in enumerate(ins.items())})), pm_renumber_slots, + init_ctx=CountCtx, walk=True) + with Context(TRACK_MATCH_STATS=0): return f.call(*ins.values(), name=name or fn.__name__) return functools.wraps(fn)(outlined) @dataclass(frozen=True) @@ -1329,6 +1298,7 @@ class KernelInfo: opts_to_apply: tuple|None = None estimates: Estimates|None = None beam: int = 0 + optimized: bool = False # the AST has been through the optimizer, skip re-optimization @property def function_name(self): return to_function_name(self.name) @@ -1336,7 +1306,7 @@ class KernelInfo: class ProgramInfo: global_size: tuple[int|float, ...] = (1, 1, 1) local_size: tuple[int, ...] = (1, 1, 1) - vars: tuple[UOp, ...] = () + params: tuple[UOp, ...] = () globals: tuple[int, ...] = () outs: tuple[int, ...] = () ins: tuple[int, ...] = () @@ -1347,28 +1317,25 @@ class ProgramInfo: local_size = tuple([sym_infer(sz, var_vals) for sz in self.local_size]) return global_size, local_size - def vals(self, var_vals:dict[str, int]) -> tuple[int, ...]: - try: return tuple(var_vals[k.expr] for k in self.vars) - except KeyError as e: raise RuntimeError(f"unbound Variable {e}") from None - @staticmethod def from_sink(sink:UOp, target:Target=Target()) -> ProgramInfo: - _vars: list[UOp] = [] + _params: list[UOp] = [] _globals: list[int] = [] outs: list[int] = [] ins: list[int] = [] global_size: list[int] = [1, 1, 1] local_size: list[int] = [1, 1, 1] for u in sink.toposort(): - if u.op is Ops.PARAM and u.addrspace == AddrSpace.ALU: _vars.append(u) + if u.op is Ops.PARAM: _params.append(u) if u.op is Ops.PARAM and u.addrspace != AddrSpace.ALU: _globals.append(u.arg.slot) if u.op in (Ops.STORE, Ops.LOAD): if (idx:=u.src[0]).op in (Ops.INDEX, Ops.SHRINK) or (u.src[0].op is Ops.CAST and (idx:=u.src[0].src[0]).op is Ops.INDEX): if (buf:=idx.src[0].buf_uop).op is Ops.PARAM: (outs if u.op is Ops.STORE else ins).append(buf.arg.slot) - if u.op is Ops.SPECIAL: (local_size if u.arg[0] == 'l' else global_size)[int(u.arg[-1])] = cast(int, u.src[0].ssimplify()) + if u.op is Ops.SPECIAL: (local_size if u.arg[0] == 'l' else global_size)[int(u.arg[-1])] = cast(int, u.src[0].substitute( + {v:v.replace(arg=replace(v.arg, slot=-1)) for v in u.src[0].toposort() if v.op is Ops.PARAM and v.is_variable}).ssimplify()) if not outs and not ins: outs = ins = _globals # if neither is inferred, default to all buffers return ProgramInfo(tuple(global_size), tuple(local_size), - tuple(sorted(_vars, key=lambda v: v.arg.slot)), tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), + tuple(sorted(_params, key=lambda p: p.arg.slot)), tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), tuple(sorted(dedup(ins))), target) # the body of a CALL is always one of these: programs (SINK/PROGRAM/LINEAR), bulk stores, and function references @@ -1407,22 +1374,21 @@ def safe_pow(x, y): python_alu: dict[Ops, Callable] = { Ops.LOG2: lambda x: math.log2(x) if x > 0 else -math.inf if x == 0 else math.nan, Ops.EXP2: safe_exp2, - Ops.SQRT: lambda x: math.sqrt(x) if x >= 0 else math.nan, Ops.RECIPROCAL: lambda x: 1/x if x != 0 else math.copysign(math.inf, x), + Ops.SQRT: lambda x: math.sqrt(x) if x >= 0 else math.nan, Ops.RECIP: lambda x: 1/x if x != 0 else math.copysign(math.inf, x), Ops.SIN: lambda x: math.sin(x) if not math.isinf(x) else math.nan, Ops.POW: safe_pow, - Ops.TRUNC: lambda x: math.trunc(x) if math.isfinite(x) else x, + Ops.TRUNC: lambda x: math.modf(x)[1] if isinstance(x, float) else x, Ops.NEG: operator.neg, Ops.ADD: operator.add, Ops.SUB: operator.sub, Ops.MUL: operator.mul, Ops.CMPNE: operator.ne, Ops.CMPLT: operator.lt, Ops.XOR: operator.xor, Ops.OR: operator.or_, Ops.AND: operator.and_, Ops.SHR: operator.rshift, Ops.SHL: operator.lshift, Ops.MAX: max, - Ops.CMOD: cmod, Ops.CDIV: cdiv, Ops.FLOORDIV: floordiv, Ops.FLOORMOD: floormod, + Ops.TRUNCMOD: cmod, Ops.TRUNCDIV: cdiv, Ops.DIV: floordiv, Ops.MOD: floormod, Ops.MULACC: lambda x,y,z: (x*y)+z, Ops.WHERE: lambda x,y,z: y if x else z, Ops.CMPEQ: operator.eq} -def exec_alu(op:Ops, dtype:DType, operands, truncate_output=True): +def exec_alu(op:Ops, dtype:DType, operands): if any(isinstance(x, tuple) for x in operands): count = max(len(x) for x in operands if isinstance(x, tuple)) return tuple([exec_alu(op, dtype, [x[i] if isinstance(x, tuple) else x for x in operands]) for i in range(count)]) if op in GroupOp.Binary and Invalid in operands: return Invalid alu = python_alu[op](*operands) - if truncate_output and (truncate_fxn:=truncate.get(dtype)) is not None: return truncate_fxn(alu) - return alu + return truncate_fxn(alu) if (truncate_fxn:=truncate.get(dtype)) is not None else alu # ***** pattern matcher ***** @@ -1436,15 +1402,13 @@ def get_location() -> tuple[str, int]: return frm.f_code.co_filename, frm.f_lineno class UPat(RandMixin): - __slots__ = ("op", "match_dtype", "match_tag", "arg", "name", "src", "is_any") + __slots__ = ("op", "match_dtype", "arg", "name", "src", "is_any") def __init__(self, op:Ops|tuple[Ops, ...]|set[Ops]|None=None, dtype:DType|tuple[DType, ...]|set[DType]|None=None, src:tuple[UPat, ...]|list[UPat]|UPat|None=None, arg:Any=None, - name:str|None=None, allow_any_len:bool=False, custom_early_reject:set[Ops]|None=None, location=None, is_any:bool=False, - tag:Any=None): + name:str|None=None, allow_any_len:bool=False, custom_early_reject:set[Ops]|None=None, location=None, is_any:bool=False): assert op is None or isinstance(op, (Ops, tuple, set)), f"op must be Ops or tuple of Ops, not {op!r}" self.op: tuple[Ops, ...]|None = (op,) if isinstance(op, Ops) else (tuple(op) if isinstance(op, set) else op) self.match_dtype: tuple[DType, ...]|None = (dtype,) if isinstance(dtype, DType) else (tuple(dtype) if isinstance(dtype, set) else dtype) - self.match_tag: tuple[Any, ...]|None = (tag,) if isinstance(tag, str) else (tuple(tag) if isinstance(tag, set) else tag) self.arg, self.name, self._in_src, self.custom_early_reject = arg, name, src, custom_early_reject self.src: Any = None self.is_any = is_any @@ -1472,9 +1436,9 @@ class UPat(RandMixin): def __reduce__(self): return UPat, (self.op, self.match_dtype, self._in_src, self.arg, self.name, not self.strict_length, self.custom_early_reject, self.location, - self.is_any, self.match_tag) + self.is_any) def named(self, name:str): - return UPat(self.op, self.match_dtype, self._in_src, self.arg, name, not self.strict_length, self.custom_early_reject, tag=self.match_tag) + return UPat(self.op, self.match_dtype, self._in_src, self.arg, name, not self.strict_length, self.custom_early_reject) @staticmethod def any(*src): return UPat(src=src, is_any=True) @@ -1491,14 +1455,14 @@ class UPat(RandMixin): @staticmethod def const(b:ConstType, dtype:DType|tuple[DType, ...]|None=None): return UPat(Ops.CONST, dtype=dtype, arg=b) @staticmethod - def custom_function(fn:str, **kwargs): return UPat(Ops.CUSTOM_FUNCTION, arg=CustomFunction(fn), **kwargs) + def custom_function(fn:str, dtype:DType=dtypes.void, **kwargs): return UPat(Ops.CUSTOM_FUNCTION, arg=CustomFunction(fn, dtype), **kwargs) # lil helper def f(self, op, **kwargs): return UPat(op, src=(self,), **kwargs) # copied from UOp def sink(*srcs:UPat|None, **kwargs): # pylint: disable=no-self-argument - return UPat(Ops.SINK, src=tuple([x for x in srcs if x is not None]), **kwargs) + return UPat(Ops.SINK, src=tuple([x for x in srcs if x is not None]) or None, **kwargs) def index(self, *srcs:UPat|None, **kwargs): return UPat(Ops.INDEX, src=(self,)+tuple(x for x in srcs if x is not None), **kwargs) def cast(self, dtype=None, **kwargs): @@ -1513,6 +1477,7 @@ class UPat(RandMixin): return UPat(Ops.REDUCE, self.match_dtype, src=(self,)+src, arg=arg, **kwargs) def broadcast(self, **kwargs): return UPat(Ops.STACK, self.match_dtype, src=self, **kwargs) def after(self, *src:UPat, **kwargs): return UPat(Ops.AFTER, self.match_dtype, (self,)+src, **kwargs) + def call(self, *src:UPat, **kwargs): return UPat(Ops.CALL, src=(self,)+src, **kwargs) def end(self, *src:UPat, **kwargs): return UPat(Ops.END, src=(self,)+src, **kwargs) def backedge(self, loop:UPat, cond:UPat, **kwargs): return UPat(Ops.BACKEDGE, src=(self, loop, cond), **kwargs) @@ -1520,12 +1485,12 @@ class UPat(RandMixin): y = self.ufix(y) return (y, self) if reverse else (self, y) def ufix(self, x): return UPat.cvar(arg=x) if not isinstance(x, UPat) else x - def __floordiv__(self, x): return self._binop(Ops.FLOORDIV, x, False) - def __rfloordiv__(self, x): return self._binop(Ops.FLOORDIV, x, True) - def mod(self, x, reverse=False): return self._binop(Ops.FLOORMOD, x, reverse) + def __floordiv__(self, x): return self._binop(Ops.DIV, x, False) + def __rfloordiv__(self, x): return self._binop(Ops.DIV, x, True) + def mod(self, x, reverse=False): return self._binop(Ops.MOD, x, reverse) def alu(self, op:Ops, *src:UPat): asrc = (self,)+src - return UPat(op, dtypes.bool if op in {Ops.CMPLT, Ops.CMPNE} else asrc[-1].match_dtype, list(asrc) if op in GroupOp.Commutative else asrc) + return UPat(op, dtypes.bool if op in GroupOp.Comparison else asrc[-1].match_dtype, list(asrc) if op in GroupOp.Commutative else asrc) def match(self:UPat, uop:UOp, store:dict[str, UOp]) -> list[dict[str, UOp]]: if self.is_any: return flatten([x.match(uop, store.copy()) for x in self.src[0]]) @@ -1533,7 +1498,6 @@ class UPat(RandMixin): (self.name is not None and store.setdefault(self.name, uop) is not uop) or \ (self.match_dtype is not None and uop.dtype not in self.match_dtype) or \ (self.arg is not None and self.arg != uop.arg) or \ - (self.match_tag is not None and uop.tag not in self.match_tag) or \ (len(uop.src) < self.required_len) or \ (self.strict_length and len(uop.src) != self.required_len): return [] if self.src is None: return [store] @@ -1638,12 +1602,12 @@ def get_process_replay_loc() -> str: return f"{frm.f_code.co_filename.split('/')[-1]}:{frm.f_lineno} {frm.f_code.co_name}" if CAPTURE_PROCESS_REPLAY: - replay_capture: list[bytes] = [] + replay_capture: list[tuple[str, bytes]] = [] import atexit, uuid @atexit.register def save_to_diskcache(): uid = uuid.uuid4() # one id per process - for i,v in enumerate(replay_capture): diskcache_put("process_replay", f"{uid}_{i}", v, prepickled=True) + for i,(k,v) in enumerate(replay_capture): diskcache_put(k, f"{uid}_{i}", v, prepickled=True) def add_trace_group(kt:TracingKey) -> None: tracked_keys.append(kt) @@ -1651,7 +1615,7 @@ def add_trace_group(kt:TracingKey) -> None: active_group:list[int] = [] active_rewrites:list[TrackedGraphRewrite] = [] -def rewrite_group(name:Callable[..., str|TracingKey]|bool=True, replay:bool=False, new_ctx:bool=True): +def rewrite_group(name:Callable[..., str|TracingKey]|bool=True, replay:str="", new_ctx:bool=True): if not new_ctx: assert not callable(name) and not replay, "name fxn and replay are only supported for new_ctx groups" def _decorator(func): def __wrapper(*args, **kwargs): @@ -1687,7 +1651,7 @@ def rewrite_group(name:Callable[..., str|TracingKey]|bool=True, replay:bool=Fals if CAPTURE_PROCESS_REPLAY and replay: # capture global context vars and all the args passed in with Context(PROCESS_REPLAY_LOC=get_process_replay_loc()): - replay_capture.append(pickle.dumps((fn, args, kwargs, ContextVar._cache, ret))) + replay_capture.append((replay, pickle.dumps((fn, args, kwargs, ContextVar._cache, ret)))) return ret return __wrapper return _decorator @@ -1759,6 +1723,13 @@ if TRACK_MATCH_STATS or PROFILE: # A pure Python sentinel, but *typed* as UOp so it fits all the dict annotations SENTINEL: Final[UOp] = cast(UOp, object()) class BottomUpGate(Exception): pass + +# common graph rewrite ctx constructors + +class CountCtx(itertools.count): pass + +class MapCtx(dict): pass + class RewriteContext: def __init__(self, pm, bpm, ctx=None, enter_calls=False): self.pm: PatternMatcher|None = pm @@ -1796,7 +1767,7 @@ class RewriteContext: # rebuild node with rewritten srcs skip = int(n.op is Ops.CALL and not self.enter_calls) new_src = n.src[:skip] + tuple(self.replace.get(x, x) for x in n.src[skip:]) - new_n = UOp(n.op, new_src, n.arg, n.tag) if new_src != n.src else n + new_n = UOp(n.op, new_src, n.arg) if new_src != n.src else n # top-down: try pm on rebuilt node, use result as-is (no re-traversal) if self.pm is not None and (rewritten:=self.pm_rewrite(new_n)) is not None: new_n = rewritten self.replace[n] = new_n @@ -1852,7 +1823,7 @@ class RewriteContext: continue else: # if srcs changed from rewrites, construct a new UOp with the new srcs - new_src_n = UOp(new_n.op, new_src, new_n.arg, new_n.tag) + new_src_n = UOp(new_n.op, new_src, new_n.arg) # trigger a rewrite of new_src_n, then after that rewrite is done, link it back to n stack.append((n, 2, new_src_n)) stack.append((new_src_n, 0, new_src_n)) @@ -1874,7 +1845,11 @@ class RewriteContext: return self.replace[root] @rewrite_group(new_ctx=False) -def graph_rewrite(sink:UOp, pm:PatternMatcher, ctx=None, bottom_up=False, name=None, bpm=None, walk=False, enter_calls=False) -> UOp: +def graph_rewrite(sink:UOp, pm:PatternMatcher, ctx=None, bottom_up=False, name=None, bpm=None, walk=False, enter_calls=False, + init_ctx:type|None=None) -> UOp: + if init_ctx is not None: + assert ctx is None + ctx = init_ctx() rewrite_ctx = RewriteContext(pm if not bottom_up else None, pm if bottom_up else bpm, ctx, enter_calls) return rewrite_ctx.walk_rewrite(sink) if walk else rewrite_ctx.unified_rewrite(sink) @@ -1890,7 +1865,6 @@ def resolve_returned_after(r:UOp, t:UOp) -> UOp|None: stores = [st for st in t.src if st.op is Ops.STORE and st.src[0].unsharded_base is r.unsharded_base] if len(stores) != 1: return None return r.after(stores[0]) if r.unsharded_base.op is Ops.PARAM else stores[0].src[1] -remove_all_tags = PatternMatcher([(UPat(GroupOp.All, name="x"), lambda x: x.replace(tag=None) if x.tag is not None else None)]) # a store's storage keeps the views and drops AFTERs (they only sequence stores) pm_drop_after = PatternMatcher([(UPat(Ops.AFTER, name="a"), lambda a: a.src[0])]) diff --git a/tinygrad_repo/tinygrad/uop/render.py b/tinygrad_repo/tinygrad/uop/render.py index 03066faac0..14925d7bfa 100644 --- a/tinygrad_repo/tinygrad/uop/render.py +++ b/tinygrad_repo/tinygrad/uop/render.py @@ -12,7 +12,7 @@ def pretty_print(x:UOp, cache=None, d=0)->str: if cache is None: dfs(x, cache:={}) if (cx:=cache.setdefault(x, [0,0,False]))[2]: return f"{' '*d}x{cx[0]}" cx[2], srcs = True, (''.join(f'\n{pretty_print(s, cache, d+2)},' for s in x.src)) - return f"{' '*d}{f'x{cx[0]}:=' * (cx[1]>1)}{type(x).__name__}({x.op}, arg={x.argstr()}{x.tagstr()}, src=({srcs}))" + return f"{' '*d}{f'x{cx[0]}:=' * (cx[1]>1)}{type(x).__name__}({x.op}, arg={x.argstr()}, src=({srcs}))" # ***** SSA wire format ***** @@ -23,7 +23,7 @@ uops_colors = {Ops.LOAD: "#ffc0c0", Ops.STORE: "#87CEEB", Ops.CONST: "#e0e0e0", **{x:"#D8F9E4" for x in GroupOp.Movement}, **{x:"#ffffc0" for x in GroupOp.ALU}, Ops.THREEFRY:"#ffff80", Ops.BUFFER: "#B0BDFF", Ops.GETADDR: "#9DB1F0", Ops.COPY: "#ff90c0", Ops.CUSTOM_FUNCTION: "#bf71b6", Ops.CALL: "#00B7C8", Ops.PARAM: "#14686F", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", - Ops.LINEAR: "#7DF4FF", Ops.ALLOC: "#C07788", + Ops.LINEAR: "#7DF4FF", Ops.ALLOC: "#C07788", Ops.PROGRAM: "#a0a0a0", Ops.ALLREDUCE: "#ff40a0", Ops.MSELECT: "#d040a0", Ops.MSTACK: "#d040a0", Ops.STAGE: "#FFC14D", Ops.REWRITE_ERROR: "#1a1b26", Ops.AFTER: "#8A7866", Ops.END: "#524C46", Ops.BACKEDGE: "#464752"} @@ -40,7 +40,6 @@ def _render_arg(x:UOp) -> str: case Ops.PARAM | Ops.BUFFER | Ops.ALLOC: a, opts = x.arg, "" if a.vmin_vmax is not None: opts += f" bounds=[{a.vmin_vmax[0]},{a.vmin_vmax[1]}]" - if a.multiple_of is not None: opts += f" multiple_of={a.multiple_of}" if a.addrspace not in (None, AddrSpace.GLOBAL): opts += f" addrspace={a.addrspace.name}" if a.device is not None: opts += " device=" + (a.device if isinstance(a.device, str) and re.fullmatch(r"[\w:]+", a.device) else repr(a.device)) @@ -78,10 +77,10 @@ def render_uir(root:UOp|list[UOp]) -> str: # for debug -syms = { Ops.ADD: "+", Ops.SUB: "-", Ops.FLOORDIV: "//", Ops.FLOORMOD: "%", Ops.SHL: "<<", Ops.SHR: ">>", - Ops.MUL: "*", Ops.CMPLT: "<", Ops.CMPNE: "!=", Ops.AND: "&", Ops.OR: "|", Ops.XOR: "^"} +syms = { Ops.ADD: "+", Ops.SUB: "-", Ops.DIV: "//", Ops.MOD: "%", Ops.SHL: "<<", Ops.SHR: ">>", + Ops.MUL: "*", Ops.CMPLT: "<", Ops.CMPNE: "!=", Ops.CMPEQ: "==", Ops.AND: "&", Ops.OR: "|", Ops.XOR: "^"} # comparison operators are not in here because they are chained in python, not left-associative -precedence = {Ops.MUL:1, Ops.FLOORDIV:1, Ops.FLOORMOD:1, Ops.ADD:2, Ops.SUB:2, Ops.SHL:3, Ops.SHR:3, Ops.AND:4, Ops.XOR:5, Ops.OR:6} +precedence = {Ops.MUL:1, Ops.DIV:1, Ops.MOD:1, Ops.ADD:2, Ops.SUB:2, Ops.SHL:3, Ops.SHR:3, Ops.AND:4, Ops.XOR:5, Ops.OR:6} def strip_binary_parens(x:UOp, left:str, right:str, code_for_op) -> str: if x.op not in precedence: return code_for_op(left, right) return code_for_op(strip_parens(left) if precedence.get(x.src[0].op,99)<=precedence[x.op] else left, strip_parens(right) if @@ -112,12 +111,10 @@ renderer = PatternMatcher([ (UPat.cvar("c").cast(), lambda c: str(c.val)), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({str(x.dtype)[7:]})({ctx[x.src[0]]})"), (UPat(Ops.NEG, name="x"), lambda ctx,x: f"(-{ctx[x.src[0]]})"), - (UPat(Ops.RECIPROCAL, name="x"), lambda ctx,x: f"(1/{ctx[x.src[0]]})"), - (UPat(Ops.MAX, name="x"), lambda ctx,x: f"max({ctx[x.src[0]]}, {ctx[x.src[1]]})"), + (UPat(Ops.RECIP, name="x"), lambda ctx,x: f"(1/{ctx[x.src[0]]})"), + (UPat((Ops.MAX, Ops.TRUNCDIV, Ops.TRUNCMOD), name="x"), lambda ctx,x: f"{x.op.name.lower()}({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.MULACC, name="x"), lambda ctx,x: f"({ctx[x.src[0]]}*{ctx[x.src[1]]}+{ctx[x.src[2]]})"), (UPat(Ops.WHERE, name="x"), lambda ctx,x: f"({ctx[x.src[1]]} if {ctx[x.src[0]]} else {ctx[x.src[2]]})"), - (UPat(Ops.CDIV, name="x"), lambda ctx,x: f"cdiv({ctx[x.src[0]]}, {ctx[x.src[1]]})"), - (UPat(Ops.CMOD, name="x"), lambda ctx,x: f"cmod({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(GroupOp.Movement, name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({render_marg(ctx, x)})"), (UPat(set(syms.keys()), name="x"), lambda ctx,x: strip_binary_parens(x, ctx[x.src[0]], ctx[x.src[1]], lambda a,b: f"({a}{syms[x.op]}{b})")), (UPat((Ops.INDEX, Ops.STAGE), name="x"), lambda x, ctx: render_index(ctx[y] for y in x.src[1:])), @@ -129,8 +126,7 @@ renderer = PatternMatcher([ ]) renderer_infer = PatternMatcher([ - (UPat(Ops.FLOORMOD, name="x"), lambda ctx,x: f"floormod({ctx[x.src[0]]}, {ctx[x.src[1]]})"), - (UPat(Ops.FLOORDIV, name="x"), lambda ctx,x: f"floordiv({ctx[x.src[0]]}, {ctx[x.src[1]]})"), + (UPat((Ops.MOD, Ops.DIV), name="x"), lambda ctx,x: f"{x.op.name.lower()}({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"{'float' if dtypes.is_float(x.dtype) else 'bool' if x.dtype is dtypes.bool else 'int'}({ctx[x.src[0]]})"), (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"bitcast({ctx[x.src[0]]}, {x.src[0].dtype!r}, {x.dtype!r})"), diff --git a/tinygrad_repo/tinygrad/uop/spec.py b/tinygrad_repo/tinygrad/uop/spec.py index 66d63d6fc0..624504b1c2 100644 --- a/tinygrad_repo/tinygrad/uop/spec.py +++ b/tinygrad_repo/tinygrad/uop/spec.py @@ -69,7 +69,7 @@ spec_shared = PatternMatcher([ (UPat((Ops.AND, Ops.OR, Ops.XOR, Ops.SHL, Ops.SHR), name="x"), lambda x: False if any(dtypes.is_float(s.dtype) for s in x.src) else None), (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat.var("c")), name="a"), lambda a,x,c: matches_dtype(c, a.dtype) or c.dtype in (dtypes.uint, dtypes.weakint) or x.base.is_invalid), - (UPat((Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD), name="x"), + (UPat((Ops.TRUNCDIV, Ops.TRUNCMOD, Ops.DIV, Ops.MOD), name="x"), lambda x: None if dtypes.is_int(x.dtype) or any(s.base.is_invalid for s in x.src) else False), (UPat(GroupOp.ALU, name="x"), lambda x: all(matches_dtype(y, x.dtype) or y.dtype == weak_dtype(x.dtype) for y in x.src)), @@ -94,11 +94,10 @@ spec_shared = PatternMatcher([ (UPat(Ops.BINARY, dtypes.uint8, src=(), name="x"), lambda x: isinstance(x.arg, bytes)), - # AFTER on Movement Op, PARAM, BUFFER, ALLOC, STAGE, or another AFTER + # AFTER on Movement Op, a define, STAGE, or another AFTER # CONST/CAST/NOOP are range bounds: RANGE(AFTER(CONST, other_range)) orders a loop after a sibling - (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement.union({Ops.PARAM, Ops.BUFFER, Ops.ALLOC, Ops.STAGE, Ops.INDEX, - Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS, - Ops.CONST, Ops.CAST, Ops.NOOP, Ops.STACK})),), + (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement|GroupOp.Defines|{Ops.STAGE, Ops.INDEX, Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS, + Ops.CONST, Ops.CAST, Ops.NOOP, Ops.STACK}),), allow_any_len=True), lambda: True), # an AFTER can wrap a scalar ALU (e.g. a computed RANGE bound) to order it after effect ops (UPat(Ops.AFTER, src=(UPat(GroupOp.ALU),), allow_any_len=True, name="x"), lambda x: x.src[0].shape == ()), @@ -119,7 +118,7 @@ spec_shared = PatternMatcher([ (UPat(Ops.BARRIER, dtypes.void), lambda: True), # assembly instruction - (UPat(Ops.INS, name="x"), lambda x: isinstance(x.arg, tuple) and len(x.arg) == 2 and isinstance(x.arg[1], DType)), + (UPat(Ops.INS, name="x"), lambda x: isinstance(x.arg, tuple) and len(x.arg) >= 2 and isinstance(x.arg[1], DType)), # LOAD(idx) / STORE(idx, val) with gates on the LOAD/STORE; a STORE's value matches its target's dtype (UPat(Ops.STORE, src=(UPat(name="t"), UPat(name="v")), allow_any_len=True), @@ -133,7 +132,7 @@ spec_shared = PatternMatcher([ # STORE: the target must be storage or a STAGE realization point (or an AFTER/BITCAST/view of one); # STAGE targets are written into the buffer the STAGE creates. INDEX stores are checked above (UPat(Ops.STORE, dtypes.void, (UPat(name="x"), UPat())), lambda x: - True if (b:=x.storage_base).op in {Ops.BUFFER, Ops.ALLOC, Ops.PARAM, Ops.STAGE} else None if b.op is Ops.INDEX else False), + True if (b:=x.storage_base).op in GroupOp.Defines|{Ops.STAGE} else None if b.op is Ops.INDEX else False), # WMMA has a (UPat(Ops.WMMA, src=(UPat(), UPat(), UPat()), name="x"), lambda x: isinstance(x.arg, tuple) and len(x.arg) == 4), @@ -143,7 +142,7 @@ def is_device(d): return isinstance(d, str) or (isinstance(d, tuple) and all(isi # these ops can exist in tensor but not programs. example: movement spec_tensor = PatternMatcher([ - (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL), src=(UPat(),), name="u"), + (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIP), src=(UPat(),), name="u"), lambda u: dtypes.is_float(u.dtype) or u.src[0].base.is_invalid), # BUFFER has bound storage; ALLOC declares storage without a runtime buffer @@ -165,7 +164,8 @@ spec_tensor = PatternMatcher([ # movement ops (UPat((Ops.RESHAPE, Ops.EXPAND), src=(UPat(), UPat(dtype=dtypes.ints+(dtypes.weakint, dtypes.void)))), lambda: True), (UPat((Ops.PAD, Ops.SHRINK), src=(UPat(), UPat(), UPat()), name="x"), lambda x: x.src[1].shape == x.src[2].shape), - (UPat(Ops.PERMUTE, name="mv", src=(UPat(),)), lambda mv: isinstance(mv.arg, tuple)), + (UPat(Ops.PERMUTE, name="mv", src=(UPat(), UPat(Ops.STACK, src=UPat.cvar(dtype=dtypes.weakint), name="s"))), + lambda mv, s: len(s.src) >= 2 and sorted(v.arg for v in s.src) == list(range(len(s.src)))), # REDUCE has arg=(op, num_axes), src[1:] are ranges after lowering (UPat(Ops.REDUCE, src=(UPat(),), allow_any_len=True, name="x"), @@ -207,7 +207,7 @@ spec_program = PatternMatcher([ (UPat(GroupOp.All-{Ops.CONST}, dtypes.weaks), lambda: False), # allow special SHRINK of a buffer or its bitcast - (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.ALLOC, Ops.AFTER)).or_bitcasted(), UPat(), UPat.cvar().or_casted())), lambda: True), + (UPat(Ops.SHRINK, src=(UPat(GroupOp.Defines|{Ops.AFTER}).or_bitcasted(), UPat(), UPat.cvar().or_casted())), lambda: True), # movement ops are not allowed in programs (UPat(GroupOp.Movement), lambda: False), @@ -228,7 +228,7 @@ spec_program = PatternMatcher([ spec_hcq = PatternMatcher([ (UPat(Ops.GETADDR, dtypes.uint64, name="x", - src=(UPat((Ops.BUFFER, Ops.ALLOC, Ops.PARAM, Ops.SHRINK, Ops.BITCAST, Ops.MSTACK, Ops.MSELECT, Ops.LINEAR)).or_after(),)), + src=(UPat(GroupOp.Defines|{Ops.SHRINK, Ops.BITCAST, Ops.MSTACK, Ops.MSELECT, Ops.LINEAR}).or_after(),)), lambda x: is_device(x.arg)), (UPat(Ops.PROGRAM, dtypes.void, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),)), lambda: True), ])+spec_shared @@ -275,6 +275,6 @@ spec_kernel_graph = PatternMatcher([ # all calls are on opaque bodies (UPat(Ops.CALL, src=(UPat(tuple(OPAQUE_CALL_BODIES)),), allow_any_len=True), lambda: True), # after on PARAM or AFTER - (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement.union({Ops.PARAM, Ops.AFTER, Ops.BUFFER, Ops.ALLOC, - Ops.MSTACK, Ops.MSELECT, Ops.BITCAST, Ops.RESHAPE})),), allow_any_len=True), lambda: True), + (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement|GroupOp.Defines|{Ops.AFTER, Ops.MSTACK, Ops.MSELECT, Ops.BITCAST}),), allow_any_len=True), + lambda: True), ]) diff --git a/tinygrad_repo/tinygrad/uop/symbolic.py b/tinygrad_repo/tinygrad/uop/symbolic.py index bd12e42fc3..f830bdf6d6 100644 --- a/tinygrad_repo/tinygrad/uop/symbolic.py +++ b/tinygrad_repo/tinygrad/uop/symbolic.py @@ -1,7 +1,7 @@ # all of symbolic lives here now import math from collections import defaultdict -from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu, promo_dtype +from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu, promo_dtype, identity_element from tinygrad.dtype import PyConst, ConstFloat, dtypes, can_lossless_cast, Invalid, bitcast, truncate from tinygrad.helpers import partition, all_same, prod, flatten, IMAGE, dedup, ceildiv from tinygrad.uop.divandmod import div_and_mod_symbolic @@ -26,15 +26,17 @@ def fold_bitcast(root:UOp, c:UOp) -> UOp|None: # the value is mathematical and may not fit: reading it as bits is the emission that pins it to the stated width return root.const_like(bitcast(truncate[c.dtype](c.val), c.dtype, root.dtype)) -# no truncate: ints stay mathematical past the fold (emission truncates); floats re-round in the mint -def fold_const_alu(a:UOp) -> UOp: return a.const_like(exec_alu(a.op, a.dtype, [const_arg(s) for s in a.src], False)) +# the mint truncates to a strong dtype, ints wrap and floats re-round +def fold_const_alu(a:UOp) -> UOp: return a.const_like(exec_alu(a.op, a.dtype, [const_arg(s) for s in a.src])) +# a fold mints a signed const only if it fits, the mint would wrap it. unsigned ring ops wrap by definition +def fits(x:UOp, v:PyConst) -> bool: return x.dtype not in dtypes.sints or x.dtype.min <= v <= x.dtype.max def _quotient_base(q:UOp, base:UOp, div:int) -> UOp|None: # the B with q == B//div and B%div == base%div, or None. only such congruence is needed to recombine, and canonicalization # moves consts freely: the quotient may be merged ((x//c + a)//div -> (x + a*c)//(c*div) for div>0) and shifted ((y + k*D)//D == y//D + k) (q, s), (num, a) = q.pop_const(), base.pop_const() - if q.op is not Ops.FLOORDIV or q.src[1].op is not Ops.CONST: return None - if div > 0 and num.op is Ops.FLOORDIV and num.src[1].op is Ops.CONST and q.src[1].val == (c:=num.src[1].val)*div: num, a, D = num.src[0], a*c, c*div + if q.op is not Ops.DIV or q.src[1].op is not Ops.CONST: return None + if div > 0 and num.op is Ops.DIV and num.src[1].op is Ops.CONST and q.src[1].val == (c:=num.src[1].val)*div: num, a, D = num.src[0], a*c, c*div elif q.src[1].val == div: D = div else: return None (x, xa), (p, pa) = num.pop_const(), q.src[0].pop_const() @@ -48,14 +50,14 @@ def fold_add_divmod_recombine(x:UOp) -> UOp|None: terms = list(x.split_uop(Ops.ADD)) for i,u in enumerate(terms): mod, mul = u.pop_const(Ops.MUL) - if mod.op is not Ops.FLOORMOD or mod.src[1].op is not Ops.CONST: continue + if mod.op is not Ops.MOD or mod.src[1].op is not Ops.CONST: continue base, div = mod.src[0], mod.src[1].val for j,v in enumerate(terms): q, scale = v.pop_const(Ops.MUL) if i == j or scale != div*mul: continue rest = [t for k,t in enumerate(terms) if k not in (i,j)] if (b:=_quotient_base(q, base, div)) is not None: return (b*mul).usum(*rest) - if q.op is Ops.FLOORMOD and q.src[1].op is Ops.CONST and (d:=q.src[1].val) > 0 and (b:=_quotient_base(q.src[0], base, div)) is not None: + if q.op is Ops.MOD and q.src[1].op is Ops.CONST and (d:=q.src[1].val) > 0 and (b:=_quotient_base(q.src[0], base, div)) is not None: return ((b % (div*d))*mul).usum(*rest) return None @@ -73,13 +75,16 @@ def const_arg(u:UOp): def lift_reduce_gate(red:UOp, cond:UOp, x:UOp, i:UOp) -> UOp|None: # a REDUCE moves inside the gate clauses without its ranges: they invalidate every lane at once, so that gate lifts out - if red.arg[1] != 0: return None + # the clauses with its ranges invalidate single lanes, an Invalid lane is the reduce's identity + op, horizontal = red.arg + if horizontal != 0: return None keep, lift = partition(cond.split_uop(Ops.AND), lambda c: any(rr in c.ranges for r in red.src[1:] for rr in r.ranges)) - inner = keep[0].uprod(*keep[1:]).where(x, i) if keep else x - return lift[0].uprod(*lift[1:]).where(red.replace(src=(inner,)+red.src[1:]), i) if lift else None + inner = keep[0].uprod(*keep[1:]).where(x, x.const_like(identity_element(op, red.dtype))) if keep else x + ret = red.replace(src=(inner,)+red.src[1:]) + return lift[0].uprod(*lift[1:]).where(ret, i) if lift else ret pm_data_invalid = PatternMatcher([ - (invalid_pat.broadcast(), lambda i: i), + (invalid_pat.broadcast(name="s"), lambda i,s: i.expand(s.shape)), (UPat(GroupOp.Unary|{Ops.CAST, Ops.BITCAST}, src=(invalid_pat,)), lambda i: i), (UPat(GroupOp.Unary|{Ops.CAST, Ops.BITCAST}, src=(invalid_gate,), name="op"), lambda cond,x,op,i: cond.where(op.replace(src=(x,)), i)), @@ -98,15 +103,15 @@ pm_data_invalid = PatternMatcher([ (a.logical_not()|cond).where(a.where(x,c), i) if not c.is_invalid else None), (UPat.var("a").where(UPat.var("b"), invalid_gate), lambda cond,i,x,a,b: (a|cond).where(a.where(b, x), i) if not b.is_invalid else None), # fold gated LOAD/STORE - (UPat(Ops.STORE, src=(UPat(Ops.INDEX, src=(UPat(), invalid_pat), allow_any_len=True).or_casted(), UPat())), lambda i: UOp(Ops.NOOP)), + (UPat(Ops.STORE, src=(UPat(Ops.INDEX, src=(UPat(), invalid_pat), allow_any_len=True).or_casted(), UPat()), name="x"), + lambda i,x: UOp(Ops.NOOP).after(*x.src)), (UPat(Ops.LOAD, src=(UPat(Ops.INDEX, src=(UPat(), invalid_pat), allow_any_len=True).or_casted(),), allow_any_len=True, name="x"), lambda x,i: x.src[1] if len(x.src) > 1 else x.const_like(0)), ]) pm_remove_invalid = PatternMatcher([ (invalid_gate.named("w"), lambda cond,x,i,w: w.replace(src=(cond,x,w.const_like(0)))), - (UPat(Ops.STACK, name="s"), lambda s: s.replace(src=tuple(UOp.const(0, s.dtype) if x.is_invalid else x for x in s.src)) - if any(x.is_invalid for x in s.src) else None), + (UPat(Ops.STACK, name="s"), lambda s: s.replace(src=tuple(UOp.const(0, s.dtype) if x.is_invalid else x for x in s.src))), ]) def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: @@ -121,21 +126,21 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ (UPat.var("x") * 1, lambda x: x), # x*1 -> x (UPat.var("x") // UPat.var("x"), lambda x: x.const_like(1)), # x//x -> 1 (UPat.var("x") // 1, lambda x: x), # x//1 -> x - (UPat.var("x") // -1, lambda x: -x), # x//-1 -> -x - ((UPat.var("x") ^ UPat.var("y")) ^ UPat.var("y"), lambda x,y: x), # (x^y)^y -> x + (UPat.var("x", dtypes.sints+(dtypes.weakint,)) // -1, lambda x: -x), # x//-1 -> -x + ((UPat.var("x") ^ UPat.var("y", dtypes.weakint)) ^ UPat.var("y"), lambda x,y: x), # (x^y)^y -> x ((UPat.var() % UPat.var("y")).named("base") % UPat.var("y"), lambda base,y: base), # (x%y)%y = -> x%y (rewritten with base for speed) # variations of (x%c)+(x//c)*c = x (UPat(Ops.ADD, dtype=dtypes.weakint, name="x"), fold_add_divmod_recombine), (UPat.var("x", dtype=dtypes.bool) & UPat.cvar("c"), lambda x,c: x if c.val else c), (UPat.var("x", dtype=dtypes.bool) | UPat.cvar("c"), lambda x,c: c if c.val else x), - (UPat.var("x", dtype=dtypes.bool) != UPat.const(False, dtypes.bool), lambda x: x), # x != False -> x + (UPat.var("x", dtype=dtypes.bool).eq(UPat.const(True, dtypes.bool)), lambda x: x), # x == True -> x (UPat(GroupOp.Idempotent, src=(UPat.var("x"), UPat.var("x"))), lambda x: x), (UPat.var("x", dtype=dtypes.bool).logical_not().logical_not(), lambda x: x), (UPat.var("x", dtype=dtypes.bool).where(UPat.const(True, dtypes.bool), UPat.const(False, dtypes.bool)), lambda x: x), (UPat.var("x", dtype=dtypes.bool).where(UPat.const(False, dtypes.bool), UPat.const(True, dtypes.bool)), lambda x: x.logical_not()), - # CAST(bool -> int) != const — CAST(True)=1, CAST(False)=0, so fold based on const value - (UPat.var("x", dtype=dtypes.bool).cast(dtypes.ints+(dtypes.weakint,)) != UPat.cvar("c"), - lambda x,c: x if c.val == 0 else x.logical_not() if c.val == 1 else x.const_like(True)), + # CAST(bool -> int) == const — CAST(True)=1, CAST(False)=0, so fold based on const value + (UPat.var("x", dtype=dtypes.bool).cast(dtypes.ints+(dtypes.weakint,)).eq(UPat.cvar("c")), + lambda x,c: x.logical_not() if c.val == 0 else x if c.val == 1 else x.const_like(False)), (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)).trunc(), lambda x: x), # ** zero folding ** (UPat.var("x") < UPat.var("x"), lambda x: x.const_like(False, dtypes.bool)), # x < x -> False @@ -144,11 +149,9 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ (UPat.var("x") & 0, lambda x: x.const_like(0)), # x&0 -> 0 # (x&mask)>>k -> x>>k when mask only clears bits below k ((UPat.var("x") & UPat.cvar("mask")) >> UPat.cvar("k"), - lambda x,mask,k: x >> k.val if mask.val | ((1 << k.val) - 1) == -1 else None), - ((UPat.var("x") & UPat.cvar("mask")) // UPat.cvar("c"), - lambda x,mask,c: x // c.val if c.val > 0 and c.val & (c.val-1) == 0 and mask.val | (c.val-1) == -1 else None), - (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) != UPat.var("x"), - lambda x: x.const_like(False, dtypes.bool)), # x != x -> False (only ints) + lambda x,mask,k: x >> k.val if x.dtype.const(mask.val | ((1 << k.val) - 1)) == x.dtype.const(-1) else None), + (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)).eq(UPat.var("x")), + lambda x: x.const_like(True, dtypes.bool)), # x == x -> True (only ints) # ** constant folding ** # canonicalize casted CONST (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.val)), @@ -162,6 +165,10 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=[casted_const, bare_const], name="a"), lambda a: a.replace(src=tuple(s.ccast(dt) if s.dtype in dtypes.weaks else s for s in a.src)) if (dt:=promo_dtype(a.src)) not in dtypes.weaks else None), + # ** two stage ALU folding ** + *((UPat.var("x").alu(op, UPat.cvar("c1")).alu(op, UPat.cvar("c2")).named("f"), + lambda f,x,c1,c2: x.alu(f.op,c1.alu(f.op,c2)) if fits(x, exec_alu(f.op, dtypes.weakint, [c1.vmin, c2.vmin])) else None) + for op in GroupOp.Associative), # bool MUL is AND, ADD/MAX is OR. prevents other rules to rewrite bool ADD/MUL incorrectly (UPat.var('x', dtype=dtypes.bool) * UPat.var('y', dtype=dtypes.bool), lambda x,y: x&y), (UPat.var('x', dtype=dtypes.bool) + UPat.var('y', dtype=dtypes.bool), lambda x,y: x|y), @@ -170,7 +177,6 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ (UPat.cvar('x', arg=0) / 0, lambda x: x.const_like(float('nan'))), # 0/0 -> nan # can be wrong if x or x2 is 0 (UPat.var("x") / UPat.var("x"), lambda x: x.const_like(1)), # x/x -> 1 - ((UPat.var("x") * UPat.var("x2")) / UPat.var("x2"), lambda x,x2: x), # (x*x2)/x2 -> x # x*0 -> 0 or 0*x -> 0 # if x is nan or inf it should render the nan value. # NOTE: this can be wrong for loaded NaN @@ -190,11 +196,11 @@ symbolic_simple = pm_data_invalid + PatternMatcher([ (UPat.var("x").alu(Ops.POW, UPat.cvar("c")), simplify_pow), # positive const ** x (UPat.cvar("c").alu(Ops.POW, UPat.var("x")), - lambda c,x: c if c.val == 1 else (x*math.log2(c.val)).exp2() if c.val > 0 and dtypes.is_float(x.dtype) else None), + lambda c,x: (x*math.log2(c.val)).exp2() if c.val > 0 and dtypes.is_float(x.dtype) else None), # unpack a uint64 packed from two uint32 (threefry) (((UPat.var(None, dtypes.uint64)<<32) | UPat.var('y', dtypes.uint32).cast(dtypes.uint64)).cast(dtypes.uint32), lambda y: y), - (((UPat.var('x', dtypes.uint32).cast(dtypes.uint64)<<32) | UPat.var(None, dtypes.uint32).cast(dtypes.uint64))>>32, - lambda x: x.cast(dtypes.uint64)), + (((UPat.var('x', dtypes.uint64)<<32) | UPat.var(None, dtypes.uint32).cast(dtypes.uint64))>>32, + lambda x: x.cast(dtypes.uint32).cast(dtypes.uint64)), # ** simple where folding ** # a conditional with the same results either way is a noop, also fold const conditionals (UPat.var().where(UPat.var("val"), UPat.var("val")), lambda val: val), @@ -241,15 +247,16 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ # TODO: make a more general or folder like simplify_valid (UPat.var("x", dtype=dtypes.bool) | UPat.var("x", dtype=dtypes.bool).logical_not(), lambda x: x.const_like(True)), # x|!x -> True # ** combine terms ** - (UPat.var("x") * UPat.cvar("c0") + UPat.var("x") * UPat.cvar("c1"), lambda x,c0,c1: x*(c0+c1)), # (x*c0)+(x*c1) -> x*(c0+c1) - ((UPat.var("y") + UPat.var("x") * UPat.cvar("c0")) + UPat.var("x") * UPat.cvar("c1"), lambda x,y,c0,c1: y+x*(c0+c1)), - (UPat.var("x") + UPat.var("x") * UPat.cvar("c"), lambda x,c: x*(c+1)), # (x+x*c)-> x*(c+1) - ((UPat.var("y") + UPat.var("x")) + UPat.var("x") * UPat.cvar("c"), lambda x,y,c: y+x*(c+1)), - ((UPat.var("y") + UPat.var("x") * UPat.cvar("c")) + UPat.var("x"), lambda x,y,c: y+x*(c+1)), + # (x*c0)+(x*c1) -> x*(c0+c1) + (UPat.var("x") * UPat.cvar("c0") + UPat.var("x") * UPat.cvar("c1"), lambda x,c0,c1: x*(c0+c1) if fits(x, c0.vmin+c1.vmin) else None), + ((UPat.var("y") + UPat.var("x") * UPat.cvar("c0")) + UPat.var("x") * UPat.cvar("c1"), + lambda x,y,c0,c1: y+x*(c0+c1) if fits(x, c0.vmin+c1.vmin) else None), + (UPat.var("x") + UPat.var("x") * UPat.cvar("c"), lambda x,c: x*(c+1) if fits(x, c.vmin+1) else None), # (x+x*c)-> x*(c+1) + ((UPat.var("y") + UPat.var("x")) + UPat.var("x") * UPat.cvar("c"), lambda x,y,c: y+x*(c+1) if fits(x, c.vmin+1) else None), + ((UPat.var("y") + UPat.var("x") * UPat.cvar("c")) + UPat.var("x"), lambda x,y,c: y+x*(c+1) if fits(x, c.vmin+1) else None), (UPat.var("x") + UPat.var("x"), lambda x: x*2), # (x+x)-> x*2 - ((UPat.var("y") + UPat.var("x")) + UPat.var("x"), lambda y,x: y+x*2), - ((UPat.var("x") / UPat.var("x2")) / UPat.var("x3"), lambda x,x2,x3: x/(x2*x3) if x2 is not x3 else None), # (x/x2)/x3 -> x/(x2*x3) - (-1 * (UPat.var("x") + UPat.cvar("c")), lambda x,c: (-x)+(-c)), # -(x+c) -> -x + -c + (((UPat.var("y") + UPat.var("x")) + UPat.var("x")).named("a"), + lambda a,y,x: y+x*2 if fits(a, 2*x.vmin) and fits(a, 2*x.vmax) else None), (UPat.cvar("y") * (UPat.var("x", dtype=dtypes.weakint) + UPat.cvar("c")), lambda x,y,c: (y*x)+(y*c)), # y*(x+c) -> y*x + y*c # ** where folding ** (UPat.var("cond", dtype=dtypes.bool).logical_not().where(UPat.var("t"), UPat.var("f")), @@ -259,6 +266,8 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ (UPat.var("gate").where(UPat.var("x"), 0) != 0, lambda gate,x: gate & (x != 0)), # a.where(b.where(c, d), d) -> (a & b).where(c, d) (UPat.var("a").where(UPat.var("b").where(UPat.var("c"), UPat.var("d")), UPat.var("d")), lambda a,b,c,d: (a&b).where(c,d)), + # a.where(b.where(c, d), c) -> (a & !b).where(d, c) + (UPat.var("a").where(UPat.var("b").where(UPat.var("c"), UPat.var("d")), UPat.var("c")), lambda a,b,c,d: (a&b.logical_not()).where(d,c)), # a.where(c, b.where(c, d)) -> (a | b).where(c, d) (UPat.var("a").where(UPat.var("c"), UPat.var("b").where(UPat.var("c"), UPat.var("d"))), lambda a,b,c,d: (a|b).where(c,d)), # alu of two where with same conds can combine, only do if true branch or false branch is const @@ -270,35 +279,37 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ # complementary zero branches under the same condition select directly (UPat.var("c").where(UPat.var("t"), 0) + UPat.var("c").where(0, UPat.var("f")), lambda c,t,f: c.where(t, f)), # ALU/variable min==max -> CONST - (UPat({Ops.CMPLT, Ops.CMPNE, Ops.FLOORDIV, Ops.FLOORMOD, Ops.PARAM, Ops.AFTER, Ops.SPECIAL}, name="x"), + (UPat({Ops.CMPLT, Ops.CMPEQ, Ops.DIV, Ops.MOD, Ops.PARAM, Ops.SPECIAL}, name="x"), lambda x: x.const_like(x.vmin) if x.dtype is not dtypes.void and x.vmin == x.vmax else None), (UPat(Ops.RANGE, src=(UPat.cvar().or_casted(),), name="x"), lambda x: x.const_like(x.vmin) if x.vmin == x.vmax else None), # max folding ((UPat.cvar("a") < UPat.var("b")).where(UPat.var("b"), UPat.cvar("c")), lambda a,b,c: UOp.maximum(a,b) if a.val == c.val else None), - ((UPat.var("a") < UPat.cvar("b")).where(UPat.cvar("c"), UPat.var("a")), lambda a,b,c: UOp.maximum(a,b) if b.val == c.val else None), - (UPat.maximum(UPat.var("x"), UPat.var("y")), lambda x,y: x if x.vmin >= y.vmax else y if x.vmax <= y.vmin else None), + # (a max(a+(c-b), c), the shift needs a signed int so the unselected arm can't wrap + ((UPat.var("a") < UPat.cvar("b")).where(UPat.cvar("c"), UPat.var("y")), lambda a,b,c,y: UOp.maximum(y,c) if (y is a and b.val == c.val) or + (dtypes.is_int(a.dtype) and not dtypes.is_unsigned(a.dtype) and y is a+(c.val-b.val)) else None), + (UPat.maximum(UPat.var("x"), UPat.var("y")), lambda x,y: x if x.vmin >= y.vmax else None), # TODO: why does this rule break beautiful_mnist? #((UPat.var("x")+UPat.var("z")).maximum(UPat.var("y")+UPat.var("z")), lambda x,y,z: x.maximum(y) + z), - # ** two stage ALU folding ** - *((UPat.var("x").alu(op, UPat.cvar("c1")).alu(op, UPat.cvar("c2")).named("f"), - lambda f,x,c1,c2: x.alu(f.op,c1.alu(f.op,c2))) for op in GroupOp.Associative), - # (x//c1)//c2 -> x//(c1*c2) for c2>0 - ((UPat.var("x") // UPat.cvar("c1")) // UPat.cvar("c2"), lambda x,c1,c2: x//(c1*c2) if c2.vmin>0 else None), + # (x//c1)//c2 -> x//(c1*c2) for c2>0 when c1*c2 fits + ((UPat.var("x") // UPat.cvar("c1")) // UPat.cvar("c2"), + lambda x,c1,c2: x//(c1*c2) if c2.vmin>0 and x.dtype.min<=c1.vmin*c2.vmin<=x.dtype.max else None), # ** lt ** # c0+x x < c1-c0 - ((UPat.cvar("c0") + UPat.var("x", dtype=dtypes.ints+(dtypes.weakint,))) < UPat.cvar("c1"), lambda x,c0,c1: x<(c1-c0)), - # x//d x0, and -> c*d x0 ((UPat.var("x", dtype=dtypes.weakint)//UPat.cvar("d")) 0 else (x>c.val*d.val) if d.val < 0 else None), + lambda x,d,c: (x 0 else None), # ** move add/mul consts to end (NOTE: this is still happening before constant folding) ** ((UPat.var("x") + UPat.cvar("c1")) + UPat.var("y"), lambda x,c1,y: (x+y)+c1 if y.op is not Ops.CONST else None), ((UPat.var("x") * UPat.cvar("c1")) * UPat.var("y"), lambda x,c1,y: (x*y)*c1 if y.op is not Ops.CONST else None), # *** rules from symbolic *** # generic lt folding (UPat.var("x", dtypes.weakint) 0. NOTE: not x < 1 means x > 0 - ((UPat.var("x", dtypes.weakint)<1).ne(True), lambda x: (newx<1).ne(True) if (newx:=canonicalize_simplex(x)) is not None else None), + ((UPat.var("x", dtypes.weakint)<1).logical_not(), lambda x: (newx<1).logical_not() if (newx:=canonicalize_simplex(x)) is not None else None), # a range mod its own upper bound is just the range (UPat(Ops.RANGE, src=UPat.var("end"), name="r")%UPat.var("end"), lambda r,end: r), (UPat(Ops.RANGE, src=UPat.var("end"), name="r")//UPat.var("end"), lambda r,end: r.const_like(0)), @@ -319,8 +330,6 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ else y.src for y in x.src[1:]]))))), # after/end with 1 src is just src[0] (UPat((Ops.AFTER, Ops.END), src=(UPat.var("s"),)), lambda s: s), - # ranges can be subbed for CONSTs, remove them from ENDs. BACKEDGE conditions are never range selectors. - (UPat(Ops.END, name="x"), lambda x: x.replace(src=(x.src[0],)+tuple(r for r in x.src[1:] if r.op is not Ops.CONST))), # the rules above key on bare CONSTs, so a redundantly committed const has to be uncast in the same fixpoint ])+div_and_mod_symbolic+pm_uncast_const @@ -331,7 +340,7 @@ def parse_valid(v:UOp) -> tuple[UOp, bool, int]|None: # if it's X >= c, returns X, False, c # a=a+1 for a const a), its negation a>=b.vmin (b<=a for a const a) - neg = v.op is Ops.CMPNE and v.src[1].op is Ops.CONST and v.src[1].val == 1 + neg = v.op is Ops.CMPEQ and v.src[1].op is Ops.CONST and v.src[1].val == 0 if (lt:=v.src[0] if neg else v).op is not Ops.CMPLT or not dtypes.is_int(lt.src[0].dtype): return None if (a:=lt.src[0]).op is Ops.CONST: return lt.src[1], neg, int(a.val) + (0 if neg else 1) return a, not neg, int(lt.src[1].vmin) if neg else int(lt.src[1].vmax)-1 @@ -352,7 +361,7 @@ def uop_given_valid(valid:UOp, uop:UOp, try_simplex=True) -> UOp: v0, v1 = (expr.vmin if v[0] is None else v[0], expr.vmax if v[1] is None else v[1]) all_candidates.append((expr, UOp.variable(f"fake{i}", v0, v1, expr.dtype))) - if try_simplex and expr.op is Ops.ADD and v0 == 1 and all(u.op in GroupOp.Irreducible for u in expr.split_uop(Ops.ADD)): + if try_simplex and expr.op is Ops.ADD and v0 == 1 and all(u.op in GroupOp.Irreducible and u.op is not Ops.CONST for u in expr.split_uop(Ops.ADD)): # For X0 + X1 + ... > 0, check whether every Xi > 0 gives the same simplified output. candidate = [(Xi, UOp.variable(f"fake{i}", 1, Xi.vmax, Xi.dtype)) for Xi in expr.split_uop(Ops.ADD)] if any(X not in uop.backward_slice_with_self for X,_ in candidate): continue @@ -386,6 +395,8 @@ def simplify_valid(valid:UOp) -> UOp|None: def reduce_mul_chain(r:UOp) -> UOp|None: if r.arg[0] not in {Ops.ADD, Ops.MAX}: return None + # max(x*m) = max(x)*m needs x*m monotone in x, and an unsigned multiply wraps + if r.arg[0] is Ops.MAX and dtypes.is_unsigned(r.dtype): return None inside, outside = [], [] for m in r.src[0].split_uop(Ops.MUL): m_parents = m.backward_slice @@ -425,7 +436,7 @@ def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: # pure index math only: a LOAD in x executes even where cond is false, so its INDEX valid must survive the assumption if x.dtype is not dtypes.weakint or x.op_in_backward_slice_with_self(Ops.INDEX): return None # Skip if x contains DIV/MOD AND IMAGE mode is enabled -> image index e.g. openpilot - if IMAGE.value > 0 and x.op_in_backward_slice_with_self(Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD): return None + if IMAGE.value > 0 and x.op_in_backward_slice_with_self(Ops.TRUNCDIV, Ops.TRUNCMOD, Ops.DIV, Ops.MOD): return None return cond.where(uop_given_valid(cond, x, try_simplex=False), i) pm_simplify_valid = PatternMatcher([ @@ -447,25 +458,24 @@ sym = symbolic+pm_simplify_valid+PatternMatcher([ # ** pow ** ((UPat(Ops.POW, name="p"), lambda p: xpow_int(p.src[0], p.src[1], p.dtype) if dtypes.is_int(p.dtype) else xpow(*p.src))), # ** load/store folding ** - (UPat.store(UPat(Ops.INDEX, name="index"), UPat.load(UPat(Ops.INDEX, name="index"))), lambda index: UOp(Ops.NOOP)), + (UPat.store(UPat(Ops.INDEX, name="index"), UPat.load(UPat(Ops.INDEX, name="index")), name="x"), lambda index,x: UOp(Ops.NOOP).after(*x.src)), (UPat.store(UPat(Ops.INDEX, name="index"), UPat.var("gate").where(UPat.var("alt"), UPat.load(UPat(Ops.INDEX, name="index")))), lambda index, gate, alt: UOp.store(index.src[0].index(index.src[1].valid(gate)), alt)), + (UPat.store(UPat(Ops.INDEX, name="index"), UPat.var("gate").where(UPat.load(UPat(Ops.INDEX, name="index")), UPat.var("alt"))), + lambda index, gate, alt: UOp.store(index.src[0].index(index.src[1].valid(gate.logical_not())), alt)), # fold gated LOAD/STORE - (UPat(Ops.STORE, src=(UPat(), invalid_pat)), lambda i: UOp(Ops.NOOP)), + (UPat(Ops.STORE, src=(UPat(), invalid_pat), name="x"), lambda i,x: UOp(Ops.NOOP).after(*x.src)), # store of where with invalid -> gated store (UPat(Ops.STORE, src=(UPat(Ops.INDEX, name="index"), UPat.var("cond").where(UPat.var("val"), invalid_pat))), lambda index, cond, val, i: UOp.store(index.src[0].index(index.src[1].valid(cond)), val)), - ((UPat.var("x") * UPat.var("x")).reciprocal(), lambda x: x.reciprocal()*x.reciprocal()), # 1/(x^c) -> (1/x)^c - ((UPat.var("x") * UPat.var("x") * UPat.var("x")).reciprocal(), lambda x: x.reciprocal()*x.reciprocal()*x.reciprocal()), - ((UPat.var("x") * UPat.cvar("c")).reciprocal(), lambda x,c: x.reciprocal()*c.reciprocal()), # 1/(x*c) -> (1/c)*(1/x) (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")), lambda x,d: 1-d), # x*/(1+x) -> 1-1/(1+x) (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")*UPat.var("y")), lambda x,y,d: y*(1-d)), (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")+UPat.var("y")), lambda x,y,d: (1-d)+x*y), # reduce mul chain, move muls after the reduce (UPat(Ops.MUL).reduce(name="r", allow_any_len=True), reduce_mul_chain), # ** combine terms (opinionated) ** - (-1 * (UPat.var("x") + UPat.var("y")), lambda x,y: (-x)+(-y)), # -(x+y) -> -x + -y # (x+y)*c -> x*c+y*c. only for int, float has inf*0=nan issue - ((UPat.var("x", dtypes.weakint) + UPat.var("y")) * UPat.cvar("c"), lambda x,y,c: x*c+y*c), + ((UPat.var("x", dtypes.weakint) + UPat.var("y")).named("a") * UPat.cvar("c"), + lambda a,x,y,c: x*c+y*c if all(fits(a, v*c.vmin) for v in (x.vmin, x.vmax, y.vmin, y.vmax)) else None), ])+pm_clean_up_group_sink diff --git a/tinygrad_repo/tinygrad/uop/upat.py b/tinygrad_repo/tinygrad/uop/upat.py index 12ee4047a5..ab8800b840 100644 --- a/tinygrad_repo/tinygrad/uop/upat.py +++ b/tinygrad_repo/tinygrad/uop/upat.py @@ -38,10 +38,6 @@ def _get_clause(self:UPat, base:UOp, depth=0) -> UOp: else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_dtype[0])), arg=("{0}.dtype is {1}", dtypes.void))) - if self.match_tag is not None: - if len(self.match_tag) > 1: - and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=frozenset(self.match_tag))), arg=("{0}.tag in {1}", dtypes.void))) - else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_tag[0])), arg=("{0}.tag == {1}", dtypes.void))) if self.src is not None: # single match if len(self.src) == 1 and isinstance(self.src[0], tuple): diff --git a/tinygrad_repo/tinygrad/uop/validate.py b/tinygrad_repo/tinygrad/uop/validate.py index cf3e6311e5..74b683f86f 100644 --- a/tinygrad_repo/tinygrad/uop/validate.py +++ b/tinygrad_repo/tinygrad/uop/validate.py @@ -42,8 +42,8 @@ def z3_shift(x:UOp, a:z3.ExprRef, b:z3.ExprRef) -> z3.ExprRef: return z3.If(b < 0, z3.FreshInt("invalid_shift", ctx=a.ctx), a) # handlers take (x, *src exprs): x provides vmin/vmax for the bounds-aware ops; anything not here falls back to python_alu -z3_alu: dict[Ops, Callable[..., z3.ExprRef]] = {Ops.CMOD: lambda _,a,b: a-z3_cdiv(a,b)*b, Ops.CDIV: lambda _,a,b: z3_cdiv(a,b), - Ops.FLOORDIV: lambda _,a,b: z3_floordiv(a,b), Ops.FLOORMOD: lambda _,a,b: a-z3_floordiv(a,b)*b, +z3_alu: dict[Ops, Callable[..., z3.ExprRef]] = {Ops.TRUNCMOD: lambda _,a,b: a-z3_cdiv(a,b)*b, Ops.TRUNCDIV: lambda _,a,b: z3_cdiv(a,b), + Ops.DIV: lambda _,a,b: z3_floordiv(a,b), Ops.MOD: lambda _,a,b: a-z3_floordiv(a,b)*b, Ops.WHERE: lambda _,c,a,b: z3.If(c,a,b), Ops.MAX: lambda _,a,b: z3.If(a UOp: @@ -64,7 +65,14 @@ def lower_weak_node(u:UOp) -> UOp|None: src = src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else s.ccast(dt) for s in src[start:]) return u.replace(src=src).cast(u.dtype) +def narrow_image_coord(c:UOp) -> UOp: + if (idx:=c.get_idx()).dtype is not dtypes.long: return c + return idx.cast(dtypes.int) if idx is c else idx.cast(dtypes.int).valid(c.get_valid()) + pm_lower_weak = PatternMatcher([ + # a long image coordinate narrows, image reads and writes take int coordinates + (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var("y"), UPat.var("x")), name="u"), lambda u,buf,y,x: + u.replace(src=(buf, narrow_image_coord(y), narrow_image_coord(x))) if is_image_shape(buf._shape) else None), # a gated long index into a small buffer narrows; its out-of-gate value is discarded (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("gate").where(UPat.var("idx", dtypes.long), UPat(Ops.CONST, arg=Invalid))), allow_any_len=True, name="u"), diff --git a/tinygrad_repo/tinygrad/viz/README.md b/tinygrad_repo/tinygrad/viz/README.md index 5b0c37bcc1..da1d61c0fd 100644 --- a/tinygrad_repo/tinygrad/viz/README.md +++ b/tinygrad_repo/tinygrad/viz/README.md @@ -8,21 +8,6 @@ to use: Use `python -m tinygrad.viz.cli` (add --json for scripting) to view the full timeline of events. -### Environment variables - -Setting `DEBUG` includes the following data in the output stream. These show up as raw `{"value": "..."}` lines when using `--json`. - -| DEBUG | Includes | -|------:|----------| -| 3 | Base AST | -| 4 | Generated source | -| 5 | Rewrite steps and kernel graph | -| 6 | All UOp graphs | -| 7 | All rewrites | - - -VIZ defaults to colored output. Set `NO_COLOR=1` to disable colors. - ### Profiling examples Get kernel times and ASTs @@ -31,6 +16,12 @@ Get kernel times and ASTs DEBUG=3 python -m tinygrad.viz.cli --json > /tmp/events.jsonl ``` +Search kernels or rewrite passes + +``` +python -m tinygrad.viz.cli -q 'schedule|linear' +``` + Select events between two markers Markers are set using the `profile_marker` helper in user code. To list them: @@ -46,48 +37,44 @@ Set `-t` to aggregate events. ### Rewrites Debugging example -First, find the rewrite you are looking for. This can be a schedule or kernel: +Query rewrites: ```bash -python -m tinygrad.viz.cli -s TINY | rg Schedule -python -m tinygrad.viz.cli -s TINY | rg E_3 +python -m tinygrad.viz.cli -q 'tensor graph' +python -m tinygrad.viz.cli -q 'devectorize2' +python -m tinygrad.viz.cli -q '^E_' ``` -List all rewrite passes: - -Rewrite pass names come from `graph_rewrite(..., name="...")` in user code. -```bash -python -m tinygrad.viz.cli -s TINY "Schedule 6 Kernels n1" --ls -``` - -Show the input graph for each pass +Select specific rewrites: ```bash -DEBUG=6 python -m tinygrad.viz.cli -s TINY "Schedule 6 Kernels n1" +python -m tinygrad.viz.cli -s TINY "Schedule 6 Kernels n1" ``` -Show all rewrites +### Environment variables -```bash -# for the entire scheduler -DEBUG=7 python -m tinygrad.viz.cli -s TINY "Schedule 6 Kernels n1" +Combine `DEBUG` with the CLI command to include additional information -# or for a specific pass -DEBUG=7 python -m tinygrad.viz.cli -s TINY "Schedule 6 Kernels n1" "earliest rewrites" -``` +| DEBUG | Includes | +|------:|----------| +| 3 | Base AST | +| 4 | Generated source | +| 5 | Rewrite steps and kernel graph | +| 6 | All UOp graphs | +| 7 | All rewrites | + +Set `NO_COLOR=1` to disable colors. # SQTT / PMC profiling (DEV=AMD only) +PMC profiling is included by default, select using: + +```bash +python -m tinygrad.viz.cli -s "gemm PMC" +``` + SQTT has additional overhead. Set VIZ=2 to include it in pkls. -Examples: - -Get all SQTT packets ```bash python -m tinygrad.viz.cli -s "kernel SQTT SE:0 PKTS" --json ``` - -Get bank conflicts: -```bash -python -m tinygrad.viz.cli -s "gemm PMC" | rg -A 16 SQC_LDS_BANK_CONFLICT -``` diff --git a/tinygrad_repo/tinygrad/viz/cli.py b/tinygrad_repo/tinygrad/viz/cli.py index 357c9b08dd..c1777dc76d 100755 --- a/tinygrad_repo/tinygrad/viz/cli.py +++ b/tinygrad_repo/tinygrad/viz/cli.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import argparse, pathlib, signal, struct, json, os, itertools, heapq +import argparse, pathlib, signal, struct, json, os, itertools, heapq, re os.environ["VIZ"] = "0" if hasattr(signal, "SIGPIPE"): signal.signal(signal.SIGPIPE, signal.SIG_DFL) from typing import Iterator @@ -66,6 +66,8 @@ def get(data:dict, key:str): match = difflib.get_close_matches(key, [ansistrip(k) for k in data], n=1, cutoff=0.6) raise RuntimeError(f'item "{key}" not found in list'+(f", did you mean {match[0]!r}?" if match else '')) +def matches(name:str, query:str) -> bool: return bool(query) and re.search(query, name, re.IGNORECASE) is not None + def fmt_top(k:dict) -> str: return f"{fmt_colored(k['name'])}{' ' * max(0, 38-ansilen(k['name']))} {time_to_str(k['dur_ms']*1e-3, w=9)} {k['count']:7d} {k['pct']:6.2f}%"+\ (" "*4+fmt_data(k['fmt']) if k['fmt'] else "") @@ -198,21 +200,23 @@ def main(args) -> None: elif not file.startswith("<") and not fxn.startswith("<"): fmt["loc"] = line yield {"device":dev, "name":fmt_colored(e["name"]), "dur_ms":e["dur"]*1e-3, "st_ms":e["st"]*1e-3, "fmt":fmt, "ref":e["ref"], "ext":"\n".join(ext)} - fmt_row = fmt_top if args.t else fmt_all + fmt_row, query = fmt_top if args.t else fmt_all, "".join(args.query) if args.query else None seen_refs:set[int] = set() def render_event(k:dict, ls=args.list) -> None: - if len(args.src) > 1 and ansistrip(k["name"]) not in args.src: return None - print(emit(k, to_str=fmt_row)) + if len(args.src) > 1 and ansistrip(k["name"]) not in args.src and query is None: return None + if (debug_steps:=query is None or matches(ansistrip(k["name"]), query)): print(emit(k, to_str=fmt_row)) if k["ref"] is not None and k["ref"] not in seen_refs: seen_refs.add(k["ref"]) for i,s in enumerate(viz_data.ctxs[k["ref"]]["steps"]): - if DEBUG >= 3 and s["name"] == "View Base AST": print_step(s) - if DEBUG >= 4 and s["name"] == "View Source": print_step(s) - if DEBUG >= 5 or ls: print(emit(" "*s["depth"]+s["name"]+(f" - {s['match_count']}" if s.get('match_count', 0) else ''))) - if DEBUG >= 6 or (DEBUG >= 5 and s["name"] == "View Kernel Graph") or (s["name"] in args.src): - print_step(s, reconstruct_matches=s["name"] in args.src) - if DEBUG >= 7: print_step(s, reconstruct_matches=True) - elif DEBUG >= 3 and k.get("ext"): print(emit(k["ext"])) + if debug_steps: + if DEBUG >= 3 and s["name"] == "View Base AST": print_step(s) + if DEBUG >= 4 and s["name"] == "View Source": print_step(s) + if DEBUG >= 5 or ls: print(emit(" "*s["depth"]+s["name"]+(f" - {s['match_count']}" if s.get('match_count', 0) else ''))) + if DEBUG >= 6 or (DEBUG >= 5 and s["name"] == "View Kernel Graph") or (s["name"] in args.src): + print_step(s, reconstruct_matches=s["name"] in args.src) + if DEBUG >= 7: print_step(s, reconstruct_matches=True) + elif query is not None and matches(s["name"], query): print_step(s, reconstruct_matches=DEBUG >= 7) + elif debug_steps and (DEBUG >= 3 and k.get("ext")): print(emit(k["ext"])) for k in (produce_top_kernels if args.t else produce_all_kernels)(): render_event(k) def get_arg_parser() -> argparse.ArgumentParser: @@ -221,6 +225,7 @@ def get_arg_parser() -> argparse.ArgumentParser: parser.add_argument("--list", "--ls", dest="list", action="store_true", help="List sources") parser.add_argument("--interval", nargs="+", metavar=("START", "END"), help="Optional start and end marker") parser.add_argument("-t", nargs="?", type=int, const=20, metavar="COUNT", help="Aggregate top kernels (optional count, default 20)") + parser.add_argument("-q", "--query", nargs="+", default=[], metavar="REGEX", help="Search query") parser.add_argument("--profile-path", type=str, metavar="PATH", help="Optional path to profile.pkl (default: latest profile)", default=temp("profile.pkl", append_user=True)) parser.add_argument("--rewrites-path", type=str, metavar="PATH", help="Optional path to rewrites.pkl (default: latest rewrites)", diff --git a/tinygrad_repo/tinygrad/viz/index.html b/tinygrad_repo/tinygrad/viz/index.html index c96063f5bb..0b665122b9 100644 --- a/tinygrad_repo/tinygrad/viz/index.html +++ b/tinygrad_repo/tinygrad/viz/index.html @@ -211,7 +211,7 @@ background-color: #0f1018; padding: 20px; z-index: 2; - position: absolute; + position: relative; height: 100%; } .metadata > * + *, .info > * + *, .rewrite-container > * + *, .ctx-list > * + * { @@ -239,6 +239,12 @@ } #custom { z-index: 1; + margin-inline: 0; + } + #profiler { + position: absolute; + inset: 0 0 auto; + width: auto; } .ctx-list-parent { width: 15%; diff --git a/tinygrad_repo/tinygrad/viz/js/index.js b/tinygrad_repo/tinygrad/viz/js/index.js index daf5e1b8d7..9ffd2381fb 100644 --- a/tinygrad_repo/tinygrad/viz/js/index.js +++ b/tinygrad_repo/tinygrad/viz/js/index.js @@ -125,8 +125,6 @@ const drawGraph = (data) => { nodes.selectAll("foreignObject.source-node").data(d => d.source != null ? [d] : []).join("foreignObject").attr("class", "source-node") .attr("x", d => -d.width/2).attr("y", d => -d.height/2).attr("width", d => d.width).attr("height", d => d.height) .each((d, i, nodes) => nodes[i].replaceChildren(codeBlock(d.source, d.lang))); - addTags(nodes.selectAll("g.tag").data(d => d.tag != null ? [d] : []).join("g").attr("class", "tag") - .attr("transform", d => `translate(${-d.width/2+8}, ${-d.height/2+8})`).datum(e => ({ text:e.tag }))); addTags(nodes.selectAll("g.addrspace").data(d => d.addrspace != null ? [d] : []).join("g").attr("class", "tag addrspace") .attr("transform", d => `translate(${d.width/2-8}, ${-d.height/2+8})`).datum(e => ({ rect:true, width:10, height:10, fill:e.addrspace, stroke:"none" }))); const CALL_TAG_WIDTH = 14; @@ -239,7 +237,7 @@ const waveColor = (op) => { if (op.includes("LDS_")) { ret = darkenHex(ret, 25) } return ret }; -const colorScheme = {TINY:new Map([["Schedule","#1b5745"],["do_to_program","#1d2e62"],["DEFAULT","#354f52"]]), +const colorScheme = {TINY:new Map([["Schedule","#1b5745"],["do_to_program","#1d2e62"],["lift","#292B62"],["DEFAULT","#354f52"]]), DEFAULT:["#2b2e39", "#2c2f3a", "#31343f", "#323544", "#2d303a", "#2e313c", "#343746", "#353847", "#3c4050", "#404459", "#444862", "#4a4e65"], BUFFER:["#342483", "#3E2E94", "#4938A4", "#5442B4", "#5E4CC2", "#674FCA"], GPC:new Map([["NONE","#1a7a2e"],["MEMORY_DEPENDENCY","#8b1a00"],["EXEC_DEPENDENCY","#006b6b"],["INST_FETCH","#7a7a00"],["SYNC","#6b006b"], diff --git a/tinygrad_repo/tinygrad/viz/js/worker.js b/tinygrad_repo/tinygrad/viz/js/worker.js index a4319633ac..dc6fab8289 100644 --- a/tinygrad_repo/tinygrad/viz/js/worker.js +++ b/tinygrad_repo/tinygrad/viz/js/worker.js @@ -57,7 +57,7 @@ const layoutUOp = (g, { graph, change }, opts) => { ctx.font = `350 ${lineHeight}px ${g.graph().font}`; if (change?.length) g.setNode("overlay", {label:"", labelWidth:0, labelHeight:0, labelX:0, className:"overlay"}); let callCount = 0; - for (const [k, {label, src, ref, color, tag, exclude, addrspace}] of Object.entries(graph)) { + for (const [k, {label, src, ref, color, exclude, addrspace}] of Object.entries(graph)) { // adjust node dims by label size (excluding escape codes) + add padding let dims, source, lang; if (label.startsWith("SOURCE\n")) { @@ -75,11 +75,11 @@ const layoutUOp = (g, { graph, change }, opts) => { dims = rectDims(width, height); } const op = label.split("\n", 1)[0]; - const callNode = op === "CALL", programNode = op === "PROGRAM"; - const collapsePorts = callNode ? [0] : programNode ? [0, 1] : null; + const callNode = op === "CALL" || op === "PROGRAM"; + const collapsePorts = op === "CALL" ? [0] : op === "PROGRAM" ? [0, 1] : null; if (callNode) callCount++; g.setNode(k, {...dims, label, labelX:0, ref, id:k, color, callNode, collapsePorts, exclude, addrspace, source, lang, - className:label.startsWith("REWRITE_ERROR") ? "err" : null, tag:tag?.length > 8 ? tag.substring(0, 8) : tag}); + className:label.startsWith("REWRITE_ERROR") ? "err" : null}); // add edges const edgeCounts = {}; for (const [_, s] of src) edgeCounts[s] = (edgeCounts[s] || 0)+1; diff --git a/tinygrad_repo/tinygrad/viz/serve.py b/tinygrad_repo/tinygrad/viz/serve.py index e82a2b1d11..11105baa7e 100755 --- a/tinygrad_repo/tinygrad/viz/serve.py +++ b/tinygrad_repo/tinygrad/viz/serve.py @@ -6,11 +6,12 @@ from dataclasses import dataclass, field from urllib.parse import parse_qs, urlparse from http.server import BaseHTTPRequestHandler from typing import Any, TypedDict, TypeVar, Generator, Callable -from tinygrad.helpers import colored, getenv, unwrap, word_wrap, TRACEMETA, ProfileEvent, ProfileRangeEvent, TracingKey, ProfilePointEvent, temp +from tinygrad.helpers import colored, getenv, unwrap, word_wrap, ProfileEvent, ProfileRangeEvent, TracingKey, ProfilePointEvent, temp from tinygrad.helpers import printable, Context, START_TIME, NO_COLOR, ansistrip from tinygrad.renderer.amd.dsl import Inst, Reg from tinygrad.renderer.amd import detect_format from tinygrad.runtime.autogen.amd.common import OpType +from tinygrad.runtime.autogen.amd.rdna3.ins import s_code_end # NOTE: using HTTPServer forces a potentially slow socket.getfqdn class TCPServerWithReuse(socketserver.TCPServer): @@ -40,7 +41,7 @@ class HTTPRequestHandler(BaseHTTPRequestHandler): except (BrokenPipeError, ConnectionResetError): source.close() from tinygrad.uop.ops import TrackedGraphRewrite, RewriteTrace, UOp, Ops, GroupOp, srender, sint, sym_infer, range_str, range_start, multirange_str -from tinygrad.uop.ops import KernelInfo, ParamArg +from tinygrad.uop.ops import KernelInfo, ParamArg, ProgramInfo from tinygrad.uop.render import render_uir, render_index, uops_colors, _inline, _render_arg from tinygrad.device import ProfileDeviceEvent, ProfileGraphEvent, ProfileGraphEntry, ProfileProgramEvent from tinygrad.dtype import dtypes, AddrSpace @@ -82,7 +83,7 @@ def load_rewrites(data:VizData) -> None: steps.append(create_step("View Source", ("/code", i, len(steps)), lin_idx, depth=0)) steps.append(create_step("View Disassembly", ("/asm", i, len(steps)), (k.ret, lin_idx), depth=0)) lin_idx = None - if s.name == "View Program": ki = _reconstruct(data, s.sink, depth=1).src[0].arg + if s.name == "View Program": ki = data.trace.uop_fields[data.trace.uop_fields[s.sink][1][0]][2] for key in k.keys: data.ref_map[key] = i data.ctxs.append({"name":k.display_name, "steps":steps, "ki":ki}) @@ -122,14 +123,16 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: # exclude RESHAPE/EXPAND that only serve to broadcast a CONST if u.op in {Ops.RESHAPE, Ops.EXPAND} and len(u.src) >= 1 and u.src[0] in excluded and u is not x: excluded.add(u) if u.op in {*GroupOp.Movement, Ops.PARAM}: excluded.update(s for s in u.src if s.op is Ops.STACK and all(x.op is Ops.CONST for x in s.src)) - if u.op in {*GroupOp.Binary, *GroupOp.Ternary} and all(s.op in {Ops.CONST, Ops.PARAM} for s in u.src): excluded.update(u.src) + if u.op in GroupOp.Broadcastable and all(s.op in {Ops.CONST, Ops.PARAM} for s in u.src): excluded.update(u.src) for u in toposort: - argst = codecs.decode(u.arg if isinstance(u.arg, str) else _render_arg(u), "unicode_escape") + argst = u.arg if u.op is Ops.SOURCE else codecs.decode(u.arg if isinstance(u.arg, str) else _render_arg(u), "unicode_escape") with soft_err(): if u.op in GroupOp.Movement and u.marg: argst = (mask_to_str if u.op in {Ops.SHRINK, Ops.PAD} else shape_to_str)(u.marg) if u.op is Ops.BINARY: argst = f"<{len(u.arg)} bytes>" if u.op is Ops.CONST and dtypes.is_float(u.dtype): argst = f"{u.val:g}" if u.op is not Ops.SOURCE: argst = word_wrap(argst.replace(':', '')) + if isinstance(u.arg, ProgramInfo): + argst = word_wrap(repr(u.arg).replace(f"params={u.arg.params!r}", "params=(\n" + '\n'.join(render_uir(p) for p in u.arg.params) + "\n)")) label = f"{str(u.op).split('.')[1]}{(chr(10)+argst) if argst else ''}" if u.dtype != dtypes.void: label += f"\n{u.dtype}" for idx,x in enumerate(u.src[:1] if u.op in {Ops.STAGE, Ops.INDEX} else (u.src if u.op is not Ops.END else [])): @@ -160,29 +163,33 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: label += "\n" ref = data.ref_map.get(canonicalize_ast(u.body)) if u.op is Ops.CALL else None if ref is not None: label += f"\ncodegen@{fmt_colored(data.ctxs[ref]['name'])}" - if TRACEMETA >= 2 and u.metadata is not None: label += "\n"+str(u.metadata) addrspace_color:str|None = None with soft_err(): addrspace_color = addrspace_colors.get(u.addrspace, None) if u.addrspace is not None else None color = uops_colors.get(u.op, "#ffffff") graph[id(u)] = {"label":label, "src":[(i,id(x)) for i,x in enumerate(u.src)], "exclude":u in excluded, "color":color, - "ref":ref, "tag":repr(u.tag) if u.tag is not None else None, "addrspace":addrspace_color} + "ref":ref, "addrspace":addrspace_color} return graph -def _reconstruct(data:VizData, a:int, depth:int|None=None) -> UOp: - if depth is None and a in data.all_uops: return data.all_uops[a] - op, src, arg, *rest = data.trace.uop_fields[a] - if depth is not None and depth <= 0: return UOp(op, (), arg, *rest) - ret = UOp(op, tuple(_reconstruct(data, s, None if depth is None else depth-1) for s in src), arg, *rest) - if depth is None: data.all_uops[a] = ret - return ret +def _reconstruct(data:VizData, a:int) -> UOp: + stack = [(a, False)] + while stack: + num, visited = stack.pop() + if num in data.all_uops: continue + if visited: + op, src, arg = data.trace.uop_fields[num] + data.all_uops[num] = UOp(op, tuple(data.all_uops[s] for s in src), arg) + else: + stack.append((num, True)) + stack.extend((s, False) for s in data.trace.uop_fields[num][1]) + return data.all_uops[a] -def get_full_rewrite(data:VizData, ctx:TrackedGraphRewrite, depth:int|None=None, update_sink=True) -> Generator[GraphRewriteDetails, None, None]: - next_sink, err = _reconstruct(data, ctx.sink, depth=depth), False +def get_full_rewrite(data:VizData, ctx:TrackedGraphRewrite, update_sink=True) -> Generator[GraphRewriteDetails, None, None]: + next_sink, err = _reconstruct(data, ctx.sink), False yield {"graph":uop_to_json(data, next_sink), "uop":tokenize_uir(data, next_sink), "change":None, "diff":None, "upat":None, "_sink":next_sink} replaces: dict[UOp, UOp] = {} for u0_num,u1_num,upat_loc,dur in ctx.matches: if err: break - replaces[u0:=_reconstruct(data, u0_num, depth=depth)] = u1 = _reconstruct(data, u1_num, depth=depth) + replaces[u0:=_reconstruct(data, u0_num)] = u1 = _reconstruct(data, u1_num) try: new_sink = next_sink.substitute(replaces, walk=ctx.walk, enter_calls=ctx.enter_calls) if update_sink else next_sink except RuntimeError: new_sink, err = UOp(Ops.REWRITE_ERROR, arg=traceback.format_exc()), True match_repr = f"# {dur*1e6:.2f} us\n"+printable(upat_loc) @@ -192,11 +199,11 @@ def get_full_rewrite(data:VizData, ctx:TrackedGraphRewrite, depth:int|None=None, "diff":[ansistrip(x) for x in diff if not x.startswith(("---","+++","@@"))]} if not ctx.bottom_up: next_sink = new_sink -def get_sink_at(upats:tuple[str, ...], viz_data:VizData, kernel_idx:int, lin_idx:int, depth:int|None=None, alt:str|None=None) -> UOp|None: +def get_sink_at(upats:tuple[str, ...], viz_data:VizData, kernel_idx:int, lin_idx:int, src:int, alt:str|None=None) -> UOp|None: for i in range(lin_idx+1, len(rewrites:=viz_data.trace.rewrites[kernel_idx])): - if (r:=rewrites[i]).name == alt: return _reconstruct(viz_data, r.sink, depth=depth) - for s in get_full_rewrite(viz_data, rewrites[lin_idx], depth=depth): - if (s["upat"] is not None and any(n in s["upat"][1] for n in upats)): return s["_sink"] + if (r:=rewrites[i]).name == alt: return _reconstruct(viz_data, viz_data.trace.uop_fields[r.sink][1][src]) + for s in get_full_rewrite(viz_data, rewrites[lin_idx]): + if (s["upat"] is not None and any(n in s["upat"][1] for n in upats)): return u if (u:=s["_sink"]).op is Ops.REWRITE_ERROR else u.src[src] return None # encoder helpers @@ -225,7 +232,6 @@ def flatten_events(profile:list[ProfileEvent], device_ts_diffs:dict[str, Decimal elif isinstance(e, ProfileGraphEvent): cpu_ts = [] for ent in e.ents: cpu_ts += [e.sigs[ent.st_id]+(diff:=cpu_ts_diff(device_ts_diffs, ent.device)), e.sigs[ent.en_id]+diff] - yield (st:=min(cpu_ts)), (et:=max(cpu_ts)), ProfileRangeEvent(f"{e.ents[0].device.split(':')[0]} Graph", f"batched {len(e.ents)}", st, et) for i,ent in enumerate(e.ents): yield (cpu_ts[i*2], cpu_ts[i*2+1], ent) # normalize event timestamps and attach kernel metadata @@ -548,14 +554,13 @@ def get_elf_section(lib:bytes, name:str): def get_arch(target:str) -> str: return "rdna3" if target.startswith("gfx11") else "rdna4" if target.startswith("gfx12") else "cdna" -def amd_decode(buf:bytes, arch:str, off:int=0) -> dict[int, Inst]: - from tinygrad.runtime.autogen.amd.rdna3.ins import s_code_end - code_end = s_code_end().to_bytes()*5 if arch.startswith("rdna") else None +def amd_decode(buf:bytes|memoryview, arch:str, off:int=0) -> dict[int, Inst]: addr_table:dict[int, Inst] = {} + code_end = s_code_end().to_bytes() offset, view = 0, memoryview(buf) while offset < len(buf): - if code_end is not None and buf.startswith(code_end, offset): break remaining = view[offset:] + if remaining[:4] == code_end: break fmt = detect_format(remaining, arch) decoded = fmt.from_bytes(remaining) addr_table[off+offset] = decoded @@ -601,6 +606,9 @@ def get_cfg(pc_table:dict[int, Inst]) -> dict: if op == "s_branch": paths[curr][nx+offset] = UNCOND else: paths[curr].update([(nx+offset, COND_TAKEN), (nx, COND_NOT_TAKEN)]) elif nx in leaders and op != "s_endpgm": paths[curr][nx] = UNCOND + return {"blocks":blocks, "paths":paths} + +def tokenize_asm(pc_table:dict[int, Inst]) -> dict: pc_tokens:dict[int, list[dict]] = {} for pc, inst in pc_table.items(): pc_tokens[pc] = tokens = [] @@ -612,12 +620,9 @@ def get_cfg(pc_table:dict[int, Inst]) -> dict: elif name in {"op","opx","opy"}: tokens.append({"st":(op_name:=val.name.lower()), "keys":[op_name], "kind":0}) elif name != "encoding" and val != f.default: tokens.append({"st":repr(val - (1 << 32) if name == "literal" and val >= (1 << 31) else val), "keys":[repr(val)], "kind":1}) - lines:list[str] = [] disasm = {pc:str(inst) for pc,inst in pc_table.items()} asm_width = max(len(asm) for asm in disasm.values()) - for pcs in blocks.values(): - for pc in pcs: lines.append(f"{disasm[pc]:<{asm_width}} # {pc:012X}") - return {"data":{"blocks":blocks, "paths":paths, "pc_tokens":pc_tokens}, "src":"\n".join(lines), "lang":"python"} + return {"data":{"pc_tokens":pc_tokens}, "src":"\n".join([f"{disasm[pc]:<{asm_width}} # {pc:012X}" for pc in pc_table]), "lang":"python"} # ** Main render function to get the complete details about a trace event @@ -627,23 +632,27 @@ def get_render(viz_data:VizData, query:str, **kwargs) -> dict: data = viz_data.ctxs[i]["steps"][j]["_data"] if fmt == "graph-rewrites": return {"value":get_full_rewrite(viz_data, viz_data.trace.rewrites[i][j], **kwargs), "content_type":"text/event-stream"} if fmt == "uops": - if (sink:=get_sink_at(("do_linearize",), viz_data, i, data, alt="View Program")) is None: return {"src":"No linear found"} + if (sink:=get_sink_at(("do_linearize",), viz_data, i, data, src=1, alt="View Program")) is None: return {"src":"No linear found"} if sink.op is Ops.REWRITE_ERROR: return {"src":sink.arg} ret:dict = {} - with soft_err(lambda err: ret.update(err)): ret["src"] = render_uir(list(sink.src[1].toposort(enter_calls=True))[:-1]) + with soft_err(lambda err: ret.update(err)): ret["src"] = render_uir(list(sink.toposort(enter_calls=True))[:-1]) return ret if fmt == "code": - if (sink:=get_sink_at(("do_render",), viz_data, i, data, depth=1, alt="View Program")) is None: return {"src":"No source found"} - return {"src":sink.arg} if sink.op is Ops.REWRITE_ERROR else {"src":sink.src[2].arg, "lang":"cpp"} + if (sink:=get_sink_at(("do_render",), viz_data, i, data, src=2, alt="View Program")) is None: return {"src":"No source found"} + if sink.op is Ops.REWRITE_ERROR: return {"src":sink.arg} + ast = unwrap(get_sink_at(("do_render",), viz_data, i, data, src=0, alt="View Program")) + extern_srcs = [u.arg for u in ast.toposort(lambda u: u.op is not Ops.LINEAR, enter_calls=True) if u.op is Ops.SOURCE] + return {"src":"\n".join([sink.arg, *reversed(extern_srcs)]), "lang":"cpp"} if fmt == "asm": ret = {} renderer, idx = data - if (sink:=get_sink_at(("do_compile","do_assemble"), viz_data, i, idx, depth=1, alt="View Program")) is None: return {"src":"No binary found"} + if (sink:=get_sink_at(("do_compile","do_assemble"), viz_data, i, idx, src=3, alt="View Program")) is None: return {"src":"No binary found"} if sink.op is Ops.REWRITE_ERROR: return {"src":sink.arg} - lib:bytes = sink.src[3].arg + lib:bytes = sink.arg if (target:=renderer.target.arch).startswith("gfx"): with soft_err(lambda err: ret.update(err)): - ret.update(get_cfg(amd_decode((text:=get_elf_section(lib, ".text")).content, get_arch(target), text.header.sh_addr))) + ret.update(tokenize_asm(pc_table:=amd_decode((text:=get_elf_section(lib, ".text")).content, get_arch(target), text.header.sh_addr))) + ret["data"].update(get_cfg(pc_table)) with soft_err(lambda err: ret.update(err)): from tinygrad.runtime.autogen import amdgpu_kd kd = amdgpu_kd.llvm_amdhsa_kernel_descriptor_t.from_buffer_copy(bytearray(get_elf_section(lib, ".rodata").content)) diff --git a/upstream-sync.json b/upstream-sync.json index 6dde723c3c..998aa0b270 100644 --- a/upstream-sync.json +++ b/upstream-sync.json @@ -43,8 +43,8 @@ { "path": "tinygrad_repo", "url": "https://github.com/tinygrad/tinygrad.git", - "commit": "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf", - "tree": "03f35edac84fbe516023911c1a135e5a577f5d95", + "commit": "d39227beebd0c3a340dc42b2cdc1e3db1c36aed9", + "tree": "b02c108aacae57aea75b417c43b1e1caa22ee609", "exclude": [ "AGENTS.md" ]