|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) <2026> NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# |
| 3 | +# SPDX-License-Identifier: Apache-2.0 |
| 4 | +from cuda.lang._enums import PrefetchLevel, CachePolicy |
| 5 | +from cuda.lang._ir.op_defs import RawNVVMIntrinsic |
| 6 | +from cuda.lang._ir.type import TensorMapTy |
| 7 | +from cuda.lang._ir.type_checking_helpers import require_pointer_type |
| 8 | +from cuda.lang._stub.prefetch import prefetch, prefetch_uniform, prefetch_tensor_map |
| 9 | +from cuda.tile._datatype import PointerInfo |
| 10 | +from cuda.tile._exception import InvalidValueError, TypeCheckingError |
| 11 | +from cuda.tile._ir.ir import Var, add_operation_variadic |
| 12 | +from cuda.tile._ir.op_impl import ImplRegistry, require_constant_enum |
| 13 | +from cuda.tile._ir.type import NONE |
| 14 | +from cuda.tile._memory_model import MemorySpace |
| 15 | + |
| 16 | +_registry = ImplRegistry() |
| 17 | +impl = _registry.impl |
| 18 | + |
| 19 | + |
| 20 | +def prefetch_impl_registry() -> ImplRegistry: |
| 21 | + return _registry |
| 22 | + |
| 23 | + |
| 24 | +@impl(prefetch) |
| 25 | +def prefetch_impl(address: Var, level: Var, eviction_priority: Var): |
| 26 | + address_ty = require_pointer_type(address) |
| 27 | + address_space = PointerInfo(address_ty.pointer_dtype).memory_space |
| 28 | + level = require_constant_enum(level, PrefetchLevel) |
| 29 | + |
| 30 | + if eviction_priority.get_type() == NONE: |
| 31 | + evict_str = "" |
| 32 | + else: |
| 33 | + eviction_priority = require_constant_enum(eviction_priority, CachePolicy) |
| 34 | + if level != PrefetchLevel.L2: |
| 35 | + raise InvalidValueError("Prefetch eviction priority is supported only for L2") |
| 36 | + priority_map = {CachePolicy.L2_EVICT_NORMAL: "normal", CachePolicy.L2_EVICT_LAST: "last"} |
| 37 | + suffix = priority_map.get(eviction_priority) |
| 38 | + if suffix is None: |
| 39 | + valid = ", ".join(x._name_ for x in priority_map.keys()) |
| 40 | + raise InvalidValueError(f"Invalid eviction priority {eviction_priority._name_}." |
| 41 | + f" Accepted values: {valid}.") |
| 42 | + if address_space != MemorySpace.GLOBAL: |
| 43 | + raise TypeCheckingError(f"Pointer in GLOBAL address space is required" |
| 44 | + f" when eviction priority is specified;" |
| 45 | + f" received a {address_space._name_} pointer instead.") |
| 46 | + evict_str = f".evict.{suffix}" |
| 47 | + |
| 48 | + address_space_map = { |
| 49 | + MemorySpace.GENERIC: "", |
| 50 | + MemorySpace.GLOBAL: ".global", |
| 51 | + MemorySpace.LOCAL: ".local", |
| 52 | + } |
| 53 | + address_space_str = address_space_map.get(address_space) |
| 54 | + if address_space_str is None: |
| 55 | + valid = ", ".join(x._name_ for x in address_space_map.keys()) |
| 56 | + raise TypeCheckingError(f"Invalid pointer address space {address_space._name_}." |
| 57 | + f" Accepted address spaces: {valid}.") |
| 58 | + |
| 59 | + level_map = {PrefetchLevel.L1: "L1", PrefetchLevel.L2: "L2"} |
| 60 | + level_str = level_map[level] |
| 61 | + |
| 62 | + add_operation_variadic( |
| 63 | + RawNVVMIntrinsic, |
| 64 | + (), |
| 65 | + intrinsic=f"llvm.nvvm.prefetch{address_space_str}.{level_str}{evict_str}", |
| 66 | + operands_=(address,) |
| 67 | + ) |
| 68 | + |
| 69 | + |
| 70 | +@impl(prefetch_uniform) |
| 71 | +def prefetch_uniform_impl(address: Var): |
| 72 | + address_ty = require_pointer_type(address) |
| 73 | + address_space = PointerInfo(address_ty.pointer_dtype).memory_space |
| 74 | + if address_space != MemorySpace.GENERIC: |
| 75 | + raise TypeCheckingError(f"Expected a pointer in GENERIC address space;" |
| 76 | + f" received {address_space._name_} instead.") |
| 77 | + |
| 78 | + add_operation_variadic( |
| 79 | + RawNVVMIntrinsic, |
| 80 | + (), |
| 81 | + intrinsic="llvm.nvvm.prefetchu.L1", |
| 82 | + operands_=(address,) |
| 83 | + ) |
| 84 | + |
| 85 | + |
| 86 | +@impl(prefetch_tensor_map) |
| 87 | +def prefetch_tensor_map_impl(tensor_map: Var): |
| 88 | + if isinstance(tensor_map.get_type(), TensorMapTy): |
| 89 | + from cuda.lang._ir.ops import tensor_map_as_opaque_ptr |
| 90 | + tensor_map = tensor_map_as_opaque_ptr(tensor_map) |
| 91 | + else: |
| 92 | + address_ty = require_pointer_type(tensor_map) |
| 93 | + address_space = PointerInfo(address_ty.pointer_dtype).memory_space |
| 94 | + valid_spaces = (MemorySpace.GENERIC, MemorySpace.CONSTANT) |
| 95 | + if address_space not in valid_spaces: |
| 96 | + valid = ", ".join(x._name_ for x in valid_spaces) |
| 97 | + raise TypeCheckingError(f"Invalid address space {address_space._name_}." |
| 98 | + f" Accepted address spaces: {valid}") |
| 99 | + add_operation_variadic( |
| 100 | + RawNVVMIntrinsic, |
| 101 | + (), |
| 102 | + intrinsic="llvm.nvvm.prefetch.tensormap", |
| 103 | + operands_=(tensor_map,) |
| 104 | + ) |
0 commit comments