Executable examples#
Read or download these programs, then run them from a compatible installed environment. Each example allocates device inputs, compiles and launches a CuTe kernel, and checks its defined results against a CPU reference. The Programming Guide explains their layout and synchronization requirements.
scan.py#
Block prefix sums with scratch reuse and a nonzero initial value.
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Scan a block tile while reusing scratch with Load and Store."""
import cutlass
import numpy as np
from cutlass import cute
from cutlass.cute.runtime import make_ptr
from cuda.bindings import driver
_BLOCK = (8, 4, 2)
_THREADS = 64
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
def run_example(items_per_thread=2):
"""Check exclusive prefix sums that start from seven across a block tile.
Load, Scan and Store share one descriptor with automatic synchronization.
The CPU reference gives each item seven plus the sum of all earlier
inputs in blocked order. The stored values come from Scan's separate
output payload.
"""
tile_size = _THREADS * items_per_thread
import cuda.coop.cutlass as coop
# docs: start cutlass-scan
@cute.kernel
def scan_tiles(
source: cute.Pointer,
destination: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
group = coop.this_block()
storage = coop.TempStorage(
sharing="shared", alignment=64, auto_sync=True
)
payload = coop.load(
group,
source,
items_per_thread=items_per_thread,
algorithm="transpose",
temp_storage=storage,
)
scanned = coop.exclusive_scan(
group, payload, initial_value=7, temp_storage=storage
)
# Scan returns a new payload. The input retains its original values.
coop.store(
group,
destination,
scanned,
algorithm="transpose",
temp_storage=storage,
)
@cute.jit
def launch(
source: cute.Pointer,
destination: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
scan_tiles(source, destination, items_per_thread).launch(
grid=1, block=_BLOCK
)
# docs: end cutlass-scan
source = np.arange(tile_size, dtype=np.int32)
destination = np.full(tile_size, -101, dtype=np.int32)
cutlass.cuda.initialize_cuda_context()
src = _check(driver.cuMemAlloc(source.nbytes))
try:
dst = _check(driver.cuMemAlloc(destination.nbytes))
try:
_check(driver.cuMemcpyHtoD(src, source.ctypes.data, source.nbytes))
_check(
driver.cuMemcpyHtoD(
dst, destination.ctypes.data, destination.nbytes
)
)
src_pointer = make_ptr(
cutlass.Int32,
int(src),
cute.AddressSpace.gmem,
assumed_align=16,
)
dst_pointer = make_ptr(
cutlass.Int32,
int(dst),
cute.AddressSpace.gmem,
assumed_align=16,
)
launch(src_pointer, dst_pointer, items_per_thread)
_check(driver.cuCtxSynchronize())
_check(
driver.cuMemcpyDtoH(
destination.ctypes.data, dst, destination.nbytes
)
)
finally:
_check(driver.cuMemFree(dst))
finally:
_check(driver.cuMemFree(src))
expected = np.concatenate(
(np.array([7], dtype=np.int32), 7 + source[:-1].cumsum(dtype=np.int32))
)
np.testing.assert_array_equal(destination, expected)
return destination
if __name__ == "__main__":
run_example()
reduce.py#
Block and logical-warp reductions with defined leader results.
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Reduce a full block, eight-thread groups, and a valid input prefix."""
import cutlass
import numpy as np
from cutlass import cute
from cutlass.cute.runtime import make_ptr
from cuda.bindings import driver
_BLOCK = (8, 4, 2)
_THREADS = 64
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
def run_example(items_per_thread=2):
"""Check full block and logical-warp results plus a scalar input prefix.
The output has one block sum, one maximum per logical group, and one
prefix sum, each stored by its group leader. The expected prefix
sum uses each thread's first item; the count is threads, not contiguous
input elements.
"""
tile_size = _THREADS * items_per_thread
import cuda.coop.cutlass as coop
# docs: start cutlass-reduce
@cute.kernel
def reduce_tiles(
source: cute.Pointer,
destination: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
tile_size = _THREADS * items_per_thread
block = coop.this_block()
thread = block.rank()
lanes = coop.this_warp().group_by(8)
inputs = cute.make_tensor(source, cute.make_layout(tile_size))
outputs = cute.make_tensor(
destination, cute.make_layout(_THREADS // 8 + 2)
)
payload = coop.ThreadData(items_per_thread)
for item in cutlass.range_constexpr(items_per_thread):
payload[item] = inputs[thread * items_per_thread + item]
total = coop.sum(block, payload)
maximum = coop.reduce(lanes, payload[0], binary_op="max")
prefix = coop.sum(block, payload[0], valid_items=23)
if thread == 0:
outputs[0] = total
outputs[_THREADS // 8 + 1] = prefix
if lanes.rank() == 0:
outputs[1 + thread // 8] = maximum
@cute.jit
def launch(
source: cute.Pointer,
destination: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
reduce_tiles(source, destination, items_per_thread).launch(
grid=1, block=_BLOCK
)
# docs: end cutlass-reduce
source = np.arange(tile_size, dtype=np.int32)
destination = np.full(_THREADS // 8 + 2, -101, dtype=np.int32)
cutlass.cuda.initialize_cuda_context()
src = _check(driver.cuMemAlloc(source.nbytes))
try:
dst = _check(driver.cuMemAlloc(destination.nbytes))
try:
_check(driver.cuMemcpyHtoD(src, source.ctypes.data, source.nbytes))
_check(
driver.cuMemcpyHtoD(
dst, destination.ctypes.data, destination.nbytes
)
)
src_pointer = make_ptr(
cutlass.Int32,
int(src),
cute.AddressSpace.gmem,
assumed_align=16,
)
dst_pointer = make_ptr(
cutlass.Int32,
int(dst),
cute.AddressSpace.gmem,
assumed_align=16,
)
launch(src_pointer, dst_pointer, items_per_thread)
_check(driver.cuCtxSynchronize())
_check(
driver.cuMemcpyDtoH(
destination.ctypes.data, dst, destination.nbytes
)
)
finally:
_check(driver.cuMemFree(dst))
finally:
_check(driver.cuMemFree(src))
expected = np.concatenate(
(
np.array([source.sum(dtype=np.int32)], dtype=np.int32),
source[::items_per_thread].reshape(-1, 8).max(axis=1),
np.array(
[
source[0 : 23 * items_per_thread : items_per_thread].sum(
dtype=np.int32
)
],
dtype=np.int32,
),
)
)
np.testing.assert_array_equal(destination, expected)
return destination
if __name__ == "__main__":
run_example()
exchange_shuffle.py#
Blocked/striped movement, scatter, and scalar/array Shuffle.
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Exchange a block payload layout and shift its elements by one position."""
import cutlass
import numpy as np
from cutlass import cute
from cutlass.cute.runtime import make_ptr
from cuda.bindings import driver
_BLOCK = (8, 4, 2)
_THREADS = 64
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
def run_example(items_per_thread=3):
"""Check a layout conversion followed by a one-item downward shift.
The CPU reference builds striped order: thread t holds source[t],
source[t + 64], and so on. It shifts that flattened sequence down by one
and appends zero. The kernel explicitly repairs Shuffle's undefined final
item before Store. The comparison checks the composition of two
operations.
"""
tile_size = _THREADS * items_per_thread
import cuda.coop.cutlass as coop
# docs: start cutlass-exchange-shuffle
@cute.kernel
def rearrange(
source: cute.Pointer,
destination: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
group = coop.this_block()
thread = group.rank()
payload = coop.load(group, source, items_per_thread=items_per_thread)
striped = coop.exchange(group, payload, mode="blocked_to_striped")
shifted = coop.shuffle(group, striped, mode="down")
# Shuffle leaves one boundary undefined; initialize it before storing.
if thread == _THREADS - 1:
shifted[items_per_thread - 1] = cutlass.Int32(0)
coop.store(group, destination, shifted)
@cute.jit
def launch(
source: cute.Pointer,
destination: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
rearrange(source, destination, items_per_thread).launch(
grid=1, block=_BLOCK
)
# docs: end cutlass-exchange-shuffle
source = np.arange(tile_size, dtype=np.int32)
destination = np.full(tile_size, -101, dtype=np.int32)
cutlass.cuda.initialize_cuda_context()
src = _check(driver.cuMemAlloc(source.nbytes))
try:
dst = _check(driver.cuMemAlloc(destination.nbytes))
try:
_check(driver.cuMemcpyHtoD(src, source.ctypes.data, source.nbytes))
_check(
driver.cuMemcpyHtoD(
dst, destination.ctypes.data, destination.nbytes
)
)
src_pointer = make_ptr(
cutlass.Int32,
int(src),
cute.AddressSpace.gmem,
assumed_align=16,
)
dst_pointer = make_ptr(
cutlass.Int32,
int(dst),
cute.AddressSpace.gmem,
assumed_align=16,
)
launch(src_pointer, dst_pointer, items_per_thread)
_check(driver.cuCtxSynchronize())
_check(
driver.cuMemcpyDtoH(
destination.ctypes.data, dst, destination.nbytes
)
)
finally:
_check(driver.cuMemFree(dst))
finally:
_check(driver.cuMemFree(src))
striped = source.reshape(items_per_thread, _THREADS).T.reshape(-1)
expected = np.concatenate((striped[1:], np.zeros(1, dtype=np.int32)))
np.testing.assert_array_equal(destination, expected)
return destination
if __name__ == "__main__":
run_example()
merge_sort.py#
Partial-tile key and pair sorting with input preservation.
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Sort a partial block tile while preserving keys and associated values."""
from contextlib import ExitStack
import cutlass
import numpy as np
from cutlass import cute
from cutlass.cute.runtime import make_ptr
from cuda.bindings import driver
_BLOCK = (8, 4, 2)
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
def run_example(items_per_thread=3):
"""Check partial sorting, pair association, and unchanged input payloads.
Five output buffers expose ascending keys, descending key/value pairs,
and both original inputs. Unique values identify their associated keys;
the pair check permits any order among equal keys. Outputs start as -999;
unchanged tails show that no Store wrote past the valid prefix.
"""
tile_size = 64 * items_per_thread
valid_items = tile_size - 7
import cuda.coop.cutlass as coop
# docs: start cutlass-merge-sort
@cute.kernel
def sort_tile(
source_keys: cute.Pointer,
source_values: cute.Pointer,
ascending: cute.Pointer,
descending: cute.Pointer,
sorted_values: cute.Pointer,
original_keys: cute.Pointer,
original_values: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
tile_size = 64 * items_per_thread
valid_items = tile_size - 7
group = coop.this_block()
keys = coop.load(
group,
source_keys,
items_per_thread=items_per_thread,
valid_items=valid_items,
oob_default=1000,
)
values = coop.load(
group,
source_values,
items_per_thread=items_per_thread,
valid_items=valid_items,
oob_default=0.0,
)
scratch = coop.TempStorage(alignment=16, auto_sync=True)
sorted_keys = coop.merge_sort_keys(
group,
keys,
valid_items=valid_items,
oob_default=1000,
temp_storage=scratch,
)
pair_keys, pair_values = coop.merge_sort_pairs(
group,
keys,
values,
descending=True,
valid_items=valid_items,
oob_default=-1000,
temp_storage=scratch,
)
coop.store(
group,
ascending,
sorted_keys,
valid_items=valid_items,
)
coop.store(
group,
descending,
pair_keys,
valid_items=valid_items,
)
coop.store(
group,
sorted_values,
pair_values,
valid_items=valid_items,
)
coop.store(
group,
original_keys,
keys,
valid_items=valid_items,
)
coop.store(
group,
original_values,
values,
valid_items=valid_items,
)
@cute.jit
def launch(
source_keys: cute.Pointer,
source_values: cute.Pointer,
ascending: cute.Pointer,
descending: cute.Pointer,
sorted_values: cute.Pointer,
original_keys: cute.Pointer,
original_values: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
sort_tile(
source_keys,
source_values,
ascending,
descending,
sorted_values,
original_keys,
original_values,
items_per_thread,
).launch(grid=1, block=_BLOCK)
# docs: end cutlass-merge-sort
rng = np.random.default_rng(42)
source_keys = rng.integers(-31, 32, size=valid_items, dtype=np.int32)
source_values = np.arange(valid_items, dtype=np.float64) + 0.25
outputs = [
np.full(tile_size, -999, dtype=dtype)
for dtype in (np.int32, np.int32, np.float64, np.int32, np.float64)
]
arrays = [source_keys, source_values, *outputs]
cutlass.cuda.initialize_cuda_context()
with ExitStack() as cleanup:
allocations = []
pointers = []
for array in arrays:
allocation = _check(driver.cuMemAlloc(array.nbytes))
cleanup.callback(
lambda ptr=allocation: _check(driver.cuMemFree(ptr))
)
allocations.append(allocation)
_check(
driver.cuMemcpyHtoD(allocation, array.ctypes.data, array.nbytes)
)
dtype = (
cutlass.Int32 if array.dtype == np.int32 else cutlass.Float64
)
pointers.append(
make_ptr(
dtype,
int(allocation),
cute.AddressSpace.gmem,
assumed_align=16,
)
)
launch(*pointers, items_per_thread)
_check(driver.cuCtxSynchronize())
for array, allocation in zip(outputs, allocations[2:]):
_check(
driver.cuMemcpyDtoH(array.ctypes.data, allocation, array.nbytes)
)
ascending, descending, sorted_values, original_keys, original_values = (
outputs
)
expected_keys = np.sort(source_keys)
np.testing.assert_array_equal(ascending[:valid_items], expected_keys)
np.testing.assert_array_equal(descending[:valid_items], expected_keys[::-1])
# Equal keys have no stability guarantee; compare complete key/value pairs.
assert sorted(
zip(
descending[:valid_items],
sorted_values[:valid_items],
)
) == sorted(zip(source_keys, source_values))
np.testing.assert_array_equal(original_keys[:valid_items], source_keys)
np.testing.assert_array_equal(original_values[:valid_items], source_values)
for output in outputs:
np.testing.assert_array_equal(output[valid_items:], -999)
return outputs
if __name__ == "__main__":
run_example()
radix.py#
Radix Sort, Rank, striped output, and bin prefixes.
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Sort signed keys and stably rank a selected digit in a block tile."""
from contextlib import ExitStack
import cutlass
import numpy as np
from cutlass import cute
from cutlass.cute.runtime import make_ptr
from cuda.bindings import driver
_BLOCK = (8, 4, 2)
_BINS = 16
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
def run_example(items_per_thread=2):
"""Check full-key Sort, stable signed-digit pairs, ranks, and prefixes.
The host flips the sign bit before extracting the high digit, then uses a
stable descending order to track ties by their original positions. Rank
must return that permutation's inverse at the original input positions.
Striped pair-sort results use a striped Store. Bin prefixes count
greater digits, and a separate output checks that input keys are unchanged.
"""
tile_size = 64 * items_per_thread
import cuda.coop.cutlass as coop
# docs: start cutlass-radix
@cute.kernel
def order_tile(
source: cute.Pointer,
full_keys: cute.Pointer,
digit_keys: cute.Pointer,
digit_positions: cute.Pointer,
ranks_out: cute.Pointer,
prefixes_out: cute.Pointer,
original_keys: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
block = coop.this_block()
positions = coop.ThreadData(items_per_thread)
keys = coop.load(block, source, items_per_thread=items_per_thread)
for item in cutlass.range_constexpr(items_per_thread):
positions[item] = (
cutlass.Int32(block.rank()) * items_per_thread + item
)
scratch = coop.TempStorage(alignment=32, auto_sync=True)
ordered = coop.radix_sort_keys(block, keys, temp_storage=scratch)
coop.store(block, full_keys, ordered)
pair_keys, pair_positions = coop.radix_sort_pairs(
block,
keys,
positions,
begin_bit=28,
end_bit=32,
descending=True,
temp_storage=scratch,
blocked_to_striped=True,
)
# Match the Store layout to the striped result registers.
coop.store(block, digit_keys, pair_keys, algorithm="striped")
coop.store(block, digit_positions, pair_positions, algorithm="striped")
prefixes = coop.ThreadData(items_per_thread=1)
ranks = coop.radix_rank_keys(
block,
keys,
begin_bit=28,
radix_bits=4,
descending=True,
exclusive_digit_prefix=prefixes,
)
# Only the first 16 bin slots are defined for this four-bit digit.
coop.store(block, prefixes_out, prefixes, valid_items=_BINS)
coop.store(block, ranks_out, ranks)
coop.store(block, original_keys, keys)
@cute.jit
def launch(
source: cute.Pointer,
full_keys: cute.Pointer,
digit_keys: cute.Pointer,
digit_positions: cute.Pointer,
ranks_out: cute.Pointer,
prefixes_out: cute.Pointer,
original_keys: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
order_tile(
source,
full_keys,
digit_keys,
digit_positions,
ranks_out,
prefixes_out,
original_keys,
items_per_thread,
).launch(grid=1, block=_BLOCK)
# docs: end cutlass-radix
source = (
np.random.default_rng(42)
.integers(0, 1 << 32, size=tile_size, dtype=np.uint32)
.view(np.int32)
)
outputs = [np.full(tile_size, -999, dtype=np.int32) for _ in range(6)]
outputs[4] = np.full(_BINS, -999, dtype=np.int32)
arrays = [source, *outputs]
cutlass.cuda.initialize_cuda_context()
with ExitStack() as cleanup:
allocations = []
pointers = []
for array in arrays:
allocation = _check(driver.cuMemAlloc(array.nbytes))
cleanup.callback(
lambda ptr=allocation: _check(driver.cuMemFree(ptr))
)
allocations.append(allocation)
_check(
driver.cuMemcpyHtoD(allocation, array.ctypes.data, array.nbytes)
)
pointers.append(
make_ptr(
cutlass.Int32,
int(allocation),
cute.AddressSpace.gmem,
assumed_align=16,
)
)
launch(*pointers, items_per_thread)
_check(driver.cuCtxSynchronize())
for array, allocation in zip(outputs, allocations[1:]):
_check(
driver.cuMemcpyDtoH(array.ctypes.data, allocation, array.nbytes)
)
full_keys, digit_keys, positions, ranks, prefixes, original_keys = outputs
# Signed radix keys invert the sign bit before selecting the digit.
digit = (source.view(np.uint32) ^ np.uint32(1 << 31)) >> np.uint32(28)
order = np.argsort(-digit.astype(np.int32), kind="stable")
expected_ranks = np.empty(tile_size, dtype=np.int32)
expected_ranks[order] = np.arange(tile_size, dtype=np.int32)
np.testing.assert_array_equal(full_keys, np.sort(source))
np.testing.assert_array_equal(digit_keys, source[order])
np.testing.assert_array_equal(positions, order)
np.testing.assert_array_equal(ranks, expected_ranks)
np.testing.assert_array_equal(original_keys, source)
expected_prefixes = np.array(
[np.count_nonzero(digit > bin_index) for bin_index in range(_BINS)],
dtype=np.int32,
)
np.testing.assert_array_equal(prefixes, expected_prefixes)
return outputs
if __name__ == "__main__":
run_example()
topk.py#
Minimum keys and maximum pairs without assumptions about ties or order.
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Select a partial tile's smallest keys and largest key/value pairs."""
from contextlib import ExitStack
import cutlass
import numpy as np
from cutlass import cute
from cutlass.cute.runtime import make_ptr
from cuda.bindings import driver
_THREADS = 64
_K = 31
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
def run_example(items_per_thread=2):
"""Check selected keys and pair identity without assuming output order.
Repeated keys allow several correct selections at the boundary. Sorting
only the selected outputs checks membership while permitting those ties.
Original-position values identify each chosen input. Comparing each
selected key's bits with the source key at that position checks the pair
and also distinguishes positive and negative zero. The kernel also stores
the original payloads to confirm that TopK left them unchanged. Sentinel
tails check that Store writes only the defined result prefix.
"""
tile_size = _THREADS * items_per_thread
valid_items = tile_size - 35
selected_items = min(_K, valid_items)
import cuda.coop.cutlass as coop
# docs: start cutlass-topk
@cute.kernel
def select_tile(
source: cute.Pointer,
smallest: cute.Pointer,
largest: cute.Pointer,
selected_positions: cute.Pointer,
original_keys: cute.Pointer,
original_positions: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
tile_size = _THREADS * items_per_thread
valid_items = tile_size - 35
selected_items = min(_K, valid_items)
block = coop.this_block()
positions = coop.ThreadData(items_per_thread)
keys = coop.load(block, source, items_per_thread=items_per_thread)
for item in cutlass.range_constexpr(items_per_thread):
positions[item] = (
cutlass.Int32(block.rank()) * items_per_thread + item
)
input_keys = keys.to_register_tensor()
input_positions = positions.to_tensor_ssa(dtype=cutlass.Int32)
scratch = coop.TempStorage(alignment=16, auto_sync=True)
min_keys = coop.topk_min_keys(
block,
input_keys,
k=_K,
valid_items=valid_items,
temp_storage=scratch,
)
max_keys, max_positions = coop.topk_max_pairs(
block,
input_keys,
input_positions,
k=_K,
valid_items=valid_items,
temp_storage=scratch,
)
# Only min(k, valid_items) output positions are defined, in any order.
coop.store(
block,
smallest,
min_keys,
valid_items=selected_items,
)
coop.store(
block,
largest,
max_keys,
valid_items=selected_items,
)
coop.store(
block,
selected_positions,
max_positions,
valid_items=selected_items,
)
coop.store(block, original_keys, keys)
coop.store(block, original_positions, positions)
@cute.jit
def launch(
source: cute.Pointer,
smallest: cute.Pointer,
largest: cute.Pointer,
selected_positions: cute.Pointer,
original_keys: cute.Pointer,
original_positions: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
select_tile(
source,
smallest,
largest,
selected_positions,
original_keys,
original_positions,
items_per_thread,
).launch(grid=1, block=_THREADS)
# docs: end cutlass-topk
source = (
np.random.default_rng(42)
.integers(-1, 2, size=tile_size)
.astype(np.float32)
)
source[2::3] = 0.0
source[1::5] = -0.0
outputs = [
np.full(tile_size, -999, dtype=dtype)
for dtype in (np.float32, np.float32, np.int32, np.float32, np.int32)
]
arrays = [source, *outputs]
cutlass.cuda.initialize_cuda_context()
with ExitStack() as cleanup:
allocations = []
pointers = []
for array in arrays:
allocation = _check(driver.cuMemAlloc(array.nbytes))
cleanup.callback(
lambda ptr=allocation: _check(driver.cuMemFree(ptr))
)
allocations.append(allocation)
_check(
driver.cuMemcpyHtoD(allocation, array.ctypes.data, array.nbytes)
)
dtype = (
cutlass.Float32 if array.dtype == np.float32 else cutlass.Int32
)
pointers.append(
make_ptr(
dtype,
int(allocation),
cute.AddressSpace.gmem,
assumed_align=16,
)
)
launch(*pointers, items_per_thread)
_check(driver.cuCtxSynchronize())
for array, allocation in zip(outputs, allocations[1:]):
_check(
driver.cuMemcpyDtoH(array.ctypes.data, allocation, array.nbytes)
)
smallest, largest, positions, original_keys, original_positions = outputs
ordered = np.sort(source[:valid_items])
np.testing.assert_array_equal(
np.sort(smallest[:selected_items]),
ordered[:selected_items],
)
np.testing.assert_array_equal(
np.sort(largest[:selected_items]),
ordered[-selected_items:],
)
selected_ids = positions[:selected_items]
assert len(np.unique(selected_ids)) == selected_items
assert np.all((selected_ids >= 0) & (selected_ids < valid_items))
# Check pair association and the sign bit of tied floating-point zeros.
np.testing.assert_array_equal(
largest[:selected_items].view(np.uint32),
source[selected_ids].view(np.uint32),
)
np.testing.assert_array_equal(
original_keys.view(np.uint32), source.view(np.uint32)
)
np.testing.assert_array_equal(
original_positions,
np.arange(tile_size, dtype=np.int32),
)
for output in outputs[:3]:
np.testing.assert_array_equal(output[selected_items:], -999)
return outputs
if __name__ == "__main__":
run_example()
reduce_batched.py#
Independent per-feature warp reductions and output placement.
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Sum each feature independently across each warp."""
import cutlass
import numpy as np
from cutlass import cute
from cutlass.cute.runtime import make_ptr
from cuda.bindings import driver
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
def run_example(items_per_thread=3):
"""Sum matching features in two independent warps and check against NumPy.
Reshaping the input as warp, lane, feature makes the CPU reference reduce
only the lane axis. The kernel writes each defined striped result once,
leaving the two warps' feature totals separate.
"""
import cuda.coop.cutlass as coop
# docs: start cutlass-reduce-batched
@cute.kernel
def feature_sums(
samples: cute.Pointer,
totals: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
warp = coop.this_warp()
features = coop.load(warp, samples, items_per_thread=items_per_thread)
sums = coop.reduce_batched(warp, features)
outputs = cute.make_tensor(
totals, cute.make_layout(2 * items_per_thread)
)
# Output slots are striped across the warp lanes.
warp_index = coop.this_block().rank() // 32
for item in cutlass.range_constexpr((items_per_thread + 31) // 32):
feature = warp.rank() + item * 32
if feature < items_per_thread:
outputs[warp_index * items_per_thread + feature] = sums[item]
@cute.jit
def launch(
samples: cute.Pointer,
totals: cute.Pointer,
items_per_thread: cutlass.Constexpr,
):
feature_sums(samples, totals, items_per_thread).launch(
grid=1, block=(8, 4, 2)
)
# docs: end cutlass-reduce-batched
samples = np.arange(64 * items_per_thread, dtype=np.int32)
totals = np.zeros(2 * items_per_thread, dtype=np.int32)
cutlass.cuda.initialize_cuda_context()
src = _check(driver.cuMemAlloc(samples.nbytes))
try:
dst = _check(driver.cuMemAlloc(totals.nbytes))
try:
_check(
driver.cuMemcpyHtoD(src, samples.ctypes.data, samples.nbytes)
)
_check(driver.cuMemcpyHtoD(dst, totals.ctypes.data, totals.nbytes))
pointers = [
make_ptr(
cutlass.Int32,
int(pointer),
cute.AddressSpace.gmem,
assumed_align=16,
)
for pointer in (src, dst)
]
compiled = cute.compile(launch, *pointers, items_per_thread)
compiled(*pointers)
_check(driver.cuCtxSynchronize())
_check(driver.cuMemcpyDtoH(totals.ctypes.data, dst, totals.nbytes))
finally:
_check(driver.cuMemFree(dst))
finally:
_check(driver.cuMemFree(src))
expected = (
samples.reshape(2, 32, items_per_thread)
.sum(axis=1, dtype=np.int32)
.ravel()
)
np.testing.assert_array_equal(totals, expected)
return totals
if __name__ == "__main__":
run_example()
tensor_memory_topk.py#
Tensor Core MMA, cooperative TopK in registers, and a TMEM round trip across dependent windows. Requires an SM100 or SM103 GPU. The Tensor Core and TopK walkthrough explains the copy layouts, synchronization, and accumulator feedback.
Download tensor_memory_topk.py
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Interleave Tensor Core matrix products and cooperative TopK selection.
Each block owns a matrix batch. Every window reads a real tcgen05 accumulator
from TMEM, selects keys with coop, writes the result back to TMEM, and uses it
as the next MMA's accumulator. The undefined TopK tail is explicitly zeroed.
Later products are uniform so an independent multiset reference can account
for TopK's unspecified order without imposing a register-to-matrix mapping.
"""
import cutlass
import cutlass.pipeline as pipeline
import cutlass.utils.blackwell_helpers as sm100_utils
import numpy as np
from cutlass import cute
from cutlass.cute.nvgpu import tcgen05
from cutlass.cute.runtime import make_ptr
import cuda.coop.cutlass as coop
from cuda.bindings import driver
_M, _N, _K = 128, 8, 16
_ITEMS = _M * _N
def _check(result):
"""Raise on Driver errors and unwrap a single returned handle or value."""
if int(result[0]):
raise RuntimeError(f"CUDA Driver call failed: {result[0]}")
return result[1] if len(result) == 2 else result[1:]
# docs: start cutlass-tensor-memory-topk
@cute.struct
class _SharedStorage:
barrier: cutlass.Int64
tmem_address: cutlass.Int32
@cute.kernel
def _windowed_topk_kernel(
output: cute.Pointer,
windows: cutlass.Int32,
keep: cutlass.Int32,
mma: cute.TiledMma,
a_layout: cute.ComposedLayout,
b_layout: cute.ComposedLayout,
):
thread, _, _ = cute.arch.thread_idx()
batch, _, _ = cute.arch.block_idx()
warp = cute.arch.make_warp_uniform(cute.arch.warp_idx())
# docs: start cutlass-tmem-storage
# Native operand buffers stay live alongside coop's separate scratch.
# CuTe places allocations from both libraries in one shared-memory layout.
smem = cutlass.memory.SmemAllocator()
storage = smem.allocate(_SharedStorage)
a = smem.allocate_tensor(
cutlass.Float16, a_layout.outer, 128, a_layout.inner
)
b = smem.allocate_tensor(
cutlass.Float16, b_layout.outer, 128, b_layout.inner
)
a_coords = mma.get_slice(0).partition_A(cute.make_identity_tensor((_M, _K)))
b_coords = mma.get_slice(0).partition_B(cute.make_identity_tensor((_N, _K)))
a_fragment = mma.make_fragment_A(a)
b_fragment = mma.make_fragment_B(b)
accumulator = mma.make_fragment_C(mma.partition_shape_C((_M, _N)))
tmem = cutlass.memory.TmemAllocator(
storage.tmem_address.ptr,
barrier_for_retrieve=pipeline.NamedBarrier(1, _M),
)
# TMEM allocations have a 32-column minimum; this tile uses eight.
tmem.allocate(32)
if warp == 0:
with cute.arch.elect_one():
cute.arch.mbarrier_init(storage.barrier.ptr, 1)
cute.arch.mbarrier_init_fence()
tmem.wait_for_alloc()
pointer = tmem.retrieve_ptr(cutlass.Float32)
accumulator = cute.make_tensor(pointer, accumulator.layout)
tmem.relinquish_alloc_permit()
# docs: end cutlass-tmem-storage
# docs: start cutlass-tmem-copy-layout
# Use the same matrix slice for the copy and both thread partitions.
# The destination partition determines the eight registers per thread.
load_copy = tcgen05.make_tmem_copy(
cute.make_copy_atom(
tcgen05.Ld32x32bOp(tcgen05.Repetition.x8), cutlass.Float32
),
accumulator[(None, None), 0, 0],
)
store_atom = cute.make_copy_atom(
tcgen05.St32x32bOp(tcgen05.Repetition.x8), cutlass.Float32
)
source = load_copy.get_slice(thread).partition_S(
accumulator[(None, None), 0, 0]
)
coordinates = mma.get_slice(0).partition_C(
cute.make_identity_tensor((_M, _N))
)
register_shape = (
load_copy.get_slice(thread)
.partition_D(coordinates[(None, None), 0, 0])
.shape
)
assert cute.size(register_shape) == _N
registers = cute.make_rmem_tensor(register_shape, cutlass.Float32)
# docs: end cutlass-tmem-copy-layout
# Record the MMA output, the selection before storing, and the TMEM reload.
result = cute.make_tensor(
output, cute.make_layout((3, _ITEMS, windows, cute.arch.grid_dim()[0]))
)
group = coop.this_block()
scratch = coop.TempStorage()
phase = cutlass.Int32(0)
for window in range(windows):
# docs: start cutlass-tmem-mma
# The first product varies by matrix coordinate. Later products are
# uniform, so their host reference is independent of TopK ordering.
# SIMT writes must become visible to the asynchronous MMA proxy.
for index in cutlass.range_constexpr(_M * _K // _M):
offset = thread + index * _M
row, k = a_coords[offset]
value = cutlass.Float16(batch + window + 1)
if window == 0:
value = cutlass.Float16((row % 17 - 8) + (k % 3))
a[offset] = value
for index in cutlass.range_constexpr(_N * _K // _M):
offset = thread + index * _M
column, k = b_coords[offset]
value = cutlass.Float16(1)
if window == 0:
value = cutlass.Float16(column % 7 - 3 + batch)
b[offset] = value
cute.arch.fence_proxy("async.shared", space="cta")
cute.arch.sync_threads()
# One warp issues MMA; all threads wait for its completion barrier.
# After the first window, C is the selection stored by the last window.
if warp == 0:
mma.set(tcgen05.Field.ACCUMULATE, window > 0)
cute.gemm(
mma,
accumulator,
a_fragment[None, None, None, 0],
b_fragment[None, None, None, 0],
accumulator,
)
with cute.arch.elect_one():
tcgen05.commit(storage.barrier.ptr)
cute.arch.mbarrier_wait(storage.barrier.ptr, phase)
phase ^= 1
# docs: end cutlass-tmem-mma
# docs: start cutlass-tmem-simt
# Native CuTe moves TMEM values into registers. ThreadData then exposes
# those per-thread values to coop without claiming ownership of TMEM.
cute.copy(load_copy, source, registers)
values = coop.ThreadData.from_register_tensor(registers)
for item in cutlass.range_constexpr(_N):
result[0, thread * _N + item, window, batch] = values[item]
selected = coop.topk_max_keys(
group, values, k=keep, temp_storage=scratch
)
# Only the selected prefix is defined. Zero the remaining positions
# before the next MMA consumes the entire accumulator.
for item in cutlass.range_constexpr(_N):
value = cutlass.Float32(0)
if thread * _N + item < keep:
value = selected[item]
values[item] = value
result[1, thread * _N + item, window, batch] = value
# docs: end cutlass-tmem-simt
# docs: start cutlass-tmem-store
# Reconstruct the native register layout and apply the inverse copy.
registers.store(values.to_tensor_ssa(like=registers))
cute.copy(store_atom, registers, source)
# Complete all stores before another warp or the next MMA uses TMEM.
cute.arch.fence_view_async_tmem_store()
cute.arch.sync_threads()
# Read back the stored selection, including the explicitly zero tail.
cute.copy(load_copy, source, registers)
roundtrip = coop.ThreadData.from_register_tensor(registers)
for item in cutlass.range_constexpr(_N):
result[2, thread * _N + item, window, batch] = roundtrip[item]
cute.arch.fence_view_async_tmem_load()
cute.arch.sync_threads()
# docs: end cutlass-tmem-store
tmem.free(pointer)
@cute.jit
def _launch(
output: cute.Pointer,
windows: cutlass.Int32,
keep: cutlass.Int32,
batches: cutlass.Constexpr,
):
mma = cute.make_tiled_mma(
tcgen05.MmaF16BF16Op(
cutlass.Float16,
cutlass.Float32,
(_M, _N, _K),
tcgen05.CtaGroup.ONE,
tcgen05.OperandSource.SMEM,
cute.nvgpu.OperandMajorMode.K,
cute.nvgpu.OperandMajorMode.K,
)
)
a_layout = sm100_utils.make_smem_layout_a(
mma, (_M, _N, _K), cutlass.Float16, 1
)
b_layout = sm100_utils.make_smem_layout_b(
mma, (_M, _N, _K), cutlass.Float16, 1
)
_windowed_topk_kernel(
output, windows, keep, mma, a_layout, b_layout
).launch(grid=batches, block=_M)
# docs: end cutlass-tensor-memory-topk
def is_supported_device():
"""Check whether the active device supports this SM100/SM103 example."""
cutlass.cuda.initialize_cuda_context()
device = _check(driver.cuCtxGetDevice())
attribute = driver.CUdevice_attribute
major = _check(
driver.cuDeviceGetAttribute(
attribute.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device
)
)
minor = _check(
driver.cuDeviceGetAttribute(
attribute.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device
)
)
return (major, minor) in {(10, 0), (10, 3)}
# docs: start cutlass-tmem-validation
def validate_result(output, keep):
"""Check exact roundtrips and each window's independent matrix reference.
TopK is unsorted, so compare multisets for the matrix product and selected
keys. Exact before/after comparisons separately check the native TMEM copy
mapping. Small integer inputs make these float32 comparisons exact.
"""
batches, windows = output.shape[:2]
np.testing.assert_array_equal(output[..., 1], output[..., 2])
np.testing.assert_array_equal(output[:, :, keep:, 1], 0)
rows = np.arange(_M)[:, None]
columns = np.arange(_N)[:, None]
k = np.arange(_K)[None, :]
a = (rows % 17 - 8 + k % 3).astype(np.float32)
for batch in range(batches):
b = np.broadcast_to(columns % 7 - 3 + batch, (_N, _K))
expected = np.sort((a @ b.T).ravel())
for window in range(windows):
if window:
expected = np.sort(expected + _K * (batch + window + 1))
np.testing.assert_array_equal(
np.sort(output[batch, window, :, 0]), expected
)
expected = np.sort(
np.concatenate(
(
expected[-keep:],
np.zeros(_ITEMS - keep, dtype=np.float32),
)
)
)
np.testing.assert_array_equal(
np.sort(output[batch, window, :, 1]), expected
)
# docs: end cutlass-tmem-validation
def run_example(batches=2, windows=4, keep=129):
"""Run independent batches of dependent MMA/TopK windows and check them.
The returned array has shape (batches, windows, 1024, 3). Its channels hold
each MMA output, the selection with a zeroed tail, and the TMEM reload.
"""
if batches < 1 or windows < 1 or not 1 <= keep <= _ITEMS:
raise ValueError(
"Require positive batches/windows and 1 <= keep <= 1024"
)
if not is_supported_device():
raise RuntimeError("This tcgen05 TMEM example requires SM100 or SM103")
output = np.full((batches, windows, _ITEMS, 3), np.nan, dtype=np.float32)
allocation = _check(driver.cuMemAlloc(output.nbytes))
try:
_check(
driver.cuMemcpyHtoD(allocation, output.ctypes.data, output.nbytes)
)
pointer = make_ptr(
cutlass.Float32,
int(allocation),
cute.AddressSpace.gmem,
assumed_align=16,
)
_launch(pointer, cutlass.Int32(windows), cutlass.Int32(keep), batches)
_check(driver.cuCtxSynchronize())
_check(
driver.cuMemcpyDtoH(output.ctypes.data, allocation, output.nbytes)
)
finally:
_check(driver.cuMemFree(allocation))
validate_result(output, keep)
return output
if __name__ == "__main__":
output = run_example()
batches, windows = output.shape[:2]
print(f"Validated {batches} batches with {windows} MMA/TopK windows each.")
Integration walkthroughs#
The debugger walkthrough
compares Direct and Transpose tile copies. It compiles once and executes
the compiled callable twice, with code excerpts and a
complete script.
The minimal CuTe DSL integration example
calls cub::BlockReduce from a CuTe kernel using the DSL’s own interfaces.
Its walkthrough explains each step with code excerpts and offers the
complete script.