Coverage for cuda/core/_rt.pyx: 33.33%
6 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-05 01:11 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-05 01:11 +0000
1# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2#
3# SPDX-License-Identifier: Apache-2.0
5# cython: show_performance_hints=False
6#
7# Performance hints are off for this module on purpose. The handle factories
8# below are declared ``except+ nogil``, and Cython reports each one as an
9# exception check that "will always require the GIL". For ``except+`` the
10# GIL is taken only inside the C++ catch handler, so the success path pays
11# nothing and the hint is noise here. Functions that only return a status
12# code are declared ``noexcept nogil`` instead, which is what the hint is
13# really about: without it, every call from a nogil block acquires the GIL
14# to run PyErr_Occurred().
16# This module compiles the C++ under _cpp/rt/ into one shared library.
17# Consumer modules cimport the functions declared in _rt.pxd. Since there is
18# only one copy of the C++ code (in this .so), all static and thread-local
19# state is shared correctly across all consumer modules.
20#
21# "rt" is short for runtime: this is cuda.core's runtime support layer
22# (resource handles, the driver function-pointer table, error reporting and
23# deferred cleanup). It is unrelated to the CUDA Runtime API (cudart).
24#
25# The cdef extern from declarations below satisfy the .pxd declarations directly,
26# without needing separate wrapper functions.
28from cpython.object cimport PyObject
29from libc.stddef cimport size_t
31from cuda.bindings cimport cydriver
32from cuda.bindings cimport cynvrtc
33from cuda.bindings cimport cynvvm
34from cuda.bindings cimport cynvjitlink
37# =============================================================================
38# C++ function declarations (non-inline, implemented under _cpp/rt/)
39#
40# These declarations satisfy the cdef function declarations in _rt.pxd.
41# Consumer modules cimport these functions and calls go through this .so.
42# =============================================================================
44cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt":
45 # Thread-local error handling
46 cydriver.CUresult get_last_error "cuda_core::rt::get_last_error" () noexcept nogil
47 cydriver.CUresult peek_last_error "cuda_core::rt::peek_last_error" () noexcept nogil
48 void clear_last_error "cuda_core::rt::clear_last_error" () noexcept nogil
50 # Non-propagating error reporting
51 void register_warning_category "cuda_core::rt::register_warning_category" (
52 PyObject* category) noexcept
53 void report_cuda_error "cuda_core::rt::report_cuda_error" (
54 const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil
55 void report_message "cuda_core::rt::report_message" (const char* message) noexcept nogil
56 void report_status_code "cuda_core::rt::report_status_code" (
57 const char* operation, long code) noexcept nogil
58 void attach_rollback_failure "cuda_core::rt::attach_rollback_failure" (
59 const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil
60 # Alias for calls made from this module: calling the pxd-declared name here
61 # would make Cython emit a conflicting static prototype for it.
62 void _attach_rollback_failure_local "cuda_core::rt::attach_rollback_failure" (
63 const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil
64 const char* take_last_error_detail "cuda_core::rt::take_last_error_detail" (
65 cydriver.CUresult status) noexcept nogil
66 void clear_last_error_detail "cuda_core::rt::clear_last_error_detail" () noexcept nogil
67 void set_context_restore_fault_for_testing "cuda_core::rt::set_context_restore_fault_for_testing" (
68 cydriver.CUresult status) noexcept nogil
70 # Context handles
71 ContextHandle create_context_handle_ref "cuda_core::rt::create_context_handle_ref" (
72 cydriver.CUcontext ctx) except+ nogil
73 ContextHandle create_context_handle_from_green_ctx "cuda_core::rt::create_context_handle_from_green_ctx" (
74 const GreenCtxHandle& h_green_ctx) except+ nogil
75 GreenCtxHandle get_context_green_ctx "cuda_core::rt::get_context_green_ctx" (
76 const ContextHandle& h) noexcept nogil
77 GreenCtxHandle create_green_ctx_handle "cuda_core::rt::create_green_ctx_handle" (
78 cydriver.CUdevResource* resources, unsigned int nbResources,
79 cydriver.CUdevice dev, unsigned int flags) except+ nogil
80 GreenCtxHandle create_green_ctx_handle_ref "cuda_core::rt::create_green_ctx_handle_ref" (
81 cydriver.CUgreenCtx ctx) except+ nogil
82 ContextHandle get_primary_context "cuda_core::rt::get_primary_context" (
83 int device_id) except+ nogil
84 ContextHandle get_current_context "cuda_core::rt::get_current_context" () except+ nogil
85 cydriver.CUresult context_synchronize "cuda_core::rt::context_synchronize" (
86 const ContextHandle& h_context) noexcept nogil
87 cydriver.CUresult context_get_stream_priority_range "cuda_core::rt::context_get_stream_priority_range" (
88 const ContextHandle& h_context,
89 int* least_priority,
90 int* greatest_priority) noexcept nogil
91 cydriver.CUresult context_get_device "cuda_core::rt::context_get_device" (
92 const ContextHandle& h_context, cydriver.CUdevice* device) noexcept nogil
93 cydriver.CUresult graph_node_set_params "cuda_core::rt::graph_node_set_params" (
94 cydriver.CUgraphNode node, cydriver.CUgraphNodeParams* params,
95 const ContextHandle& h_context, cydriver.CUresult* restore_status) noexcept nogil
97 # Stream handles
98 StreamHandle create_stream_handle "cuda_core::rt::create_stream_handle" (
99 const ContextHandle& h_ctx, unsigned int flags, int priority) except+ nogil
100 StreamHandle create_stream_handle_ref "cuda_core::rt::create_stream_handle_ref" (
101 cydriver.CUstream stream) except+ nogil
102 StreamHandle create_stream_handle_with_owner "cuda_core::rt::create_stream_handle_with_owner" (
103 cydriver.CUstream stream, object owner) except+ nogil
104 void initialize_deferred_cleanup "cuda_core::rt::initialize_deferred_cleanup" () except+
105 void retry_deferred_cleanup "cuda_core::rt::retry_deferred_cleanup" () noexcept
106 cydriver.CUresult launch_host_function "cuda_core::rt::launch_host_function" (
107 cydriver.CUstream stream, cydriver.CUhostFn fn, void* user_data,
108 OpaqueHandle fn_owner, OpaqueHandle data_owner) noexcept nogil
109 ContextHandle get_stream_context "cuda_core::rt::get_stream_context" (
110 const StreamHandle& h) noexcept nogil
111 StreamHandle get_legacy_stream "cuda_core::rt::get_legacy_stream" () except+ nogil
112 StreamHandle get_per_thread_stream "cuda_core::rt::get_per_thread_stream" () except+ nogil
113 StreamHandle create_context_bound_legacy_stream "cuda_core::rt::create_context_bound_legacy_stream" (
114 const ContextHandle& h_context) except+ nogil
116 # Event handles (note: _create_event_handle* are internal due to C++ overloading)
117 EventHandle create_event_handle "cuda_core::rt::create_event_handle" (
118 const ContextHandle& h_ctx, unsigned int flags,
119 bint timing_enabled, bint is_blocking_sync,
120 bint ipc_enabled, int device_id) except+ nogil
121 EventHandle create_event_handle_for_stream "cuda_core::rt::create_event_handle_for_stream" (
122 cydriver.CUstream stream, unsigned int flags) except+ nogil
123 EventHandle create_event_handle_ref "cuda_core::rt::create_event_handle_ref" (
124 cydriver.CUevent event) except+ nogil
125 EventHandle create_event_handle_ipc "cuda_core::rt::create_event_handle_ipc" (
126 const cydriver.CUipcEventHandle& ipc_handle, bint is_blocking_sync) except+ nogil
128 # Event metadata getters
129 bint get_event_timing_enabled "cuda_core::rt::get_event_timing_enabled" (
130 const EventHandle& h) noexcept nogil
131 bint get_event_is_blocking_sync "cuda_core::rt::get_event_is_blocking_sync" (
132 const EventHandle& h) noexcept nogil
133 bint get_event_ipc_enabled "cuda_core::rt::get_event_ipc_enabled" (
134 const EventHandle& h) noexcept nogil
135 int get_event_device_id "cuda_core::rt::get_event_device_id" (
136 const EventHandle& h) noexcept nogil
137 ContextHandle get_event_context "cuda_core::rt::get_event_context" (
138 const EventHandle& h) noexcept nogil
140 # Memory pool handles
141 MemoryPoolHandle create_mempool_handle "cuda_core::rt::create_mempool_handle" (
142 const cydriver.CUmemPoolProps& props) except+ nogil
143 MemoryPoolHandle create_mempool_handle_ref "cuda_core::rt::create_mempool_handle_ref" (
144 cydriver.CUmemoryPool pool) except+ nogil
145 MemoryPoolHandle get_device_mempool "cuda_core::rt::get_device_mempool" (
146 int device_id) except+ nogil
147 MemoryPoolHandle create_mempool_handle_ipc "cuda_core::rt::create_mempool_handle_ipc" (
148 int fd, cydriver.CUmemAllocationHandleType handle_type) except+ nogil
150 # Device pointer handles
151 DevicePtrHandle deviceptr_alloc_from_pool "cuda_core::rt::deviceptr_alloc_from_pool" (
152 size_t size, const MemoryPoolHandle& h_pool, const StreamHandle& h_stream) except+ nogil
153 DevicePtrHandle deviceptr_alloc_async "cuda_core::rt::deviceptr_alloc_async" (
154 size_t size, const StreamHandle& h_stream) except+ nogil
155 cydriver.CUresult deviceptr_alloc_raw "cuda_core::rt::deviceptr_alloc_raw" (
156 cydriver.CUdeviceptr* ptr, size_t size, const ContextHandle& h_context) noexcept nogil
157 DevicePtrHandle deviceptr_alloc_host "cuda_core::rt::deviceptr_alloc_host" (size_t size) except+ nogil
158 DevicePtrHandle deviceptr_create_ref "cuda_core::rt::deviceptr_create_ref" (
159 cydriver.CUdeviceptr ptr) except+ nogil
160 DevicePtrHandle deviceptr_create_with_owner "cuda_core::rt::deviceptr_create_with_owner" (
161 cydriver.CUdeviceptr ptr, object owner) except+ nogil
162 DevicePtrHandle deviceptr_create_mapped_graphics "cuda_core::rt::deviceptr_create_mapped_graphics" (
163 cydriver.CUdeviceptr ptr,
164 const GraphicsResourceHandle& h_resource,
165 const StreamHandle& h_stream) except+ nogil
167 # MR deallocation callback
168 void register_mr_dealloc_callback "cuda_core::rt::register_mr_dealloc_callback" (
169 MRDeallocCallback cb) noexcept
170 DevicePtrHandle deviceptr_create_with_mr "cuda_core::rt::deviceptr_create_with_mr" (
171 cydriver.CUdeviceptr ptr, size_t size, object mr) except+ nogil
173 DevicePtrHandle deviceptr_import_ipc "cuda_core::rt::deviceptr_import_ipc" (
174 const MemoryPoolHandle& h_pool, const void* export_data, const StreamHandle& h_stream) except+ nogil
175 StreamHandle deallocation_stream "cuda_core::rt::deallocation_stream" (
176 const DevicePtrHandle& h) noexcept nogil
177 cydriver.CUresult set_deallocation_stream "cuda_core::rt::set_deallocation_stream" (
178 const DevicePtrHandle& h, const StreamHandle& h_stream) except+ nogil
180 # Virtual memory management (VMM_DESIGN.md)
181 MemAllocationHandle create_mem_allocation_handle "cuda_core::rt::create_mem_allocation_handle" (
182 size_t size, const cydriver.CUmemAllocationProp& prop,
183 const cydriver.CUmemAccessDesc* descs, size_t count) except+ nogil
184 size_t mem_allocation_size "cuda_core::rt::mem_allocation_size" (
185 const MemAllocationHandle& h) noexcept nogil
186 bint mem_allocation_is_imported "cuda_core::rt::mem_allocation_is_imported" (
187 const MemAllocationHandle& h) noexcept nogil
188 MemAllocationHandle import_mem_allocation_handle "cuda_core::rt::import_mem_allocation_handle" (
189 void* os_handle, cydriver.CUmemAllocationHandleType handle_type, size_t size,
190 const cydriver.CUmemAccessDesc* descs, size_t count) except+ nogil
191 VaReservationHandle create_va_reservation_handle "cuda_core::rt::create_va_reservation_handle" (
192 size_t size, size_t alignment, cydriver.CUdeviceptr hint) except+ nogil
193 size_t va_reservation_size "cuda_core::rt::va_reservation_size" (
194 const VaReservationHandle& h) noexcept nogil
195 long long live_owned_mempool_count "cuda_core::rt::live_owned_mempool_count" () noexcept nogil
196 long long live_va_reservation_count "cuda_core::rt::live_va_reservation_count" () noexcept nogil
197 VaMappingHandle create_va_mapping_handle "cuda_core::rt::create_va_mapping_handle" (
198 cydriver.CUdeviceptr ptr, const MemAllocationHandle& h_alloc,
199 const VaReservationHandle& h_res) except+ nogil
200 size_t va_mapping_size "cuda_core::rt::va_mapping_size" (const VaMappingHandle& h) noexcept nogil
201 MemAllocationHandle va_mapping_allocation "cuda_core::rt::va_mapping_allocation" (
202 const VaMappingHandle& h) noexcept nogil
203 VmmRangeHandle create_vmm_range "cuda_core::rt::create_vmm_range" (
204 const vector[VaMappingHandle]& mappings) except+ nogil
205 VmmRangeHandle vmm_range "cuda_core::rt::vmm_range" (const DevicePtrHandle& h) noexcept nogil
206 vector[VaMappingHandle] vmm_range_mappings "cuda_core::rt::vmm_range_mappings" (
207 const VmmRangeHandle& range) except+ nogil
208 size_t vmm_range_total "cuda_core::rt::vmm_range_total" (const VmmRangeHandle& range) noexcept nogil
209 DevicePtrHandle deviceptr_create_vmm "cuda_core::rt::deviceptr_create_vmm" (
210 cydriver.CUdeviceptr base, const VmmRangeHandle& range) except+ nogil
212 # Library handles
213 LibraryHandle create_library_handle_from_file "cuda_core::rt::create_library_handle_from_file" (
214 const char* path) except+ nogil
215 LibraryHandle create_library_handle_from_data "cuda_core::rt::create_library_handle_from_data" (
216 const void* data) except+ nogil
217 LibraryHandle create_library_handle_ref "cuda_core::rt::create_library_handle_ref" (
218 cydriver.CUlibrary library) except+ nogil
220 # Kernel handles
221 KernelHandle create_kernel_handle "cuda_core::rt::create_kernel_handle" (
222 const LibraryHandle& h_library, const char* name) except+ nogil
223 KernelHandle create_kernel_handle_ref "cuda_core::rt::create_kernel_handle_ref" (
224 cydriver.CUkernel kernel) except+ nogil
225 LibraryHandle get_kernel_library "cuda_core::rt::get_kernel_library" (
226 const KernelHandle& h) noexcept nogil
228 # Graph handles
229 GraphHandle create_graph_handle "cuda_core::rt::create_graph_handle" (
230 cydriver.CUgraph graph) except+ nogil
231 GraphHandle create_child_graph_handle "cuda_core::rt::create_child_graph_handle" (
232 cydriver.CUgraph child_graph, const GraphHandle& h_parent,
233 cydriver.CUgraphNode owner_node) except+ nogil
235 # Graph node attachments
236 OpaqueHandle make_opaque_py "cuda_core::rt::make_opaque_py" (object obj) except+
237 OpaqueHandle make_opaque_malloc "cuda_core::rt::make_opaque_malloc" (void* buf) except+
238 cydriver.CUresult graph_get_attachment "cuda_core::rt::graph_get_attachment" (
239 const GraphHandle& h_graph, cydriver.CUgraphNode node,
240 OpaqueHandle* owner0, OpaqueHandle* owner1) except+
241 cydriver.CUresult graph_prepare_attachment "cuda_core::rt::graph_prepare_attachment" (
242 const GraphHandle& h_graph, OpaqueHandle owner0, OpaqueHandle owner1,
243 PreparedAttachment* out_prepared) except+
244 cydriver.CUresult graph_commit_attachment "cuda_core::rt::graph_commit_attachment" (
245 PreparedAttachment& prepared, cydriver.CUgraphNode node) except+
246 cydriver.CUresult graph_clone_attachments "cuda_core::rt::graph_clone_attachments" (
247 const GraphHandle& h_clone, const GraphHandle& h_source) except+
248 cydriver.CUresult graph_prepare_child_graph_update "cuda_core::rt::graph_prepare_child_graph_update" (
249 const GraphHandle& h_parent, const GraphHandle& h_old_child,
250 cydriver.CUgraphNode owner_node, const GraphHandle& h_source,
251 PreparedChildGraphUpdate* out_prepared) except+
252 cydriver.CUresult graph_commit_child_graph_update "cuda_core::rt::graph_commit_child_graph_update" (
253 PreparedChildGraphUpdate& prepared, GraphHandle* out_child) except+
254 void invalidate_child_graph_state "cuda_core::rt::invalidate_child_graph_state" (
255 const GraphHandle& h_parent, cydriver.CUgraphNode owner_node) noexcept
256 void invalidate_root_graph_state "cuda_core::rt::invalidate_root_graph_state" (
257 const GraphHandle& h_root) noexcept
259 # Graph exec handles
260 GraphExecHandle create_graph_exec_handle "cuda_core::rt::create_graph_exec_handle" (
261 const GraphHandle& h_source,
262 cydriver.CUDA_GRAPH_INSTANTIATE_PARAMS* params) except+
263 cydriver.CUresult graph_exec_update "cuda_core::rt::graph_exec_update" (
264 const GraphExecHandle& h_exec,
265 const GraphHandle& h_source,
266 cydriver.CUgraphExecUpdateResultInfo* result_info) except+
267 cydriver.CUresult graph_prepare_exec_attachment "cuda_core::rt::graph_prepare_exec_attachment" (
268 const GraphExecHandle& h_exec,
269 OpaqueHandle owner0,
270 OpaqueHandle owner1,
271 PreparedExecAttachment* out_prepared) except+
272 void graph_commit_exec_attachment "cuda_core::rt::graph_commit_exec_attachment" (
273 PreparedExecAttachment& prepared) noexcept
275 # Graph node handles
276 GraphNodeHandle create_graph_node_handle "cuda_core::rt::create_graph_node_handle" (
277 cydriver.CUgraphNode node, const GraphHandle& h_graph) except+ nogil
278 GraphHandle graph_node_get_graph "cuda_core::rt::graph_node_get_graph" (
279 const GraphNodeHandle& h) noexcept nogil
280 void invalidate_graph_node "cuda_core::rt::invalidate_graph_node" (
281 const GraphNodeHandle& h) noexcept nogil
283 # Graphics resource handles
284 GraphicsResourceHandle create_graphics_resource_handle "cuda_core::rt::create_graphics_resource_handle" (
285 cydriver.CUgraphicsResource resource) except+ nogil
287 # NVRTC Program handles
288 NvrtcProgramHandle create_nvrtc_program_handle "cuda_core::rt::create_nvrtc_program_handle" (
289 cynvrtc.nvrtcProgram prog) except+ nogil
290 NvrtcProgramHandle create_nvrtc_program_handle_ref "cuda_core::rt::create_nvrtc_program_handle_ref" (
291 cynvrtc.nvrtcProgram prog) except+ nogil
293 # NVVM Program handles
294 NvvmProgramHandle create_nvvm_program_handle "cuda_core::rt::create_nvvm_program_handle" (
295 cynvvm.nvvmProgram prog) except+ nogil
296 NvvmProgramHandle create_nvvm_program_handle_ref "cuda_core::rt::create_nvvm_program_handle_ref" (
297 cynvvm.nvvmProgram prog) except+ nogil
299 # nvJitLink handles
300 NvJitLinkHandle create_nvjitlink_handle "cuda_core::rt::create_nvjitlink_handle" (
301 cynvjitlink.nvJitLinkHandle handle) except+ nogil
302 NvJitLinkHandle create_nvjitlink_handle_ref "cuda_core::rt::create_nvjitlink_handle_ref" (
303 cynvjitlink.nvJitLinkHandle handle) except+ nogil
305 # cuLink handles
306 CuLinkHandle create_culink_handle "cuda_core::rt::create_culink_handle" (
307 cydriver.CUlinkState state) except+ nogil
308 CuLinkHandle create_culink_handle_ref "cuda_core::rt::create_culink_handle_ref" (
309 cydriver.CUlinkState state) except+ nogil
311 # File descriptor handles
312 FileDescriptorHandle create_fd_handle "cuda_core::rt::create_fd_handle" (
313 int fd) except+ nogil
314 FileDescriptorHandle create_fd_handle_ref "cuda_core::rt::create_fd_handle_ref" (
315 int fd) except+ nogil
317 # Array / mipmapped-array / texture / surface handles (PR #467)
318 OpaqueArrayHandle create_array_handle "cuda_core::rt::create_array_handle" (
319 const ContextHandle& h_context, const cydriver.CUDA_ARRAY3D_DESCRIPTOR& desc) except+ nogil
320 OpaqueArrayHandle create_array_handle_ref "cuda_core::rt::create_array_handle_ref" (
321 cydriver.CUarray arr) except+ nogil
322 OpaqueArrayHandle create_array_handle_owning "cuda_core::rt::create_array_handle_owning" (
323 cydriver.CUarray arr) except+ nogil
324 ContextHandle get_array_context "cuda_core::rt::get_array_context" (
325 const OpaqueArrayHandle& h) noexcept nogil
326 OpaqueArrayHandle create_array_level_handle "cuda_core::rt::create_array_level_handle" (
327 const MipmappedArrayHandle& h_mip, unsigned int level) except+ nogil
328 MipmappedArrayHandle create_mipmapped_array_handle "cuda_core::rt::create_mipmapped_array_handle" (
329 const ContextHandle& h_context, const cydriver.CUDA_ARRAY3D_DESCRIPTOR& desc,
330 unsigned int num_levels) except+ nogil
331 ContextHandle get_mipmapped_array_context "cuda_core::rt::get_mipmapped_array_context" (
332 const MipmappedArrayHandle& h) noexcept nogil
333 TexObjectHandle create_tex_object_handle_array "cuda_core::rt::create_tex_object_handle_array" (
334 const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res,
335 const cydriver.CUDA_TEXTURE_DESC& tex, const OpaqueArrayHandle& h_backing) except+ nogil
336 TexObjectHandle create_tex_object_handle_mipmap "cuda_core::rt::create_tex_object_handle_mipmap" (
337 const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res,
338 const cydriver.CUDA_TEXTURE_DESC& tex, const MipmappedArrayHandle& h_backing) except+ nogil
339 TexObjectHandle create_tex_object_handle_linear "cuda_core::rt::create_tex_object_handle_linear" (
340 const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res,
341 const cydriver.CUDA_TEXTURE_DESC& tex, const DevicePtrHandle& h_backing) except+ nogil
342 SurfObjectHandle create_surf_object_handle "cuda_core::rt::create_surf_object_handle" (
343 const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res,
344 const OpaqueArrayHandle& h_backing) except+ nogil
347initialize_deferred_cleanup()
350def _set_context_restore_fault_for_testing(int status):
351 """Make the next context restoration on this thread fail with ``status``.
353 Test hook for the context save/restore paths in the handle layer. The
354 injected failure leaves the target context current, exactly as a failing
355 ``cuCtxSetCurrent`` would, so callers must restore the context themselves.
356 """
357 set_context_restore_fault_for_testing(<cydriver.CUresult>status) 1abcdefg
360def _attach_rollback_failure_for_testing(int status):
361 """Attach a failed CUDA call to the exception being handled, or report it.
363 Test hook for ``attach_rollback_failure()``. Called inside an ``except``
364 block it adds a note to the exception being handled (Python 3.11+); anywhere
365 else it emits a ``CUDAWarning``.
366 """
367 _attach_rollback_failure_local( 1h
368 b"cuTestOperation", <cydriver.CUresult>status, b"failed while testing")