Coverage for cuda/core/_memory/_device_memory_resource.pyx: 81.08%

74 statements  

« prev     ^ index     » next       coverage.py v7.14.1, created at 2026-06-13 01:38 +0000

1# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 

2# 

3# SPDX-License-Identifier: Apache-2.0 

4  

5from __future__ import annotations 

6  

7from cuda.bindings cimport cydriver 

8from cuda.core._memory._memory_pool cimport ( 

9 _MemPool, MP_init_create_pool, MP_raise_release_threshold, 

10) 

11from cuda.core._memory cimport _ipc 

12from cuda.core._memory._ipc cimport IPCAllocationHandle 

13from cuda.core._resource_handles cimport ( 

14 as_cu, 

15 get_device_mempool, 

16 get_last_error, 

17) 

18from cuda.core._utils.cuda_utils cimport ( 

19 check_or_create_options, 

20 HANDLE_RETURN, 

21) 

22from dataclasses import dataclass 

23import multiprocessing 

24import platform # no-cython-lint 

25import uuid 

26  

27from cuda.core._memory._peer_access_utils import PeerAccessibleBySetProxy, replace_peer_accessible_by 

28from cuda.core._utils.cuda_utils import check_multiprocessing_start_method 

29  

30from typing import TYPE_CHECKING 

31  

32if TYPE_CHECKING: 

33 from cuda.core._device import Device 

34  

35__all__ = ['DeviceMemoryResource', 'DeviceMemoryResourceOptions'] 

36  

37  

38@dataclass 

39cdef class DeviceMemoryResourceOptions: 

40 """Customizable :obj:`~_memory.DeviceMemoryResource` options. 

41  

42 Attributes 

43 ---------- 

44 ipc_enabled : bool, optional 

45 Specifies whether to create an IPC-enabled memory pool. When set to 

46 True, the memory pool and its allocations can be shared with other 

47 processes. (Default to False) 

48  

49 max_size : int, optional 

50 Maximum pool size. When set to 0, defaults to a system-dependent value. 

51 (Default to 0) 

52 """ 

53 ipc_enabled : bool = False 

54 max_size : int = 0 

55  

56  

57cdef class DeviceMemoryResource(_MemPool): 

58 """ 

59 A device memory resource managing a stream-ordered memory pool. 

60  

61 Parameters 

62 ---------- 

63 device_id : Device | int 

64 Device or Device ordinal for which a memory resource is constructed. 

65  

66 options : DeviceMemoryResourceOptions 

67 Memory resource creation options. 

68  

69 If set to `None`, the memory resource uses the driver's current 

70 stream-ordered memory pool for the specified `device_id`. If no memory 

71 pool is set as current, the driver's default memory pool for the device 

72 is used. 

73  

74 If not set to `None`, a new memory pool is created, which is owned by 

75 the memory resource. 

76  

77 When using an existing (current or default) memory pool, the returned 

78 device memory resource does not own the pool (`is_handle_owned` is 

79 `False`), and closing the resource has no effect. 

80  

81 Notes 

82 ----- 

83 To create an IPC-Enabled memory resource (MR) that is capable of sharing 

84 allocations between processes, specify ``ipc_enabled=True`` in the initializer 

85 option. Sharing an allocation is a two-step procedure that involves 

86 mapping a memory resource and then mapping buffers owned by that resource. 

87 These steps can be accomplished in several ways. 

88  

89 An IPC-enabled memory resource can allocate memory buffers but cannot 

90 receive shared buffers. Mapping an MR to another process creates a "mapped 

91 memory resource" (MMR). An MMR cannot allocate memory buffers and can only 

92 receive shared buffers. MRs and MMRs are both of type 

93 :class:`DeviceMemoryResource` and can be distinguished via 

94 :attr:`DeviceMemoryResource.is_mapped`. 

95  

96 An MR is shared via an allocation handle accessed through the 

97 :attr:`DeviceMemoryResource.allocation_handle` property. The allocation 

98 handle has a platform-specific interpretation; however, memory IPC is 

99 currently only supported for Linux, and in that case allocation handles 

100 are file descriptors. After sending an allocation handle to another 

101 process, it can be used to create an MMR by invoking 

102 :meth:`DeviceMemoryResource.from_allocation_handle`. 

103  

104 Buffers can be shared as serializable descriptors accessed through the 

105 :attr:`Buffer.ipc_descriptor` property. In a receiving process, a shared 

106 buffer is created by invoking :meth:`Buffer.from_ipc_descriptor` with an 

107 MMR and buffer descriptor, where the MMR corresponds to the MR that 

108 created the described buffer. 

109  

110 To help manage the association between memory resources and buffers, a 

111 registry is provided. Every MR has a unique identifier (UUID). MMRs can be 

112 registered by calling :meth:`DeviceMemoryResource.register` with the UUID 

113 of the corresponding MR. Registered MMRs can be looked up via 

114 :meth:`DeviceMemoryResource.from_registry`. When registering MMRs in this 

115 way, the use of buffer descriptors can be avoided. Instead, buffer objects 

116 can themselves be serialized and transferred directly. Serialization embeds 

117 the UUID, which is used to locate the correct MMR during reconstruction. 

118  

119 IPC-enabled memory resources interoperate with the :mod:`multiprocessing` 

120 module to provide a simplified interface. This approach can avoid direct 

121 use of allocation handles, buffer descriptors, MMRs, and the registry. When 

122 using :mod:`multiprocessing` to spawn processes or send objects through 

123 communication channels such as :class:`multiprocessing.Queue`, 

124 :class:`multiprocessing.Pipe`, or :class:`multiprocessing.Connection`, 

125 :class:`Buffer` objects may be sent directly, and in such cases the process 

126 for creating MMRs and mapping buffers will be handled automatically. 

127  

128 For greater efficiency when transferring many buffers, one may also send 

129 MRs and buffers separately. When an MR is sent via :mod:`multiprocessing`, 

130 an MMR is created and registered in the receiving process. Subsequently, 

131 buffers may be serialized and transferred using ordinary :mod:`pickle` 

132 methods. The reconstruction procedure uses the registry to find the 

133 associated MMR. 

134 """ 

135  

136 def __cinit__(self, *args, **kwargs) -> None: 

137 self._dev_id = cydriver.CU_DEVICE_INVALID 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

138  

139 def __init__( 

140 self, 

141 device_id: Device | int, 

142 options: DeviceMemoryResourceOptions | dict[str, object] | None = None 

143 ) -> None: 

144 _DMR_init(self, device_id, options) 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

145  

146 def __reduce__(self) -> tuple[object, ...]: 

147 return DeviceMemoryResource.from_registry, (self.uuid,) 1adbc

148  

149 @staticmethod 

150 def from_registry(uuid: uuid.UUID) -> DeviceMemoryResource: # no-cython-lint 

151 """ 

152 Obtain a registered mapped memory resource. 

153  

154 Raises 

155 ------ 

156 RuntimeError 

157 If no mapped memory resource is found in the registry. 

158 """ 

159 return <DeviceMemoryResource>(_ipc.MP_from_registry(uuid)) 

160  

161 def register(self, uuid: uuid.UUID) -> DeviceMemoryResource: # no-cython-lint 

162 """ 

163 Register a mapped memory resource. 

164  

165 Returns 

166 ------- 

167 The registered mapped memory resource. If one was previously registered 

168 with the given key, it is returned. 

169 """ 

170 return <DeviceMemoryResource>(_ipc.MP_register(self, uuid)) 

171  

172 @classmethod 

173 def from_allocation_handle( 

174 cls, device_id: Device | int, alloc_handle: int | IPCAllocationHandle 

175 ) -> DeviceMemoryResource: 

176 """Create a device memory resource from an allocation handle. 

177  

178 Construct a new `DeviceMemoryResource` instance that imports a memory 

179 pool from a shareable handle. The memory pool is marked as owned, and 

180 the resource is associated with the specified `device_id`. 

181  

182 Parameters 

183 ---------- 

184 device_id : int | Device 

185 The ID of the device or a Device object for which the memory 

186 resource is created. 

187  

188 alloc_handle : int | IPCAllocationHandle 

189 The shareable handle of the device memory resource to import. If an 

190 integer is supplied, it must represent a valid platform-specific 

191 handle. It is the caller's responsibility to close that handle. 

192  

193 Returns 

194 ------- 

195 A new device memory resource instance with the imported handle. 

196 """ 

197 cdef DeviceMemoryResource mr = <DeviceMemoryResource>( 

198 _ipc.MP_from_allocation_handle(cls, alloc_handle)) 

199 from .._device import Device 

200 mr._dev_id = Device(device_id).device_id 

201 return mr 

202  

203 @property 

204 def allocation_handle(self) -> IPCAllocationHandle: 

205 """Shareable handle for this memory pool (requires IPC). 

206  

207 The handle can be used to share the memory pool with other processes. 

208 The handle is cached in this `MemoryResource` and owned by it. 

209 """ 

210 if not self.is_ipc_enabled: 2dca d cce ecfctcucgchcvcicjcwckclcmcxcncocpcf g qcrcsch i j k b c u l v m n

211 raise RuntimeError("Memory resource is not IPC-enabled") 2a d ccu

212 return self._ipc_data._alloc_handle 2dca cce ecfctcucgchcvcicjcwckclcmcxcncocpcf g qcrcsch i j k b c l v m n

213  

214 @property 

215 def device_id(self) -> int: 

216 """The associated device ordinal.""" 

217 return self._dev_id 2T U V W X Y Z 0 dca d cce ecfcgchcicjckclcmcncocpcf g qcrcsch i j k b c ] ^ r s q ` l m n w x yczcy z A B C AcD E F BcG CcH DcI J EcK FcL GcM HcN IcJcO P KcQ R S

218  

219 @property 

220 def peer_accessible_by(self) -> PeerAccessibleBySetProxy: 

221 """ 

222 Get or set the devices that can access allocations from this memory 

223 pool. Access can be modified at any time and affects all allocations 

224 from this memory pool. 

225  

226 Returns a set-like proxy of :obj:`~_device.Device` objects that manages 

227 peer access. Inputs are accepted as either :obj:`~_device.Device` 

228 objects or device-ordinal :class:`int` values. 

229  

230 Examples 

231 -------- 

232 >>> dmr = DeviceMemoryResource(0) 

233 >>> dmr.peer_accessible_by = {1} # grant access to device 1 

234 >>> assert 1 in dmr.peer_accessible_by 

235 >>> dmr.peer_accessible_by.add(2) # update access to include device 2 

236 >>> dmr.peer_accessible_by = [] # revoke peer access 

237 """ 

238 return PeerAccessibleBySetProxy(self) 1_t

239  

240 @peer_accessible_by.setter 

241 def peer_accessible_by(self, devices) -> None: 

242 replace_peer_accessible_by(self, devices) 1t

243  

244 @property 

245 def is_device_accessible(self) -> bool: 

246 """Return True. This memory resource provides device-accessible buffers.""" 

247 return True 2r s q w x yczcy z A B C AcD E F BcG CcH DcI J EcK FcL GcM HcN IcJcO P KcQ R S

248  

249 @property 

250 def is_host_accessible(self) -> bool: 

251 """Return False. This memory resource does not provide host-accessible buffers.""" 

252 return False 1rsq

253  

254  

255cdef inline _DMR_init(DeviceMemoryResource self, device_id, options): 

256 from .._device import Device 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

257 cdef int dev_id = Device(device_id).device_id 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

258 cdef DeviceMemoryResourceOptions opts = check_or_create_options( 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

259 DeviceMemoryResourceOptions, options, "DeviceMemoryResource options", 

260 keep_none=True 

261 ) 

262 cdef bint ipc_enabled = False 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

263 cdef size_t max_size = 0 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

264  

265 self._dev_id = dev_id 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

266  

267 if opts is not None: 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

268 ipc_enabled = opts.ipc_enabled 21 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c q t { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu l v m n obpb

269 if ipc_enabled and not _ipc.is_supported(): 21 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c q t { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu l v m n obpb

270 raise RuntimeError(f"IPC is not available on {platform.system()}") 

271 max_size = opts.max_size 21 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c q t { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu l v m n obpb

272  

273 if opts is None: 2qbrbsbtbubT U V W X Y Z 0 1 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c ] ^ vbwbxbybzbr s _ q o t ` { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu AbBbl v m n CbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@bobpb[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

274 self._h_pool = get_device_mempool(dev_id) 2qbrbsbtbubT U V W X Y Z 0 ] ^ vbwbxbybzbr s _ o ` AbBbCbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@b[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

275 if not self._h_pool: 2qbrbsbtbubT U V W X Y Z 0 ] ^ vbwbxbybzbr s _ o ` AbBbCbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@b[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

276 HANDLE_RETURN(get_last_error()) 

277 raise RuntimeError( 

278 f"Failed to initialize DeviceMemoryResource for device {dev_id}: " 

279 "cuda-core returned an empty memory pool handle without recording a CUDA error. " 

280 "This is an internal cuda-core error; please report it with your CUDA driver, " 

281 "CUDA Toolkit, and cuda-python versions." 

282 ) 

283 self._mempool_owned = False 2qbrbsbtbubT U V W X Y Z 0 ] ^ vbwbxbybzbr s _ o ` AbBbCbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@b[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

284 MP_raise_release_threshold(self) 2qbrbsbtbubT U V W X Y Z 0 ] ^ vbwbxbybzbr s _ o ` AbBbCbDbEbFbGbHbIbJbKbLbMbNbObPbQbRbSbTbUbVbWbXbYbZb0b1b2b3b4b5b6b7b8b9b!b#b$b%b'b(b)b*b+b,b-b.b/b:b;b=b?b@b[b]b^b_b`b{b|b}bw x y z A B C D E F G H I J K L M N O P ~bacbcQ R S

285 else: 

286 MP_init_create_pool( 21 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c q t { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu l v m n obpb

287 self, 

288 cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE, 

289 dev_id, 

290 cydriver.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED, 

291 ipc_enabled, 

292 max_size, 21 a 2 d 3 e 4 5 6 7 8 9 ! # $ % ' ( ) * + , - . / : ; = f g ? @ [ h i j k b c q t { | } ~ abbbcbdbebfbgbhbibjbkblbmbnbu l v m n obpb

293 ) 

294  

295  

296# Note: this is referenced in instructions to debug nvbug 5698116. 

297cpdef str DMR_mempool_get_access(DeviceMemoryResource dmr, int device_id): 

298 """ 

299 Probes peer access from the given device using cuMemPoolGetAccess. 

300  

301 Parameters 

302 ---------- 

303 device_id : int or Device 

304 The device to query access for. 

305  

306 Returns 

307 ------- 

308 str 

309 Access permissions: "rw" for read-write, "r" for read-only, "" for no access. 

310 """ 

311 from .._device import Device 1o

312  

313 cdef int dev_id = Device(device_id).device_id 1o

314 cdef cydriver.CUmemAccess_flags flags 

315 cdef cydriver.CUmemLocation location 

316  

317 location.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE 1o

318 location.id = dev_id 1o

319  

320 with nogil: 1o

321 HANDLE_RETURN(cydriver.cuMemPoolGetAccess(&flags, as_cu(dmr._h_pool), &location)) 1o

322  

323 if flags == cydriver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE: 1o

324 return "rw" 1o

325 elif flags == cydriver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READ: 

326 return "r" 

327 else: 

328 return "" 

329  

330  

331def _deep_reduce_device_memory_resource(mr) -> tuple[object, ...]: 

332 check_multiprocessing_start_method() 2dca d cce ecfcgchcicjckclcmcncocpcf g qcrcsch i j k b c l m n

333 from .._device import Device 2dca d cce ecfcgchcicjckclcmcncocpcf g qcrcsch i j k b c l m n

334 device = Device(mr.device_id) 2dca d cce ecfcgchcicjckclcmcncocpcf g qcrcsch i j k b c l m n

335 alloc_handle = mr.allocation_handle 2dca d cce ecfcgchcicjckclcmcncocpcf g qcrcsch i j k b c l m n

336 return mr.from_allocation_handle, (device, alloc_handle) 2dca cce ecfcgchcicjckclcmcncocpcf g qcrcsch i j k b c l m n

337  

338  

339multiprocessing.reduction.register(DeviceMemoryResource, _deep_reduce_device_memory_resource)