TRT Utils#

class AuxStreamSet#

RAII owner for the non-blocking auxiliary streams passed to IExecutionContext::setAuxStreams(). Declare it before the context member so the context is destroyed first (the streams must outlive it).

Public Functions

AuxStreamSet() = default#
inline ~AuxStreamSet() noexcept#
AuxStreamSet(AuxStreamSet const&) = delete#
AuxStreamSet &operator=(AuxStreamSet const&) = delete#
inline AuxStreamSet(AuxStreamSet &&other) noexcept#
inline AuxStreamSet &operator=(AuxStreamSet &&other) noexcept#
inline void add(cudaStream_t stream)#
inline size_t size() const noexcept#

Number of streams currently held.

inline cudaStream_t *data() noexcept#

Pointer to the contiguous stream storage, for passing to setAuxStreams(). Only valid until the next add(); read it after all add() calls.

struct DlDeleter#

Custom deleter for dynamic library handles.

Handles proper cleanup of dynamically loaded libraries using dlclose.

Public Functions

inline void operator()(void *handle) const noexcept#

Delete operator for library handles.

Parameters:

handle – Library handle to close

std::optional<std::pair<cudaGraph_t, cudaGraphExec_t>> trt_edgellm::captureTRTCudaGraph(
nvinfer1::IExecutionContext *context,
cudaStream_t stream,
)#

Capture a TensorRT CUDA graph from an execution context and stream.

Returns:

Pair of graph and graph exec on success, std::nullopt on failure

void trt_edgellm::setNonBlockingAuxStreams(
nvinfer1::IExecutionContext *context,
nvinfer1::ICudaEngine const *engine,
AuxStreamSet &out,
)#

Create non-blocking auxiliary streams for the context, register them via IExecutionContext::setAuxStreams(), and append them to out (which owns them). No-op when the engine reports zero aux streams.

std::string trt_edgellm::dimsToString(
nvinfer1::Dims const &dims,
) noexcept#

Convert TensorRT dimensions to a string representation.

bool trt_edgellm::hasDynamicDims(
nvinfer1::Dims const &dims,
) noexcept#

Return true when a TensorRT dimensions object contains runtime dimensions.

bool trt_edgellm::dimsEqual(
nvinfer1::Dims const &lhs,
nvinfer1::Dims const &rhs,
) noexcept#

Return true when two TensorRT dimensions objects have the same rank and extent.

bool trt_edgellm::isEngineInput(
nvinfer1::ICudaEngine const &engine,
std::string const &tensorName,
) noexcept#

Return true when tensorName exists in the engine I/O list and is an input.

std::string trt_edgellm::printEngineInfo(
nvinfer1::ICudaEngine const *engine,
int32_t profileIndex,
) noexcept#

Print the engine information for a specific profile index.

bool trt_edgellm::engineHasOutputTensor(
nvinfer1::ICudaEngine const *engine,
char const *tensorName,
) noexcept#

Whether the engine exposes an output binding with the given name. Use this instead of getTensorIOMode(name) when probing for an optional binding — getTensorIOMode logs a spurious TensorRT ERROR for unknown names.

Pre:

engine and tensorName are non-null.

std::unique_ptr<nvinfer1::ICudaEngine> trt_edgellm::deserializeCudaEngineFromFile(
nvinfer1::IRuntime &runtime,
std::filesystem::path const &enginePath,
)#

Deserialize a TensorRT engine plan from disk without mapping the full plan into process memory.