Qwen ViT Runner#
-
class QwenViTRunner : public trt_edgellm::rt::MultimodalRunner#
Runner for Qwen-VL vision encoder.
This class handles the preprocessing and inference of Qwen-VL vision encoder,
Subclassed by trt_edgellm::rt::MuseGlimmerViTRunner, trt_edgellm::rt::Qwen25VLViTRunner, trt_edgellm::rt::Qwen3VLViTRunner
Per-model strategy hooks. Base implementations = Qwen2-VL (the simplest model: image-only, 2D resize,
rotary+cu_seqlens only, incrementing pad, spatial-only MRoPE advance). Subclasses override what they add.
Public Functions
- QwenViTRunner(
- std::string const &engineDir,
- int32_t llmMaxBatchSize,
- int32_t llmMaxSequenceLength,
- cudaStream_t stream,
- std::string const &checkpointDir = ""
Constructor for QwenViTRunner.
- Parameters:
engineDir – [in] Directory containing the TensorRT engine files
llmMaxBatchSize – [in] Maximum batch size from LLM engine
llmMaxSequenceLength – [in] Maximum sequence length from LLM engine
stream – [in] CUDA stream for execution
checkpointDir – [in] HF checkpoint directory used by runtime weight loading
- Throws:
std::runtime_error – if engine directory does not contain engine files, or if buffer allocation fails
json::type_error – if JSON configuration contains unexpected datatypes
-
~QwenViTRunner() noexcept = default#
-
void initialize(cudaStream_t stream)#
Load config + allocate buffers. Called by the factory AFTER construction (not from the constructor) so the per-model virtual hooks dispatch to the subclass.
- Throws:
std::runtime_error – on failure.
- virtual bool preprocess(
- rt::LLMGenerationRequest const &request,
- std::vector<std::vector<int32_t>> &batchedInputIds,
- tokenizer::Tokenizer const *tokenizer,
- rt::OptionalOutputTensor mropeCosSinOut,
- cudaStream_t stream,
- bool imageOnly = false,
- bool skipEncoderWork = false
Preprocess multimodal input including images and text.
- Parameters:
request – [in] LLM generation request containing images and text
batchedInputIds – [inout] Batched input token IDs after preprocessing
tokenizer – [in] Tokenizer for text processing
mropeCosSinOut – [out] MRope cos/sin output cache (required when !imageOnly)
stream – [in] CUDA stream for execution
- Throws:
std::runtime_error – if sequence length is invalid, or a CUDA error occurs
- Returns:
True if preprocessing succeeded, false otherwise
-
void initializeDataParallel()#
Allocate reusable TP2 planning and preprocessing metadata before accepting requests.
- VisionDpPlan const *planDataParallel(
- rt::LLMGenerationRequest const &request
Plan contiguous TP2 image shards without modifying runner tensors.
- Returns:
Null for unsupported inputs; otherwise a borrowed plan valid until the next planning call.
- bool preprocessDataParallel(
- rt::LLMGenerationRequest const &request,
- int32_t rank,
- VisionDpPlan const &plan,
- std::vector<std::vector<int32_t>> &batchedInputIds,
- tokenizer::Tokenizer const *tokenizer,
- rt::OptionalOutputTensor mropeCosSinOut,
- cudaStream_t stream
Preprocess one image shard using a plan produced by planDataParallel(request). Text/MRoPE cover the full request; the caller must gather visual outputs before LLM prefill.
- virtual bool preprocessSystemPrompt(
- std::string const &systemPrompt,
- tokenizer::Tokenizer const *tokenizer,
- rt::OptionalOutputTensor mropeCosSinOut,
- cudaStream_t stream
Encode the system prompt and generate ND-RoPE parameters for the system prompt for KVCache saving.
- Parameters:
systemPrompt – [in] System prompt string
tokenizer – [in] Tokenizer for text processing
mropeCosSinOut – [out] MRope cos/sin output cache (required for non-empty system prompts)
stream – [in] CUDA stream for execution
- Returns:
True if preprocessing succeeded, false otherwise
- virtual void setMropeYarnFrequencies(
- rt::Tensor const &invFreq,
- float mscale
Use YaRN-scaled frequencies for the MRope cos/sin this runner writes.
- Parameters:
invFreq – Device tensor of rotaryDim/2 inverse frequencies; the caller keeps it alive
mscale – Scale applied to cos/sin
- Throws:
std::runtime_error – if the runner does not support YaRN
-
virtual bool infer(cudaStream_t stream) noexcept override#
Run inference on the vision encoder.
- Parameters:
stream – [in] CUDA stream for execution
- Returns:
True if inference succeeded, false otherwise
- virtual bool validateAndFillConfig(
- std::string const &engineDir
Validate and load configuration from JSON file.
- Parameters:
engineDir – [in] Path to engine directory
- Throws:
json::type_error – if JSON configuration contains unexpected datatypes
- Returns:
True if configuration is valid and loaded successfully, false otherwise
-
virtual bool allocateBuffer(cudaStream_t stream) override#
Allocate buffers for inference.
- Throws:
std::runtime_error – if a CUDA operation fails
- Returns:
True if allocation succeeded, false otherwise
- virtual std::vector<std::reference_wrapper<rt::Tensor>> getDeepstackFeatures(
Get deepstack features for Qwen3-VL.
- Returns:
Optional input tensors vector containing deepstack features
- inline std::vector<int64_t> const &getMropeRopeDeltasPerBatch(
Get MRoPE rope deltas for each batch from the last preprocess/infer run.
- Returns:
Vector of length (batch size): delta = maxMropePositionId + 1 - inputIdSize per batch; empty if not yet set.
-
struct LlmVisionBlock#
What the LLM token stream & MRoPE see for one vision span (HF
get_rope_indexllm_grid_*view).Public Members
-
int64_t numTokens = {0}#
LLM tokens this span inserts (llmGridT * llmGridH * llmGridW)
-
int64_t llmGridT = {0}#
HF llm_grid_t (== vit.gridT)
-
int64_t llmGridH = {0}#
HF llm_grid_h (vit.gridH / mergeSize)
-
int64_t llmGridW = {0}#
HF llm_grid_w (vit.gridW / mergeSize)
-
double secondPerGrid = {1.0}#
temporalPatchSize / fps, kept as float; each family’s MRoPE decides how to map it onto its int64 position ids
-
int64_t numTokens = {0}#
-
struct VitFrameGrid#
What the ViT engine sees for one vision span (HF
grid_thw; cu_seqlens / rotary / fast-pos-emb / window).Public Functions
-
inline bool operator==(VitFrameGrid const &o) const#
Value equality: this is exactly the ViT-input cache key (cu_seqlens / rotary / extra inputs).
Public Members
-
int64_t gridT = {0}#
HF grid_t — frames (each emits one per-frame gridH*gridW cu_seqlens block)
-
int64_t gridH = {0}#
HF grid_h — patch-grid height (resizedHeight / patchSize)
-
int64_t gridW = {0}#
HF grid_w — patch-grid width (resizedWidth / patchSize)
-
int64_t patchStart = {0}#
Absolute ViT patch offset where this span begins (rotary / fast-pos base)
-
inline bool operator==(VitFrameGrid const &o) const#
-
struct VisionSpan#
A VisionSpan is one block of vision tokens — an image, or one video frame-group from a single <|vision_start|> — held in both views: the LLM/MRoPE token view (llm) and the ViT patch view (vit).
-
struct VisionDpPlan#
Full-request geometry and contiguous TP2 image shards shared by preprocessing and gathering.
Public Members
-
std::vector<VisionSpan> spans#
-
std::vector<int64_t> spansPerRequest#
-
size_t splitIndex = {0}#
First image assigned to rank 1.
-
std::array<int64_t, 2> tokenCounts = {}#
Visual token counts in rank order.
-
std::vector<VisionSpan> spans#
-
struct QwenViTConfig#
Configuration for Qwen-VL vision encoder.
Public Members
-
int64_t maxHW = {0}#
Maximum height * width.
-
int64_t minHW = {0}#
Minimum height * width.
-
int64_t inputDim = {0}#
Input dimension.
-
int64_t vitPosEmbDim = {0}#
Vision transformer position embedding dimension.
-
int64_t outHiddenSize = {0}#
Output hidden dimension size.
-
int32_t vocabSize = {0}#
Vocabulary size.
-
int32_t visionStartTokenId = {0}#
Token ID for vision start.
-
int32_t visionEndTokenId = {0}#
Token ID for vision end.
-
int32_t imageTokenId = {0}#
Token ID for image placeholder.
-
int32_t videoTokenId = {0}#
Token ID for video placeholder.
-
float mropeTheta = {0}#
Multi-dimensional RoPE theta parameter.
-
int64_t patchSize = {0}#
Patch size in pixels.
-
int64_t temporalPatchSize = {0}#
Temporal patch size for video.
-
int64_t mergeSize = {0}#
Merge size for patches.
-
int32_t mropeSectionH = {0}#
MRoPE section: number of frequency pairs for height.
-
int32_t mropeSectionW = {0}#
MRoPE section: number of frequency pairs for width.
-
bool mropeInterleaved = {false}#
Whether MRoPE uses interleaved layout (from config)
-
int64_t minImageTokensPerImage = {0}#
Minimum image tokens generated by each image. Used for resizing.
-
int64_t maxImageTokensPerImage = {0}#
Maximum image tokens generated by each image. Used for resizing.
-
int64_t maxNumImages = {0}#
Max cu_seqlens entries (images / video temporal groups) per request.
-
std::vector<float> imageMean = {}#
Image normalization mean values (RGB)
-
std::vector<float> imageStd = {}#
Image normalization standard deviation values (RGB)
-
int64_t maxHW = {0}#