Sample configuration reference#
This reference is generated directly from configuration files under each
installed top-level sample’s yaml/ directory in agent-samples/ or
model-server-samples/, and from configuration files
beside a direct capability subproject. Adding a file in either location enrolls
it automatically. Values are sample- and hardware-specific examples, not
universal defaults. Keep field guidance beside the value as a source comment;
the generated page preserves those comments verbatim.
Refer to the generated Python API reference for public typed model configuration fields. Operational choices, credentials, and deployment workflows remain in the handwritten guides.
To change a sample parameter:
Start in
agent-samples/<sample>/ormodel-server-samples/<sample>/and find the owning file in that sample’s README or published guide.Edit the checked-in YAML or JSON value, preserving its documented type. Resolve relative paths from the file that declares them unless the sample guide documents different precedence.
Restart the sample process that owns the file. Sample configuration is not hot-reloaded.
If the edit changes a persistent model server, stop and restart the shared model stack. Refer to Customizing model servers for that workflow.
lab-instrument-monitoring#
yaml/device_io_hub.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Development-only placeholders. For deployed setups, replace both values or
# remove them and set LIVEKIT_API_KEY and LIVEKIT_API_SECRET in the environment.
api_key: devkey
api_secret: devsecret-xr-livekit-prototype-2026
room_name: xr-room
lk_port_ws: 7880
lk_port_tcp: 7881
lk_port_udp: 7882
enable_web_server: true
web_server_port: 8080
web_client_dir: ../../../client-samples/web
enable_token_server: true
yaml/device_map.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
devices:
qr_code:
device-1: Device1
device-2: Device2
device-3: Device3
device-4: Device4
device-5: Device5
aruco:
"0": Device1
"1": Device2
"2": Device3
"3": Device4
"4": Device5
yaml/lab_instrument_monitoring_worker.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
voice_gate_yaml: voice_gate.yaml
device_map_yaml: device_map.yaml
artifacts_dir: ../artifacts
capture_marker_scans: false
web_events_host: 127.0.0.1
web_events_port: 8092
web_events_max_events: 5000
monitor_interval_s: 5.0
instrument_monitor_interval_s: 5.0
instrument_state_interval_s: 10.0
instrument_lost_after_s: 30.0
monitor_history_size: 20
frame_max_age_s: 5.0
frame_timeout_s: 5.0
silero_threshold: 0.5
silence_duration: 0.4
min_speech: 0.1
idle_timeout_secs: 0
yaml/models.json#
{
"models": {
"llm": {
"adapter": {"preset": "nemotron_omni"},
"endpoint": {"base_url": "http://localhost:8108"}
},
"vlm": {
"adapter": {"preset": "cosmos3_nano_reasoner"},
"endpoint": {"base_url": "http://localhost:8100"}
},
"stt": {
"adapter": {"preset": "parakeet_stt"},
"endpoint": {"base_url": "http://localhost:8103"}
},
"tts": {
"adapter": {"preset": "pocket_tts"},
"endpoint": {"base_url": "http://localhost:8105"}
}
}
}
yaml/voice_gate.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Every final microphone transcript is recorded before this gate. Only accepted
# turns are sent to the foreground agent.
magic_phrases:
- agent
- hey agent
listening_chime: true
followup_grace_s: 5.0
simple-vlm-example#
yaml/device_io_hub.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# DeviceIOHub configuration for the simple-vlm-example sample.
# Paths are relative to this file's directory.
# See services/device-io-hub/device_io_hub.yaml for all available options.
# ── LiveKit server ────────────────────────────────────────────────────────────
# Development-only placeholders. For deployed setups, replace both values or
# remove them and set LIVEKIT_API_KEY and LIVEKIT_API_SECRET in the environment.
api_key: devkey
api_secret: devsecret-xr-livekit-prototype-2026
room_name: xr-room
lk_port_ws: 7880
lk_port_tcp: 7881
lk_port_udp: 7882
# Cloud VMs behind NAT may need STUN-based public-IP discovery. Also skip
# validation when the NAT does not support LiveKit's public-IP self-ping.
# lk_use_external_ip: true
# lk_skip_external_ip_validation: true
# ── Web client server ─────────────────────────────────────────────────────────
enable_web_server: true
web_server_port: 8080
web_client_dir: ../../../client-samples/web
# ── Optional: token server ────────────────────────────────────────────────────
enable_token_server: true
# Change if another service holds port 8000 (e.g. iphlpsvc under WSL2 mirrored
# networking; see the note in services/device-io-hub/device_io_hub.yaml).
# token_server_port: 8000
yaml/media_capture.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Opt-in server-side capture for Simple VLM. Each participant gets a
# timestamped bundle with fast-start H.264/AAC-LC .mp4 video, retained raw media,
# and the exact inbound/outbound data timeline.
hub_sub_addr: ipc:///tmp/xr_hub_pub
hub_push_addr: ipc:///tmp/xr_hub_in
out_dir: ~/.local/share/xr-ai/captures/simple-vlm-example
profile: demo
session_mode: participant
sample_fps: 30.0
bitrate: 6000000
gpu_id: 0
frame_queue_size: 2
encoder_workers: 2
audio_sample_rate: 48000
overlay_seconds: 12.0
overlay_lines: 4
max_total_bytes: 10737418240
yaml/models.json#
{
"models": {
"stt": {
"adapter": {"preset": "parakeet_stt"},
"endpoint": {"base_url": "http://localhost:8103"}
},
"vlm": {
"adapter": {"preset": "cosmos3_nano_reasoner"},
"endpoint": {"base_url": "http://localhost:8100"}
},
"tts": {
"adapter": {"preset": "pocket_tts"},
"endpoint": {"base_url": "http://localhost:8105"}
}
}
}
yaml/simple_vlm_example_worker.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Simple VLM example worker configuration.
# Auto-discovered by the launcher and passed as --config to the worker process.
# Path to the voice-gate config (resolved relative to this file's
# directory). Owns the magic-phrase / follow-up / listening-chime knobs.
voice_gate_yaml: voice_gate.yaml
# The worker uses its packaged prompts/system.txt by default. Set
# system_prompt or system_prompt_file here only to override it.
# Frame freshness — tune if the agent sees stale frames.
frame_max_age_s: 5.0 # frame older than this is considered stale
frame_timeout_s: 5.0 # give up waiting for a fresh frame after this many seconds
# Voice Activity Detection — Silero VAD (ONNX backend).
# Tune if STT triggers too early or misses speech.
silero_threshold: 0.5 # Silero speech-probability gate (0..1); lower = more sensitive
silence_duration: 0.4 # seconds of silence that ends an utterance
min_speech: 0.1 # minimum seconds of speech before STT is called
# ── Idle timeout ──────────────────────────────────────────────────────────
# Auto-cancel the voice pipeline after this many seconds with no user/bot
# speech. DISABLED by default (0 = off) — a quiet session stays connected
# indefinitely, which is what XR sessions usually want. Set a positive value
# (e.g. 300 for 5 min) to opt in; the worker then tears the pipeline down
# after that long idle. Applied by VoiceAgent.
idle_timeout_secs: 0
yaml/voice_gate.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Voice-gate (speech-input opt-in) configuration for simple-vlm-example.
# Consumed by xr_ai_voicegate.VoiceGate inside the unified pipecat pipeline.
#
# Magic phrases are sentence-boundary opt-ins: an STT transcript must begin
# with one of these or contain one after '.', '?', or '!' followed by
# whitespace or a closing quote. Matching is case-insensitive. Text through
# the matched phrase is stripped before the agent sees the query; commas and
# other mid-sentence mentions do not match. The same stripped tail is used for
# early global-STOP detection; the complete final transcript still determines
# whether a stop-like opening is a global stop or a scoped agent command.
magic_phrases:
- "agent"
- "hey agent"
# Audible "I heard you" feedback. When true (and magic_phrases is
# non-empty), a short two-tone chime plays the moment a phrase matches —
# before STT+VLM finish — so the user knows the agent is processing.
listening_chime: false
# Follow-up grace window. After a successful magic-phrase match, the
# next utterance from the same participant within this many seconds
# bypasses the gate so "hey agent" → pause → "what am I looking at?"
# works as one natural interaction. Set to 0 to disable.
followup_grace_s: 5.0
tea-making-sample#
yaml/device_io_hub.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Development-only placeholders. For deployed setups, replace both values or
# remove them and set LIVEKIT_API_KEY and LIVEKIT_API_SECRET in the environment.
api_key: devkey
api_secret: devsecret-xr-livekit-prototype-2026
room_name: xr-room
lk_port_ws: 7880
lk_port_tcp: 7881
lk_port_udp: 7882
enable_web_server: true
web_server_port: 8080
web_client_dir: ../../../client-samples/web
enable_token_server: true
yaml/models.local.json#
{
"models": {
"llm": {
"adapter": {"preset": "nemotron_omni"},
"endpoint": {"base_url": "http://localhost:8108"}
},
"vlm": {
"category": "vlm",
"adapter": {
"kind": "openai_compat",
"model_name": "llm",
"reasoning_field": "reasoning_content",
"capabilities": {"streaming": true, "vision": true},
"default_extras": {
"max_tokens": 128,
"temperature": 0.0,
"chat_template_kwargs": {"enable_thinking": false}
}
},
"endpoint": {"base_url": "http://localhost:8108"}
},
"stt": {
"adapter": {"preset": "parakeet_stt"},
"endpoint": {"base_url": "http://localhost:8103"}
},
"embedding": {
"adapter": {"preset": "nemotron_embedding"},
"endpoint": {"base_url": "http://localhost:8109"}
},
"tts": {
"adapter": {"preset": "pocket_tts"},
"endpoint": {"base_url": "http://localhost:8105"}
}
}
}
yaml/rag_service.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
endpoint: tcp://0.0.0.0:8340
documents_dir: ../rag-documents
models_config: models.local.json
embedding_role: embedding
cache_dir: ../../../models/tea-making-rag-cache
chunk_size: 700
overlap: 100
embedding_dim: 768
batch_size: 16
min_score: 0.3
yaml/tea_making_worker.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# The launcher materializes these paths for runtime path resolution.
models_config: models.local.json
workflow_config: workflow.yaml
voice_gate_yaml: voice_gate.yaml
# Choose one to replace the idle and explicit background-action instructions.
# Active tea guidance comes from the focused step policies in workflow.yaml.
# foreground_prompt: <inline shared foreground instructions>
# foreground_prompt_file: ../worker/tea_making_worker/prompts/foreground_prompt.txt
rag_endpoint: tcp://127.0.0.1:8340
artifacts_dir: ../artifacts
web_events_host: 127.0.0.1
web_events_port: 8092
web_events_max_events: 5000
frame_max_age_s: 3.0
frame_timeout_s: 5.0
vlm_timeout_s: 15.0
change_watch_interval_s: 2.0
transcript_summary_interval_s: 120.0
video_log_interval_s: 2.0
background_history_size: 50
change_watch_default_instruction: >-
important changes involving people, objects, actions, or possible hazards
silero_threshold: 0.3
silence_duration: 1.2
min_speech: 0.15
idle_timeout_secs: 0
yaml/voice_gate.always-on.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
magic_phrases: []
listening_chime: false
followup_grace_s: 0.0
yaml/voice_gate.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
magic_phrases:
- agent
- hey agent
listening_chime: true
followup_grace_s: 5.0
yaml/workflow.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
task:
name: tea making
start_step: identify
foreground_prompt: This workflow guides tea making.
complete_message: Tea guidance complete. Enjoy your tea.
state:
tea_name:
type: string
description: Tea name from the current caption.
target_temperature_c:
type: number
description: Brewing temperature in Celsius.
steep_duration_s:
type: integer
description: Steep duration in seconds.
guidance_source:
type: string
description: Package or retrieved source.
tea_ready:
type: boolean
description: True only when all preceding fields are supported.
water_filled:
type: boolean
description: Whether water is visibly in the heating vessel.
initial: false
heating_started:
type: boolean
description: Whether a heater-on indicator or unit-bearing temperature was seen.
initial: false
steeping_started_at_us:
type: integer
description: Start time returned by clock__now.
steeping_started:
type: boolean
description: Whether tea is immersed and its start time is recorded.
initial: false
steeping_complete:
type: boolean
description: Whether the fresh timer result is expired.
initial: false
steps:
- id: identify
title: Identify the tea
reads: []
writes: [tea_name, target_temperature_c, steep_duration_s, guidance_source, tea_ready]
trigger:
function: current_view
interval_s: 1.5
arguments:
question: >-
Read the front label brand and tea/blend name, plus visible brewing
temperature/time. Ignore slogans, claims, bag count, weight, and
badges. Transcribe exactly; never infer. If no brand or tea/blend is
legible, say label unreadable.
result_field: answer
evidence:
pattern: "^accepted$"
consecutive: 1
agent:
prompt: >-
Identify tea only from the current caption, never state or RAG.
If the tea name is unreadable, commit empty updates; do not retrieve or
write placeholder values. Prefer
visible values. If temperature/time is missing,
call rag_lookup once with "<exact name> water temperature steep time",
top_k 2; accept only a section naming it. If every writable field is
supported, commit all with tea_ready true. Otherwise commit nothing;
never write tea_ready false.
tools: [rag_lookup]
voice:
prompt: >-
Tea identity comes only from state tea_name or current_view, never RAG.
tea_ready means the tea instructions were identified, not that the
drink has been prepared.
If the exact name is unknown, inspect the current frame. Retrieve only
missing temperature or duration after the exact name is known. Never
replace package values. Answer only what was asked.
tools: [current_view, rag_lookup]
complete_when: {tea_ready: true}
next: fill_water
state_on_skip:
tea_name: generic tea
target_temperature_c: 93
steep_duration_s: 180
guidance_source: generic fallback
tea_ready: true
messages:
enter: Hold the tea package or tag in view so I can read it.
complete: I found the tea instructions. Say next when ready.
skip: I will use generic tea defaults.
- id: fill_water
title: Fill the heating vessel
reads: [tea_name, target_temperature_c, steep_duration_s]
writes: [water_filled]
trigger:
function: current_view
interval_s: 2
arguments:
question: >-
Describe the visible interior and contents of the kettle or pot.
State whether water is visible inside and what supports that reading.
If the contents are unclear, say so. Never infer.
result_field: answer
evidence:
pattern: "^accepted$"
consecutive: 2
agent:
prompt: >-
Judge the fresh caption semantically without requiring particular
words. Set water_filled true when it reasonably shows water inside the
vessel. Otherwise commit empty updates and message.
tools: []
voice:
prompt: >-
Give instructions or verify with current_view. A user report is not
visual confirmation. Confirm filled only from state water_filled or a
current view showing the vessel interior and concrete water cues;
otherwise say it is not yet visually confirmed.
tools: [current_view]
complete_when: {water_filled: true}
next: heat_water
state_on_skip: {water_filled: true}
messages:
enter: Fill a kettle or pot with fresh water and keep it in view.
complete: The vessel has water. Say next when ready.
skip: I will assume the vessel is filled.
- id: heat_water
title: Heat the water
reads: [tea_name, target_temperature_c, steep_duration_s]
writes: [heating_started]
trigger:
function: current_view
interval_s: 2
arguments:
question: >-
Describe the visible kettle or thermometer display and transcribe its
current numeric reading and unit. If either is unclear, say so. Do not
confuse a printed target temperature with the current reading.
result_field: answer
evidence:
pattern: "^accepted$"
consecutive: 2
agent:
prompt: >-
This step detects that heating is underway; it does not wait for the tea
target temperature. Ignore target_temperature_c when deciding this
step. For every fresh current reading with an explicit Celsius or
Fahrenheit unit, first call temperature__threshold with that exact
reading and a threshold_c of 50. If its above result is true, commit
heating_started true. If above is false, commit empty updates and
message with rejected evidence. For a target value, ambiguous unit,
missing reading, or unclear current-versus-target value, do not call the
tool; commit empty updates and message with unknown evidence so it does
not erase a prior confirmed reading.
tools: [temperature__threshold]
voice:
prompt: >-
Explain target from state. For a reading, call current_view and
repeat the visible number and unit; do not verify. For hot-enough
checks, call current_view, then temperature__verify only with an exact
number and Celsius or Fahrenheit unit. Report result. If missing,
say the reading is unavailable.
tools: [current_view, temperature__verify]
complete_when: {heating_started: true}
next: start_steeping
state_on_skip: {heating_started: true}
messages:
enter: Heat the water to {{ target_temperature_c | temperature_c }}.
complete: Heating is underway. Wait for the water to reach the target temperature.
skip: I will assume heating has started.
- id: start_steeping
title: Start steeping
reads: [tea_name, target_temperature_c, steep_duration_s]
writes: [steeping_started_at_us, steeping_started]
trigger:
function: current_view
interval_s: 2
arguments:
question: >-
Look for two separately visible facts in one vessel: a water surface
or level; and a tea bag, infuser, or leaves entering or immersed in
that water. If either water or tea-water contact is not visible, say
it is unconfirmed. Note hazards.
result_field: answer
evidence:
pattern: "^accepted$"
consecutive: 2
agent:
prompt: >-
Judge when steeping starts, not when it finishes. Treat unconfirmed,
unknown, unclear, or missing visual evidence as a hard stop: do not call
clock__now, and commit empty updates and message with unknown evidence.
Only when the fresh caption explicitly confirms both visible liquid and
tea visibly contacting that liquid inside the same vessel, first call
clock__now. Nearby, above, or outside the liquid does not count. Then
commit exactly its epoch_us as steeping_started_at_us with
steeping_started true. For any other caption, do not infer contact from
context or prior state; commit empty updates and message. Never invent or
replace a start time.
tools: [clock__now]
voice:
prompt: >-
Give instructions or verify with current_view. A report or tea item in
a vessel is not confirmation. Require both visible water and visible
tea-water contact. The observation loop, not this answer, starts the
timer.
tools: [current_view]
complete_when: {steeping_started: true}
next: steep_timer
complete_on_skip: true
state_on_skip: {steeping_started: true}
messages:
enter: Put the tea in the water; I will start the timer when I see it immersed.
complete: Steeping timer started. Say next to monitor it.
skip: Steeping detection and the timer were skipped.
- id: steep_timer
title: Wait for steeping
reads:
[tea_name, target_temperature_c, steeping_started_at_us, steep_duration_s]
writes: [steeping_complete]
trigger:
function: clock__timer
interval_s: 1
arguments:
started_at_us: $state.steeping_started_at_us
duration_s: $state.steep_duration_s
agent:
prompt: >-
Copy the fresh clock__timer expired boolean: set steeping_complete true
only when expired is true. While false, commit empty updates/message.
Never infer from elapsed text, wall time, or prior state.
tools: []
voice:
prompt: >-
Call clock__timer for every timer or completion question. Report only
its fresh remaining time or completion; never calculate from state,
prior answers, or wall-clock time.
tools: [clock__timer]
complete_when: {steeping_complete: true}
next: null
state_on_skip: {steeping_complete: true}
messages:
enter: Steep for {{ steep_duration_s | duration }}.
complete: Steeping is complete. Remove the tea, then say next to finish.
skip: Timer skipped; remove the tea when it tastes right.
xr-render-demo#
scene/scene_service.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Sample-local scene process reference configuration.
# LOVR_BIN in the environment takes priority over lovr_bin.
# lovr_bin: /home/you/hub/lovr/build/bin/lovr
xr_app_dir: ./lovr
endpoint: tcp://0.0.0.0:8320
scene_socket: ipc:///tmp/xr_render_scene
cloudxr_env_file: ~/.cloudxr/run/cloudxr.env
yaml/cloudxr_runtime.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# CloudXR runtime configuration.
# Full docs: python -m isaacteleop.cloudxr --help
# Directory where CloudXR stores runtime state, certs, and logs (default: ~/.cloudxr).
cloudxr_install_dir: ~/.cloudxr
# Set to true to accept the NVIDIA CloudXR EULA non-interactively.
# EULA: https://github.com/NVIDIA/IsaacTeleop/blob/main/deps/cloudxr/CLOUDXR_LICENSE
# Once accepted the marker is written to <cloudxr_install_dir>/run/eula_accepted
# and this flag is no longer checked.
accept_eula: true
# CloudXR env overrides applied before the native service starts.
# NV_DEVICE_PROFILE selects the client type:
# auto-webrtc: WebRTC / web XR clients (default)
# auto-native: native iOS / visionOS clients (set this to use the native apps)
# Device-specific values also accepted: apple-vision-pro | ipad-pro | quest3
cloudxr_env:
NV_DEVICE_PROFILE: auto-webrtc
# Physical GPU (nvidia-smi index) the CloudXR compositor pins to. Translated
# to PCI bus selectors for Vulkan, CUDA, and Mesa, set on this process's
# environment, and appended to cloudxr.env so the scene process picks them up
# when it sources the file (and LOVR inherits from the scene spawn env).
gpu_index: 0
# ── Network ports (CloudXR owns these — do not conflict with LiveKit) ─────────
# CloudXR native service: localhost:49100 (internal, not exposed)
# WSS proxy (TLS): 0.0.0.0:48322 (XR clients connect here)
yaml/device_io_hub.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# DeviceIOHub configuration for the xr-render-demo sample.
# Paths are relative to this file's directory.
# See services/device-io-hub/device_io_hub.yaml for all available options.
# ── LiveKit server ────────────────────────────────────────────────────────────
# Development-only placeholders. For deployed setups, replace both values or
# remove them and set LIVEKIT_API_KEY and LIVEKIT_API_SECRET in the environment.
api_key: devkey
api_secret: devsecret-xr-livekit-prototype-2026
room_name: xr-room
lk_port_ws: 7880
lk_port_tcp: 7881
lk_port_udp: 7882
# Cloud VMs behind NAT may need STUN-based public-IP discovery. Also skip
# validation when the NAT does not support LiveKit's public-IP self-ping.
# lk_use_external_ip: true
# lk_skip_external_ip_validation: true
# ── Web client server ─────────────────────────────────────────────────────────
# The page serves both stacks: LiveKit (agent audio/data over the WSS proxy on
# this port) AND CloudXR (one-tap Launch XR → WebXR over wss://<host>:48322).
# Because both live in the same JS context, entering XR does not drop the
# agent connection.
enable_web_server: true
web_server_port: 8080
web_client_dir: ../../../client-samples/web-xr
# ── Optional: token server ────────────────────────────────────────────────────
enable_token_server: true
# Change if another service holds port 8000 (e.g. iphlpsvc under WSL2 mirrored
# networking; see the note in services/device-io-hub/device_io_hub.yaml).
# token_server_port: 8000
# ── Video recording ──────────────────────────────────────────────────────────
# Disabled: xr-render-demo obtains current frames through LiveFrameSource.
# Enable this and set recordings_dir in video_memory_service.yaml for history.
video_recording:
enabled: false
yaml/media_capture.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Opt-in server-side capture for the XR render demo. Each participant gets
# a timestamped bundle with composited H.264/AAC-LC .mp4 video, retained raw
# media, and the exact inbound/outbound data timeline.
hub_sub_addr: ipc:///tmp/xr_hub_pub
hub_push_addr: ipc:///tmp/xr_hub_in
out_dir: ~/.local/share/xr-ai/captures/xr-render-demo
profile: demo
session_mode: participant
sample_fps: 30.0
bitrate: 6000000
gpu_id: 0
frame_queue_size: 2
encoder_workers: 2
audio_sample_rate: 48000
overlay_seconds: 12.0
overlay_lines: 4
max_total_bytes: 10737418240
yaml/models.json#
{
"models": {
"llm": {
"adapter": {"preset": "nemotron_omni"},
"endpoint": {"base_url": "http://localhost:8108"}
},
"agent_llm": {
"adapter": {"preset": "nemotron_omni"},
"endpoint": {"base_url": "http://localhost:8108"}
},
"stt": {
"adapter": {"preset": "parakeet_stt"},
"endpoint": {"base_url": "http://localhost:8103"}
},
"tts": {
"adapter": {"preset": "pocket_tts"},
"endpoint": {"base_url": "http://localhost:8105"}
},
"vlm": {
"adapter": {"preset": "cosmos3_nano_reasoner"},
"endpoint": {"base_url": "http://localhost:8100"}
}
}
}
yaml/openxr_service.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
endpoint: tcp://0.0.0.0:8330
cloudxr_env_file: ~/.cloudxr/run/cloudxr.env
# Test hook: the eval live drivers need a simulated head pose; flip to true
# while running them. The injected pose overrides real tracking for every
# client until cleared, so it must never ship enabled.
allow_sim_pose: false
yaml/video_memory_service.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
endpoint: tcp://0.0.0.0:8310
out_dir: /tmp/xr_video_queries/xr-render-demo
gpu_id: 0
yaml/voice_gate.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# xr-render-demo voice gate configuration.
#
# magic_phrases — wake-word gate. Empty list (the current default) DISABLES
# the gate: every STT transcript is dispatched to the agent
# (always-on, no wake word). To require a "hey agent" / "agent"
# wake word instead, set:
# magic_phrases:
# - "agent"
# - "hey agent"
# (matches the simple-vlm-example default). Strict-prefix match.
# listening_chime — when true AND magic_phrases is non-empty, a short two-tone
# chime plays on fresh magic-phrase matches. Inert while
# magic_phrases is empty (no match can occur).
# followup_grace_s — seconds after a phrase match during which the next
# utterance from the same participant bypasses the gate.
magic_phrases: [] # always-on; see note above to enable a wake word
listening_chime: true
followup_grace_s: 5.0
yaml/xr_render_demo_worker.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# xr-render-demo worker configuration.
# Paths resolve relative to this file's directory.
# ── Voice gate ────────────────────────────────────────────────────────────────
# Magic-phrase opt-in gate; see voice_gate.yaml for the schema. Ships with
# `magic_phrases: []` (always-on) to preserve the pre-migration behavior;
# operators populate the list to enable wake-word gating.
voice_gate_yaml: voice_gate.yaml
# ── Native capability services ───────────────────────────────────────────────
scene_endpoint: tcp://127.0.0.1:8320
openxr_endpoint: tcp://127.0.0.1:8330
video_memory_endpoint: tcp://127.0.0.1:8310
# Past-moment questions ("what was I holding a moment ago?") need recorded
# video: enable video_recording in device_io_hub.yaml and set recordings_dir
# in video_memory_service.yaml, then turn this on. Off, vision_agent answers
# from the live camera only.
video_history_enabled: false
text_memory_dir: /dev/shm/xr-ai/text-memory
# ── Voice activity detection (Silero VAD, ONNX backend) ──────────────────────
# silence_duration: seconds of trailing silence that ends an utterance.
# min_speech: minimum cumulative speech before STT fires (filters short blips).
# silero_threshold: speech-probability gate (0..1). Lower = more sensitive
# (catches quieter/faster speech). Default 0.5 often misses the onset of a
# word; 0.3 captures it earlier.
silence_duration: 0.8
min_speech: 0.15
silero_threshold: 0.3
# ── Idle timeout ──────────────────────────────────────────────────────────
# Auto-cancel the voice pipeline after this many seconds with no user/bot
# speech. DISABLED by default (0 = off) — a quiet session stays connected
# indefinitely, which is what XR sessions usually want. Set a positive value
# (e.g. 300 for 5 min) to opt in; the worker then tears the pipeline down
# after that long idle. VoiceAgent owns the timeout.
idle_timeout_secs: 0
model-servers#
yaml/96G_blackwell/embedding_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
model: nvidia/llama-nemotron-embed-1b-v2
host: "0.0.0.0"
port: 8109
served_model_name: embed
model_cache: ../../../../models
max_num_seqs: 32
max_model_len: 8192
gpu_memory_utilization: 0.08
enforce_eager: true
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
yaml/96G_blackwell/nemotron_omni_llm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Nemotron-3-Nano-Omni — single Blackwell 96 GB.
# NVFP4 is selected automatically. Leave moe_backend unset so vLLM can select
# a compatible kernel for the checkpoint and hardware. This standalone Omni
# stack leaves the remainder of the GPU for STT, rendering, and runtime overhead.
host: "0.0.0.0"
port: 8108
hf_token: ""
model_cache: ../../../../models
max_num_seqs: 8
tensor_parallel_size: 1
max_model_len: 32768
gpu_memory_utilization: 0.35
enforce_eager: false
video_pruning_rate: 0.5
video_fps: 2
video_num_frames: 64
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
yaml/96G_blackwell/nim_llm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted Nemotron-3-Nano-Omni NIM (serves both the `llm` and
# `agent_llm` roles). This matches the model family used by the default local
# deployment. Swap in any compatible NIM from https://catalog.ngc.nvidia.com.
# For sample configuration, refer to
# https://nvidia.github.io/xr-ai/latest/guides/customizing-model-servers.html
# Verify startup and memory use on the target 96 GB Blackwell GPU before
# deployment.
image: nvcr.io/nim/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:2.0.4-variant
http_port: 8110
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
NIM_MAX_MODEL_LEN: "32768"
# Match the local Omni deployment's reasoning and tool-call parsers.
NIM_PASSTHROUGH_ARGS: "--gpu-memory-utilization 0.4 --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3"
yaml/96G_blackwell/nim_stt_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Riva ASR (parakeet). Workers reach the gRPC port via the
# riva_grpc model kind; http_port serves /v1/health/ready only.
# UNTESTED ESTIMATE for this GPU profile; validated values are dual_48G_ada's.
image: nvcr.io/nim/nvidia/parakeet-tdt-0.6b-v2:1.2.0
http_port: 9010
grpc_port: 50051
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
# Single-process inference: the default multiprocessing mode runs two
# CUDA processes; one is plenty for demo traffic. ~14 GB VRAM either way.
NIM_USE_MULTIPROCESSING_FOR_INFERENCE: "false"
yaml/96G_blackwell/nim_tts_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Riva TTS (magpie). Workers reach the gRPC port via the
# riva_grpc model kind; http_port serves /v1/health/ready only.
# UNTESTED ESTIMATE for this GPU profile; validated values are dual_48G_ada's.
image: nvcr.io/nim/nvidia/magpie-tts-multilingual:1.9.0
http_port: 9011
grpc_port: 50052
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
yaml/96G_blackwell/nim_vlm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Cosmos3-Nano Reasoner VLM.
# Swap in any vision NIM from https://catalog.ngc.nvidia.com, but check its
# engine accepts a full image prompt: the nemotron-nano-vl NIM builds a
# 3200-token input cap on this class of GPU, below its own ~3330-token
# per-image cost, so every image request 500s.
# For sample configuration, refer to
# https://nvidia.github.io/xr-ai/latest/guides/customizing-model-servers.html
# UNTESTED ESTIMATE for this GPU profile; validated values are dual_48G_ada's.
image: nvcr.io/nim/nvidia/cosmos3-reasoner:1.7.0
http_port: 8100
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
# Nano is the smaller model served as nvidia/cosmos3-nano-reasoner.
NIM_MODEL_SIZE: "nano"
NIM_MAX_MODEL_LEN: "16384"
# Fraction of total GPU memory despite the PERCENT name (0..1]; leaves
# headroom for whatever shares the GPU.
NIM_KVCACHE_PERCENT: "0.2"
yaml/96G_blackwell/pocket_tts_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Shared GPU Pocket TTS service. bill_boerst is a CC0 Voice-Zero recording.
voice: bill_boerst
language: english
device: cuda
cuda_visible_devices: "0"
port: 8105
startup_timeout_s: 600
model_cache: ../../../../models
yaml/96G_blackwell/stt_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# STT server — single Blackwell 96 GB.
model: nvidia/parakeet-tdt-0.6b-v3
device: auto
port: 8103
host: "0.0.0.0"
startup_timeout_s: 600
model_cache: ../../../../models
yaml/96G_blackwell/vlm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# VLM server — single Blackwell 96 GB.
#
# vlm is THIRD in load order: STT → Omni → vlm.
# free_at_startup ≈ 96 − 34 (Omni) − 2 (STT) ≈ 60 GiB.
# 0.23 × 96 = 22.1 GiB cap: 16.8 GiB measured Reasoner weights plus
# multimodal encoder profiling, runtime overhead, and KV cache.
# hf_overrides selects the Reasoner runtime; details are documented in
# docs/source/components/ai-services.md#per-server-notes.
model: nvidia/Cosmos3-Nano
hf_overrides:
architectures:
- Cosmos3ForConditionalGeneration
hf_token: ""
port: 8100
host: "0.0.0.0"
model_cache: ../../../../models
gpu_memory_utilization: 0.23
max_num_seqs: 4
enforce_eager: true
async_scheduling: true
mm_encoder_tp_mode: data
max_videos_per_prompt: 0
# Raise max_model_len too if multi-image turns outgrow the context window.
max_images_per_prompt: 4
# vLLM runtime: pip (this wrapper's venv) or docker (NGC nvcr.io/nvidia/vllm).
# See services/vlm-server/vlm_server.yaml for the full toggle docs.
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
yaml/dual_48G_ada/embedding_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
model: nvidia/llama-nemotron-embed-1b-v2
host: "0.0.0.0"
port: 8109
served_model_name: embed
model_cache: ../../../../models
cuda_visible_devices: "0"
max_num_seqs: 32
max_model_len: 8192
gpu_memory_utilization: 0.08
enforce_eager: true
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
yaml/dual_48G_ada/nemotron_omni_llm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Nemotron-3-Nano-Omni — dual 48 GB Ada configuration.
# Cosmos remains on GPU 0; FP8 Omni replaces Nano on GPU 1 beside STT.
host: "0.0.0.0"
port: 8108
hf_token: ""
model_cache: ../../../../models
cuda_visible_devices: "1"
max_num_seqs: 8
tensor_parallel_size: 1
max_model_len: 32768
gpu_memory_utilization: 0.78
enforce_eager: false
video_pruning_rate: 0.5
video_fps: 2
video_num_frames: 64
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
yaml/dual_48G_ada/nim_llm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted Nemotron-3-Nano-Omni NIM (serves both the `llm` and
# `agent_llm` roles). This matches the model family used by the default local
# deployment. Swap in any compatible NIM from https://catalog.ngc.nvidia.com.
# For sample configuration, refer to
# https://nvidia.github.io/xr-ai/latest/guides/customizing-model-servers.html
# RTX Ada is not listed for this Omni NIM; verify startup and memory use on
# target hardware before deployment.
image: nvcr.io/nim/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:2.0.4-variant
http_port: 8110
nim_cache: ../../../../models/nim
cuda_visible_devices: "1"
env:
NIM_MAX_MODEL_LEN: "32768"
# Match the local Omni deployment's reasoning and tool-call parsers.
NIM_PASSTHROUGH_ARGS: "--gpu-memory-utilization 0.8 --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3"
yaml/dual_48G_ada/nim_stt_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Riva ASR (parakeet). Workers reach the gRPC port via the
# riva_grpc model kind; http_port serves /v1/health/ready only.
image: nvcr.io/nim/nvidia/parakeet-tdt-0.6b-v2:1.2.0
http_port: 9010
grpc_port: 50051
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
# Single-process inference: the default multiprocessing mode runs two
# CUDA processes; one is plenty for demo traffic. ~14 GB VRAM either way.
NIM_USE_MULTIPROCESSING_FOR_INFERENCE: "false"
yaml/dual_48G_ada/nim_tts_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Riva TTS (magpie). Workers reach the gRPC port via the
# riva_grpc model kind; http_port serves /v1/health/ready only.
image: nvcr.io/nim/nvidia/magpie-tts-multilingual:1.9.0
http_port: 9011
grpc_port: 50052
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
yaml/dual_48G_ada/nim_vlm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Cosmos3-Nano Reasoner VLM.
# Swap in any vision NIM from https://catalog.ngc.nvidia.com, but check its
# engine accepts a full image prompt: the nemotron-nano-vl NIM builds a
# 3200-token input cap on this class of GPU, below its own ~3330-token
# per-image cost, so every image request 500s.
# For sample configuration, refer to
# https://nvidia.github.io/xr-ai/latest/guides/customizing-model-servers.html
# GPU 0 with the embedding server and a demo's CloudXR + LOVR; the
# Nemotron NIM and local STT share GPU 1.
image: nvcr.io/nim/nvidia/cosmos3-reasoner:1.7.0
http_port: 8100
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
# Nano is the smaller model served as nvidia/cosmos3-nano-reasoner.
NIM_MODEL_SIZE: "nano"
NIM_MAX_MODEL_LEN: "16384"
# Fraction of total GPU memory despite the PERCENT name (0..1]; leaves
# headroom for whatever shares the GPU.
NIM_KVCACHE_PERCENT: "0.7"
yaml/dual_48G_ada/pocket_tts_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Shared GPU Pocket TTS service. bill_boerst is a CC0 Voice-Zero recording.
voice: bill_boerst
language: english
device: cuda
cuda_visible_devices: "0"
port: 8105
startup_timeout_s: 600
model_cache: ../../../../models
yaml/dual_48G_ada/stt_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# STT server configuration — nvidia/parakeet-tdt-0.6b-v3.
# Auto-discovered by xr-ai-launcher and passed as --config to stt_server.
# OpenAI-compatible API at http://localhost:8103/v1
# NeMo model name.
# nvidia/parakeet-tdt-0.6b-v3 — 0.6B TDT model; CC-BY-4.0; ~1.5 GB VRAM; English only
# nvidia/parakeet-rnnt-1.1b — stronger, larger; same CC-BY-4.0 license
model: nvidia/parakeet-tdt-0.6b-v3
# Inference device: "cuda", "cpu", or "auto" (CUDA if available, else CPU).
device: auto
# GPU 1 only — VLM, embedding, and Pocket TTS share GPU 0.
cuda_visible_devices: "1"
port: 8103
host: "0.0.0.0"
startup_timeout_s: 600
# NeMo + HuggingFace weight cache — resolved relative to this YAML file.
model_cache: ../../../../models
yaml/dual_48G_ada/vlm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# VLM server configuration — Cosmos3 Nano Reasoner.
# Auto-discovered by xr-ai-launcher and passed as --config to vlm_server.
# OpenAI-compatible API at http://localhost:8100/v1 (accepts image_url in messages)
# hf_overrides selects the Reasoner runtime; details are documented in
# docs/source/components/ai-services.md#per-server-notes.
# Cosmos-Reason1 compatibility uses nvidia/Cosmos-Reason1-7B with the
# cosmos_vlm client preset.
model: nvidia/Cosmos3-Nano
hf_overrides:
architectures:
- Cosmos3ForConditionalGeneration
# HuggingFace token — optional for this public model and required for gated
# alternatives.
# !! Do not commit this file with a real token !!
hf_token: ""
port: 8100
host: "0.0.0.0"
cuda_visible_devices: "0" # share GPU 0 with embeddings; Nemotron/STT use GPU 1
model_cache: ../../../../models
# 0.47 × 47 = 22.1 GiB budget: 16.8 GiB Reasoner weights plus multimodal
# encoder profiling, runtime overhead, and KV cache.
# Single-image queries have short context so the KV budget is adequate.
gpu_memory_utilization: 0.47
enforce_eager: true
async_scheduling: true
mm_encoder_tp_mode: data
max_videos_per_prompt: 0
# Raise max_model_len too if multi-image turns outgrow the context window.
max_images_per_prompt: 4
# vLLM runtime: pip (this wrapper's venv) or docker (NGC nvcr.io/nvidia/vllm).
# See services/vlm-server/vlm_server.yaml for the full toggle docs.
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
yaml/models.default.json#
{
"models": {
"stt": {
"adapter": {
"preset": "parakeet_stt"
},
"endpoint": {
"base_url": "http://localhost:8103",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "stt"
}
},
"tts": {
"adapter": {
"preset": "pocket_tts"
},
"endpoint": {
"base_url": "http://localhost:8105",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "tts"
}
},
"llm": {
"adapter": {
"preset": "nemotron_omni"
},
"endpoint": {
"base_url": "http://localhost:8108",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "omni"
}
},
"agent_llm": {
"adapter": {
"preset": "nemotron_omni"
},
"endpoint": {
"base_url": "http://localhost:8108",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "omni"
}
},
"vlm": {
"adapter": {
"preset": "cosmos3_nano_reasoner"
},
"endpoint": {
"base_url": "http://localhost:8100",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "vlm"
}
},
"embedding": {
"adapter": {
"preset": "nemotron_embedding"
},
"endpoint": {
"base_url": "http://localhost:8109",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "embedding"
}
}
}
}
yaml/models.vlm_llm_nim.json#
{
"models": {
"stt": {
"adapter": {
"preset": "parakeet_stt"
},
"endpoint": {
"base_url": "http://localhost:8103",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "stt"
}
},
"tts": {
"adapter": {
"preset": "pocket_tts"
},
"endpoint": {
"base_url": "http://localhost:8105",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "tts"
}
},
"llm": {
"adapter": {
"preset": "nemotron_omni",
"model_name": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning",
"reasoning_field": "reasoning"
},
"endpoint": {
"base_url": "http://localhost:8110",
"readiness": "health",
"health_path": "/v1/health/ready"
},
"deployment": {
"ownership": "managed",
"service": "llm-nim",
"credentials": [
"NGC_API_KEY"
]
}
},
"vlm": {
"category": "vlm",
"adapter": {
"kind": "openai_compat",
"model_name": "nvidia/cosmos3-nano-reasoner",
"capabilities": {
"streaming": true,
"vision": true,
"video": true
}
},
"endpoint": {
"base_url": "http://localhost:8100",
"readiness": "health",
"health_path": "/v1/health/ready"
},
"deployment": {
"ownership": "managed",
"service": "vlm-nim",
"credentials": [
"NGC_API_KEY"
]
}
},
"embedding": {
"adapter": {
"preset": "nemotron_embedding"
},
"endpoint": {
"base_url": "http://localhost:8109",
"readiness": "health"
},
"deployment": {
"ownership": "managed",
"service": "embedding"
}
}
}
}
yaml/spark/embedding_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
model: nvidia/llama-nemotron-embed-1b-v2
host: "0.0.0.0"
port: 8109
served_model_name: embed
model_cache: ../../../../models
max_num_seqs: 32
max_model_len: 8192
gpu_memory_utilization: 0.08
enforce_eager: true
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
# Prefetch and sync the snapshot before CUDA initialization, then retry once
# only if the driver rejects a CUDA allocation without an OOM-killed container.
spark_uma: true
yaml/spark/nemotron_omni_llm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Nemotron-3-Nano-Omni — DGX Spark (GB10).
# The NVFP4 model shares unified memory with the rest of the stack. vLLM's
# fractional profiler is unreliable while cold checkpoint reads change the
# filesystem page cache, so reserve 2 GiB explicitly for the 32K hybrid
# Mamba/attention cache. NGC 26.08 limits Nemotron Nano V3 on DGX Spark to
# four concurrent sequences.
# gpu_memory_utilization remains the vLLM startup admission threshold; it does
# not size the cache when the byte value is set.
host: "0.0.0.0"
port: 8108
hf_token: ""
model_cache: ../../../../models
max_num_seqs: 4
tensor_parallel_size: 1
max_model_len: 32768
gpu_memory_utilization: 0.25
kv_cache_memory_bytes: 2147483648
enforce_eager: false
video_pruning_rate: 0.5
video_fps: 2
video_num_frames: 64
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
# Prefetch and sync the snapshot before CUDA initialization, then retry once
# only if the driver rejects a CUDA allocation without an OOM-killed container.
spark_uma: true
yaml/spark/nim_llm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted Nemotron-3-Nano-Omni NIM (serves both the `llm` and
# `agent_llm` roles). This matches the model family used by the default local
# deployment. Swap in any compatible NIM from https://catalog.ngc.nvidia.com.
# For sample configuration, refer to
# https://nvidia.github.io/xr-ai/latest/guides/customizing-model-servers.html
# GB10 is not listed for this Omni NIM; verify startup and memory use on target
# hardware before deployment.
image: nvcr.io/nim/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:2.0.4-variant
http_port: 8110
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
NIM_MAX_MODEL_LEN: "32768"
# Match the local Omni deployment's reasoning and tool-call parsers.
NIM_PASSTHROUGH_ARGS: "--gpu-memory-utilization 0.35 --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3"
yaml/spark/nim_stt_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Riva ASR (parakeet). Workers reach the gRPC port via the
# riva_grpc model kind; http_port serves /v1/health/ready only.
# UNTESTED ESTIMATE for this GPU profile; validated values are dual_48G_ada's.
image: nvcr.io/nim/nvidia/parakeet-tdt-0.6b-v2:1.2.0
http_port: 9010
grpc_port: 50051
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
# Single-process inference: the default multiprocessing mode runs two
# CUDA processes; one is plenty for demo traffic. ~14 GB VRAM either way.
NIM_USE_MULTIPROCESSING_FOR_INFERENCE: "false"
yaml/spark/nim_tts_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Riva TTS (magpie). Workers reach the gRPC port via the
# riva_grpc model kind; http_port serves /v1/health/ready only.
# UNTESTED ESTIMATE for this GPU profile; validated values are dual_48G_ada's.
image: nvcr.io/nim/nvidia/magpie-tts-multilingual:1.9.0
http_port: 9011
grpc_port: 50052
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
yaml/spark/nim_vlm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Self-hosted NIM: Cosmos3-Nano Reasoner VLM.
# Swap in any vision NIM from https://catalog.ngc.nvidia.com, but check its
# engine accepts a full image prompt: the nemotron-nano-vl NIM builds a
# 3200-token input cap on this class of GPU, below its own ~3330-token
# per-image cost, so every image request 500s.
# For sample configuration, refer to
# https://nvidia.github.io/xr-ai/latest/guides/customizing-model-servers.html
# UNTESTED ESTIMATE for this GPU profile; validated values are dual_48G_ada's.
image: nvcr.io/nim/nvidia/cosmos3-reasoner:1.7.0
http_port: 8100
nim_cache: ../../../../models/nim
cuda_visible_devices: "0"
env:
# Nano is the smaller model served as nvidia/cosmos3-nano-reasoner.
NIM_MODEL_SIZE: "nano"
NIM_MAX_MODEL_LEN: "16384"
# Fraction of total GPU memory despite the PERCENT name (0..1]; leaves
# headroom for whatever shares the GPU.
NIM_KVCACHE_PERCENT: "0.15"
yaml/spark/pocket_tts_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Shared GPU Pocket TTS service. bill_boerst is a CC0 Voice-Zero recording.
voice: bill_boerst
language: english
device: cuda
cuda_visible_devices: "0"
port: 8105
startup_timeout_s: 600
model_cache: ../../../../models
yaml/spark/stt_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# STT server — DGX Spark (1× Blackwell, 128 GB).
model: nvidia/parakeet-tdt-0.6b-v3
device: auto
port: 8103
host: "0.0.0.0"
startup_timeout_s: 600
model_cache: ../../../../models
yaml/spark/vlm_server.yaml#
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# VLM server — DGX Spark (GB10, ~120 GiB GPU-visible unified memory).
#
# vlm is THIRD in load order: STT → Omni → vlm.
# free_at_startup ≈ 120 − 30 (Omni) − 2 (STT) ≈ 88 GiB.
# Cosmos3 requires about 1.12 GiB of KV cache for one 8,192-token request.
# Pin 1.5 GiB because vLLM's fractional memory profiler can misattribute
# filesystem page-cache changes as model overhead on unified-memory systems.
# Keep four sequences for concurrent shorter requests. The fixed cache is
# shared, so concurrent maximum-length requests can queue, preempt, or
# recompute when their aggregate token demand exceeds its capacity.
# The 0.20 utilization remains vLLM's startup admission threshold; the fixed
# byte value controls the KV-cache allocation.
# hf_overrides selects the Reasoner runtime; details are documented in
# docs/source/components/ai-services.md#per-server-notes.
model: nvidia/Cosmos3-Nano
hf_overrides:
architectures:
- Cosmos3ForConditionalGeneration
hf_token: ""
port: 8100
host: "0.0.0.0"
model_cache: ../../../../models
gpu_memory_utilization: 0.20
kv_cache_memory_bytes: 1610612736
max_num_seqs: 4
enforce_eager: true
async_scheduling: true
mm_encoder_tp_mode: data
max_videos_per_prompt: 0
# Raise max_model_len too if multi-image turns outgrow the context window.
max_images_per_prompt: 4
# vLLM runtime: pip (this wrapper's venv) or docker (NGC nvcr.io/nvidia/vllm).
# See services/vlm-server/vlm_server.yaml for the full toggle docs.
vllm_backend: docker
vllm_image: nvcr.io/nvidia/vllm:26.08-py3
# Prefetch and sync the snapshot before CUDA initialization, then retry once
# only if the driver rejects a CUDA allocation without an OOM-killed container.
spark_uma: true