# Copyright 2026 NVIDIA CORPORATION
# SPDX-License-Identifier: Apache-2.0
#
# NVSentinel values for the mock-GPU thermal-margin demo on Kind.
#
# Enables the DCGM-based GPU Health Monitor (GpuThermalMarginWatch) plus the
# remediation pipeline (MongoDB store -> Fault Quarantine -> Node Drainer) so a
# GPU thermal-margin violation is detected AND remediated (worker cordoned +
# drained), then auto-recovered on cooldown. Syslog/CSP monitors are off
# (detection is via DCGM field 153, not kernel logs).
#
# Prometheus Operator and cluster NetworkPolicies are not installed in this
# demo, so podMonitor and the metrics-access NetworkPolicy are disabled.

global:
  dryRun: false

  # Core (system) modules are pinned to the control-plane so the Node Drainer
  # does not evict the pipeline that is doing the draining when it cordons and
  # drains the GPU worker. The GPU Health Monitor DaemonSet still runs on the
  # GPU worker via its own tolerations.
  systemNodeSelector:
    node-role.kubernetes.io/control-plane: ""
  systemNodeTolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule

  dcgm:
    mode: operator-service
    enabled: true
    service:
      endpoint: "nvidia-dcgm.gpu-operator.svc"
      port: 5555

  # Datastore: the chart's own MongoDB, deployed via the Percona operator (see
  # the mongodb-store overrides at the bottom of this file). The operator names
  # the replica-set service <cluster>-rs0, hence mongodb-rs0. Percona serves
  # requireTLS and the chart issues the client certificate through cert-manager,
  # so the connection carries tls=true.
  datastore:
    provider: "mongodb"
    connection:
      host: "mongodb-rs0.nvsentinel.svc.cluster.local"
      port: 27017
      database: "HealthEventsDatabase"
      collection: "HealthEvents"
      tokenCollection: "ResumeTokens"
      extraParams:
        replicaSet: "rs0"
        tls: "true"
    options:
      maxConnections: "25"
      maxIdleConnections: "10"
      connectionMaxLifetime: "1h"
      pollInterval: "5s"


  # Health monitors
  gpuHealthMonitor:
    enabled: true
  syslogHealthMonitor:
    enabled: false
  cspHealthMonitor:
    enabled: false
  kubernetesObjectMonitor:
    enabled: false
  nicHealthMonitor:
    enabled: false

  # Remediation pipeline, backed by the chart's own MongoDB store.
  mongodbStore:
    enabled: true
  faultQuarantine:
    enabled: true
  nodeDrainer:
    enabled: true
  faultRemediation:
    enabled: false
  janitor:
    enabled: false
  janitorProvider:
    enabled: false
  healthEventsAnalyzer:
    enabled: false

  # Housekeeping modules kept minimal
  labeler:
    enabled: true
  # The metadata-collector DaemonSet reads each GPU's signed slowdown T.Limit
  # offset (NVML field 194, NVML_FI_DEV_TEMPERATURE_SLOWDOWN_TLIMIT) once and
  # writes it to global.metadataPath (gpu_metadata.json). The GPU Health
  # Monitor needs that offset to arm GpuThermalMarginWatch; without it the
  # thermal-margin check stays inactive ("missing slowdown TLIMIT threshold
  # metadata"). The nvml-mock exposes field 194 so this works on the mock GPUs.
  metadataCollector:
    enabled: true
  eventExporter:
    enabled: false
  preflight:
    enabled: false

# No Prometheus Operator in this demo.
podMonitor:
  enabled: false

# The metrics-access NetworkPolicy can block cross-namespace traffic on Kind's
# NetworkPolicy-enforcing CNI; disable it for the demo.
networkPolicy:
  enabled: false

# --- Subchart overrides -------------------------------------------------------

# labeler: this demo runs the GPU Operator with the real driver disabled (the
# nvml-mock supplies libnvidia-ml.so instead), so there is no
# nvidia-driver-daemonset pod for the labeler to detect and it never sets
# nvsentinel.dgxc.nvidia.com/driver.installed=true. assumeDriverInstalled makes
# the labeler set that label on every node GFD marks nvidia.com/gpu.present, which the
# metadata-collector DaemonSet requires in its node selector. (This is the same
# knob NVIDIA documents for hosts with pre-baked drivers.)
labeler:
  assumeDriverInstalled: true

# gpu-health-monitor: suppress the mock's baseline NVLink effective-BER
# "failure". The mock reports DCGM_FR_NVLINK_EFFECTIVE_BER_THRESHOLD on every GPU
# at boot; that is a mock-data artifact (recommendedAction CONTACT_SUPPORT, not
# the fault under test). If left active it stays a "failing check" forever and
# keeps the node quarantined even after the GPU cools back down, so the
# recovery/uncordon path never completes. Suppressing it lets all checks clear
# once the thermal margin recovers.
gpu-health-monitor:
  dcgmHealthCheck:
    suppressedErrorCodes:
      - DCGM_FR_NVLINK_EFFECTIVE_BER_THRESHOLD
  # GpuThermalMarginWatch (DCGM field 153 vs the per-GPU slowdown T.Limit offset
  # from gpu_metadata.json) is the fault this demo exercises. By default the
  # watch is enabled but runs dry (gpuTempLimitStoreOnly=true), so its events
  # are emitted STORE_ONLY and never cordon. Turn store-only OFF so a closing
  # thermal margin drives the full detect -> cordon -> drain pipeline; when the
  # GPU cools and the margin re-opens the watch clears and the node uncordons.
  dcgmFieldsMonitoring:
    gpuTempLimitMonitoringEnabled: true
    gpuTempLimitStoreOnly: false

# node-drainer: evict user-namespace pods immediately on drain. The default mode
# is "AllowCompletion", which waits for each pod to finish gracefully (respecting
# terminationGracePeriodSeconds). The demo's sample GPU workload is a long-running
# pod that never completes on its own, so under the default it would never be
# evicted and you would not see it reschedule onto the healthy worker. "Immediate"
# makes the drain -> reschedule step observable. Real clusters typically keep
# "AllowCompletion" for graceful eviction.
node-drainer:
  userNamespaces:
    - name: "*"
      mode: "Immediate"

# fault-quarantine: disable the circuit breaker for this tiny demo cluster. The
# breaker trips when >= 50% of GPU nodes are cordoned within a 5m window; with
# only two GPU workers a single legitimate cordon already meets that threshold
# and halts ALL further processing (including the uncordon on recovery), leaving
# the node stuck. Disabling it keeps the detect -> cordon -> cooldown -> uncordon
# loop observable. Leave it enabled (the default) on real, larger clusters.
fault-quarantine:
  circuitBreaker:
    enabled: false

# mongodb-store: deploy the datastore with the Percona Server for MongoDB
# operator instead of the chart's default Bitnami MongoDB. The Bitnami path is
# amd64-only (bitnamilegacy/* images) and its containers run Bitnami-specific
# startup scripts, so it cannot come up on arm64; every image on the Percona
# path (percona-server-mongodb, the operator, mongosh, mongodb_exporter) is
# published for linux/amd64 and linux/arm64. Both flags must be set explicitly:
# they gate which chart dependencies are pulled in.
mongodb-store:
  useBitnami: false
  usePerconaOperator: true

  psmdb-db:
    # Percona refuses a replica set smaller than three members unless the size
    # check is explicitly waived. NVSentinel only needs change streams, which a
    # single-member replica set provides, and three members (3 x 1 CPU / 1.5Gi)
    # do not fit a laptop-sized Kind cluster.
    unsafeFlags:
      replsetSize: true
    replsets:
      rs0:
        size: 1
        resources:
          requests:
            cpu: "250m"
            memory: "512Mi"
          limits:
            cpu: "1"
            memory: "2Gi"
        volumeSpec:
          pvc:
            resources:
              requests:
                storage: 2Gi
        # No Prometheus Operator in this demo, so the metrics sidecar is dead weight.
        sidecars: []