Scale Triton from zero replicas on RepositoryIndex

.github/workflows/ci-full.yaml

@@ -63,7 +63,7 @@ jobs:
 
       - name: Envoy proxy ready
         run: |
-          kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 120s -n cms
+          kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 180s -n cms
 
       - name: Prometheus ready
         run: |
@@ -87,10 +87,22 @@ jobs:
           kubectl wait --for condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 180s -n cms
           kubectl wait --for condition=Ready so -l app.kubernetes.io/component=keda --timeout 180s -n cms
 
-      - name: Triton server ready
+      - name: Triton idle at zero replicas
         run: |
-          kubectl describe pod -l app.kubernetes.io/component=triton -n cms
-          kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=triton --timeout 500s -n cms
+          echo "Waiting for Triton Deployment spec.replicas=0..."
+          for i in $(seq 1 36); do
+            replicas=$(kubectl get deploy -l app.kubernetes.io/component=triton -n cms -o jsonpath='{.items[0].spec.replicas}')
+            echo "Triton spec.replicas=${replicas:-unset}"
+            if [ "${replicas}" = "0" ]; then
+              kubectl get deploy,pod -l app.kubernetes.io/component=triton -n cms
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "Triton did not scale to 0 replicas"
+          kubectl describe deploy -l app.kubernetes.io/component=triton -n cms
+          kubectl get so,hpa -n cms -o yaml
+          exit 1
 
       - name: Validate Deployment
         run: |
@@ -99,7 +111,7 @@ jobs:
       - name: Run Perf Analyzer Job
         run: |
           kubectl apply -f tests/perf-analyzer-job-ci.yaml
-          kubectl wait --for=condition=complete job/perf-analyzer-job -n cms --timeout=300s || \
+          kubectl wait --for=condition=complete job/perf-analyzer-job -n cms --timeout=800s || \
           (echo "Perf-analyzer job did not complete in time or failed." && exit 1)
 
           POD_NAME=$(kubectl get pods -n cms -l job-name=perf-analyzer-job -o jsonpath="{.items[0].metadata.name}")

.github/workflows/ci-local.sh

@@ -45,15 +45,15 @@ helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add opentelemetry https://open-telemetry.github.io/opentelemetry-helm-charts
 helm repo update
 helm dependency build ./helm/supersonic
-helm upgrade --install supersonic ./helm/supersonic --values values/values-cms-ci.yaml -n cms
+helm upgrade --install supersonic ./helm/supersonic --values values/values-minimal-full.yaml -n cms
 
 # 8. Wait for components to become ready
 
 echo "Waiting for CVMFS pods to be ready..."
 kubectl wait --for=condition=Ready pod --all -n cvmfs-csi --timeout 120s
 
 echo "Waiting for Envoy proxy pods to be ready..."
-kubectl wait --for=condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 120s -n cms
+kubectl wait --for=condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 180s -n cms
 
 echo "Waiting for Prometheus pods to be ready..."
 kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=prometheus --timeout 120s -n cms
@@ -69,21 +69,34 @@ echo "Waiting for Tempo pods to be ready..."
 kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=tempo --timeout 300s -n cms
 
 
-echo "Waiting for Triton server pods to be ready..."
-kubectl wait --for=condition=Ready pod -l app.kubernetes.io/component=triton --timeout 300s -n cms
-
 echo "Waiting for KEDA Autoscaler to be ready..."
 kubectl wait --for=condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 120s -n cms
 kubectl wait --for=condition=Ready so -l app.kubernetes.io/component=keda --timeout 120s -n cms
 
+echo "Waiting for Triton Deployment spec.replicas=0..."
+for i in $(seq 1 36); do
+  replicas=$(kubectl get deploy -l app.kubernetes.io/component=triton -n cms -o jsonpath='{.items[0].spec.replicas}')
+  echo "Triton spec.replicas=${replicas:-unset}"
+  if [ "${replicas}" = "0" ]; then
+    break
+  fi
+  if [ "$i" -eq 36 ]; then
+    echo "Triton did not scale to 0 replicas"
+    kubectl describe deploy -l app.kubernetes.io/component=triton -n cms
+    kubectl get so,hpa -n cms -o yaml
+    exit 1
+  fi
+  sleep 5
+done
+
 # 9. Validate the Deployment
 echo "Validating Deployment in 'cms' namespace..."
 kubectl get all -n cms
 
 # 10. Run Perf Analyzer Job
 echo "Running Perf Analyzer Job..."
 kubectl apply -f tests/perf-analyzer-job-ci.yaml
-kubectl wait --for=condition=complete job/perf-analyzer-job -n cms --timeout=180s || {
+kubectl wait --for=condition=complete job/perf-analyzer-job -n cms --timeout=800s || {
   echo "Perf-analyzer job did not complete in time or failed."
   exit 1
 }

README.md

@@ -19,7 +19,7 @@ applications in large high energy physics (HEP) and multi-messenger astrophysics
 Currently, SuperSONIC supports the following functionality:
 - GPU inference-as-a-service via [Nvidia Triton Inference Server](https://developer.nvidia.com/triton-inference-server)
 - Load balancing across many GPUs via [Envoy Proxy](envoyproxy.io)
-- Load-based autoscaling via [KEDA](keda.sh)
+- Load-based autoscaling via [KEDA](keda.sh), including scale from zero replicas on `RepositoryIndex`
 - Monitoring via [Prometheus](https://prometheus.io), [Grafana](https://grafana.com), and [OpenTelemetry](https://opentelemetry.io/docs/collector/)
 - Rate limiting
 - Token-based authentication

docs/.values-table.md

@@ -5,7 +5,12 @@
 | nameOverride | string | `""` | Unique identifier of SuperSONIC instance (equal to release name by default) |
 | serverLoadMetric | string | `""` | A metric used by both KEDA autoscaler and Envoy's prometheus-based rate limiter. # Default metric (inference queue latency) is defined in templates/_helpers.tpl |
 | serverLoadThreshold | int | `100` | Threshold for the metric |
-| triton.replicas | int | `1` | Number of Triton server instances (if autoscaling is disabled) |
+| scaleFromZero | object | `{"admissionImage":"python:3.11-slim","enabled":false,"holdMinReplicasSeconds":300,"readyTimeoutSeconds":300}` | On RepositoryIndex, scale Triton to at least one replica and return the index only after Envoy has a healthy Triton upstream. Requires keda.enabled, keda.minReplicaCount: 0, and envoy.enabled. |
+| scaleFromZero.enabled | bool | `false` | Enable scale from zero |
+| scaleFromZero.readyTimeoutSeconds | int | `300` | Seconds to wait for a healthy Triton upstream before rejecting RepositoryIndex |
+| scaleFromZero.holdMinReplicasSeconds | int | `300` | Seconds to keep KEDA minReplicaCount at 1 after the last RepositoryIndex |
+| scaleFromZero.admissionImage | string | `"python:3.11-slim"` | Image for the admission sidecar on the Envoy pod |
+| triton.replicas | int | `1` | Number of Triton server instances. Unused when scaleFromZero.enabled is true. |
 | triton.image | string | `"nvcr.io/nvidia/tritonserver:26.04-py3-min"` | Docker image for the Triton server |
 | triton.command | list | `["/bin/sh","-c"]` | Command and arguments to run in Triton container |
 | triton.args[0] | string | `"/opt/tritonserver/bin/tritonserver \\\n--model-repository=/tmp/ \\\n--log-verbose=0 \\\n--exit-timeout-secs=60\n"` |  |
@@ -19,9 +24,9 @@
 | triton.service.labels | object | `{}` |  |
 | triton.service.annotations | object | `{}` |  |
 | triton.service.ports | list | `[{"name":"http","port":8000,"protocol":"TCP","targetPort":8000},{"name":"grpc","port":8001,"protocol":"TCP","targetPort":8001},{"name":"metrics","port":8002,"protocol":"TCP","targetPort":8002}]` | Ports for communication with Triton servers |
-| triton.readinessProbe | object | `{"command":["/bin/sh","-c","curl -sf http://localhost:8000/v2/health/ready > /dev/null && [ ! -f /tmp/shutdown ]"],"failureThreshold":10,"initialDelaySeconds":10,"periodSeconds":10,"reset":false,"successThreshold":1,"timeoutSeconds":5}` | Custom readiness probe configuration |
+| triton.readinessProbe | object | `{"command":["/bin/sh","-c","curl -sf http://localhost:8000/v2/health/ready > /dev/null && [ ! -f /tmp/shutdown ]"],"failureThreshold":10,"initialDelaySeconds":10,"periodSeconds":10,"reset":false,"successThreshold":1,"timeoutSeconds":15}` | Custom readiness probe configuration |
 | triton.readinessProbe.reset | bool | `false` | If true, will reset settings to k8s defaults (other readinessProbe settings will be ignored) |
-| triton.startupProbe | object | `{"failureThreshold":12,"httpGet":{"path":"/v2/health/ready","port":"http"},"initialDelaySeconds":0,"periodSeconds":10,"reset":false}` | Custom startup probe configuration |
+| triton.startupProbe | object | `{"failureThreshold":24,"httpGet":{"path":"/v2/health/ready","port":"http"},"initialDelaySeconds":0,"periodSeconds":10,"reset":false,"timeoutSeconds":15}` | Custom startup probe configuration |
 | triton.startupProbe.reset | bool | `false` | If true, will reset settings to k8s defaults (other startupProbe settings will be ignored) |
 | envoy.enabled | bool | `true` | Enable Envoy Proxy |
 | envoy.replicas | int | `1` | Number of Envoy Proxy pods in Deployment |
@@ -44,8 +49,9 @@
 | envoy.rate_limiter.listener_level.max_tokens | int | `5` | Maximum number of simultaneous connections to the Envoy Proxy. Each new connection takes a "token" from the "bucket" which initially contains ``max_tokens`` tokens. |
 | envoy.rate_limiter.listener_level.tokens_per_fill | int | `1` | ``tokens_per_fill`` tokens are added to the "bucket" every ``fill_interval``, allowing new connections to be established. |
 | envoy.rate_limiter.listener_level.fill_interval | string | `"12s"` | For example, adding a new token every 12 seconds allows 5 new connections every minute. |
-| envoy.rate_limiter.prometheus_based | object | `{"enabled":false,"luaConfig":"cfg/envoy-filter.lua"}` | This rate limiter rejects new connections based on metric extracted from Prometheus (e.g. inference queue latency). The metric is taken from parameter ``prometheus.serverLoadMetric``, and the threshold is set by ``prometheus.serverLoadThreshold``. These parameters are the same as those used by the KEDA autoscaler. |
+| envoy.rate_limiter.prometheus_based | object | `{"enabled":false,"luaConfig":"cfg/envoy-filter.lua"}` | This rate limiter rejects new connections based on metric extracted from Prometheus (e.g. inference queue latency). The metric is taken from parameter ``serverLoadMetric``, and the threshold is set by ``serverLoadThreshold``. These parameters are the same as those used by the KEDA autoscaler. |
 | envoy.rate_limiter.prometheus_based.enabled | bool | `false` | Enable rate limiter |
+| envoy.rate_limiter.prometheus_based.luaConfig | string | `"cfg/envoy-filter.lua"` | Envoy Lua filter (prometheus rate limiter and scaleFromZero) |
 | envoy.loadBalancerPolicy | string | `"LEAST_REQUEST"` | Envoy load balancer policy. Options: ROUND_ROBIN, LEAST_REQUEST, RING_HASH, RANDOM, MAGLEV |
 | envoy.auth.enabled | bool | `false` | Enable authentication in Envoy proxy |
 | envoy.auth.jwt_issuer | string | `""` |  |
@@ -54,10 +60,12 @@
 | envoy.auth.url | string | `""` |  |
 | envoy.auth.port | int | `443` |  |
 | envoy.tracing_sampling_rate | float | `0.01` |  |
-| keda.enabled | bool | `false` | Enable autoscaling (requires Prometheus to also be enabled). Autoscaling will be based on the metric is taken from parameter ``prometheus.serverLoadMetric``, new Triton servers will spawn if the metric exceedds the threshold set by ``prometheus.serverLoadThreshold``. |
-| keda.minReplicaCount | int | `1` | Minimum and maximum number of Triton servers. Warning: if min=0 and desired Prometheus metric is empty, the first server will never start |
+| keda.enabled | bool | `false` | Enable autoscaling (requires Prometheus to also be enabled). Autoscaling will be based on the metric from parameter ``serverLoadMetric``; new Triton servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. |
+| keda.minReplicaCount | int | `1` | Minimum and maximum number of Triton servers. Set minReplicaCount to 0 when scaleFromZero.enabled is true. With scaleFromZero, upgrades keep the live ScaledObject minReplicaCount. |
 | keda.maxReplicaCount | int | `2` |  |
-| keda.zeroIdleReplicas | bool | `false` | If set to true, the server will release all GPUs when idle. Be careful: if the scaling metric is extracted from Triton servers, it will be unavailable, and scaling from 0 to 1 will never happen. |
+| keda.pollingInterval | int | `30` | How often KEDA polls Prometheus |
+| keda.cooldownPeriod | int | `120` | Period to wait after the last trigger is inactive before scaling to minReplicaCount |
+| keda.zeroIdleReplicas | bool | `false` | If set to true, KEDA sets idleReplicaCount to 0. This cannot scale from 0 back to 1 when the load metric comes from Triton. Use scaleFromZero.enabled with keda.minReplicaCount: 0 instead. |
 | keda.scaleUp.stabilizationWindowSeconds | int | `60` |  |
 | keda.scaleUp.periodSeconds | int | `60` |  |
 | keda.scaleUp.stepsize | int | `1` |  |
@@ -84,7 +92,7 @@
 | prometheus.server.service.servicePort | int | `9090` |  |
 | prometheus.server.configMapOverrideName | string | `"prometheus-config"` |  |
 | prometheus.server.ingress | object | `{"annotations":{},"enabled":false,"hostName":"","hosts":[],"ingressClassName":"","tls":[{"hosts":[]}]}` | Ingress configuration for Prometheus |
-| prometheus.server.ingress.hostName | string | `""` | If this parameter is set in values.yaml, the `hosts` and `tls` parameters can be ommitted, as they will be set by the installer plugin |
+| prometheus.server.ingress.hostName | string | `""` | If this parameter is set in values.yaml, the `hosts` and `tls` parameters can be omitted, as they will be set by the installer plugin |
 | prometheus.serviceAccounts.server.create | bool | `false` |  |
 | prometheus.serviceAccounts.server.name | string | `"supersonic-prometheus-sa"` |  |
 | prometheus.rbac.create | bool | `false` |  |
@@ -120,7 +128,7 @@
 | grafana.resources.requests.cpu | string | `"100m"` |  |
 | grafana.resources.requests.memory | string | `"128Mi"` |  |
 | grafana.ingress | object | `{"annotations":{},"enabled":false,"hostName":"","hosts":[],"ingressClassName":"","path":"/","pathType":"ImplementationSpecific","tls":[{"hosts":[]}]}` | Ingress configuration for Grafana |
-| grafana.ingress.hostName | string | `""` | If this parameter is set in values.yaml, the `hosts` and `tls` parameters can be ommitted, as they will be set by the installer plugin |
+| grafana.ingress.hostName | string | `""` | If this parameter is set in values.yaml, the `hosts` and `tls` parameters can be omitted, as they will be set by the installer plugin |
 | tempo.enabled | bool | `false` |  |
 | tempo.tempo.metricsGenerator.enabled | bool | `true` |  |
 | tempo.tempo.metricsGenerator.remoteWriteUrl | string | `"http://supersonic-prometheus-server:9090/api/v1/write"` |  |

docs/configuration-guide.rst

@@ -71,7 +71,7 @@ Triton version must be specified in the ``triton.image`` parameter in the values
 
      ## -- OR --
      ## Option 3: mount CVMFS via hostPath (CVMFS must be already mounted on the nodes)
-     storageType: "cvmfs"
+     storageType: "cvmfs-hostPath"
 
      ## -- OR --
      ## Option 4: mount an NFS storage volume
@@ -163,6 +163,11 @@ default configuration completely (the configuration file must be supplied as a C
        configmap_name: external-envoy-config
        configmap_key: envoy.yaml
 
+.. warning::
+
+   ``scaleFromZero.enabled`` cannot be used with ``envoy.external_config.load_from_configmap``.
+   Scale-from-zero injects Envoy clusters and a Lua filter that an external ConfigMap would replace.
+
 5. (Optional) Configure Rate Limiting in Envoy Proxy
 ======================================================
 
@@ -343,7 +348,9 @@ can be enabled via the ``keda.enabled`` parameter in the values file.
    Please contact cluster administrators if this step of installation fails.
 
 The parameters ``keda.minReplicaCount`` and ``keda.maxReplicaCount`` define the range in which
-the number of Triton servers can scale.
+the number of Triton servers can scale. ``keda.pollingInterval`` is how often KEDA queries
+Prometheus, and ``keda.cooldownPeriod`` is how long the load metric must stay below the
+threshold before KEDA scales down to ``minReplicaCount``.
 
 Additional optional parameters can control how quickly the autoscaler reacts to changes in the Prometheus metric:
 
@@ -355,6 +362,9 @@ Additional optional parameters can control how quickly the autoscaler reacts to
      minReplicaCount: 1
      maxReplicaCount: 10
 
+     pollingInterval: 30
+     cooldownPeriod: 120
+
      scaleUp:
        stabilizationWindowSeconds: 120
        periodSeconds: 30
@@ -364,6 +374,46 @@ Additional optional parameters can control how quickly the autoscaler reacts to
        periodSeconds: 30
        stepsize: 1
 
+To keep **zero** Triton replicas when idle, set ``keda.minReplicaCount`` to ``0`` and enable
+``scaleFromZero``. Envoy stays running. On a ``RepositoryIndex`` request (the first RPC
+used by CMS SONIC clients), SuperSONIC scales Triton to one replica and returns the index
+only after Envoy has a healthy Triton upstream. KEDA then scales from 1 to
+``maxReplicaCount`` using the Prometheus load metric. After
+``scaleFromZero.holdMinReplicasSeconds`` with no further ``RepositoryIndex`` requests,
+the ScaledObject minimum returns to ``keda.minReplicaCount``, and KEDA can scale back to zero.
+
+.. code-block:: yaml
+
+   envoy:
+     enabled: true
+
+   keda:
+     enabled: true
+     minReplicaCount: 0
+     maxReplicaCount: 10
+
+   scaleFromZero:
+     enabled: true
+     readyTimeoutSeconds: 300
+     holdMinReplicasSeconds: 300
+
+.. warning::
+
+   The client deadline for ``RepositoryIndex`` must cover Triton startup. If no healthy
+   upstream is available within ``scaleFromZero.readyTimeoutSeconds``, the index request
+   is rejected.
+
+``triton.replicas`` is unused when ``scaleFromZero`` is enabled; KEDA owns the replica
+count. Helm upgrades keep the live ScaledObject ``minReplicaCount`` so they do not
+interrupt the hold at 1 replica. ``scaleFromZero`` requires ``keda.enabled`` and
+``envoy.enabled``, and cannot be used with an external Envoy ConfigMap.
+
+Do not set ``keda.zeroIdleReplicas: true`` together with ``minReplicaCount: 0``.
+``zeroIdleReplicas`` sets KEDA ``idleReplicaCount`` to 0 and cannot scale from 0 back to 1
+when the load metric is scraped from Triton. Use ``scaleFromZero`` for that.
+
+An example is ``values/values-geddes-cms.yaml``.
+
 11. (Optional) Configure Metrics Collector for Running ``perf_analyzer``
 =========================================================================
 

docs/getting-started.rst

@@ -9,6 +9,8 @@ Pre-requisites
    2. `Helm <https://helm.sh>`_
    3. Access to an existing `Prometheus <https://prometheus.io>`_ instance in the cluster, or sufficient permissions to deploy a custom instance (preferred).
    4. If using autoscaling, you may need to ask cluster administrators to install CustomResourceDefinitions for `KEDA <https://keda.sh>`_.
+      To keep zero Triton replicas when idle, enable ``scaleFromZero`` as described in the
+      `configuration guide <configuration-guide>`_.
 
 Installation
 ~~~~~~~~~~~~~~

helm/supersonic/README.md

@@ -19,7 +19,7 @@ applications in large high energy physics (HEP) and multi-messenger astrophysics
 Currently, SuperSONIC supports the following functionality:
 - GPU inference-as-a-service via [Nvidia Triton Inference Server](https://developer.nvidia.com/triton-inference-server)
 - Load balancing across many GPUs via [Envoy Proxy](envoyproxy.io)
-- Load-based autoscaling via [KEDA](keda.sh)
+- Load-based autoscaling via [KEDA](keda.sh), including scale from zero replicas on `RepositoryIndex`
 - Monitoring via [Prometheus](https://prometheus.io), [Grafana](https://grafana.com), and [OpenTelemetry](https://opentelemetry.io/docs/collector/)
 - Rate limiting
 - Token-based authentication