diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8318f0..7ec8559 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: - name: Block-root package delivery run: tests/block_root_package_delivery.sh + - name: Sandbox-runner liveness checks + run: tests/sandbox_runner_healthcheck.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh index fdb03de..c639848 100755 --- a/docker/sandbox-runner-healthcheck.sh +++ b/docker/sandbox-runner-healthcheck.sh @@ -2,16 +2,84 @@ set -euo pipefail fd_limit="${SANDBOX_RUNNER_FD_LIVENESS_LIMIT:-40000}" +clock_skew_limit_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-0}" +clock_skew_jitter_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS:-2}" timeout_seconds="${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-5}" +manifest_clock_tolerance_seconds=30 port="${PORT:-2000}" -url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/runtimes}" +url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/health}" -case "$fd_limit" in - ''|*[!0-9]*) - echo "invalid SANDBOX_RUNNER_FD_LIVENESS_LIMIT: $fd_limit" >&2 +validate_non_negative_integer() { + local name="$1" + local value="$2" + + case "$value" in + 0|[1-9]|[1-9][0-9]*) ;; + *) + echo "invalid ${name}: ${value} (expected a canonical non-negative integer)" >&2 + exit 2 + ;; + esac +} + +validate_non_negative_integer SANDBOX_RUNNER_FD_LIVENESS_LIMIT "$fd_limit" +validate_non_negative_integer \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \ + "$clock_skew_limit_seconds" +validate_non_negative_integer \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \ + "$clock_skew_jitter_seconds" +validate_non_negative_integer \ + SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS \ + "$timeout_seconds" + +if [ "$timeout_seconds" -eq 0 ]; then + echo "SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be greater than zero" >&2 + exit 2 +fi + +if [ "$clock_skew_limit_seconds" -gt 0 ]; then + if [ "${#clock_skew_limit_seconds}" -gt 2 ] || \ + [ "$clock_skew_limit_seconds" -ge "$manifest_clock_tolerance_seconds" ]; then + echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2 exit 2 - ;; -esac + fi + + remaining_tolerance_seconds=$((manifest_clock_tolerance_seconds - clock_skew_limit_seconds)) + if [ "${#timeout_seconds}" -gt 2 ] || \ + [ "$timeout_seconds" -ge "$remaining_tolerance_seconds" ]; then + echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2 + exit 2 + fi +fi + +if [ "$clock_skew_limit_seconds" -gt 0 ] && ! command -v date >/dev/null 2>&1; then + echo "sandbox-runner clock-skew check requires date" >&2 + exit 2 +fi + +effective_clock_skew_limit_seconds="$clock_skew_limit_seconds" +if [ "$clock_skew_limit_seconds" -gt 0 ] && [ "$clock_skew_jitter_seconds" -gt 0 ]; then + if [ "${#clock_skew_jitter_seconds}" -gt 2 ] || \ + [ "$clock_skew_jitter_seconds" -ge "$clock_skew_limit_seconds" ]; then + echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS must be less than SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS" >&2 + exit 2 + fi + if ! command -v cksum >/dev/null 2>&1; then + echo "sandbox-runner clock-skew jitter requires cksum" >&2 + exit 2 + fi + + # Pods from one rollout have nearly identical ages and drift rates. Give + # each pod a stable threshold within the configured range to reduce the + # chance that same-age replicas restart together. + jitter_key="${HOSTNAME:-sandbox-runner}" + jitter_checksum=$(printf '%s' "$jitter_key" | cksum) + jitter_checksum="${jitter_checksum%% *}" + effective_clock_skew_limit_seconds=$(( + clock_skew_limit_seconds - (jitter_checksum % (clock_skew_jitter_seconds + 1)) + )) +fi if [ "$fd_limit" -gt 0 ]; then fd_count=$(find /proc/1/fd -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d '[:space:]') @@ -21,4 +89,57 @@ if [ "$fd_limit" -gt 0 ]; then fi fi -curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null +if [ "$clock_skew_limit_seconds" -eq 0 ]; then + curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null + exit 0 +fi + +host_before_seconds=$(date -u +%s) +response_headers=$(curl -fsS --max-time "$timeout_seconds" --dump-header - --output /dev/null "$url") +host_after_seconds=$(date -u +%s) + +guest_date=$(printf '%s\n' "$response_headers" | awk ' + tolower($1) == "date:" { + sub(/\r$/, "") + sub(/^[^:]*:[[:space:]]*/, "") + value = $0 + } + END { print value } +') + +if [ -z "$guest_date" ]; then + echo "sandbox-runner unhealthy: guest response is missing the HTTP Date header" >&2 + exit 1 +fi + +http_date_pattern='^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), [0-9]{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} GMT$' +if ! [[ "$guest_date" =~ $http_date_pattern ]]; then + echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2 + exit 1 +fi + +if ! guest_seconds=$(date -u -d "$guest_date" +%s 2>/dev/null); then + echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2 + exit 1 +fi + +# The response can take time to arrive, so compare guest time with the host +# interval that enclosed the request instead of with a single sample. This +# prevents probe latency from looking like backward clock drift. +if [ "$guest_seconds" -lt "$host_before_seconds" ]; then + skew_seconds=$((guest_seconds - host_before_seconds)) +elif [ "$guest_seconds" -gt "$host_after_seconds" ]; then + skew_seconds=$((guest_seconds - host_after_seconds)) +else + skew_seconds=0 +fi + +absolute_skew_seconds="$skew_seconds" +if [ "$absolute_skew_seconds" -lt 0 ]; then + absolute_skew_seconds=$((-absolute_skew_seconds)) +fi + +if [ "$absolute_skew_seconds" -ge "$effective_clock_skew_limit_seconds" ]; then + echo "sandbox-runner unhealthy: guest clock skew is ${skew_seconds}s, pod limit is ${effective_clock_skew_limit_seconds}s (configured maximum ${clock_skew_limit_seconds}s)" >&2 + exit 1 +fi diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index 8a82a61..3484a61 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -27,6 +27,22 @@ Split runtime deployments: {{- if and (not $packagesFromPvc) (not .Values.workerSandbox.kvmEnabled) }} {{- fail "workerSandbox.packages.source=image requires workerSandbox.kvmEnabled=true because the baked runner boots from a libkrun block root image" }} {{- end }} +{{- $clockSkewLimitValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds }} +{{- $clockSkewJitterValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds }} +{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewLimitValue) }} +{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be a non-negative integer" }} +{{- end }} +{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewJitterValue) }} +{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be a non-negative integer" }} +{{- end }} +{{- $clockSkewLimit := int $clockSkewLimitValue }} +{{- $clockSkewJitter := int $clockSkewJitterValue }} +{{- if or (lt $clockSkewLimit 0) (ge $clockSkewLimit 25) }} +{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be between 0 and 24 so the 5-second healthcheck timeout stays below the 30-second execution-manifest tolerance" }} +{{- end }} +{{- if or (lt $clockSkewJitter 0) (and (gt $clockSkewLimit 0) (ge $clockSkewJitter $clockSkewLimit)) }} +{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds when the clock-skew guard is enabled" }} +{{- end }} {{- range .Values.workerSandbox.sandboxExtraEnv }} {{- $name := .name | default "" }} {{- if and $name (or (regexMatch "(?i)(SECRET|TOKEN|PASSWORD|KEY)" $name) (hasPrefix "REDIS_" $name) (hasPrefix "AWS_" $name) (hasPrefix "S3_" $name) (hasPrefix "MINIO_" $name) (hasPrefix "CODEAPI_" $name) (eq $name "FILE_SERVER_URL") (eq $name "TOOL_CALL_SERVER_URL")) }} @@ -258,6 +274,10 @@ spec: value: {{ .Values.workerSandbox.launcher.filterVsockEnotconn | quote }} - name: SANDBOX_RUNNER_FD_LIVENESS_LIMIT value: {{ .Values.workerSandbox.sandboxRunner.fdLivenessLimit | quote }} + - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS + value: {{ $clockSkewLimit | quote }} + - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS + value: {{ $clockSkewJitter | quote }} - name: EGRESS_GATEWAY_URL value: "http://{{ include "codeapi.fullname" . }}-egress-gateway:{{ .Values.egressGateway.service.port }}" - name: SANDBOX_LOG_LEVEL @@ -362,14 +382,15 @@ spec: - /usr/local/bin/sandbox-runner-healthcheck.sh initialDelaySeconds: 60 periodSeconds: 30 - timeoutSeconds: 5 + timeoutSeconds: 7 readinessProbe: - httpGet: - path: /api/v2/health - port: sandbox + exec: + command: + - /usr/local/bin/sandbox-runner-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 5 + timeoutSeconds: 7 + failureThreshold: 1 resources: {{- $sandboxResources := deepCopy .Values.workerSandbox.resources }} {{- if $useKvmDevicePlugin }} diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index c926e76..2385814 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -200,6 +200,15 @@ workerSandbox: # Restart a sandbox-runner before launcher/VMM host-side fd retention can # exhaust the process file descriptor limit. Set 0 to disable. fdLivenessLimit: 40000 + # Restart a sandbox-runner when its microVM guest wall clock differs from + # the host by this many seconds. This must remain comfortably below the + # 30-second execution-manifest clock tolerance; the chart reserves the + # five-second probe timeout and accepts at most 24. Set 0 to disable. + clockSkewLivenessLimitSeconds: 10 + # Subtract a stable 0..N-second offset from each pod's clock-skew limit so + # replicas created together do not all recycle at once. Must be less than + # clockSkewLivenessLimitSeconds. + clockSkewLivenessJitterSeconds: 2 inheritSharedScheduling: true nodeSelector: {} tolerations: [] diff --git a/tests/block_root_package_delivery.sh b/tests/block_root_package_delivery.sh index c549533..20ec6ea 100755 --- a/tests/block_root_package_delivery.sh +++ b/tests/block_root_package_delivery.sh @@ -36,6 +36,23 @@ assert_not_contains() { fi } +assert_env_value() { + local file="$1" + local name="$2" + local expected="$3" + local message="$4" + if ! awk -v name="$name" -v expected="$expected" ' + $0 ~ "- name: " name { + getline + if ($0 ~ "value: \\\"" expected "\\\"") found = 1 + } + END { exit found ? 0 : 1 } + ' "$file"; then + echo "$message" >&2 + exit 1 + fi +} + assert_compose_mode() { local compose_file="$1" local service="$2" @@ -196,6 +213,25 @@ assert_not_contains \ "$TMP_DIR/helm-image.yaml" \ 'app.kubernetes.io/component: package-init' \ "default Helm render must not create package-init" +assert_env_value \ + "$TMP_DIR/helm-image.yaml" \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \ + 10 \ + "default Helm render must keep clock skew below the manifest tolerance" +assert_env_value \ + "$TMP_DIR/helm-image.yaml" \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \ + 2 \ + "default Helm render must stagger clock-skew recycling across replicas" +assert_contains \ + "$TMP_DIR/helm-image.yaml" \ + '^[[:space:]]+failureThreshold: 1$' \ + "sandbox-runner readiness must fail immediately when clock skew is detected" + +if [ "$(grep -Fc -- '- /usr/local/bin/sandbox-runner-healthcheck.sh' "$TMP_DIR/helm-image.yaml")" -lt 2 ]; then + echo "sandbox-runner liveness and readiness must both check guest clock skew" >&2 + exit 1 +fi helm template codeapi "$TMP_DIR/chart" \ --set executionManifest.privateKey=test \ @@ -243,6 +279,74 @@ assert_contains \ 'value: "0"' \ "fdLivenessLimit=0 must reach the healthcheck instead of reverting to 40000" +helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=0 \ + > "$TMP_DIR/helm-no-clock-skew-liveness.yaml" + +assert_env_value \ + "$TMP_DIR/helm-no-clock-skew-liveness.yaml" \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \ + 0 \ + "clockSkewLivenessLimitSeconds=0 must reach the healthcheck instead of reverting to 10" + +if helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=25 \ + > "$TMP_DIR/invalid-clock-skew-limit.yaml" 2> "$TMP_DIR/invalid-clock-skew-limit.log"; then + echo "clock-skew liveness limit must stay below manifest tolerance" >&2 + exit 1 +fi + +assert_contains \ + "$TMP_DIR/invalid-clock-skew-limit.log" \ + 'clockSkewLivenessLimitSeconds must be between 0 and 24' \ + "clock-skew limit validation failed for an unexpected reason" + +if helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=29.5 \ + > "$TMP_DIR/fractional-clock-skew-limit.yaml" 2> "$TMP_DIR/fractional-clock-skew-limit.log"; then + echo "clock-skew liveness limit must reject fractional values" >&2 + exit 1 +fi + +assert_contains \ + "$TMP_DIR/fractional-clock-skew-limit.log" \ + 'clockSkewLivenessLimitSeconds must be a non-negative integer' \ + "fractional clock-skew limit validation failed for an unexpected reason" + +if helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=10 \ + > "$TMP_DIR/invalid-clock-skew-jitter.yaml" 2> "$TMP_DIR/invalid-clock-skew-jitter.log"; then + echo "clock-skew liveness jitter must stay below the configured limit" >&2 + exit 1 +fi + +assert_contains \ + "$TMP_DIR/invalid-clock-skew-jitter.log" \ + 'clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds' \ + "clock-skew jitter validation failed for an unexpected reason" + +if helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=1.5 \ + > "$TMP_DIR/fractional-clock-skew-jitter.yaml" 2> "$TMP_DIR/fractional-clock-skew-jitter.log"; then + echo "clock-skew liveness jitter must reject fractional values" >&2 + exit 1 +fi + +assert_contains \ + "$TMP_DIR/fractional-clock-skew-jitter.log" \ + 'clockSkewLivenessJitterSeconds must be a non-negative integer' \ + "fractional clock-skew jitter validation failed for an unexpected reason" + if helm template codeapi "$TMP_DIR/chart" \ --set executionManifest.privateKey=test \ --set executionManifest.publicKey=test \ diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh new file mode 100755 index 0000000..583ff7f --- /dev/null +++ b/tests/sandbox_runner_healthcheck.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +mkdir -p "$TMP_DIR/bin" + +cat > "$TMP_DIR/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${TEST_CURL_FAIL:-false}" == "true" ]]; then + exit 22 +fi + +if [[ -n "${TEST_EXPECTED_URL:-}" && "${!#}" != "$TEST_EXPECTED_URL" ]]; then + printf 'unexpected healthcheck URL: %s\n' "${!#}" >&2 + exit 2 +fi + +printf 'HTTP/1.1 200 OK\r\n' +if [[ -n "${TEST_GUEST_DATE:-}" ]]; then + printf 'Date: %s\r\n' "$TEST_GUEST_DATE" +fi +printf '\r\n' +EOF + +cat > "$TMP_DIR/bin/date" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -eq 2 && "$1" == "-u" && "$2" == "+%s" ]]; then + if [[ -e "${TEST_HOST_SAMPLE_STATE:?}" ]]; then + printf '%s\n' "${TEST_HOST_AFTER_SECONDS:?}" + else + : > "$TEST_HOST_SAMPLE_STATE" + printf '%s\n' "${TEST_HOST_BEFORE_SECONDS:?}" + fi + exit 0 +fi + +if [[ "$#" -eq 4 && "$1" == "-u" && "$2" == "-d" && "$4" == "+%s" ]]; then + if [[ "${TEST_GUEST_DATE_INVALID:-false}" == "true" ]]; then + exit 1 + fi + printf '%s\n' "${TEST_GUEST_SECONDS:?}" + exit 0 +fi + +printf 'unexpected date arguments: %q\n' "$*" >&2 +exit 2 +EOF + +cat > "$TMP_DIR/bin/cksum" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +cat >/dev/null +printf '%s 1\n' "${TEST_CKSUM:-0}" +EOF + +chmod +x "$TMP_DIR/bin/curl" "$TMP_DIR/bin/date" "$TMP_DIR/bin/cksum" + +run_check() { + rm -f "$TMP_DIR/host-sample" + env \ + PATH="$TMP_DIR/bin:$PATH" \ + SANDBOX_RUNNER_FD_LIVENESS_LIMIT=0 \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS=0 \ + TEST_HOST_BEFORE_SECONDS=100 \ + TEST_HOST_AFTER_SECONDS=100 \ + TEST_HOST_SAMPLE_STATE="$TMP_DIR/host-sample" \ + TEST_GUEST_DATE='Tue, 04 Aug 2026 08:00:00 GMT' \ + TEST_EXPECTED_URL='http://127.0.0.1:2000/api/v2/health' \ + "$@" \ + bash "$ROOT/docker/sandbox-runner-healthcheck.sh" +} + +run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + TEST_GUEST_SECONDS=91 + +# A guest timestamp inside the host interval is healthy even when either +# individual host sample differs by more than the configured limit. +run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=1 \ + TEST_HOST_BEFORE_SECONDS=100 \ + TEST_HOST_AFTER_SECONDS=104 \ + TEST_GUEST_SECONDS=102 + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + TEST_GUEST_SECONDS=90 2> "$TMP_DIR/backward.log"; then + echo "healthcheck accepted guest clock at the backward-skew limit" >&2 + exit 1 +fi +grep -F 'guest clock skew is -10s, pod limit is 10s (configured maximum 10s)' "$TMP_DIR/backward.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + TEST_GUEST_SECONDS=110 2> "$TMP_DIR/forward.log"; then + echo "healthcheck accepted guest clock at the forward-skew limit" >&2 + exit 1 +fi +grep -F 'guest clock skew is 10s, pod limit is 10s (configured maximum 10s)' "$TMP_DIR/forward.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS=2 \ + TEST_CKSUM=2 \ + TEST_GUEST_SECONDS=92 2> "$TMP_DIR/jitter.log"; then + echo "healthcheck did not apply its deterministic per-pod threshold" >&2 + exit 1 +fi +grep -F 'guest clock skew is -8s, pod limit is 8s (configured maximum 10s)' "$TMP_DIR/jitter.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + TEST_GUEST_DATE= \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/missing.log"; then + echo "healthcheck accepted a response without a Date header" >&2 + exit 1 +fi +grep -F 'guest response is missing the HTTP Date header' "$TMP_DIR/missing.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + TEST_GUEST_DATE_INVALID=true \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/malformed.log"; then + echo "healthcheck accepted a malformed Date header" >&2 + exit 1 +fi +grep -F 'guest returned an invalid HTTP Date header' "$TMP_DIR/malformed.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + TEST_GUEST_DATE=now \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/non-http-date.log"; then + echo "healthcheck accepted a GNU-date-parseable value that is not an HTTP-date" >&2 + exit 1 +fi +grep -F 'guest returned an invalid HTTP Date header: now' "$TMP_DIR/non-http-date.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + TEST_CURL_FAIL=true \ + TEST_GUEST_SECONDS=100; then + echo "healthcheck accepted a failed guest request" >&2 + exit 1 +fi + +run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=0 \ + TEST_GUEST_DATE= \ + TEST_GUEST_SECONDS=100 + +# Compose and standalone Docker callers only observe health status, so the +# clock-skew guard stays disabled unless an orchestrator explicitly opts in. +run_check \ + TEST_GUEST_DATE= \ + TEST_GUEST_SECONDS=100 + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=invalid \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/invalid.log"; then + echo "healthcheck accepted an invalid clock-skew limit" >&2 + exit 1 +fi +grep -F 'invalid SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS: invalid' "$TMP_DIR/invalid.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=08 \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/leading-zero.log"; then + echo "healthcheck accepted a non-canonical clock-skew limit" >&2 + exit 1 +fi +grep -F 'expected a canonical non-negative integer' "$TMP_DIR/leading-zero.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=9223372036854775807 \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/overflow-limit.log"; then + echo "healthcheck accepted a clock-skew limit that could overflow arithmetic" >&2 + exit 1 +fi +grep -F 'plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/overflow-limit.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=25 \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/unsafe-limit.log"; then + echo "healthcheck accepted a limit without request-latency headroom" >&2 + exit 1 +fi +grep -F 'plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/unsafe-limit.log" >/dev/null + +if run_check \ + SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \ + SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS=0 \ + TEST_GUEST_SECONDS=100 2> "$TMP_DIR/zero-timeout.log"; then + echo "healthcheck accepted a zero request timeout" >&2 + exit 1 +fi +grep -F 'SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be greater than zero' "$TMP_DIR/zero-timeout.log" >/dev/null + +echo "sandbox-runner healthcheck checks passed"